From 2527390a1b6722f5eb948aab34e57b4c0596da5b Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 1 Jul 2022 14:27:17 +0900 Subject: [PATCH 001/275] =?UTF-8?q?refactor:=20=E3=83=AD=E3=83=BC=E3=82=AB?= =?UTF-8?q?=E3=83=AB=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=92git?= =?UTF-8?q?=E5=AF=BE=E8=B1=A1=E5=A4=96=E3=81=AB=E8=A8=AD=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 8 ++++- ecs/dataimport/dataimport/main.py | 49 +++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e4f6d9fb..3c4e16ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ lambda/mbj-newdwh2021-staging-NoticeToSlack/package-lock.json lambda/mbj-newdwh2021-staging-NoticeToSlack/node_modules/* lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json -lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* \ No newline at end of file +lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* +__pycache__/ +.env +launch.json +settings.json +Pipfile +Pipfile.lock \ No newline at end of file diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 730a7be8..5540f886 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -1,4 +1,5 @@ from datetime import datetime +from pickle import FALSE import boto3 import pymysql from pymysql.constants import CLIENT @@ -23,6 +24,7 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, + 'importManner': 12, } LINE_FEED_CODE = { 'CR': '\r', @@ -30,6 +32,10 @@ LINE_FEED_CODE = { 'CRLF': '\r\n', } DIRECTORY_SETTINGS = '/settings/' +TRUNCATE_SRC_TABLE_SYMBOL = 'truncate_src_table:' +TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' +TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' +INVALID_CONFIG_EXCEPTION_MESSAGE = f'個別設定ファイルのインポート方法に不備がありました。　インポート方法は "{TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT}" のように設定してください' # クラス変数 s3_client = boto3.client('s3') @@ -161,6 +167,18 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # ⑦ ロードスキーマのデータを蓄積スキーマにUPSERTする print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-08 - ロードスキーマ（{settings_list[SETTINGS_ITEM["loadSchemaName"]]}）のデータを蓄積スキーマ（{settings_list[SETTINGS_ITEM["storageSchemaName"]]}）に登録します') + + # インポート方法判断 + try: + if truncate_judge(settings_list): + with conn.cursor() as cur: + sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["storageSchemaName"]]}' + cur.execute(sql_truncate) + print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} E-MAIN-20 - {settings_list[SETTINGS_ITEM["storageSchemaName"]]} をTRUNCATEしました') + + except InvalidConfigException as e: + print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-01 - エラー内容：{e}') + error(bucket_name, target_data_source, target_file_name, log_info) # SQL文生成 sql = f'INSERT INTO {settings_list[SETTINGS_ITEM["storageSchemaName"]]} (' @@ -272,3 +290,34 @@ def connection_close(conn, bucket_name, target_data_source, target_file_name, lo except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) + + +def truncate_judge(settings_list): + """TRUNCATE処理対応判定 + Args: + settings_list (list): 個別設定ファイル + Raises: + InvalidConfigException: 個別設定ファイルのインポート方法の設定ミス + Returns: + Bool: Truncate対象の場合True。Truncate対象でない場合False + """ + + # upsert判定 + if len(settings_list) < 13: + return False + if not settings_list[SETTINGS_ITEM["importManner"]]: + return False + + # インポート方法設定チェック + if not settings_list[SETTINGS_ITEM["importManner"]].startswith(TRUNCATE_SRC_TABLE_SYMBOL): + raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) + import_manner_splitted_list = settings_list[SETTINGS_ITEM["importManner"]].split(':') + if len(import_manner_splitted_list) != 2: + raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) + if not import_manner_splitted_list[1] != settings_list[SETTINGS_ITEM["storageSchemaName"]]: + raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) + return True + + +class InvalidConfigException(Exception): + pass From 53fb21f976b030baf3aa4768fa4fbe70cba8b287 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 1 Jul 2022 14:27:17 +0900 Subject: [PATCH 002/275] =?UTF-8?q?Revert=20"refactor:=20=E3=83=AD?= =?UTF-8?q?=E3=83=BC=E3=82=AB=E3=83=AB=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E3=82=92git=E5=AF=BE=E8=B1=A1=E5=A4=96=E3=81=AB=E8=A8=AD?= =?UTF-8?q?=E5=AE=9A"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit e97c26ac147ec4458e396bdd1ba72ba1e9d72ccb. --- .gitignore | 8 +---- ecs/dataimport/dataimport/main.py | 49 ------------------------------- 2 files changed, 1 insertion(+), 56 deletions(-) diff --git a/.gitignore b/.gitignore index 3c4e16ef..e4f6d9fb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,4 @@ lambda/mbj-newdwh2021-staging-NoticeToSlack/package-lock.json lambda/mbj-newdwh2021-staging-NoticeToSlack/node_modules/* lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json -lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* -__pycache__/ -.env -launch.json -settings.json -Pipfile -Pipfile.lock \ No newline at end of file +lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* \ No newline at end of file diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 5540f886..730a7be8 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -1,5 +1,4 @@ from datetime import datetime -from pickle import FALSE import boto3 import pymysql from pymysql.constants import CLIENT @@ -24,7 +23,6 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, - 'importManner': 12, } LINE_FEED_CODE = { 'CR': '\r', @@ -32,10 +30,6 @@ LINE_FEED_CODE = { 'CRLF': '\r\n', } DIRECTORY_SETTINGS = '/settings/' -TRUNCATE_SRC_TABLE_SYMBOL = 'truncate_src_table:' -TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' -TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' -INVALID_CONFIG_EXCEPTION_MESSAGE = f'個別設定ファイルのインポート方法に不備がありました。　インポート方法は "{TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT}" のように設定してください' # クラス変数 s3_client = boto3.client('s3') @@ -167,18 +161,6 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # ⑦ ロードスキーマのデータを蓄積スキーマにUPSERTする print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-08 - ロードスキーマ（{settings_list[SETTINGS_ITEM["loadSchemaName"]]}）のデータを蓄積スキーマ（{settings_list[SETTINGS_ITEM["storageSchemaName"]]}）に登録します') - - # インポート方法判断 - try: - if truncate_judge(settings_list): - with conn.cursor() as cur: - sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["storageSchemaName"]]}' - cur.execute(sql_truncate) - print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} E-MAIN-20 - {settings_list[SETTINGS_ITEM["storageSchemaName"]]} をTRUNCATEしました') - - except InvalidConfigException as e: - print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-01 - エラー内容：{e}') - error(bucket_name, target_data_source, target_file_name, log_info) # SQL文生成 sql = f'INSERT INTO {settings_list[SETTINGS_ITEM["storageSchemaName"]]} (' @@ -290,34 +272,3 @@ def connection_close(conn, bucket_name, target_data_source, target_file_name, lo except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) - - -def truncate_judge(settings_list): - """TRUNCATE処理対応判定 - Args: - settings_list (list): 個別設定ファイル - Raises: - InvalidConfigException: 個別設定ファイルのインポート方法の設定ミス - Returns: - Bool: Truncate対象の場合True。Truncate対象でない場合False - """ - - # upsert判定 - if len(settings_list) < 13: - return False - if not settings_list[SETTINGS_ITEM["importManner"]]: - return False - - # インポート方法設定チェック - if not settings_list[SETTINGS_ITEM["importManner"]].startswith(TRUNCATE_SRC_TABLE_SYMBOL): - raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) - import_manner_splitted_list = settings_list[SETTINGS_ITEM["importManner"]].split(':') - if len(import_manner_splitted_list) != 2: - raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) - if not import_manner_splitted_list[1] != settings_list[SETTINGS_ITEM["storageSchemaName"]]: - raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) - return True - - -class InvalidConfigException(Exception): - pass From ca26597a086f84644c1427b4ec34cb68a2e1b3f2 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 1 Jul 2022 14:27:17 +0900 Subject: [PATCH 003/275] =?UTF-8?q?refactor:=20=E3=83=AD=E3=83=BC=E3=82=AB?= =?UTF-8?q?=E3=83=AB=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=92git?= =?UTF-8?q?=E5=AF=BE=E8=B1=A1=E5=A4=96=E3=81=AB=E8=A8=AD=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e4f6d9fb..3c4e16ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ lambda/mbj-newdwh2021-staging-NoticeToSlack/package-lock.json lambda/mbj-newdwh2021-staging-NoticeToSlack/node_modules/* lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json -lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* \ No newline at end of file +lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* +__pycache__/ +.env +launch.json +settings.json +Pipfile +Pipfile.lock \ No newline at end of file From 9057515df95716f634c401c94f1fe0e6454722e7 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 1 Jul 2022 14:27:17 +0900 Subject: [PATCH 004/275] =?UTF-8?q?feat:CRM=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E9=80=A3=E6=90=BA=E3=80=80=E3=83=87=E3=83=BC=E3=82=BF=E5=8F=96?= =?UTF-8?q?=E8=BE=BC=E3=81=AE=E5=85=A8=E4=BB=B6=E7=BD=AE=E3=81=8D=E6=8F=9B?= =?UTF-8?q?=E3=81=88=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 49 +++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 730a7be8..5540f886 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -1,4 +1,5 @@ from datetime import datetime +from pickle import FALSE import boto3 import pymysql from pymysql.constants import CLIENT @@ -23,6 +24,7 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, + 'importManner': 12, } LINE_FEED_CODE = { 'CR': '\r', @@ -30,6 +32,10 @@ LINE_FEED_CODE = { 'CRLF': '\r\n', } DIRECTORY_SETTINGS = '/settings/' +TRUNCATE_SRC_TABLE_SYMBOL = 'truncate_src_table:' +TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' +TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' +INVALID_CONFIG_EXCEPTION_MESSAGE = f'個別設定ファイルのインポート方法に不備がありました。　インポート方法は "{TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT}" のように設定してください' # クラス変数 s3_client = boto3.client('s3') @@ -161,6 +167,18 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # ⑦ ロードスキーマのデータを蓄積スキーマにUPSERTする print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-08 - ロードスキーマ（{settings_list[SETTINGS_ITEM["loadSchemaName"]]}）のデータを蓄積スキーマ（{settings_list[SETTINGS_ITEM["storageSchemaName"]]}）に登録します') + + # インポート方法判断 + try: + if truncate_judge(settings_list): + with conn.cursor() as cur: + sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["storageSchemaName"]]}' + cur.execute(sql_truncate) + print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} E-MAIN-20 - {settings_list[SETTINGS_ITEM["storageSchemaName"]]} をTRUNCATEしました') + + except InvalidConfigException as e: + print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-01 - エラー内容：{e}') + error(bucket_name, target_data_source, target_file_name, log_info) # SQL文生成 sql = f'INSERT INTO {settings_list[SETTINGS_ITEM["storageSchemaName"]]} (' @@ -272,3 +290,34 @@ def connection_close(conn, bucket_name, target_data_source, target_file_name, lo except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) + + +def truncate_judge(settings_list): + """TRUNCATE処理対応判定 + Args: + settings_list (list): 個別設定ファイル + Raises: + InvalidConfigException: 個別設定ファイルのインポート方法の設定ミス + Returns: + Bool: Truncate対象の場合True。Truncate対象でない場合False + """ + + # upsert判定 + if len(settings_list) < 13: + return False + if not settings_list[SETTINGS_ITEM["importManner"]]: + return False + + # インポート方法設定チェック + if not settings_list[SETTINGS_ITEM["importManner"]].startswith(TRUNCATE_SRC_TABLE_SYMBOL): + raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) + import_manner_splitted_list = settings_list[SETTINGS_ITEM["importManner"]].split(':') + if len(import_manner_splitted_list) != 2: + raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) + if not import_manner_splitted_list[1] != settings_list[SETTINGS_ITEM["storageSchemaName"]]: + raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) + return True + + +class InvalidConfigException(Exception): + pass From f2650c1ac89f0317f8edcbf7889d4567aabc93d4 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 4 Jul 2022 14:40:57 +0900 Subject: [PATCH 005/275] =?UTF-8?q?feat:=20CRM=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E5=8F=96=E5=BE=97=E3=83=95=E3=82=A1=E3=83=BC=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=82=B3=E3=83=9F=E3=83=83=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Dockerfile | 12 ++ ecs/crm-datafetch/Pipfile | 4 + ecs/crm-datafetch/Pipfile.lock | 0 ecs/crm-datafetch/main.py | 181 ++++++++++++++++++ ecs/crm-datafetch/src/__init__.py | 0 ecs/crm-datafetch/src/chk.py | 0 ecs/crm-datafetch/src/conv.py | 0 ecs/crm-datafetch/src/csvbk.py | 0 ecs/crm-datafetch/src/date.py | 0 ecs/crm-datafetch/src/end.py | 0 ecs/crm-datafetch/src/fetch.py | 0 ecs/crm-datafetch/src/pre.py | 96 ++++++++++ ecs/crm-datafetch/src/resbk.py | 0 ecs/crm-datafetch/src/upd.py | 0 ecs/crm-datafetch/src/upld.py | 0 ecs/crm-datafetch/src/utils/__init__.py | 0 ecs/crm-datafetch/src/utils/data_retention.py | 169 ++++++++++++++++ ecs/crm-datafetch/src/utils/json_parser.py | 28 +++ ecs/crm-datafetch/src/utils/logger.py | 33 ++++ ecs/crm-datafetch/src/utils/s3_access.py | 48 +++++ .../crm/object_info/crm_object_list_diff.json | 41 ++++ s3/config/crm/task_settings/task_settings.env | 16 ++ 22 files changed, 628 insertions(+) create mode 100644 ecs/crm-datafetch/Dockerfile create mode 100644 ecs/crm-datafetch/Pipfile create mode 100644 ecs/crm-datafetch/Pipfile.lock create mode 100644 ecs/crm-datafetch/main.py create mode 100644 ecs/crm-datafetch/src/__init__.py create mode 100644 ecs/crm-datafetch/src/chk.py create mode 100644 ecs/crm-datafetch/src/conv.py create mode 100644 ecs/crm-datafetch/src/csvbk.py create mode 100644 ecs/crm-datafetch/src/date.py create mode 100644 ecs/crm-datafetch/src/end.py create mode 100644 ecs/crm-datafetch/src/fetch.py create mode 100644 ecs/crm-datafetch/src/pre.py create mode 100644 ecs/crm-datafetch/src/resbk.py create mode 100644 ecs/crm-datafetch/src/upd.py create mode 100644 ecs/crm-datafetch/src/upld.py create mode 100644 ecs/crm-datafetch/src/utils/__init__.py create mode 100644 ecs/crm-datafetch/src/utils/data_retention.py create mode 100644 ecs/crm-datafetch/src/utils/json_parser.py create mode 100644 ecs/crm-datafetch/src/utils/logger.py create mode 100644 ecs/crm-datafetch/src/utils/s3_access.py create mode 100644 s3/config/crm/object_info/crm_object_list_diff.json create mode 100644 s3/config/crm/task_settings/task_settings.env diff --git a/ecs/crm-datafetch/Dockerfile b/ecs/crm-datafetch/Dockerfile new file mode 100644 index 00000000..e84b79b3 --- /dev/null +++ b/ecs/crm-datafetch/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.8 + +ENV TZ="Asia/Tokyo" + +WORKDIR /usr/src/app +COPY Pipfile Pipfile.lock ./ +RUN pip install --no-cache-dir -r requirements.txt && \ + cp /usr/share/zoneinfo/Asia/Tokyo /etc/localtime +COPY main.py ./ +COPY crm-datafetch /usr/src + +CMD [ "python", "./main.py" ] diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile new file mode 100644 index 00000000..ec9569f7 --- /dev/null +++ b/ecs/crm-datafetch/Pipfile @@ -0,0 +1,4 @@ +[packages] +boto3 +simple-salesforce +tenacity \ No newline at end of file diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/main.py b/ecs/crm-datafetch/main.py new file mode 100644 index 00000000..0170f413 --- /dev/null +++ b/ecs/crm-datafetch/main.py @@ -0,0 +1,181 @@ +# モジュールロード +import os +import sys +from datetime import datetime +import src.pre as pre # データ取得準備処理 +import src.chk as chk # オブジェクト情報形式チェック処理 +import src.date as date # データ取得期間設定処理 +import src.fetch as fetch # CRMデータ取得処理 +import src.resbk as resbk # CRM電文データバックアップ処理 +import src.conv as conv # CSV変換処理 +import src.csvbk as csvbk # CSVバックアップ処理 +import src.upld as upld # CSVアップロード処理 +import src.upd as upd # 前回取得日時ファイル更新 +import src.end as end # 取得処理実施結果アップロード処理 + +from src.utils.logger import Logger +import src.utils.data_retention as data_retention + + +# 変数 + +# logger設定 +logger = Logger().get_logger() + + +# 処理 +def main() -> None: + """コントロール処理 + + 各処理を呼び出すコントローラー + + """ + + try: + # ① CRMデータ取得処理開始ログを出力する + logger.info('I-CTRL-01 CRMデータ取得処理を開始します') + + # ② データ取得準備処理を呼び出す + try: + logger.info('I-CTRL-02 データ取得準備処理呼び出し') + crm_object_info_var = data_retention.CrmObjectInfoVar() + crm_object_info_dict, execute_datetime, process_result_per_object = pre( + crm_object_info_var) + + except Exception as e: + logger.error('') + raise CustomException(e) + + # ③ object_infoのobjectsキーの値の件数分ループする + logger.info('I-CTRL-03 取得対象オブジェクトのループ処理開始') + + for object_info in crm_object_info_dict.get('objects'): + try: + + # 1. オブジェクト情報形式チェック処理を呼び出す + logger.info('I-CTRL-04 オブジェクト情報形式チェック処理呼び出し') + + try: + chk() + + except Exception as e: + logger.error('') + + # 2. 処理対象のオブジェクト名をログ出力する + logger.info('I-CTRL-05 ') + + # 3. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する + logger.info('I-CTRL-06 ') + + # 4. データ取得期間設定処理を呼び出す + logger.info('I-CTRL-07 ') + try: + + except Exception as e: + logger.error('') + + # 5. CRMデータ取得処理を呼び出す + logger.info('I-CTRL-08 ') + try: + + except Exception as e: + logger.error('') + + # 6. CRMデータのジェネレータをループする + # 6-1. 出力ファイル名を生成する + logger.info('I-CTRL-10 ') + try: + + except Exception as e: + logger.error('') + + # 6-2. CRM電文データバックアップ処理を呼び出す + logger.info('I-CTRL-11 ') + try: + + except Exception as e: + logger.error('') + + # 6-3. CSV変換処理を呼び出す + logger.info('I-CTRL-12 ') + try: + + except Exception as e: + logger.error('') + + # 6-4. CSVバックアップ処理を呼び出す + logger.info('I-CTRL-13 ') + try: + + except Exception as e: + logger.error('') + + # 6-5. CSVアップロード処理を呼び出す + logger.info('I-CTRL-14 ') + try: + + except Exception as e: + logger.error('') + + # 7. 前回取得日時ファイル更新処理を呼びだす + logger.info('I-CTRL-15 ') + try: + + except Exception as e: + logger.error('') + + # 8. 1オブジェクトのアップロードが完了した旨をログに出力する + logger.info('I-CTRL-16 ') + + except GetObjectException as e: + logger.error( + '[取得対象のオブジェクト名] の[処理名]でエラーが発生しました 次のオブジェクトの処理に移行します') + + except Exception as e: + logger.error( + '[取得対象のオブジェクト名] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します') + + # ④ すべてのオブジェクトの処理が完了したことと、オブジェクト毎の処理結果をログに出力する + try: + logger.info(f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{実行結果JSON}]') + + except Exception as e: + logger.error('') + raise CustomException() + + # ⑤ 取得処理実施結果アップロード処理を呼び出す + try: + logger.info('I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始') + + except Exception as e: + logger.error('') + raise CustomException() + + # ⑥ CRMデータ取得処理終了ログを出力する + try: + logger.info('I-CTRL-19 CRMデータ取得処理を終了します') + + except Exception as e: + logger.error('') + raise + + except CustomException as e: + logger.error('I-ERR-01 [処理名]でエラーが発生したため、処理を終了します') + + except Exception as e: + logger.error('I-ERR-02 予期せぬエラーが発生したため、処理を終了します') + + +# カスタムExceptionクラス +class CustomException(Exception, metaclass=ABCMeta): + def __init__(self, id, arg): + self.arg = arg + self.id = id + + +class GetObjectException(CustomException): + pass + + +if __name__ == '__main__': + main() diff --git a/ecs/crm-datafetch/src/__init__.py b/ecs/crm-datafetch/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/chk.py b/ecs/crm-datafetch/src/chk.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/conv.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/csvbk.py b/ecs/crm-datafetch/src/csvbk.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/date.py b/ecs/crm-datafetch/src/date.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/end.py b/ecs/crm-datafetch/src/end.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/pre.py b/ecs/crm-datafetch/src/pre.py new file mode 100644 index 00000000..546cbc37 --- /dev/null +++ b/ecs/crm-datafetch/src/pre.py @@ -0,0 +1,96 @@ +# モジュールロード +from datetime import datetime +from src.utils.logger import Logger +import src.utils.data_retention as data_retention +import src.utils.s3_access as s3_access +import src.utils.json_parser as json_parser + + +# 変数 +# .000ZはUTCを表す。ミリ秒までの考慮は不要なので固定で指定 +YYYYMMDDTHHMMSSTZ = '%Y-%m-%dT%H:%M:%S.000Z' +CRM_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.000%z' +YYYYMMDDHHMMSS = '%Y-%m-%d %H:%M:%S' + + +# logger設定 +logger = Logger().get_logger() + + +# 処理 + + +def pre(crm_object_info_var): + try: + # ① データ取得準備処理の開始ログを出力する + logger.info('I-PRE-01 データ取得準備処理を開始します') + + # ② 取得処理開始年月日時分秒を控える + execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) + logger.info(f'I-PRE-02 データ取得処理開始日時：{execute_datetime}') + + # ⑤ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する + + try: + crm_object_info_var = data_retention.CrmObjectInfoVar() + s3_fullpath = f's3://{crm_object_info_var.bucket}/{crm_object_info_var.file_name_key}' + logger.debug(f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{s3_fullpath}]') + + get_file_s3 = s3_access.GetFileS3() + crm_object_info_json = get_file_s3.get_file_from_s3(crm_object_info_var.bucket, crm_object_info_var.file_name_key) + + logger.debug('D-PRE-04 CRM_取得オブジェクト情報ファイルの取得成功しました') + + except Exception as e: + logger.error(f'E-PRE-01 CRM_取得オブジェクト情報ファイルが存在しません　ファイル名：[{s3_fullpath}]', e) + raise e + + + # ⑥ CRM_取得オブジェクト情報ファイルをパースし、メモリ上に展開する + try: + logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします') + + json_exclusde_comment = json_parser.JsonExculudeComment() + crm_object_info_dict: dict = json_exclusde_comment(crm_object_info_json) + + logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました') + + except Exception as e: + logger.error(f'E-PRE-02 CRM_取得オブジェクト情報ファイルのパースに失敗しました　エラー内容：[{e}]') + raise e + + # ⑦ メモリ上のCRM_取得オブジェクト情報のキーobjectsの形式チェックを行う + try: + logger.debug('D-PRE-07 CRM_取得オブジェクト情報ファイルの形式チェックを開始します') + + if 'objects' not in crm_object_info_dict: + logger.error(f'E-PRE-03 CRM_取得オブジェクト情報ファイル「objects」キーは必須です　ファイル名：[{s3_fullpath}]') + raise e + + if not isinstance(crm_object_info_dict["objects"], list): + logger.error(f'E-PRE-04 CRM_取得オブジェクト情報ファイル「objects」キーの値は配列でなければなりません　ファイル名：[{s3_fullpath}]') + raise e + + logger.debug('D-PRE-07 CRM_取得オブジェクト情報ファイルの形式チェック 正常終了') + + except Exception as e: + logger.error(f'CRM_取得オブジェクト情報ファイル「objects」キーの値は配列でなければなりません　ファイル名：[{s3_fullpath}]', e) + raise e + + # ⑧ 処理結果出力用のマップを初期化 + process_result_per_object = {} + for sfdc_object in crm_object_info_dict.get('objects'): + object_name = sfdc_object.get('object_name') + process_result_per_object[object_name] = 'fail' + + + + # ⑨ データ取得準備処理の終了ログを出力する + logger.info('I-PRE-09 データ取得準備処理を終了します') + + # ⑩ 次の処理へ移行する + return(crm_object_info_dict, execute_datetime, process_result_per_object) + + except Exception as e: + logger.error(f'データ取得準備処理で想定外のエラーが発生しました　エラー内容：[{e}]', e) + raise e diff --git a/ecs/crm-datafetch/src/resbk.py b/ecs/crm-datafetch/src/resbk.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upd.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/upld.py b/ecs/crm-datafetch/src/upld.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/utils/__init__.py b/ecs/crm-datafetch/src/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/utils/data_retention.py b/ecs/crm-datafetch/src/utils/data_retention.py new file mode 100644 index 00000000..89d7d4c3 --- /dev/null +++ b/ecs/crm-datafetch/src/utils/data_retention.py @@ -0,0 +1,169 @@ +# モジュールロード +import dataclasses +#from fileinput import filename +import os +#from tracemalloc import DomainFilter + +# 変数 +# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +LOG_LEVEL = os.environ["LOG_LEVEL"] + +CRM_AUTH_TIMEOUT = os.environ["CRM_AUTH_TIMEOUT"] # CRMへの認証処理のタイムアウト秒数 +# CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ["CRM_AUTH_MAX_RETRY_ATTEMPT"] +# CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_INTERVAL = os.environ["CRM_AUTH_RETRY_INTERVAL"] +# CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = os.environ["CRM_AUTH_RETRY_MIN_INTERVAL"] +# CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = os.environ["CRM_AUTH_RETRY_MAX_INTERVAL"] + +# CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = os.environ["CRM_GET_RECORD_COUNT_TIMEOUT"] +# CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ["CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT"] +# CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_INTERVAL"] +# CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL"] +# CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL"] + +# CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_TIMEOUT = os.environ["CRM_FETCH_RECORD_TIMEOUT"] +# CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ["CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT"] +# CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_INTERVAL"] +# CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MIN_INTERVAL"] +# CRMのレコード取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MAX_INTERVAL"] + + +CRM_AUTH_DOMAIN = os.environ["CRM_AUTH_DOMAIN"] # CRMのAPI実行のための認証エンドポイントのドメイン +CRM_USER_NAME = os.environ["CRM_USER_NAME"] # CRMのAPI実行用ユーザ名 +CRM_USER_PASSWORD = os.environ["CRM_USER_PASSWORD"] # CRMのAPI実行用ユーザパスワード +# CRMのAPI実行用ユーザのセキュリティトークン +CRM_USER_SECURITY_TOKEN = os.environ["CRM_USER_SECURITY_TOKEN"] + +# CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_CONFIG_BUCKET = os.environ["CRM_CONFIG_BUCKET"] +CRM_BACKUP_BUCKET = os.environ["CRM_BACKUP_BUCKET"] # CRMのバックアップデータを格納するバケット名 +IMPORT_DATA_BUCKET = os.environ["IMPORT_DATA_BUCKET"] # CRMの取込データを格納するバケット名 + +# TASK_SETTING_FOLDER = os.environ["TASK_SETTING_FOLDER"] # CRM取得処理タスクの設定ファイルを格納するフォルダパス +# TASK_SETTING_FILENAME = os.environ["TASK_SETTING_FILENAME"] # CRM取得処理タスクの設定ファイルのファイル名 + +# CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FOLDER = os.environ["OBJECT_INFO_FOLDER"] +# CRM取得対象オブジェクトの情報のファイル名 +OBJECT_INFO_FILENAME = os.environ["OBJECT_INFO_FILENAME"] + +# CRMデータ取得結果を格納するフォルダパス +FETCH_RESULT_FOLDER = os.environ["FETCH_RESULT_FOLDER"] +# CRMデータ取得結果を格納するファイル名 +FETCH_RESULT_FILENAME = os.environ["FETCH_RESULT_FILENAME"] + +# CRMからの最終取得日時ファイルを格納するフォルダパス +LAST_FETCH_DATE_FOLDER = os.environ["LAST_FETCH_DATE_FOLDER"] +# CRMから取得し、取込用に変換したデータを格納するフォルダ +CRM_IMPORT_DATA_FOLDER = os.environ["CRM_IMPORT_DATA_FOLDER"] + +# CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +LAST_FETCH_DATE_BACKUP_FOLDER = os.environ["LAST_FETCH_DATE_BACKUP_FOLDER"] +# CRMから取得した生データのバックアップを格納するフォルダパス +FETCH_DATA_BACKUP_FOLDER = os.environ["FETCH_DATA_BACKUP_FOLDER"] +# CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ +CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ["CRM_IMPORT_DATA_BACKUP_FOLDER"] + + +# 処理 +@dataclasses.dataclass +class LogLevel: + loglevel: str = LOG_LEVEL + + +@dataclasses.dataclass +class CrmAuthVar: + auth_domain: str = CRM_AUTH_DOMAIN + user_name: str = CRM_USER_NAME + user_password: str = CRM_USER_PASSWORD + security_token: str = CRM_USER_SECURITY_TOKEN + timeout: int = CRM_AUTH_TIMEOUT + max_retry_attempt: int = CRM_AUTH_MAX_RETRY_ATTEMPT + retry_interval: int = CRM_AUTH_RETRY_INTERVAL + retry_min_interval: int = CRM_AUTH_RETRY_MIN_INTERVAL + retry_max_interval: int = CRM_AUTH_RETRY_MAX_INTERVAL + + +@dataclasses.dataclass +class CrmGetRecordCountVar: + timeout: int = CRM_GET_RECORD_COUNT_TIMEOUT + max_retry_attempt: int = CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT + retry_interval: int = CRM_GET_RECORD_COUNT_RETRY_INTERVAL + retry_min_interval: int = CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL + retry_max_interval: int = CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL + + +@dataclasses.dataclass +class CrmFetchRecordVar: + timeout: int = CRM_FETCH_RECORD_TIMEOUT + max_retry_attempt: int = CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT + retry_interval: int = CRM_FETCH_RECORD_RETRY_INTERVAL + retry_min_interval: int = CRM_FETCH_RECORD_RETRY_MIN_INTERVAL + retry_max_interval: int = CRM_FETCH_RECORD_RETRY_MAX_INTERVAL + + +@dataclasses.dataclass +class CrmBucketVar: + config_bucket: str = CRM_CONFIG_BUCKET + backup_bucket: str = CRM_BACKUP_BUCKET + data_bucket: str = IMPORT_DATA_BUCKET + + +@dataclasses.dataclass +class CrmObjectInfoVar: + bucket: str = CRM_CONFIG_BUCKET + folder: str = OBJECT_INFO_FOLDER + file_name: str = OBJECT_INFO_FILENAME + file_name_key: str = f'{folder}/{file_name}' + + +@dataclasses.dataclass +class FetchResultVar: + folder: str = FETCH_RESULT_FOLDER + file_name: str = FETCH_RESULT_FILENAME + file_name_key: str = f'{folder}/{file_name}' + + +@dataclasses.dataclass +class FetchResultVar: + folder: str = FETCH_RESULT_FOLDER + file_name: str = FETCH_RESULT_FILENAME + file_name_key: str = f'{folder}/{file_name}' + + +@dataclasses.dataclass +class LastFetchDateVar: + folder: str = LAST_FETCH_DATE_FOLDER + + +@dataclasses.dataclass +class CrmImportDataVar: + folder: str = CRM_IMPORT_DATA_FOLDER + + +@dataclasses.dataclass +class LastFetchDateBackupVar: + folder: str = LAST_FETCH_DATE_BACKUP_FOLDER + + +@dataclasses.dataclass +class FetchDataBackupVar: + folder: str = FETCH_DATA_BACKUP_FOLDER + + +@dataclasses.dataclass +class CrmImportDataBackupVar: + folder: str = CRM_IMPORT_DATA_BACKUP_FOLDER diff --git a/ecs/crm-datafetch/src/utils/json_parser.py b/ecs/crm-datafetch/src/utils/json_parser.py new file mode 100644 index 00000000..b6d288a9 --- /dev/null +++ b/ecs/crm-datafetch/src/utils/json_parser.py @@ -0,0 +1,28 @@ +# モジュールロード +import re +import json + + +# 変数 +SYMBOL = ['#', '/'] + + +# 処理 +class JsonExculudeComment: + def __init__(self) -> None: + pass + + def json_exclude_comment(self, json_str) -> dict: + for symbol in SYMBOL: + # コメントアウトシンボルを含む部分を置き換える正規表現 + replace_comment_regex = rf'\s(?!\"){symbol}[\s\S]*?.*' + + json_str = self.json_regex_parse(replace_comment_regex, json_str) + + result = json.loads(json_str) + + return result + + def json_regex_parse(replace_comment_regex, json_str) -> str: + json_without_comment = re.sub(replace_comment_regex, '', json_str) + return json_without_comment diff --git a/ecs/crm-datafetch/src/utils/logger.py b/ecs/crm-datafetch/src/utils/logger.py new file mode 100644 index 00000000..5ec076fd --- /dev/null +++ b/ecs/crm-datafetch/src/utils/logger.py @@ -0,0 +1,33 @@ +import logging +import os + +LOG_LEVEL = { + 'critical': logging.CRITICAL, + 'error': logging.ERROR, + 'warn': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG +} + + +class Logger(): + __logger: logging.Logger + + def __init__(self): + # logger設定 + self.__logger = logging.getLogger() + formatter = logging.Formatter( + '[%(levelname)s]\t%(asctime)s\t%(message)s\n', + '%Y-%m-%d %H:%M:%S' + ) + handler = logging.StreamHandler() + self.__logger.addHandler(handler) + for handler in self.__logger.handlers: + handler.setFormatter(formatter) + level = logging.getLevelName(os.environ.get('LOG_LEVEL', 'WARN').upper()) + if not isinstance(level, int): + level = logging.INFO + self.__logger.setLevel(level) + + def get_logger(self) -> logging.Logger: + return self.__logger diff --git a/ecs/crm-datafetch/src/utils/s3_access.py b/ecs/crm-datafetch/src/utils/s3_access.py new file mode 100644 index 00000000..7ffb697d --- /dev/null +++ b/ecs/crm-datafetch/src/utils/s3_access.py @@ -0,0 +1,48 @@ +# モジュールロード +import boto3 +from tenacity import retry, stop_after_attempt +from tenacity.wait import wait_exponential + +# 変数 +s3_client = boto3.client('s3') +s3_resource = boto3.resource('s3') + +# 処理 + + +class UploadFileS3: + def __init__(self, bucket_name, local_file_path, file_name_key) -> None: + self.bucket_name: str = bucket_name + self.local_file_path: str = local_file_path + self.file_name_key: str = file_name_key + + def upload_file_to_s3(self) -> None: + bucket = s3_resource.Bucket(self.bucket_name) + bucket.upload_file(self.local_file_path, self.file_name_key) + + +class GetFileS3: + def __init__(self, bucket_name, file_name_key, multiplier=5, min=5, max=50, count=3) -> None: + self.bucket_name: str = bucket_name + self.file_name_key: str = file_name_key + self.multiplier: int = multiplier + self.min: int = min + self.max: int = max + self.count: int = count + + def get_file_from_s3(self) -> str: + bucket = s3_resource.Bucket(self.bucket_name) + response = bucket.Object(self.file_name_key).get() + body = response['Body'].read() + + return body.decode('utf-8') + + def get_file_from_s3_with_retry(self) -> str: + + @retry(wait=wait_exponential(multiplier=self.multiplier, min=self.min, max=self.max), stop=stop_after_attempt(self.count)) + def get_file_from_s3_with_retry_deco(*args, **kwargs) -> str: + return self.get_file_from_s3(self.bucket_name, self.file_name_key) + + response = get_file_from_s3_with_retry_deco(self) + + return response diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json new file mode 100644 index 00000000..78ec31a7 --- /dev/null +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -0,0 +1,41 @@ +{ + "objects": [ + { + # これはコメント行です + "object_name": "Account", + "columns": [ + "Id", + "Name", + "SystemModStamp", + "NumberOfEmployees", + "IsDeleted" + ] + }, + { + "object_name": "Territory2", + "is_skip": false, + "can_update_last_update": false, + "upload_file_name": "CRM_Territory2_All_{execute_datetime}_{split_csv_suffix_number:03}", + "last_fetch_datetime_file_name": "Territory2.json", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "SystemModStamp" + ] + }, + { + "object_name": "AccountShare", + "is_skip": false, + "datetime_column": "LastModifiedDate", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "LastModifiedDate" + ] + } + ] + } + + \ No newline at end of file diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env new file mode 100644 index 00000000..d48ff51a --- /dev/null +++ b/s3/config/crm/task_settings/task_settings.env @@ -0,0 +1,16 @@ +LOG_LEVEL=INFO +CRM_AUTH_TIMEOUT=100 +CRM_AUTH_MAX_RETRY_ATTEMPT=3 +CRM_AUTH_RETRY_INTERVAL=5 +CRM_AUTH_RETRY_MIN_INTERVAL=5 +CRM_AUTH_RETRY_MAX_INTERVAL=50 +CRM_GET_RECORD_COUNT_TIMEOUT=300 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=3 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL=5 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL=5 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL=50 +CRM_FETCH_RECORD_TIMEOUT=300 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3 +CRM_FETCH_RECORD_RETRY_INTERVAL=5 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 \ No newline at end of file From 8fa4f7df8ddcc817c1a0864085daf3326450f21a Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 4 Jul 2022 19:55:55 +0900 Subject: [PATCH 006/275] =?UTF-8?q?refactor:=E3=83=9F=E3=82=B9=E3=81=AB?= =?UTF-8?q?=E3=82=88=E3=82=8B=E9=87=8D=E8=A4=87=E8=A1=8C=E3=81=AE=E5=89=8A?= =?UTF-8?q?=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 5540f886..7e5cfdf9 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -34,7 +34,6 @@ LINE_FEED_CODE = { DIRECTORY_SETTINGS = '/settings/' TRUNCATE_SRC_TABLE_SYMBOL = 'truncate_src_table:' TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' -TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT = f'{TRUNCATE_SRC_TABLE_SYMBOL}[蓄積スキーマのテーブル名]' INVALID_CONFIG_EXCEPTION_MESSAGE = f'個別設定ファイルのインポート方法に不備がありました。　インポート方法は "{TRUNCATE_SRC_TABLE_IDENTIFY_SYMBOL_FORMAT}" のように設定してください' # クラス変数 From 012a88490f42c2e42c0f6b76c17393d7efc3ecf5 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Thu, 7 Jul 2022 02:24:04 +0900 Subject: [PATCH 007/275] =?UTF-8?q?fix:=E3=82=AB=E3=83=B3=E3=83=9E?= =?UTF-8?q?=E5=89=8A=E9=99=A4=E5=87=A6=E7=90=86=E3=81=AE=E8=A8=AD=E5=AE=9A?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E4=BF=AE=E6=AD=A3=E3=81=AB?= =?UTF-8?q?=E4=BC=B4=E3=81=86=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 7e5cfdf9..44d63705 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -24,7 +24,14 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, - 'importManner': 12, + 'reserved0': 12, + 'importManner': 13, + 'reserved1': 14, + 'reserved2': 15, + 'reserved3': 16, + 'reserved4': 17, + 'reserved5': 18, + 'reserved6': 19 } LINE_FEED_CODE = { 'CR': '\r', @@ -80,13 +87,18 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf cur.execute(f'SET time_zone = "+9:00"') print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-03 - タイムゾーンを変更しました') - # ④ 個別設定ファイルのロードスキーマのテーブル名に記載されているテーブルをTRUNCATEする + # 個別設定ファイルの読み込み settings_obj = s3_resource.Object(bucket_name, settings_key) settings_response = settings_obj.get() settings_list = [] for line in io.TextIOWrapper(io.BytesIO(settings_response["Body"].read()), encoding='utf-8'): settings_list.append(line.rstrip('\n')) + # 予約行挿入のためsetting_listとSETTINGS_ITEMの要素数を揃える + for _ in range(len(SETTINGS_ITEM) - len(settings_list)): + settings_list.append('') + + # ロードスキーマのTRUNCATE with conn.cursor() as cur: sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["loadSchemaName"]]}' cur.execute(sql_truncate) @@ -302,8 +314,6 @@ def truncate_judge(settings_list): """ # upsert判定 - if len(settings_list) < 13: - return False if not settings_list[SETTINGS_ITEM["importManner"]]: return False From b64090f06b183433826481948923a416c86fd880 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Thu, 7 Jul 2022 11:32:28 +0900 Subject: [PATCH 008/275] =?UTF-8?q?fix:I-MAIN-20=E3=81=AE=E3=82=A8?= =?UTF-8?q?=E3=83=A9=E3=83=BC=E3=82=B3=E3=83=BC=E3=83=89=E3=81=AE=E3=83=9F?= =?UTF-8?q?=E3=82=B9=E3=81=AE=E4=BF=AE=E6=AD=A3=E3=80=81=20=E3=82=A4?= =?UTF-8?q?=E3=83=B3=E3=83=9D=E3=83=BC=E3=83=88=E6=96=B9=E6=B3=95=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=83=BC=E3=83=96=E3=83=AB=E6=8C=87=E5=AE=9A=E3=81=AE?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E3=83=9F=E3=82=B9=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 44d63705..ff1818bd 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -185,7 +185,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf with conn.cursor() as cur: sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["storageSchemaName"]]}' cur.execute(sql_truncate) - print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} E-MAIN-20 - {settings_list[SETTINGS_ITEM["storageSchemaName"]]} をTRUNCATEしました') + print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-20 - {settings_list[SETTINGS_ITEM["storageSchemaName"]]} をTRUNCATEしました') except InvalidConfigException as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-01 - エラー内容：{e}') @@ -323,7 +323,7 @@ def truncate_judge(settings_list): import_manner_splitted_list = settings_list[SETTINGS_ITEM["importManner"]].split(':') if len(import_manner_splitted_list) != 2: raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) - if not import_manner_splitted_list[1] != settings_list[SETTINGS_ITEM["storageSchemaName"]]: + if import_manner_splitted_list[1] != settings_list[SETTINGS_ITEM["storageSchemaName"]]: raise InvalidConfigException(INVALID_CONFIG_EXCEPTION_MESSAGE) return True From d212a101c53dee18927791e7f22c60523f92a96a Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 8 Jul 2022 11:32:18 +0900 Subject: [PATCH 009/275] =?UTF-8?q?refactor:=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index ff1818bd..05e5c353 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -1,5 +1,4 @@ from datetime import datetime -from pickle import FALSE import boto3 import pymysql from pymysql.constants import CLIENT @@ -87,6 +86,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf cur.execute(f'SET time_zone = "+9:00"') print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-03 - タイムゾーンを変更しました') + # ④ 個別設定ファイルのロードスキーマのテーブル名に記載されているテーブルをTRUNCATEする # 個別設定ファイルの読み込み settings_obj = s3_resource.Object(bucket_name, settings_key) settings_response = settings_obj.get() From 48cb17e8fe245452e5408f58d7d1ecd54afcdca0 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 8 Jul 2022 13:03:48 +0900 Subject: [PATCH 010/275] =?UTF-8?q?docs:=20README=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0=E3=80=82=E4=B8=BB=E3=81=AB=E7=92=B0=E5=A2=83=E6=A7=8B?= =?UTF-8?q?=E7=AF=89=E3=81=AB=E9=96=A2=E3=81=99=E3=82=8B=E3=81=93=E3=81=A8?= =?UTF-8?q?=E3=82=92=E8=A8=98=E8=BC=89=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/README.md | 48 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 ecs/dataimport/README.md diff --git a/ecs/dataimport/README.md b/ecs/dataimport/README.md new file mode 100644 index 00000000..8502b98e --- /dev/null +++ b/ecs/dataimport/README.md @@ -0,0 +1,48 @@ +# データ取り込み処理 + +## 概要 + +データ取り込みバケット(`mbj-newdwh2021-<環境名>-data`)に配置されたデータファイルを、設定に基づいてデータベースに登録する処理を行う +処理順序等の詳細は設計書を参照のこと + +## 環境構築 + +### 事前準備 + +- [Wiki - Python環境構築](https://nds-tyo.backlog.com/alias/wiki/1874930)の「pipenvの導入」まで完了していること + +### Python仮想環境にパッケージをインストール + +- このドキュメントと同じ階層でコマンドラインを開き、以下のコマンドを実行する + + ```sh + pipenv install -r requirements.txt + ``` + +- 以降、依存モジュールの追加が発生した場合に、`requirements.txt`に追記した上で、上記のコマンドを実行すること + +## ローカルでの実行手順 + +- 当ディレクトリ内に`.vscode/launch.json`を作成し、以下のコードを貼り付ける + - 既にある場合は作成不要 + + ```json + { + "version": "0.2.0", + "configurations": [ + { + "name": "Python: データ取り込み処理", + "type": "python", + "request": "launch", + "program": "ecs/dataimport/controller.py", + "console": "integratedTerminal", + "justMyCode": true, + "envFile": "${workspaceFolder}/.env", + } + ] + } + ``` + +- 当ディレクトリ内に`.env`ファイルを、作成し、環境変数を設定する + - 設定する環境変数は設計書を参照のこと +- F5キーを押し、処理を実行する From 0fa358d53e41dcabd98992dad0122629ef0ca023 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 8 Jul 2022 13:21:51 +0900 Subject: [PATCH 011/275] =?UTF-8?q?refactor:=E4=B8=8B=E7=94=B0=E3=81=95?= =?UTF-8?q?=E3=82=93=E3=81=AE=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC=E6=8C=87?= =?UTF-8?q?=E6=91=98=E5=8F=8D=E6=98=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 5 +-- .vscode/launch.json | 25 +++++++++++ ecs/Dockerfile/Pipfile | 13 ++++++ ecs/Dockerfile/Pipfile.lock | 85 +++++++++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 ecs/Dockerfile/Pipfile create mode 100644 ecs/Dockerfile/Pipfile.lock diff --git a/.gitignore b/.gitignore index 3c4e16ef..15860e8b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,4 @@ lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* __pycache__/ .env -launch.json -settings.json -Pipfile -Pipfile.lock \ No newline at end of file +settings.json \ No newline at end of file diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..753ba49f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,25 @@ +{ + // IntelliSense を使用して利用可能な属性を学べます。 + // 既存の属性の説明をホバーして表示します。 + // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: データ取り込みローカル実行", + "type": "python", + "request": "launch", + // windowsだと\区切りかも + "program": "ecs\\dataimport\\dataimport\\controller.py", + "console": "integratedTerminal", + "justMyCode": true, + "envFile": "${workspaceFolder}/.env" + }, + { + "name": "Python: Attach using Process Id", + "type": "python", + "request": "attach", + "processId": "${command:pickProcess}", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/ecs/Dockerfile/Pipfile b/ecs/Dockerfile/Pipfile new file mode 100644 index 00000000..89112f6f --- /dev/null +++ b/ecs/Dockerfile/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +boto3 = "*" +pymysql = "*" + +[dev-packages] + +[requires] +python_version = "3.9" diff --git a/ecs/Dockerfile/Pipfile.lock b/ecs/Dockerfile/Pipfile.lock new file mode 100644 index 00000000..12a98015 --- /dev/null +++ b/ecs/Dockerfile/Pipfile.lock @@ -0,0 +1,85 @@ +{ + "_meta": { + "hash": { + "sha256": "7c69c0c237f231fcd67984f1d1b171c2eebfe00ed1877894bbbe77e201862057" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "boto3": { + "hashes": [ + "sha256:490f5e88f5551b33ae3019a37412158b76426d63d1fb910968ade9b6a024e5fe", + "sha256:e284705da36faa668c715ae1f74ebbff4320dbfbe3a733df3a8ab076d1ed1226" + ], + "index": "pypi", + "version": "==1.24.14" + }, + "botocore": { + "hashes": [ + "sha256:bb56fa77b8fa1ec367c2e16dee62d60000451aac5140dcce3ebddc167fd5c593", + "sha256:df1e9b208ff93daac7c645b0b04fb6dccd7f20262eae24d87941727025cbeece" + ], + "markers": "python_version >= '3.7'", + "version": "==1.27.14" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "pymysql": { + "hashes": [ + "sha256:41fc3a0c5013d5f039639442321185532e3e2c8924687abe6537de157d403641", + "sha256:816927a350f38d56072aeca5dfb10221fe1dc653745853d30a216637f5d7ad36" + ], + "index": "pypi", + "version": "==1.0.2" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.8.2" + }, + "s3transfer": { + "hashes": [ + "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", + "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" + ], + "markers": "python_version >= '3.7'", + "version": "==0.6.0" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.16.0" + }, + "urllib3": { + "hashes": [ + "sha256:44ece4d53fb1706f667c9bd1c648f5469a2ec925fcf3a776667042d645472c14", + "sha256:aabaf16477806a5e1dd19aa41f8c2b7950dd3c746362d7e3223dbe6de6ac448e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'", + "version": "==1.26.9" + } + }, + "develop": {} +} From 014385379c00eb77805f0ac2d826c66683d00ced Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Thu, 14 Jul 2022 14:51:24 +0900 Subject: [PATCH 012/275] =?UTF-8?q?feat:crm=5Fhistory=E3=81=AE=E6=96=B0?= =?UTF-8?q?=E8=A6=8F=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 rds_mysql/stored_procedure/crm_history.sql diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql new file mode 100644 index 00000000..cf2ca1d8 --- /dev/null +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -0,0 +1,24 @@ +-- A5M2で実行時にSQL区切り文字を「;」以外にすること + +CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) +BEGIN + SET @new_history_save = 'UPDATE @@target_table SET start_datetime = @@target_column, end_datetime = "9999-12-31 00:00:00" WHERE start_datetime IS NULL AND end_datetime IS NULL'; + SET @new_history_save = REPLACE(@new_history_save, "@@target_table", target_table); + SET @new_history_save = REPLACE(@new_history_save, "@@target_column", target_column); + PREPARE new_history_save_stmt from @new_history_save; + EXECUTE new_history_save_stmt; + + SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id'; + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); + PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; + EXECUTE make_history_tmp_create_stmt; + + SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NULL'; + SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); + PREPARE update_end_datetime_stmt from @update_end_datetime; + EXECUTE update_end_datetime_stmt; + + DROP TEMPORARY TABLE make_history_tmp; +END \ No newline at end of file From a89762eb0d0231de2e487becdd99443f3912d2e7 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 15 Jul 2022 09:35:55 +0900 Subject: [PATCH 013/275] =?UTF-8?q?refactor:=E3=82=B3=E3=83=A1=E3=83=B3?= =?UTF-8?q?=E3=83=88=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index cf2ca1d8..3247da51 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -2,23 +2,27 @@ CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN + -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = 'UPDATE @@target_table SET start_datetime = @@target_column, end_datetime = "9999-12-31 00:00:00" WHERE start_datetime IS NULL AND end_datetime IS NULL'; SET @new_history_save = REPLACE(@new_history_save, "@@target_table", target_table); SET @new_history_save = REPLACE(@new_history_save, "@@target_column", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; + -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id'; SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; EXECUTE make_history_tmp_create_stmt; + -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NULL'; SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; + -- ②-3 「②-1」で作成した一時テーブルを削除する DROP TEMPORARY TABLE make_history_tmp; END \ No newline at end of file From e24d299cb8acc449db88523810f575702aa7082e Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 15 Jul 2022 10:44:06 +0900 Subject: [PATCH 014/275] =?UTF-8?q?fix:=E4=B8=80=E6=99=82=E3=83=86?= =?UTF-8?q?=E3=83=BC=E3=83=96=E3=83=AB=E4=BD=9C=E6=88=90=E6=99=82=E3=81=AE?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E3=83=9F=E3=82=B9=E3=82=92=E4=BF=AE=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 3247da51..caeb09c0 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -10,7 +10,7 @@ BEGIN EXECUTE new_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 - SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id'; + SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id HAVING count(id) = 2'; SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; From d165cf20ba8a84f6152eadd9d442d2a656342b11 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 15 Jul 2022 14:23:49 +0900 Subject: [PATCH 015/275] =?UTF-8?q?fix:=20=E2=91=A1-2=E3=81=AE=E6=9D=A1?= =?UTF-8?q?=E4=BB=B6=E3=83=9F=E3=82=B9=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index caeb09c0..7c183714 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -17,7 +17,7 @@ BEGIN EXECUTE make_history_tmp_create_stmt; -- ②-2 「②-1」で取得した全件に更新処理を行う - SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NULL'; + SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NOT NULL'; SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; From a7426900b73be088bb4905938774894354f90166 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 19 Jul 2022 20:16:40 +0900 Subject: [PATCH 016/275] =?UTF-8?q?feat:=E6=A9=9F=E8=83=BD=E5=85=A8?= =?UTF-8?q?=E8=88=AC=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Dockerfile | 7 + ecs/crm-datafetch/main.py | 181 +--------------- .../src/{utils => aws}/__init__.py | 0 ecs/crm-datafetch/src/aws/s3.py | 107 ++++++++++ ecs/crm-datafetch/src/chk.py | 30 +++ ecs/crm-datafetch/src/config/objects.py | 195 ++++++++++++++++++ ecs/crm-datafetch/src/constants.py | 73 +++++++ ecs/crm-datafetch/src/conv.py | 33 +++ ecs/crm-datafetch/src/converter/converter.py | 138 +++++++++++++ ecs/crm-datafetch/src/csvbk.py | 34 +++ ecs/crm-datafetch/src/date.py | 65 ++++++ ecs/crm-datafetch/src/end.py | 37 ++++ ecs/crm-datafetch/src/environments.py | 73 +++++++ ecs/crm-datafetch/src/error/__init__.py | 0 ecs/crm-datafetch/src/error/exceptions.py | 45 ++++ ecs/crm-datafetch/src/fetch.py | 71 +++++++ ecs/crm-datafetch/src/parser/__init__.py | 0 ecs/crm-datafetch/src/parser/json_parse.py | 15 ++ ecs/crm-datafetch/src/pre.py | 153 +++++++------- ecs/crm-datafetch/src/process.py | 150 ++++++++++++++ ecs/crm-datafetch/src/resbk.py | 35 ++++ ecs/crm-datafetch/src/salesforce/__init__.py | 0 .../src/salesforce/salesforce_api.py | 78 +++++++ ecs/crm-datafetch/src/upd.py | 49 +++++ ecs/crm-datafetch/src/upld.py | 43 ++++ ecs/crm-datafetch/src/util/__init__.py | 0 ecs/crm-datafetch/src/util/dict_checker.py | 64 ++++++ ecs/crm-datafetch/src/util/logger.py | 29 +++ ecs/crm-datafetch/src/utils/data_retention.py | 169 --------------- ecs/crm-datafetch/src/utils/json_parser.py | 28 --- ecs/crm-datafetch/src/utils/logger.py | 33 --- ecs/crm-datafetch/src/utils/s3_access.py | 48 ----- 32 files changed, 1443 insertions(+), 540 deletions(-) rename ecs/crm-datafetch/src/{utils => aws}/__init__.py (100%) create mode 100644 ecs/crm-datafetch/src/aws/s3.py create mode 100644 ecs/crm-datafetch/src/config/objects.py create mode 100644 ecs/crm-datafetch/src/constants.py create mode 100644 ecs/crm-datafetch/src/converter/converter.py create mode 100644 ecs/crm-datafetch/src/environments.py create mode 100644 ecs/crm-datafetch/src/error/__init__.py create mode 100644 ecs/crm-datafetch/src/error/exceptions.py create mode 100644 ecs/crm-datafetch/src/parser/__init__.py create mode 100644 ecs/crm-datafetch/src/parser/json_parse.py create mode 100644 ecs/crm-datafetch/src/process.py create mode 100644 ecs/crm-datafetch/src/salesforce/__init__.py create mode 100644 ecs/crm-datafetch/src/salesforce/salesforce_api.py create mode 100644 ecs/crm-datafetch/src/util/__init__.py create mode 100644 ecs/crm-datafetch/src/util/dict_checker.py create mode 100644 ecs/crm-datafetch/src/util/logger.py delete mode 100644 ecs/crm-datafetch/src/utils/data_retention.py delete mode 100644 ecs/crm-datafetch/src/utils/json_parser.py delete mode 100644 ecs/crm-datafetch/src/utils/logger.py delete mode 100644 ecs/crm-datafetch/src/utils/s3_access.py diff --git a/ecs/crm-datafetch/Dockerfile b/ecs/crm-datafetch/Dockerfile index e84b79b3..88e588e1 100644 --- a/ecs/crm-datafetch/Dockerfile +++ b/ecs/crm-datafetch/Dockerfile @@ -6,6 +6,13 @@ WORKDIR /usr/src/app COPY Pipfile Pipfile.lock ./ RUN pip install --no-cache-dir -r requirements.txt && \ cp /usr/share/zoneinfo/Asia/Tokyo /etc/localtime + +RUN \ + apt update -y && \ + # パッケージのセキュリティアップデートのみを適用するコマンド + apt install -y unattended-upgrades && \ + unattended-upgrades + COPY main.py ./ COPY crm-datafetch /usr/src diff --git a/ecs/crm-datafetch/main.py b/ecs/crm-datafetch/main.py index 0170f413..b0edf545 100644 --- a/ecs/crm-datafetch/main.py +++ b/ecs/crm-datafetch/main.py @@ -1,181 +1,4 @@ -# モジュールロード -import os -import sys -from datetime import datetime -import src.pre as pre # データ取得準備処理 -import src.chk as chk # オブジェクト情報形式チェック処理 -import src.date as date # データ取得期間設定処理 -import src.fetch as fetch # CRMデータ取得処理 -import src.resbk as resbk # CRM電文データバックアップ処理 -import src.conv as conv # CSV変換処理 -import src.csvbk as csvbk # CSVバックアップ処理 -import src.upld as upld # CSVアップロード処理 -import src.upd as upd # 前回取得日時ファイル更新 -import src.end as end # 取得処理実施結果アップロード処理 - -from src.utils.logger import Logger -import src.utils.data_retention as data_retention - - -# 変数 - -# logger設定 -logger = Logger().get_logger() - - -# 処理 -def main() -> None: - """コントロール処理 - - 各処理を呼び出すコントローラー - - """ - - try: - # ① CRMデータ取得処理開始ログを出力する - logger.info('I-CTRL-01 CRMデータ取得処理を開始します') - - # ② データ取得準備処理を呼び出す - try: - logger.info('I-CTRL-02 データ取得準備処理呼び出し') - crm_object_info_var = data_retention.CrmObjectInfoVar() - crm_object_info_dict, execute_datetime, process_result_per_object = pre( - crm_object_info_var) - - except Exception as e: - logger.error('') - raise CustomException(e) - - # ③ object_infoのobjectsキーの値の件数分ループする - logger.info('I-CTRL-03 取得対象オブジェクトのループ処理開始') - - for object_info in crm_object_info_dict.get('objects'): - try: - - # 1. オブジェクト情報形式チェック処理を呼び出す - logger.info('I-CTRL-04 オブジェクト情報形式チェック処理呼び出し') - - try: - chk() - - except Exception as e: - logger.error('') - - # 2. 処理対象のオブジェクト名をログ出力する - logger.info('I-CTRL-05 ') - - # 3. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する - logger.info('I-CTRL-06 ') - - # 4. データ取得期間設定処理を呼び出す - logger.info('I-CTRL-07 ') - try: - - except Exception as e: - logger.error('') - - # 5. CRMデータ取得処理を呼び出す - logger.info('I-CTRL-08 ') - try: - - except Exception as e: - logger.error('') - - # 6. CRMデータのジェネレータをループする - # 6-1. 出力ファイル名を生成する - logger.info('I-CTRL-10 ') - try: - - except Exception as e: - logger.error('') - - # 6-2. CRM電文データバックアップ処理を呼び出す - logger.info('I-CTRL-11 ') - try: - - except Exception as e: - logger.error('') - - # 6-3. CSV変換処理を呼び出す - logger.info('I-CTRL-12 ') - try: - - except Exception as e: - logger.error('') - - # 6-4. CSVバックアップ処理を呼び出す - logger.info('I-CTRL-13 ') - try: - - except Exception as e: - logger.error('') - - # 6-5. CSVアップロード処理を呼び出す - logger.info('I-CTRL-14 ') - try: - - except Exception as e: - logger.error('') - - # 7. 前回取得日時ファイル更新処理を呼びだす - logger.info('I-CTRL-15 ') - try: - - except Exception as e: - logger.error('') - - # 8. 1オブジェクトのアップロードが完了した旨をログに出力する - logger.info('I-CTRL-16 ') - - except GetObjectException as e: - logger.error( - '[取得対象のオブジェクト名] の[処理名]でエラーが発生しました 次のオブジェクトの処理に移行します') - - except Exception as e: - logger.error( - '[取得対象のオブジェクト名] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します') - - # ④ すべてのオブジェクトの処理が完了したことと、オブジェクト毎の処理結果をログに出力する - try: - logger.info(f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{実行結果JSON}]') - - except Exception as e: - logger.error('') - raise CustomException() - - # ⑤ 取得処理実施結果アップロード処理を呼び出す - try: - logger.info('I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始') - - except Exception as e: - logger.error('') - raise CustomException() - - # ⑥ CRMデータ取得処理終了ログを出力する - try: - logger.info('I-CTRL-19 CRMデータ取得処理を終了します') - - except Exception as e: - logger.error('') - raise - - except CustomException as e: - logger.error('I-ERR-01 [処理名]でエラーが発生したため、処理を終了します') - - except Exception as e: - logger.error('I-ERR-02 予期せぬエラーが発生したため、処理を終了します') - - -# カスタムExceptionクラス -class CustomException(Exception, metaclass=ABCMeta): - def __init__(self, id, arg): - self.arg = arg - self.id = id - - -class GetObjectException(CustomException): - pass - +import src.process as process if __name__ == '__main__': - main() + process.main() diff --git a/ecs/crm-datafetch/src/utils/__init__.py b/ecs/crm-datafetch/src/aws/__init__.py similarity index 100% rename from ecs/crm-datafetch/src/utils/__init__.py rename to ecs/crm-datafetch/src/aws/__init__.py diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py new file mode 100644 index 00000000..e5235016 --- /dev/null +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -0,0 +1,107 @@ +import boto3 +import json +from src.environments import ( + CRM_CONFIG_BUCKET, + CRM_BACKUP_BUCKET, + IMPORT_DATA_BUCKET, + OBJECT_INFO_FOLDER, + OBJECT_INFO_FILENAME, + CRM_IMPORT_DATA_FOLDER, + CRM_IMPORT_DATA_BACKUP_FOLDER, + LAST_FETCH_DATE_FOLDER, + RESPONSE_JSON_BACKUP_FOLDER +) +from src.constants import ( + AWS_RESOURCE_S3, + AWS_CLINET_S3, + S3_RESPONSE_BODY +) + + +class S3Clinet: + def __init__(self) -> None: + self.__s3_client = boto3.client(AWS_CLINET_S3) + + def copy_object(self, src_bucket, src_key, dest_bucket, dest_key): + self.__s3_client.copy( + {"Bucket": src_bucket, "key": src_key}, dest_bucket, dest_key) + return + + +class S3Resource: + def __init__(self, bucket_name: str) -> None: + self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) + self.__s3_bucket = self.__s3_resource.Bucket(bucket_name) + + def get_object(self, object_key: str) -> str: + response = self.__s3_bucket.Object(object_key).get() + body = response[S3_RESPONSE_BODY].read() + + return body.decode('utf-8') + + def put_object(self, object_key: str, data) -> str: + s3_object = self.__s3_bucket.Object(object_key) + s3_object.put(Body=data.encode('utf-8'), ContentEncoding='utf-8') + return + + +class S3ResourceNonBucket: + def __init__(self) -> None: + self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) + + def copy(self, src_bucket, src_key, dest_bucket, dest_key): + copy_source = {'Bucket': src_bucket, 'Key': src_key} + self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) + + return + + +class ConfigBucket: + __s3_resource: S3Resource = None + + def __init__(self) -> None: + self.__s3_resource = S3Resource(CRM_CONFIG_BUCKET) + + def get_object_info_file(self): + return self.__s3_resource.get_object(f'{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}') + + def get_last_fetch_datetime_file(self, file_name): + return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_name}') + + def put_last_fetch_datetime_file(self, file_name, data): + self.__s3_resource.put_object( + f'{LAST_FETCH_DATE_FOLDER}/{file_name}', data) + return + + +class DataBucket: + __s3_resource: S3Resource = None + + def __init__(self) -> None: + self.__s3_resource = S3Resource(IMPORT_DATA_BUCKET) + + def put_csv(self, file_name, data): + object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_name}' + self.__s3_resource.put_object(object_key, data) + return + + +class BackupBucket: + __s3_resource: S3Resource = None + + def __init__(self) -> None: + self.__s3_resource = S3Resource(CRM_BACKUP_BUCKET) + + def put_response_json(self, file_name, data): + object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_name}' + self.__s3_resource.put_object(object_key, json.dumps(data)) + return + + def put_csv_bk(self, file_name, data): + object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_name}' + self.__s3_resource.put_object(object_key, data) + return + + def put_result_json(self, file_name, data): + object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_name}' + self.__s3_resource.put_object(object_key, json.dumps(data)) diff --git a/ecs/crm-datafetch/src/chk.py b/ecs/crm-datafetch/src/chk.py index e69de29b..1342243e 100644 --- a/ecs/crm-datafetch/src/chk.py +++ b/ecs/crm-datafetch/src/chk.py @@ -0,0 +1,30 @@ +from src.util.logger import Logger +from src.constants import( + CHK_JP_NAME +) +from src.error.exceptions import( + InvalidConfigException +) +from src.config.objects import TargetObject + + +logger = Logger().get_logger() + + +def check_object_info(object_info, execute_datetime): + # ① オブジェクト情報形式チェック処理開始ログを出力する + logger.info('I-CHK-01 オブジェクト情報形式チェック処理を開始します') + + try: + # ② オブジェクト情報形式チェック + target_object = TargetObject(object_info, execute_datetime) + + except Exception as e: + raise InvalidConfigException( + 'E-CHK-01', CHK_JP_NAME, f'オブジェクト情報形式チェック処理が失敗しました　エラー内容：[{e}]') + + # ③ チェック処理終了ログを出力する + logger.info('I-CHK-02 オブジェクト情報形式チェック処理を終了します') + + # ④ 次の処理へ移行する + return target_object diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py new file mode 100644 index 00000000..edce5b33 --- /dev/null +++ b/ecs/crm-datafetch/src/config/objects.py @@ -0,0 +1,195 @@ +from src.constants import ( + DATE_PATTERN_YYYYMMDDTHHMMSSTZ +) +from src.util.dict_checker import DictCheck + + +class FetchTargetObjects(): + def __init__(self, object_info_file_dict) -> None: + self.__dict_check = DictCheck() + self.__objects = object_info_file_dict + self.__key = 'objects' + self.check_key_objects() + self.__i = 0 + + def __iter__(self): + return self + + def __next__(self): + if self.__i == len(self.__objects[self.__key]): + raise StopIteration() + value = self.__objects['objects'][self.__i] + self.__i += 1 + return value + + def check_key_objects(self) -> None: + __check_key = self.__key + __check_type = list + self.__dict_check.check_key_exist_and_value_type( + self.__objects, __check_key, __check_type) + + +class TargetObject(): + def __init__(self, object_info, execute_datetime) -> None: + self.__dict_check = DictCheck() + self.__object_info = object_info + self.execute_datetime = execute_datetime + self.check_key_object_name() + self.check_key_columns() + self.check_key_is_skip() + self.check_key_is_update_last_fetch_datetime() + self.check_key_last_fetch_datetime_file_name() + self.check_key_upload_file_name() + self.object_name = self.__object_info['object_name'] + self.columns = self.__object_info['columns'] + self.is_skip = self.set_is_skip() + self.is_update_last_fetch_datetime = self.set_is_update_last_fetch_datetime() + self.last_fetch_datetime_file_name = self.set_fetch_datetime_file_name() + self.upload_file_name = self.set_upload_file_name() + + def check_key_object_name(self) -> None: + ''' + オブジェクト名チェック + ''' + __check_key = 'object_name' + __check_type = str + self.__dict_check.check_key_exist_and_value_type( + self.__object_info, __check_key, __check_type) + + return + + def check_key_columns(self) -> None: + ''' + カラム情報チェック + ''' + __key = 'columns' + __type = list + self.__dict_check.check_key_exist_and_value_type( + self.__object_info, __key, __type) + + return + + def check_key_is_skip(self,) -> None: + ''' + スキップフラグ型チェック + ''' + __check_key = 'is_skip' + __check_type = bool + self.__dict_check.check_key_exist_case_value_type( + self.__object_info, __check_key, __check_type) + + return + + def check_key_is_update_last_fetch_datetime(self) -> None: + ''' + 前回取得日時ファイル更新フラグチェック + ''' + __check_key = 'is_update_last_fetch_datetime' + __check_type = bool + self.__dict_check.check_key_exist_case_value_type( + self.__object_info, __check_key, __check_type) + + return + + def check_key_last_fetch_datetime_file_name(self) -> None: + ''' + 前回取得日時ファイル名型チェック + ''' + __check_key = 'last_fetch_datetime_file_name' + __check_type = str + self.__dict_check.check_key_exist_case_value_type( + self.__object_info, __check_key, __check_type) + + return + + def check_key_upload_file_name(self) -> None: + ''' + アップロードファイル名称型チェック + ''' + __check_key = 'upload_file_name' + __check_type = str + self.__dict_check.check_key_exist_case_value_type( + self.__object_info, __check_key, __check_type) + + return + + def set_is_skip(self) -> bool: + ''' + スキップフラグ設定 + ''' + __check_key = 'is_skip' + if self.__dict_check.check_key_exist(self.__object_info, __check_key): + return self.__object_info[__check_key] + else: + return False + + def set_is_update_last_fetch_datetime(self) -> bool: + ''' + 前回取得日時ファイル更新フラグ設定 + ''' + __check_key = 'is_update_last_fetch_datetime' + if self.__dict_check.check_key_exist(self.__object_info, __check_key): + return self.__object_info[__check_key] + else: + return False + + def set_fetch_datetime_file_name(self) -> str: + ''' + 前回取得日時ファイル名設定 + ''' + __check_key = 'last_fetch_datetime_file_name' + if self.__dict_check.check_key_exist(self.__object_info, __check_key): + return self.__object_info[__check_key] + else: + return self.__object_info['object_name'] + '.json' + + def set_upload_file_name(self) -> str: + ''' + アップロードファイル名称設定 + ''' + __check_key = 'upload_file_name' + if self.__dict_check.check_key_exist(self.__object_info, __check_key): + return self.__object_info[__check_key].format(execute_datetime=self.execute_datetime) + else: + return 'CRM_' + self.__object_info['object_name'] + '_' + self.execute_datetime + + +class LastFetchDatetime(): + def __init__(self, last_fetch_datetime_file_name, last_fetch_datetime_file_dict, execute_datetime) -> None: + self.__dict_check = DictCheck() + self.execute_datetime = execute_datetime + self.__last_fetch_datetime_file_dict = last_fetch_datetime_file_dict + self.last_fetch_datetime_file_name = last_fetch_datetime_file_name + self.check_key_last_fetch_datetime_from + self.check_key_last_fetch_datetime_to + self.last_fetch_datetime_from = self.__last_fetch_datetime_file_dict[ + 'last_fetch_datetime_from'] + self.last_fetch_datetime_to = self.set_last_fetch_datetime_to() + + def check_key_last_fetch_datetime_from(self) -> None: + ''' + データ取得開始日時チェック + ''' + __check_key = 'last_fetch_datetime_from' + __regex_str = DATE_PATTERN_YYYYMMDDTHHMMSSTZ + self.__dict_check.check_key_exsit_and_regex( + self.__last_fetch_datetime_file_dict, __check_key, __regex_str) + + def check_key_last_fetch_datetime_to(self) -> None: + ''' + データ取得終了日時チェック + ''' + __check_key = 'last_fetch_datetime_to' + __regex_str = DATE_PATTERN_YYYYMMDDTHHMMSSTZ + self.__dict_check.check_key_exsit_case_regex( + self.__last_fetch_datetime_file_dict, __check_key, __regex_str) + + def set_last_fetch_datetime_to(self) -> str: + ''' + データ取得終了日時設定 + ''' + __check_key = 'last_fetch_datetime_to' + if self.__dict_check.check_key_exist(self.__last_fetch_datetime_file_dict, __check_key): + return self.__last_fetch_datetime_file_dict[__check_key].format(execute_datetime=self.execute_datetime) + else: + return self.execute_datetime diff --git a/ecs/crm-datafetch/src/constants.py b/ecs/crm-datafetch/src/constants.py new file mode 100644 index 00000000..fcd6b4a3 --- /dev/null +++ b/ecs/crm-datafetch/src/constants.py @@ -0,0 +1,73 @@ +# environments(task settings file) +LOG_LEVEL = "LOG_LEVEL" # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 +CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = 'CRM_AUTH_RETRY_MIN_INTERVAL' # CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = 'CRM_AUTH_RETRY_MAX_INTERVAL' # CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = 'CRM_GET_RECORD_COUNT_TIMEOUT' # CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = 'CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT' # CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_INTERVAL' # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL' # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL' # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_TIMEOUT = 'CRM_FETCH_RECORD_TIMEOUT' # CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = 'CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT' # CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_RETRY_INTERVAL = 'CRM_FETCH_RECORD_RETRY_INTERVAL' # CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MIN_INTERVAL' # CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MAX_INTERVAL' # CRMのレコード取得処理のリトライ時の最大待ち秒数 + +# environments(ECS Task Enviroment) +CRM_AUTH_DOMAIN = 'CRM_AUTH_DOMAIN' # CRMのAPI実行のための認証エンドポイントのドメイン +CRM_USER_NAME = 'CRM_USER_NAME' # CRMのAPI実行用ユーザ名 +CRM_USER_PASSWORD = 'CRM_USER_PASSWORD' # CRMのAPI実行用ユーザパスワード +CRM_USER_SECURITY_TOKEN = 'CRM_USER_SECURITY_TOKEN' # CRMのAPI実行用ユーザのセキュリティトークン +CRM_CONFIG_BUCKET = 'CRM_CONFIG_BUCKET' # CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_BACKUP_BUCKET = 'CRM_BACKUP_BUCKET' # CRMのバックアップデータを格納するバケット名 +IMPORT_DATA_BUCKET = 'IMPORT_DATA_BUCKET' # CRMの取込データを格納するバケット名 +OBJECT_INFO_FOLDER = 'OBJECT_INFO_FOLDER' # CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FILENAME = 'OBJECT_INFO_FILENAME' # CRM取得対象オブジェクトの情報のファイル名 +PROCESS_RESULT_FOLDER = 'PROCESS_RESULT_FOLDER' # CRMデータ取得結果を格納するフォルダパス +PROCESS_RESULT_FILENAME = 'PROCESS_RESULT_FILENAME' # CRMデータ取得結果を格納するファイル名 +LAST_FETCH_DATE_FOLDER = 'LAST_FETCH_DATE_FOLDER' # CRMからの最終取得日時ファイルを格納するフォルダパス +CRM_IMPORT_DATA_FOLDER ='CRM_IMPORT_DATA_FOLDER' # CRMから取得し、取込用に変換したデータを格納するフォルダ +LAST_FETCH_DATE_BACKUP_FOLDER = 'LAST_FETCH_DATE_BACKUP_FOLDER' # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +RESPONSE_JSON_BACKUP_FOLDER = 'RESPONSE_JSON_BACKUP_FOLDER' # CRMから取得した生データのバックアップを格納するフォルダパス +CRM_IMPORT_DATA_BACKUP_FOLDER = 'CRM_IMPORT_DATA_BACKUP_FOLDER' # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ + +# 時刻フォーマット +# .000ZはUTCを表す。ミリ秒までの考慮は不要なので固定で指定 +YYYYMMDDTHHMMSSTZ = '%Y-%m-%dT%H:%M:%S.000Z' +CRM_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.000%z' +YYYYMMDDHHMMSS = '%Y-%m-%d %H:%M:%S' + +# aws +AWS_RESOURCE_S3 = 's3' +AWS_CLINET_S3 = 's3' +S3_RESPONSE_BODY = 'Body' + + +# 正規表現チェック +EXCLUDE_SYMBOL = ['#', '/'] +DATE_PATTERN_YYYYMMDDTHHMMSSTZ = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9].000Z' + +# logger +LOG_FORMAT = '[%(levelname)s]\t%(asctime)s\t%(message)s\n' +LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' +LOG_LEVEL_INFO = 'INFO' + +# 処理名 +PROCESS_JP_NAME = 'コントロール処理' +PRE_JP_NAME = 'データ取得準備処理' +CHK_JP_NAME = 'オブジェクト情報形式チェック処理' +DATE_JP_NAME = 'データ取得期間設定処理' +FETCH_JP_NAME = 'CRMデータ取得処理' +RESBK_JP_NAME = 'CRM電文データバックアップ処理' +CONV_JP_NAME = 'CSV変換処理' +CSVBK_JP_NAME = 'CSVバックアップ処理' +UPLD_JP_NAME = 'CSVアップロード処理' +UPD_JP_NAME = '前回取得日時ファイル更新' +END_JP_NAME = '取得処理実施結果アップロード処理' + +# CSVチェック +CSV_TRUE_VALUE = '1' +CSV_FALSE_VALUE = '0' \ No newline at end of file diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/conv.py index e69de29b..88b86ae8 100644 --- a/ecs/crm-datafetch/src/conv.py +++ b/ecs/crm-datafetch/src/conv.py @@ -0,0 +1,33 @@ +from datetime import datetime +from src.util.logger import Logger +from src.constants import ( + CONV_JP_NAME, +) +from src.error.exceptions import( + DataConvertException +) +from src.converter.converter import CSVStringConverter + + +logger = Logger().get_logger() + + +def convert_crm_csvdata(target_object, sf_object_jsons): + # ① CSV変換処理の開始ログを出力する + logger.info(f'I-CONV-01 [{target_object.object_name}] のCSV変換処理を開始します') + + try: + # ② CSV変換 + csv_object = CSVStringConverter(target_object, sf_object_jsons) + + logger.debug(f'D-CONV-02 [{target_object.object_name}] のCSV変換処理 正常終了') + + except Exception as e: + raise DataConvertException( + 'E-CONV-01', CONV_JP_NAME, f'[{target_object.object_name}] CSV変換に失敗しました エラー内容:[{e}]') + + # ③ CSV変換処理の終了ログを出力する + logger.info(f'I-CONV-03 [{target_object.object_name}] のCSV変換処理を終了します') + + # ④ 次の処理へ移行する + return csv_object diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py new file mode 100644 index 00000000..1b785c35 --- /dev/null +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -0,0 +1,138 @@ +import re +import io +import csv +from datetime import datetime +from src.constants import( + CSV_TRUE_VALUE, + CSV_FALSE_VALUE, + CRM_DATETIME_FORMAT, + YYYYMMDDHHMMSS +) + + +class CSVStringConverter: + def __init__(self, target_object, sf_object_jsons) -> None: + self.__target_object = target_object + self.__sf_object_jsons = sf_object_jsons + self.__extracted_sf_object_jsons = self.extract_sf_object_jsons() + self.csv_data = self.convert_to_csv() + self.csv_buffer = self.write_csv() + + def extract_sf_object_jsons(self) -> list: + try: + extracted_sf_object_jsons = [] + for sf_object_json in self.__sf_object_jsons: + extracted_sf_object_jsons.append( + self.extract_necessary_props_from(sf_object_json)) + + return extracted_sf_object_jsons + + except Exception as e: + raise Exception('必要なjsonのデータ抽出に失敗しました') + + def extract_necessary_props_from(self, sf_object_json) -> dict: + try: + clone_sf_object = {**sf_object_json} + + del clone_sf_object['attributes'] + + uppercase_key_sf_object = { + k.upper(): v for k, v in clone_sf_object.items()} + + return uppercase_key_sf_object + + except Exception as e: + raise Exception('必要なjsonのデータ成形に失敗しました') + + def convert_to_csv(self) -> list: + try: + columns = self.__target_object.columns + csv_data = [] + for i, json_object in enumerate(self.__extracted_sf_object_jsons, 1): + csv_row = [] + for column in columns: + v = json_object[column.upper()] + + converted_value = CSVStringConverterFactory( + v).value_convert() + + csv_row.append(converted_value) + + csv_data.append(csv_row) + return csv_data + + except Exception as e: + raise Exception( + f'CSV変換に失敗しました カラム名：[{column}] 行番号: [{i}] エラー内容:[{e}]') + + def write_csv(self) -> str: + try: + with io.StringIO(newline='') as string_stream: + writer = csv.writer(string_stream, delimiter=',', lineterminator='\r\n', + doublequote=True, quotechar='"', quoting=csv.QUOTE_ALL, strict=True) + writer.writerow(self.__target_object.columns) + writer.writerows(self.csv_data) + csv_value = string_stream.getvalue() + + return csv_value + + except Exception as e: + raise Exception('csvデータの取得に失敗しました') + + +class NoneValueConverter: + def __init__(self, convert_value) -> None: + self.__convert_value = convert_value + self.value = self.convert_value() + + def convert_value(self) -> str: + return '' + + +class BooleanConverter: + def __init__(self, convert_value) -> None: + self.__convert_value = convert_value + self.value = self.convert_value() + + def convert_value(self) -> bool: + return CSV_TRUE_VALUE if self.__convert_value is True else CSV_FALSE_VALUE + + +class DatatimeConverter: + def __init__(self, convert_value) -> None: + self.__convert_value = convert_value + self.value = self.convert_value() + + def convert_value(self) -> str: + return datetime.strptime(self.__convert_value, CRM_DATETIME_FORMAT).strftime(YYYYMMDDHHMMSS) + + +class FloatConverter: + def __init__(self, convert_value) -> None: + self.__convert_value = convert_value + self.value = self.convert_value() + + def convert_value(self) -> int: + return int(self.__convert_value) + + +class CSVStringConverterFactory: + def __init__(self, v) -> None: + self.__value = v + + def value_convert(self): + + converted_value = self.__value + + if self.__value is None: + converted_value = NoneValueConverter(self.__value).value + # 指数表記で取得できるパターン。指数表記を整数表記に変換する。 + elif type(self.__value) == float: + converted_value = FloatConverter(self.__value).value + # SQLの真偽値に対応するために変換する + elif type(self.__value) == bool: + converted_value = BooleanConverter(self.__value).value + elif type(self.__value) == str and re.fullmatch(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.000\+0000', self.__value): + converted_value = DatatimeConverter(self.__value).value + + return converted_value diff --git a/ecs/crm-datafetch/src/csvbk.py b/ecs/crm-datafetch/src/csvbk.py index e69de29b..db490c63 100644 --- a/ecs/crm-datafetch/src/csvbk.py +++ b/ecs/crm-datafetch/src/csvbk.py @@ -0,0 +1,34 @@ +from src.util.logger import Logger +from src.aws.s3 import BackupBucket +from src.constants import ( + CSVBK_JP_NAME +) +from src.error.exceptions import( + FileUploadException +) + + +logger = Logger().get_logger() + + +def backup_crm_csvdata(target_object, date_path, csv_object): + # ① CSVバックアップ処理の開始ログを出力する + logger.info( + f'I-CSVBK-01 [{target_object.object_name}] のCSVデータのバックアップ処理を開始します ファイル名：[{target_object.upload_file_name}.csv]') + + try: + # ② CRMバックアップ保管用バケットに、変換後のCSVデータのバックアップを保管する + backup_bucket = BackupBucket() + backup_bucket.put_csv_bk( + f'{date_path}/{target_object.upload_file_name}.csv', csv_object.csv_buffer) + + except Exception as e: + raise FileUploadException( + 'E-CSVBK-01', CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') + + # ③ CSVバックアップ処理の終了ログを出力する + logger.info( + f'I-CSVBK-03 [{target_object.object_name}] のCSVデータのバックアップ処理を終了します') + + # ④ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/date.py b/ecs/crm-datafetch/src/date.py index e69de29b..8fff0d94 100644 --- a/ecs/crm-datafetch/src/date.py +++ b/ecs/crm-datafetch/src/date.py @@ -0,0 +1,65 @@ +from src.util.logger import Logger +from src.constants import( + DATE_JP_NAME +) +from src.environments import ( + CRM_CONFIG_BUCKET, + LAST_FETCH_DATE_FOLDER +) +from src.error.exceptions import( + FileNotFoundException, + InvalidConfigException +) +from src.parser.json_parse import JsonParser +from src.config.objects import LastFetchDatetime +from src.aws.s3 import ConfigBucket + + +logger = Logger().get_logger() + + +def set_datetime_period(target_object, execute_datetime): + # ① データ取得期間設定処理の開始ログを出力する + logger.info( + f'I-DATE-01 [{target_object.object_name}] のデータ取得期間設定処理を開始します') + + try: + # ② S3 設定ファイル保管用バケットから、前回取得日時ファイルを取得する + logger.info( + f'I-DATE-02 前回取得日時ファイルの取得開始します ファイルパス：[s3://{CRM_CONFIG_BUCKET}/{LAST_FETCH_DATE_FOLDER}/{target_object.last_fetch_datetime_file_name}]') + + s3_config_bucket = ConfigBucket() + last_fetch_datetime_file_json = s3_config_bucket.get_last_fetch_datetime_file( + target_object.last_fetch_datetime_file_name) + + logger.info(f'I-DATE-03 前回取得日時ファイルの取得成功しました') + + except Exception as e: + raise FileNotFoundException( + 'E-DATE-01', DATE_JP_NAME, f'前回取得日時ファイルが存在しません　ファイル名：[{target_object.last_fetch_datetime_file_name}] エラー内容：[{e}]') + + try: + # ③ 取得した前回取得日時ファイルの形式チェックを行う + # ④ データの取得期間を設定する + logger.debug(f'D-DATE-04 前回取得日時ファイルの形式チェックを開始します') + + json_parser = JsonParser(last_fetch_datetime_file_json) + last_fetch_datetime_file_dict = json_parser.json_parser() + + last_fetch_datetime = LastFetchDatetime(target_object.last_fetch_datetime_file_name, + last_fetch_datetime_file_dict, execute_datetime) + + logger.debug(f'D-DATE-05 前回取得日時ファイルの形式チェック 正常終了') + logger.info( + f'I-DATE-06 取得範囲 From: [{last_fetch_datetime.last_fetch_datetime_from}] To: [{last_fetch_datetime.last_fetch_datetime_to}]') + + except Exception as e: + raise InvalidConfigException( + 'E-DATE-02', DATE_JP_NAME, f'前回取得日時ファイルの形式チェック処理が失敗しました エラー内容：[{e}]') + + # ⑤ データ取得準備処理の終了ログを出力する + logger.info( + f'I-DATE-07 [{target_object.object_name}] のデータ取得期間設定処理を終了します') + + # ⑥ 次の処理へ移行する + return last_fetch_datetime diff --git a/ecs/crm-datafetch/src/end.py b/ecs/crm-datafetch/src/end.py index e69de29b..e2b274b3 100644 --- a/ecs/crm-datafetch/src/end.py +++ b/ecs/crm-datafetch/src/end.py @@ -0,0 +1,37 @@ +from src.util.logger import Logger +from src.constants import ( + END_JP_NAME +) +from src.environments import( + PROCESS_RESULT_FILENAME +) +from src.error.exceptions import( + FileUploadException +) +from src.aws.s3 import BackupBucket + + +logger = Logger().get_logger() + + +def updload_result_data(process_result, date_path): + # ① 取得処理実施結果アップロード処理のログを出力する + logger.info( + f'I-END-01 取得処理実施結果アップロード処理を開始します') + + try: + # ② CRMバックアップ保管用バケットに、取得処理実施結果のJSONデータを保管する + backup_bucket = BackupBucket() + backup_bucket.put_result_json( + f'{date_path}/{PROCESS_RESULT_FILENAME}', process_result) + + logger.debug(f'D-END-02 取得処理実施結果アップロード 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-END-01', END_JP_NAME, f'取得処理実施結果のアップロードに失敗しました　ファイル名：[{PROCESS_RESULT_FILENAME}] エラー内容：[{e}]') + + # ③ 取得処理実施結果アップロード処理の終了ログを出力する + logger.info(f'I-END-03 取得処理実施結果アップロード処理を終了します') + + return diff --git a/ecs/crm-datafetch/src/environments.py b/ecs/crm-datafetch/src/environments.py new file mode 100644 index 00000000..8d63cc64 --- /dev/null +++ b/ecs/crm-datafetch/src/environments.py @@ -0,0 +1,73 @@ +import os + +from src.constants import ( + LOG_LEVEL, + LOG_LEVEL_INFO, + CRM_AUTH_TIMEOUT, + CRM_AUTH_MAX_RETRY_ATTEMPT, + CRM_AUTH_RETRY_INTERVAL, + CRM_AUTH_RETRY_MIN_INTERVAL, + CRM_AUTH_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_TIMEOUT, + CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, + CRM_AUTH_DOMAIN, + CRM_USER_NAME, + CRM_USER_PASSWORD, + CRM_USER_SECURITY_TOKEN, + CRM_CONFIG_BUCKET, + CRM_BACKUP_BUCKET, + IMPORT_DATA_BUCKET, + OBJECT_INFO_FOLDER, + OBJECT_INFO_FILENAME, + PROCESS_RESULT_FOLDER, + PROCESS_RESULT_FILENAME, + LAST_FETCH_DATE_FOLDER, + CRM_IMPORT_DATA_FOLDER, + LAST_FETCH_DATE_BACKUP_FOLDER, + RESPONSE_JSON_BACKUP_FOLDER, + CRM_IMPORT_DATA_BACKUP_FOLDER +) + +# environments(task settings file) +LOG_LEVEL = os.environ.get(LOG_LEVEL, LOG_LEVEL_INFO) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +CRM_AUTH_TIMEOUT = os.environ["CRM_AUTH_TIMEOUT"] # CRMへの認証処理のタイムアウト秒数 +CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ["CRM_AUTH_MAX_RETRY_ATTEMPT"] # CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_RETRY_INTERVAL = os.environ["CRM_AUTH_RETRY_INTERVAL"] # CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = os.environ["CRM_AUTH_RETRY_MIN_INTERVAL"] # CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = os.environ["CRM_AUTH_RETRY_MAX_INTERVAL"] # CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = os.environ["CRM_GET_RECORD_COUNT_TIMEOUT"] # CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ["CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT"] # CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_INTERVAL"] # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL"] # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL"] # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_TIMEOUT = os.environ["CRM_FETCH_RECORD_TIMEOUT"] # CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ["CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT"] # CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_INTERVAL"] # CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MIN_INTERVAL"] # CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MAX_INTERVAL"] # CRMのレコード取得処理のリトライ時の最大待ち秒数 + +# environments(ECS Task Enviroment) +CRM_AUTH_DOMAIN = os.environ["CRM_AUTH_DOMAIN"] # CRMのAPI実行のための認証エンドポイントのドメイン +CRM_USER_NAME = os.environ["CRM_USER_NAME"] # CRMのAPI実行用ユーザ名 +CRM_USER_PASSWORD = os.environ["CRM_USER_PASSWORD"] # CRMのAPI実行用ユーザパスワード +CRM_USER_SECURITY_TOKEN = os.environ["CRM_USER_SECURITY_TOKEN"] # CRMのAPI実行用ユーザのセキュリティトークン +CRM_CONFIG_BUCKET = os.environ["CRM_CONFIG_BUCKET"] # CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_BACKUP_BUCKET = os.environ["CRM_BACKUP_BUCKET"] # CRMのバックアップデータを格納するバケット名 +IMPORT_DATA_BUCKET = os.environ["IMPORT_DATA_BUCKET"] # CRMの取込データを格納するバケット名 +OBJECT_INFO_FOLDER = os.environ["OBJECT_INFO_FOLDER"] # CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FILENAME = os.environ["OBJECT_INFO_FILENAME"] # CRM取得対象オブジェクトの情報のファイル名 +PROCESS_RESULT_FOLDER = os.environ["PROCESS_RESULT_FOLDER"] # CRMデータ取得結果を格納するフォルダパス +PROCESS_RESULT_FILENAME = os.environ["PROCESS_RESULT_FILENAME"] # CRMデータ取得結果を格納するファイル名 +LAST_FETCH_DATE_FOLDER = os.environ["LAST_FETCH_DATE_FOLDER"] # CRMからの最終取得日時ファイルを格納するフォルダパス +CRM_IMPORT_DATA_FOLDER = os.environ["CRM_IMPORT_DATA_FOLDER"] # CRMから取得し、取込用に変換したデータを格納するフォルダ +LAST_FETCH_DATE_BACKUP_FOLDER = os.environ["LAST_FETCH_DATE_BACKUP_FOLDER"] # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +RESPONSE_JSON_BACKUP_FOLDER = os.environ["FETCH_DATA_BACKUP_FOLDER"] # CRMから取得した生データのバックアップを格納するフォルダパス +CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ["CRM_IMPORT_DATA_BACKUP_FOLDER"] # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ diff --git a/ecs/crm-datafetch/src/error/__init__.py b/ecs/crm-datafetch/src/error/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/error/exceptions.py b/ecs/crm-datafetch/src/error/exceptions.py new file mode 100644 index 00000000..528c262d --- /dev/null +++ b/ecs/crm-datafetch/src/error/exceptions.py @@ -0,0 +1,45 @@ +from abc import ABCMeta + + +class MeDaCaCRMDataFetchException(Exception, metaclass=ABCMeta): + """MeDaCaシステム固有のカスタムエラークラス""" + + def __init__(self, error_id: str, func_name, message) -> None: + super().__init__(message) + self.func_name = func_name + self.error_id = error_id + + +class FileNotFoundException(MeDaCaCRMDataFetchException): + """S3のファイルが見つからない場合の例外""" + pass + + +class FileUploadException(MeDaCaCRMDataFetchException): + """S3のファイルアップロード失敗の例外""" + pass + + +class InvalidConfigException(MeDaCaCRMDataFetchException): + """Configのバリデーションチェック失敗の例外""" + pass + + +class DataConvertException(MeDaCaCRMDataFetchException): + """データ変換が失敗した場合の例外""" + pass + + +class TimeOutException(MeDaCaCRMDataFetchException): + """タイムアウトが発生した場合の例外""" + pass + + +class RetryExceededException(MeDaCaCRMDataFetchException): + """リトライ処理が発生した場合の例外""" + pass + + +class SalesforceAPIException(MeDaCaCRMDataFetchException): + """SalseforceのAPI実行失敗が発生した場合の例外""" + pass diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch.py index e69de29b..bac313fd 100644 --- a/ecs/crm-datafetch/src/fetch.py +++ b/ecs/crm-datafetch/src/fetch.py @@ -0,0 +1,71 @@ +from src.util.logger import Logger +from src.constants import( + FETCH_JP_NAME +) +from src.error.exceptions import( + SalesforceAPIException, + DataConvertException +) +from src.salesforce.salesforce_api import( + SalesForceCount, + SalesForceData +) + + +logger = Logger().get_logger() + + +def fetch_crm_data(target_object, last_fetch_datetime): + # ① CRMデータ取得処理開始ログを出力する + logger.info( + f'I-FETCH-01 [{target_object.object_name}] のCRMからのデータ取得処理を開始します') + + object_name = target_object.object_name + columns = ','.join(target_object.columns) + last_fetch_datetime_from = last_fetch_datetime.last_fetch_datetime_from + last_fetch_datetime_to = last_fetch_datetime.last_fetch_datetime_to + + try: + # ② 取得対象オブジェクトの取得期間内のレコード件数を取得する + logger.info(f'I-FETCH-02 [{object_name}] の件数取得を開始します') + + saleforce_count = SalesForceCount() + record_count = saleforce_count.fetch_sf_count_retry( + object_name, last_fetch_datetime_from, last_fetch_datetime_to) + + logger.info(f'I-FETCH-03 [{object_name}] の件数：[{record_count}]') + + except Exception as e: + raise SalesforceAPIException( + 'E-FETCH-01', FETCH_JP_NAME, f'[{object_name}] の件数取得に失敗しました [{e}]') + + try: + # ③ 取得対象オブジェクトのレコードを取得する + logger.info(f'I-FETCH-04 [{object_name}] のレコード取得を開始します') + + saleforce_data = SalesForceData() + record_generator = saleforce_data.fetch_sf_data_retry( + columns, object_name, last_fetch_datetime_from, last_fetch_datetime_to) + + except Exception as e: + raise SalesforceAPIException( + 'E-FETCH-02', FETCH_JP_NAME, f'[{object_name}] のレコード取得に失敗しました [{e}]') + + try: + # ④ 取得対象オブジェクトをJSONに変換 + logger.info(f'I-FETCH-05 [{object_name}] のレコードをJSONに変換します') + + sf_object_jsons = [] + + for record in record_generator: + sf_object_jsons.append(record) + + except Exception as e: + raise DataConvertException( + 'E-FETCH-03', FETCH_JP_NAME, f'[{object_name}] のレコードのJSON変換に失敗しました [{e}]') + + # ⑤ CRMデータ取得処理終了ログを出力する + logger.info(f'I-FETCH-06 [{object_name}] のレコード取得が成功しました') + + # ⑥ 次の処理へ移行する + return sf_object_jsons diff --git a/ecs/crm-datafetch/src/parser/__init__.py b/ecs/crm-datafetch/src/parser/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parse.py new file mode 100644 index 00000000..4273125d --- /dev/null +++ b/ecs/crm-datafetch/src/parser/json_parse.py @@ -0,0 +1,15 @@ +import re +import json +from src.constants import EXCLUDE_SYMBOL + +class JsonParser(): + def __init__(self, json_str) -> None: + self.__json_str = json_str + + def json_parser(self) -> dict: + for symbol in EXCLUDE_SYMBOL: + # コメントアウトシンボルを含む部分を置き換える正規表現 + replace_comment_regex = rf'\s(?!\"){symbol}[\s\S]*?.*' + self.__json_str = re.sub(replace_comment_regex, '', self.__json_str) + + return json.loads(self.__json_str) diff --git a/ecs/crm-datafetch/src/pre.py b/ecs/crm-datafetch/src/pre.py index 546cbc37..50071fe1 100644 --- a/ecs/crm-datafetch/src/pre.py +++ b/ecs/crm-datafetch/src/pre.py @@ -1,96 +1,83 @@ -# モジュールロード from datetime import datetime -from src.utils.logger import Logger -import src.utils.data_retention as data_retention -import src.utils.s3_access as s3_access -import src.utils.json_parser as json_parser +from src.util.logger import Logger +from src.constants import( + PRE_JP_NAME, + YYYYMMDDTHHMMSSTZ +) +from src.environments import( + CRM_CONFIG_BUCKET, + OBJECT_INFO_FOLDER, + OBJECT_INFO_FILENAME +) +from src.error.exceptions import( + FileNotFoundException, + InvalidConfigException +) +from src.aws.s3 import ConfigBucket +from src.parser.json_parse import JsonParser +from src.config.objects import FetchTargetObjects -# 変数 -# .000ZはUTCを表す。ミリ秒までの考慮は不要なので固定で指定 -YYYYMMDDTHHMMSSTZ = '%Y-%m-%dT%H:%M:%S.000Z' -CRM_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.000%z' -YYYYMMDDHHMMSS = '%Y-%m-%d %H:%M:%S' - - -# logger設定 logger = Logger().get_logger() -# 処理 +def prepare_get_data(): + # ① データ取得準備処理の開始ログを出力する + logger.info('I-PRE-01 データ取得準備処理を開始します') + + # ② 取得処理開始年月日時分秒を控える + execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) + date_path = execute_datetime.rstrip('000Z').translate( + str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) + + logger.info(f'I-PRE-02 データ取得処理開始日時：{execute_datetime}') -def pre(crm_object_info_var): try: - # ① データ取得準備処理の開始ログを出力する - logger.info('I-PRE-01 データ取得準備処理を開始します') + # ③ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する + object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' + logger.debug( + f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') - # ② 取得処理開始年月日時分秒を控える - execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) - logger.info(f'I-PRE-02 データ取得処理開始日時：{execute_datetime}') + config_bucket = ConfigBucket() + object_info_file_json = config_bucket.get_object_info_file() - # ⑤ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する - - try: - crm_object_info_var = data_retention.CrmObjectInfoVar() - s3_fullpath = f's3://{crm_object_info_var.bucket}/{crm_object_info_var.file_name_key}' - logger.debug(f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{s3_fullpath}]') - - get_file_s3 = s3_access.GetFileS3() - crm_object_info_json = get_file_s3.get_file_from_s3(crm_object_info_var.bucket, crm_object_info_var.file_name_key) - - logger.debug('D-PRE-04 CRM_取得オブジェクト情報ファイルの取得成功しました') - - except Exception as e: - logger.error(f'E-PRE-01 CRM_取得オブジェクト情報ファイルが存在しません　ファイル名：[{s3_fullpath}]', e) - raise e - - - # ⑥ CRM_取得オブジェクト情報ファイルをパースし、メモリ上に展開する - try: - logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします') - - json_exclusde_comment = json_parser.JsonExculudeComment() - crm_object_info_dict: dict = json_exclusde_comment(crm_object_info_json) - - logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました') - - except Exception as e: - logger.error(f'E-PRE-02 CRM_取得オブジェクト情報ファイルのパースに失敗しました　エラー内容：[{e}]') - raise e - - # ⑦ メモリ上のCRM_取得オブジェクト情報のキーobjectsの形式チェックを行う - try: - logger.debug('D-PRE-07 CRM_取得オブジェクト情報ファイルの形式チェックを開始します') - - if 'objects' not in crm_object_info_dict: - logger.error(f'E-PRE-03 CRM_取得オブジェクト情報ファイル「objects」キーは必須です　ファイル名：[{s3_fullpath}]') - raise e - - if not isinstance(crm_object_info_dict["objects"], list): - logger.error(f'E-PRE-04 CRM_取得オブジェクト情報ファイル「objects」キーの値は配列でなければなりません　ファイル名：[{s3_fullpath}]') - raise e - - logger.debug('D-PRE-07 CRM_取得オブジェクト情報ファイルの形式チェック 正常終了') - - except Exception as e: - logger.error(f'CRM_取得オブジェクト情報ファイル「objects」キーの値は配列でなければなりません　ファイル名：[{s3_fullpath}]', e) - raise e - - # ⑧ 処理結果出力用のマップを初期化 - process_result_per_object = {} - for sfdc_object in crm_object_info_dict.get('objects'): - object_name = sfdc_object.get('object_name') - process_result_per_object[object_name] = 'fail' - - - - # ⑨ データ取得準備処理の終了ログを出力する - logger.info('I-PRE-09 データ取得準備処理を終了します') - - # ⑩ 次の処理へ移行する - return(crm_object_info_dict, execute_datetime, process_result_per_object) + logger.debug('D-PRE-04 CRM_取得オブジェクト情報ファイルの取得成功しました') except Exception as e: - logger.error(f'データ取得準備処理で想定外のエラーが発生しました　エラー内容：[{e}]', e) - raise e + raise FileNotFoundException( + 'E-PRE-01', PRE_JP_NAME, f'CRM_取得オブジェクト情報ファイルが存在しません ファイル名：[{OBJECT_INFO_FILENAME}] エラー内容：[{e}]') + + try: + # ④ CRM_取得オブジェクト情報ファイルをパースし、メモリ上に展開する + logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします') + + json_parser = JsonParser(object_info_file_json) + object_info_file_dict = json_parser.json_parser() + + logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました') + + except Exception as e: + raise InvalidConfigException( + 'E-PRE-02', PRE_JP_NAME, f'CRM_取得オブジェクト情報ファイルのパースに失敗しました エラー内容：[{e}]') + + # ⑤ メモリ上のCRM_取得オブジェクト情報のキーobjectsの形式チェックを行う + try: + logger.debug('D-PRE-07 CRM_取得オブジェクト情報ファイルの形式チェックを開始します') + + fetch_target_objects = FetchTargetObjects(object_info_file_dict) + + logger.debug('D-PRE-08 CRM_取得オブジェクト情報ファイルの形式チェック 正常終了') + + except Exception as e: + raise InvalidConfigException( + 'E-PRE-03', PRE_JP_NAME, f'CRM_取得オブジェクト情報ファイルの形式チェックに失敗しました ファイル名：[{OBJECT_INFO_FILENAME}] エラー内容：[{e}]') + + # ⑥ 処理結果出力用のマップを初期化 + process_result = {} + + # ⑦ データ取得準備処理の終了ログを出力する + logger.info('I-PRE-09 データ取得準備処理を終了します') + + # ⑧ 次の処理へ移行する + return(fetch_target_objects, execute_datetime, date_path, process_result) diff --git a/ecs/crm-datafetch/src/process.py b/ecs/crm-datafetch/src/process.py new file mode 100644 index 00000000..5ccbd716 --- /dev/null +++ b/ecs/crm-datafetch/src/process.py @@ -0,0 +1,150 @@ +from src.pre import prepare_get_data # データ取得準備処理 +from src.chk import check_object_info # オブジェクト情報形式チェック処理 +from src.date import set_datetime_period # データ取得期間設定処理 +from src.fetch import fetch_crm_data # CRMデータ取得処理 +from src.resbk import backup_crm_data # CRM電文データバックアップ処理 +from src.conv import convert_crm_csvdata # CSV変換処理 +from src.csvbk import backup_crm_csvdata # CSVバックアップ処理 +from src.upld import copy_crm_csvdata # CSVアップロード処理 +from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 +from src.end import updload_result_data # 取得処理実施結果アップロード処理 +from src.util.logger import Logger +from src.error.exceptions import( + MeDaCaCRMDataFetchException +) + +logger = Logger().get_logger() + + +def main() -> None: + try: + # ① CRMデータ取得処理開始ログを出力する + logger.info('I-CTRL-01 CRMデータ取得処理を開始します') + + fetch_target_objects = None # オブジェクト情報ファイル用オブジェクト + execute_datetime = None # 実行日次文字列 + date_path = None # 実行日次のパス文字列 + process_result = None # オブジェクトごとの実行結果JSON + + # ② データ取得準備処理を呼び出す + logger.info('I-CTRL-02 データ取得準備処理呼び出し') + + fetch_target_objects, execute_datetime, date_path, process_result = prepare_get_data() + + # ③ object_infoのobjectsキーの値の件数分ループする + logger.info('I-CTRL-03 取得対象オブジェクトのループ処理開始') + + for object_info in fetch_target_objects: + try: + # 1. オブジェクト処理結果の初期化 + target_object = None # オブジェクトごとの情報格納用オブジェクト + last_fetch_datetime = None # オブジェクトごとの取得日付格納用オブジェクト + sf_object_jsons = None # オブジェクトごとのSalesforce取得変数JSON + csv_object = None # CSVオブジェクト + process_result[object_info.get('object_name')] = 'fail' # オブジェクト処理結果 + + logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：{object_info}') + + # 2. オブジェクト情報形式チェック処理を呼び出す + logger.info('I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') + + target_object = check_object_info(object_info, execute_datetime) + + # 3. 処理対象のオブジェクト名をログ出力する + logger.info( + f'I-CTRL-06 [{target_object.object_name}]のデータ取得を開始します') + + # 4. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する + if target_object.is_skip is True: + logger.info( + f'I-CTRL-07 [{target_object.object_name}]のデータ取得処理をスキップします') + continue + + # 5. データ取得期間設定処理を呼び出す + logger.info( + f'I-CTRL-08 [{target_object.object_name}]のデータ取得期間設定処理呼び出し') + + last_fetch_datetime = set_datetime_period(target_object, execute_datetime) + + # 6. CRMデータ取得処理を呼び出す + logger.info( + f'I-CTRL-09 [{target_object.object_name}]のデータ取得処理呼び出し') + + sf_object_jsons = fetch_crm_data(target_object, last_fetch_datetime) + + # 7. 出力ファイル名をログ出力する + logger.info( + f'I-CTRL-10 [{target_object.object_name}] の出力ファイル名は [{target_object.upload_file_name}]となります') + + # 8. CRM電文データバックアップ処理を呼び出す + logger.info( + f'I-CTRL-11 [{target_object.object_name}] CRM電文データバックアップ処理呼び出し') + backup_crm_data(target_object.object_name, sf_object_jsons, date_path) + + # 9. CSV変換処理を呼び出す + logger.info( + f'I-CTRL-12 [{target_object.object_name}] CSV変換処理呼び出し') + csv_object = convert_crm_csvdata(target_object, sf_object_jsons) + + # 10. CSVバックアップ処理を呼び出す + logger.info( + f'I-CTRL-13 [{target_object.object_name}] CSVデータバックアップ処理呼び出し') + backup_crm_csvdata(target_object, date_path, csv_object) + + # 11. CSVアップロード処理を呼び出す + logger.info( + f'I-CTRL-14 [{target_object.object_name}] CSVデータアップロード処理呼び出し') + copy_crm_csvdata(target_object, date_path) + + # 12. 前回取得日時ファイル更新処理を呼びだす + logger.info( + f'I-CTRL-15 [{target_object.object_name}] 前回取得日時ファイル更新処理呼び出し') + updload_last_fetch_datetime(target_object, last_fetch_datetime) + + # 13. オブジェクト処理結果の更新 + process_result[target_object.object_name] = 'success' + + # 14. オブジェクトのアップロードが完了した旨をログに出力する + logger.info(f'I-CTRL-16 [{target_object.object_name}] 処理正常終了') + + except MeDaCaCRMDataFetchException as e: + logger.error(f'{e.error_id} {e}') + logger.exception( + f'I-ERR-03 [{object_info.get("object_name")}] の[{e.func_name}]でエラーが発生しました 次のオブジェクトの処理に移行します') + continue + + except Exception as e: + logger.exception( + f'I-ERR-04 [{object_info.get("object_name")}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e) + continue + + # ④ すべてのオブジェクトの処理が完了したことと、オブジェクト毎の処理結果をログに出力する + logger.info(f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result}]') + + # ⑤ 取得処理実施結果アップロード処理を呼び出す + logger.info('I-CTRL-19 CRM_取得処理実施結果ファイルアップロード処理開始') + updload_result_data(process_result, date_path) + + # ⑥ 最終結果をチェックし、チェック結果をログに出力 + if not all([v == 'success' for v in process_result.values()]): + logger.error('E-CTRL-01 一部のデータ取得に失敗しています。') + else: + logger.info('I-CTRL-18 すべてのデータの取得に成功しました。') + + # ⑦ CRMデータ取得処理終了ログを出力する + logger.info('I-CTRL-20 CRMデータ取得処理を終了します') + + return exit(0) + + except MeDaCaCRMDataFetchException as e: + logger.error(f'{e.error_id} {e}') + logger.exception(f'I-ERR-01 [{e.func_name}]でエラーが発生したため、処理を終了します') + return exit(0) + + except Exception as e: + logger.exception('I-ERR-02 予期せぬエラーが発生したため、処理を終了します', e) + return exit(0) + + +if __name__ == '__main__': + main() diff --git a/ecs/crm-datafetch/src/resbk.py b/ecs/crm-datafetch/src/resbk.py index e69de29b..02b3c53b 100644 --- a/ecs/crm-datafetch/src/resbk.py +++ b/ecs/crm-datafetch/src/resbk.py @@ -0,0 +1,35 @@ +from src.util.logger import Logger +from src.constants import( + RESBK_JP_NAME +) +from src.error.exceptions import( + FileUploadException +) +from src.aws.s3 import BackupBucket + + +logger = Logger().get_logger() + + +def backup_crm_data(object_name, sf_object_jsons, date_path): + # ① CRM電文データバックアップ処理の開始ログを出力する + logger.info(f'I-RESBK-01 [{object_name}] のCRM電文データバックアップ処理を開始します') + + try: + # ② CRMバックアップ保管用バケットに、CRMから取得したJSONの電文データのバックアップを保管する + file_name = f'{date_path}/{object_name}.json' + + backup_bucket = BackupBucket() + backup_bucket.put_response_json(file_name, sf_object_jsons) + + logger.debug(f'D-RESBK-02 [{object_name}] のJSONデータバックアップ 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-RESBK-01', RESBK_JP_NAME, f'[{object_name}] 電文データのバックアップに失敗しました ファイル名：[{object_name}.json] エラー内容:[{e}]') + + # ③ CRM電文データバックアップ処理の終了ログを出力する + logger.info(f'I-RESBK-03 [{object_name}] のCRM電文データバックアップ処理を終了します') + + # ④ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/salesforce/__init__.py b/ecs/crm-datafetch/src/salesforce/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py new file mode 100644 index 00000000..ad466782 --- /dev/null +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -0,0 +1,78 @@ +from tenacity import retry, stop_after_attempt, stop_after_delay +from tenacity.wait import wait_exponential +from simple_salesforce import Salesforce +from src.util.async_retry import TimeoutManager, AsyncRetry +from src.environments import( + CRM_AUTH_DOMAIN, + CRM_USER_NAME, + CRM_USER_PASSWORD, + CRM_USER_SECURITY_TOKEN, + CRM_AUTH_TIMEOUT, + CRM_AUTH_MAX_RETRY_ATTEMPT, + CRM_AUTH_RETRY_INTERVAL, + CRM_AUTH_RETRY_MIN_INTERVAL, + CRM_AUTH_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_TIMEOUT, # CRMのレコード取得処理のタイムアウト秒数 + CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + CRM_FETCH_RECORD_RETRY_MAX_INTERVAL +) + +FETCH_SOQL = """SELECT {column_or_expression} FROM {object_name} + WHERE SystemModStamp > {last_update_datetime_from} + AND SystemModStamp <= {last_update_datetime_to} +""" + + +class SalesfoeceApi(): + def __init__(self) -> None: + self.__sf = Salesforce(username=CRM_USER_NAME, password=CRM_USER_PASSWORD, + security_token=CRM_USER_SECURITY_TOKEN, + domain=CRM_AUTH_DOMAIN + ) + + def sf_query(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + return self.__sf.query(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) + + def sf_query_all_iter(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + return self.__sf.query_all_iter(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) + + +class SalesForceCount(): + def fetch_sf_count(self, object_name, last_update_datetime_from, last_update_datetime_to): + count_soql = FETCH_SOQL.format( + column_or_expression='COUNT(Id)', + object_name=object_name, + last_update_datetime_from=last_update_datetime_from, + last_update_datetime_to=last_update_datetime_to + ) + self.__sf = SalesfoeceApi() + count_res = self.__sf.sf_query( + count_soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) + return count_res.get('records')[0].get('expr0') + + @retry(wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT) | stop_after_delay(CRM_GET_RECORD_COUNT_TIMEOUT)) + def fetch_sf_count_retry(self, object_name, last_update_datetime_from, last_update_datetime_to): + return self.fetch_sf_count(object_name, last_update_datetime_from, last_update_datetime_to) + + +class SalesForceData(): + def fetch_sf_data(self, columns, object_name, last_update_datetime_from, last_update_datetime_to): + soql = FETCH_SOQL.format( + column_or_expression=columns, + object_name=object_name, + last_update_datetime_from=last_update_datetime_from, + last_update_datetime_to=last_update_datetime_to + ) + self.__sf = SalesfoeceApi() + return self.__sf.sf_query_all_iter(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + + @retry(wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT) | stop_after_delay(CRM_FETCH_RECORD_TIMEOUT)) + def fetch_sf_data_retry(self, columns, object_name, last_update_datetime_from, last_update_datetime_to): + return self.fetch_sf_data(columns, object_name, last_update_datetime_from, last_update_datetime_to) diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upd.py index e69de29b..4e3cb90d 100644 --- a/ecs/crm-datafetch/src/upd.py +++ b/ecs/crm-datafetch/src/upd.py @@ -0,0 +1,49 @@ +import json +from src.util.logger import Logger +from src.aws.s3 import ConfigBucket +from src.constants import ( + UPD_JP_NAME +) +from src.error.exceptions import( + FileUploadException +) + + +logger = Logger().get_logger() + + +def updload_last_fetch_datetime(target_object, last_fetch_datetime): + # ① 前回取得日時ファイル更新処理の開始ログを出力する + logger.info( + f'I-UPD-01 [{target_object.object_name}] の前回取得日時ファイルの更新処理を開始します') + + try: + if target_object.is_update_last_fetch_datetime == False: + # ② オブジェクト情報.is_update_last_fetch_datetimeがfalseの場合、以降の処理をスキップする + logger.info( + f'I-UPD-02 [{target_object.object_name}] の前回取得日時ファイルの更新処理をスキップします') + else: + # ③ 前回取得日時ファイル.last_fetch_datetime_fromに取得処理開始年月日時分秒を設定する + # 前回取得日時ファイル.last_fetch_datetime_toに空文字を設定する + last_fetch_datetime_dict = { + 'last_fetch_datetime_from': last_fetch_datetime.last_fetch_datetime_to, + 'last_fetch_datetime_to': '' + } + + config_bucket = ConfigBucket() + config_bucket.put_last_fetch_datetime_file( + target_object.last_fetch_datetime_file_name, json.dumps(last_fetch_datetime_dict)) + + logger.info( + f'D-UPD-03 [{target_object.object_name}] の前回取得日時ファイル更新処理 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-UPD-01', UPD_JP_NAME, f'[{target_object.object_name}] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[{target_object.last_fetch_datetime_file_name}] エラー内容:[{e}]') + + # ④ 前回取得日時ファイル更新処理の終了ログを出力する + logger.info( + f'I-UPD-04 [{target_object.object_name}] の前回取得日時ファイルの更新処理を終了します') + + # ⑤ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/upld.py b/ecs/crm-datafetch/src/upld.py index e69de29b..57e33e25 100644 --- a/ecs/crm-datafetch/src/upld.py +++ b/ecs/crm-datafetch/src/upld.py @@ -0,0 +1,43 @@ +from src.util.logger import Logger +from src.constants import ( + UPLD_JP_NAME +) +from src.environments import( + CRM_BACKUP_BUCKET, + IMPORT_DATA_BUCKET, + CRM_IMPORT_DATA_FOLDER, + CRM_IMPORT_DATA_BACKUP_FOLDER, +) +from src.error.exceptions import( + FileUploadException +) +from src.aws.s3 import S3ResourceNonBucket + + +logger = Logger().get_logger() + + +def copy_crm_csvdata(target_object, date_path): + # ① CSVデータアップロード処理の開始ログを出力する + logger.info( + f'I-UPLD-01 [{target_object.object_name}] のCSVデータアップロード処理を開始します ファイル名：[{target_object.upload_file_name}.csv]') + + try: + # ② CRMバックアップ保管用バケットに保管した変換後のCSVデータをデータ取込バケットにコピーする + s3_resource_non_bucket = S3ResourceNonBucket() + s3_resource_non_bucket.copy(CRM_BACKUP_BUCKET, f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{date_path}/{target_object.upload_file_name}.csv', + IMPORT_DATA_BUCKET, f'{CRM_IMPORT_DATA_FOLDER}/{target_object.upload_file_name}.csv') + + logger.debug( + f'D-UPLD-02 [{target_object.object_name}] のCSVデータアップロード 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-UPLD-01', UPLD_JP_NAME, f'[{target_object.object_name}] CSVデータのアップロードに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') + + # ③ CSVデータアップロード処理の終了ログを出力する + logger.info( + f'I-UPLD-03 [{target_object.object_name}] のCSVデータのアップロード処理を終了します') + + # ④ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/util/__init__.py b/ecs/crm-datafetch/src/util/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py new file mode 100644 index 00000000..a8bfed62 --- /dev/null +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -0,0 +1,64 @@ +import re + + +class DictCheck: + def __init__(self) -> None: + pass + + def check_key_exist(self, object_dict: dict, check_key: str) -> bool: + ''' + 辞書型キー存在チェック + ''' + return True if check_key in object_dict and object_dict[check_key] != '' else False + + def check_value_type(self, object_dict: dict, check_key: str, check_type: type) -> bool: + ''' + 辞書型バリュー型チェック + ''' + return True if isinstance(object_dict[check_key], check_type) else False + + def check_regex(self, regex_str: str, object_dict: dict, check_key: str) -> bool: + ''' + 辞書型バリュー正規表現チェック + ''' + return True if re.fullmatch(regex_str, object_dict[check_key]) else False + + def check_key_exist_and_value_type(self, object_dict: dict, check_key: str, check_type: type) -> None: + ''' + 辞書型キー存在チェック＆バリュー型チェック + ''' + if not self.check_key_exist(object_dict, check_key): + raise Exception(f'「{check_key}」キーは必須です') + + elif not self.check_value_type(object_dict, check_key, check_type): + raise Exception(f'「{check_key}」キーの値は「{check_type}」でなければなりません') + + def check_key_exist_case_value_type(self, object_dict: dict, check_key: str, check_type: type): + ''' + 辞書型キー存在した場合のバリュー型チェック + ''' + if not self.check_key_exist(object_dict, check_key): + pass + + elif not self.check_value_type(object_dict, check_key, check_type): + raise Exception(f'「{check_key}」キーの値は「{check_type}」でなければなりません') + + def check_key_exsit_and_regex(self, object_dict: dict, check_key: str, regex_str: str): + ''' + 辞書型キー存在チェック＆バリュー正規表現チェック + ''' + if not self.check_key_exist(object_dict, check_key): + raise Exception(f'「{check_key}」キーは必須です') + + elif not self.check_regex(regex_str, object_dict, check_key): + raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') + + def check_key_exsit_case_regex(self, object_dict: dict, check_key: str, regex_str: str): + ''' + 辞書型キー存在した場合のバリュー正規表現チェック + ''' + if not self.check_key_exist(object_dict, check_key): + pass + + elif not self.check_regex(regex_str, object_dict, check_key): + raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py new file mode 100644 index 00000000..d9892da8 --- /dev/null +++ b/ecs/crm-datafetch/src/util/logger.py @@ -0,0 +1,29 @@ +import logging +from src.environments import LOG_LEVEL + + +class Logger(): + __logger: logging.Logger + + def __init__(self): + self.__logger = logging.getLogger() + + level = logging.getLevelName(LOG_LEVEL) + if not isinstance(level, int): + level = logging.INFO + self.__logger.setLevel(level) + + if not self.__logger.hasHandlers(): + handler = logging.StreamHandler() + self.__logger.addHandler(handler) + + formatter = logging.Formatter( + '[%(levelname)s]\t%(asctime)s\t%(message)s\n', + '%Y-%m-%d %H:%M:%S' + ) + + for handler in self.__logger.handlers: + handler.setFormatter(formatter) + + def get_logger(self) -> logging.Logger: + return self.__logger diff --git a/ecs/crm-datafetch/src/utils/data_retention.py b/ecs/crm-datafetch/src/utils/data_retention.py deleted file mode 100644 index 89d7d4c3..00000000 --- a/ecs/crm-datafetch/src/utils/data_retention.py +++ /dev/null @@ -1,169 +0,0 @@ -# モジュールロード -import dataclasses -#from fileinput import filename -import os -#from tracemalloc import DomainFilter - -# 変数 -# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する -LOG_LEVEL = os.environ["LOG_LEVEL"] - -CRM_AUTH_TIMEOUT = os.environ["CRM_AUTH_TIMEOUT"] # CRMへの認証処理のタイムアウト秒数 -# CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ["CRM_AUTH_MAX_RETRY_ATTEMPT"] -# CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_INTERVAL = os.environ["CRM_AUTH_RETRY_INTERVAL"] -# CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ["CRM_AUTH_RETRY_MIN_INTERVAL"] -# CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ["CRM_AUTH_RETRY_MAX_INTERVAL"] - -# CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ["CRM_GET_RECORD_COUNT_TIMEOUT"] -# CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ["CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT"] -# CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_INTERVAL"] -# CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL"] -# CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL"] - -# CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ["CRM_FETCH_RECORD_TIMEOUT"] -# CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ["CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT"] -# CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_INTERVAL"] -# CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MIN_INTERVAL"] -# CRMのレコード取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MAX_INTERVAL"] - - -CRM_AUTH_DOMAIN = os.environ["CRM_AUTH_DOMAIN"] # CRMのAPI実行のための認証エンドポイントのドメイン -CRM_USER_NAME = os.environ["CRM_USER_NAME"] # CRMのAPI実行用ユーザ名 -CRM_USER_PASSWORD = os.environ["CRM_USER_PASSWORD"] # CRMのAPI実行用ユーザパスワード -# CRMのAPI実行用ユーザのセキュリティトークン -CRM_USER_SECURITY_TOKEN = os.environ["CRM_USER_SECURITY_TOKEN"] - -# CRMデータ取得用の設定ファイルを格納するバケット名 -CRM_CONFIG_BUCKET = os.environ["CRM_CONFIG_BUCKET"] -CRM_BACKUP_BUCKET = os.environ["CRM_BACKUP_BUCKET"] # CRMのバックアップデータを格納するバケット名 -IMPORT_DATA_BUCKET = os.environ["IMPORT_DATA_BUCKET"] # CRMの取込データを格納するバケット名 - -# TASK_SETTING_FOLDER = os.environ["TASK_SETTING_FOLDER"] # CRM取得処理タスクの設定ファイルを格納するフォルダパス -# TASK_SETTING_FILENAME = os.environ["TASK_SETTING_FILENAME"] # CRM取得処理タスクの設定ファイルのファイル名 - -# CRM取得対象オブジェクトの情報を格納するフォルダパス -OBJECT_INFO_FOLDER = os.environ["OBJECT_INFO_FOLDER"] -# CRM取得対象オブジェクトの情報のファイル名 -OBJECT_INFO_FILENAME = os.environ["OBJECT_INFO_FILENAME"] - -# CRMデータ取得結果を格納するフォルダパス -FETCH_RESULT_FOLDER = os.environ["FETCH_RESULT_FOLDER"] -# CRMデータ取得結果を格納するファイル名 -FETCH_RESULT_FILENAME = os.environ["FETCH_RESULT_FILENAME"] - -# CRMからの最終取得日時ファイルを格納するフォルダパス -LAST_FETCH_DATE_FOLDER = os.environ["LAST_FETCH_DATE_FOLDER"] -# CRMから取得し、取込用に変換したデータを格納するフォルダ -CRM_IMPORT_DATA_FOLDER = os.environ["CRM_IMPORT_DATA_FOLDER"] - -# CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ["LAST_FETCH_DATE_BACKUP_FOLDER"] -# CRMから取得した生データのバックアップを格納するフォルダパス -FETCH_DATA_BACKUP_FOLDER = os.environ["FETCH_DATA_BACKUP_FOLDER"] -# CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ -CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ["CRM_IMPORT_DATA_BACKUP_FOLDER"] - - -# 処理 -@dataclasses.dataclass -class LogLevel: - loglevel: str = LOG_LEVEL - - -@dataclasses.dataclass -class CrmAuthVar: - auth_domain: str = CRM_AUTH_DOMAIN - user_name: str = CRM_USER_NAME - user_password: str = CRM_USER_PASSWORD - security_token: str = CRM_USER_SECURITY_TOKEN - timeout: int = CRM_AUTH_TIMEOUT - max_retry_attempt: int = CRM_AUTH_MAX_RETRY_ATTEMPT - retry_interval: int = CRM_AUTH_RETRY_INTERVAL - retry_min_interval: int = CRM_AUTH_RETRY_MIN_INTERVAL - retry_max_interval: int = CRM_AUTH_RETRY_MAX_INTERVAL - - -@dataclasses.dataclass -class CrmGetRecordCountVar: - timeout: int = CRM_GET_RECORD_COUNT_TIMEOUT - max_retry_attempt: int = CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT - retry_interval: int = CRM_GET_RECORD_COUNT_RETRY_INTERVAL - retry_min_interval: int = CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL - retry_max_interval: int = CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL - - -@dataclasses.dataclass -class CrmFetchRecordVar: - timeout: int = CRM_FETCH_RECORD_TIMEOUT - max_retry_attempt: int = CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT - retry_interval: int = CRM_FETCH_RECORD_RETRY_INTERVAL - retry_min_interval: int = CRM_FETCH_RECORD_RETRY_MIN_INTERVAL - retry_max_interval: int = CRM_FETCH_RECORD_RETRY_MAX_INTERVAL - - -@dataclasses.dataclass -class CrmBucketVar: - config_bucket: str = CRM_CONFIG_BUCKET - backup_bucket: str = CRM_BACKUP_BUCKET - data_bucket: str = IMPORT_DATA_BUCKET - - -@dataclasses.dataclass -class CrmObjectInfoVar: - bucket: str = CRM_CONFIG_BUCKET - folder: str = OBJECT_INFO_FOLDER - file_name: str = OBJECT_INFO_FILENAME - file_name_key: str = f'{folder}/{file_name}' - - -@dataclasses.dataclass -class FetchResultVar: - folder: str = FETCH_RESULT_FOLDER - file_name: str = FETCH_RESULT_FILENAME - file_name_key: str = f'{folder}/{file_name}' - - -@dataclasses.dataclass -class FetchResultVar: - folder: str = FETCH_RESULT_FOLDER - file_name: str = FETCH_RESULT_FILENAME - file_name_key: str = f'{folder}/{file_name}' - - -@dataclasses.dataclass -class LastFetchDateVar: - folder: str = LAST_FETCH_DATE_FOLDER - - -@dataclasses.dataclass -class CrmImportDataVar: - folder: str = CRM_IMPORT_DATA_FOLDER - - -@dataclasses.dataclass -class LastFetchDateBackupVar: - folder: str = LAST_FETCH_DATE_BACKUP_FOLDER - - -@dataclasses.dataclass -class FetchDataBackupVar: - folder: str = FETCH_DATA_BACKUP_FOLDER - - -@dataclasses.dataclass -class CrmImportDataBackupVar: - folder: str = CRM_IMPORT_DATA_BACKUP_FOLDER diff --git a/ecs/crm-datafetch/src/utils/json_parser.py b/ecs/crm-datafetch/src/utils/json_parser.py deleted file mode 100644 index b6d288a9..00000000 --- a/ecs/crm-datafetch/src/utils/json_parser.py +++ /dev/null @@ -1,28 +0,0 @@ -# モジュールロード -import re -import json - - -# 変数 -SYMBOL = ['#', '/'] - - -# 処理 -class JsonExculudeComment: - def __init__(self) -> None: - pass - - def json_exclude_comment(self, json_str) -> dict: - for symbol in SYMBOL: - # コメントアウトシンボルを含む部分を置き換える正規表現 - replace_comment_regex = rf'\s(?!\"){symbol}[\s\S]*?.*' - - json_str = self.json_regex_parse(replace_comment_regex, json_str) - - result = json.loads(json_str) - - return result - - def json_regex_parse(replace_comment_regex, json_str) -> str: - json_without_comment = re.sub(replace_comment_regex, '', json_str) - return json_without_comment diff --git a/ecs/crm-datafetch/src/utils/logger.py b/ecs/crm-datafetch/src/utils/logger.py deleted file mode 100644 index 5ec076fd..00000000 --- a/ecs/crm-datafetch/src/utils/logger.py +++ /dev/null @@ -1,33 +0,0 @@ -import logging -import os - -LOG_LEVEL = { - 'critical': logging.CRITICAL, - 'error': logging.ERROR, - 'warn': logging.WARNING, - 'info': logging.INFO, - 'debug': logging.DEBUG -} - - -class Logger(): - __logger: logging.Logger - - def __init__(self): - # logger設定 - self.__logger = logging.getLogger() - formatter = logging.Formatter( - '[%(levelname)s]\t%(asctime)s\t%(message)s\n', - '%Y-%m-%d %H:%M:%S' - ) - handler = logging.StreamHandler() - self.__logger.addHandler(handler) - for handler in self.__logger.handlers: - handler.setFormatter(formatter) - level = logging.getLevelName(os.environ.get('LOG_LEVEL', 'WARN').upper()) - if not isinstance(level, int): - level = logging.INFO - self.__logger.setLevel(level) - - def get_logger(self) -> logging.Logger: - return self.__logger diff --git a/ecs/crm-datafetch/src/utils/s3_access.py b/ecs/crm-datafetch/src/utils/s3_access.py deleted file mode 100644 index 7ffb697d..00000000 --- a/ecs/crm-datafetch/src/utils/s3_access.py +++ /dev/null @@ -1,48 +0,0 @@ -# モジュールロード -import boto3 -from tenacity import retry, stop_after_attempt -from tenacity.wait import wait_exponential - -# 変数 -s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') - -# 処理 - - -class UploadFileS3: - def __init__(self, bucket_name, local_file_path, file_name_key) -> None: - self.bucket_name: str = bucket_name - self.local_file_path: str = local_file_path - self.file_name_key: str = file_name_key - - def upload_file_to_s3(self) -> None: - bucket = s3_resource.Bucket(self.bucket_name) - bucket.upload_file(self.local_file_path, self.file_name_key) - - -class GetFileS3: - def __init__(self, bucket_name, file_name_key, multiplier=5, min=5, max=50, count=3) -> None: - self.bucket_name: str = bucket_name - self.file_name_key: str = file_name_key - self.multiplier: int = multiplier - self.min: int = min - self.max: int = max - self.count: int = count - - def get_file_from_s3(self) -> str: - bucket = s3_resource.Bucket(self.bucket_name) - response = bucket.Object(self.file_name_key).get() - body = response['Body'].read() - - return body.decode('utf-8') - - def get_file_from_s3_with_retry(self) -> str: - - @retry(wait=wait_exponential(multiplier=self.multiplier, min=self.min, max=self.max), stop=stop_after_attempt(self.count)) - def get_file_from_s3_with_retry_deco(*args, **kwargs) -> str: - return self.get_file_from_s3(self.bucket_name, self.file_name_key) - - response = get_file_from_s3_with_retry_deco(self) - - return response From 9a62a7eb5869afe60280b135e006835feecc04e0 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Wed, 20 Jul 2022 09:01:48 +0900 Subject: [PATCH 017/275] =?UTF-8?q?fix:=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E4=BF=AE=E6=AD=A3=E3=80=81=E4=BE=8B?= =?UTF-8?q?=E5=A4=96=E5=87=A6=E7=90=86=E3=81=AE=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 54 +++++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 7c183714..f9057a5e 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,28 +1,70 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること - CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN + -- 例外処理 + -- エラーが発生した場合に一時テーブルの削除を実施 + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + DROP TEMPORARY TABLE IF EXISTS make_history_tmp; + SIGNAL SQLSTATE '45000' + SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; + END; + -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する - SET @new_history_save = 'UPDATE @@target_table SET start_datetime = @@target_column, end_datetime = "9999-12-31 00:00:00" WHERE start_datetime IS NULL AND end_datetime IS NULL'; + SET @new_history_save = ' + UPDATE @@target_table + SET + start_datetime = @@target_column + , end_datetime = "9999-12-31 00:00:00" + WHERE + start_datetime IS NULL + AND end_datetime IS NULL + '; SET @new_history_save = REPLACE(@new_history_save, "@@target_table", target_table); SET @new_history_save = REPLACE(@new_history_save, "@@target_column", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 - SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id HAVING count(id) = 2'; + SET @make_history_tmp_create = ' + CREATE TEMPORARY TABLE make_history_tmp + SELECT + id + , MIN(@@target_column) AS min_start_datetime + , MAX(start_datetime) AS max_start_datetime + FROM + @@target_table + WHERE + end_datetime = "9999-12-31 00:00:00" + GROUP BY + id + HAVING + count(id) = 2 + '; SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; EXECUTE make_history_tmp_create_stmt; -- ②-2 「②-1」で取得した全件に更新処理を行う - SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NOT NULL'; + SET @update_end_datetime = ' + UPDATE @@target_table tt + LEFT JOIN make_history_tmp mht + ON tt.id = mht.id + AND tt.start_datetime = mht.min_start_datetime + SET + start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND + WHERE + mht.id IS NOT NULL + '; SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); + SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; -- ②-3 「②-1」で作成した一時テーブルを削除する DROP TEMPORARY TABLE make_history_tmp; + END \ No newline at end of file From 085d561371a60e9fd8031485489ce6e8004d51f2 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Wed, 20 Jul 2022 11:26:58 +0900 Subject: [PATCH 018/275] =?UTF-8?q?refactor:=E5=8D=8A=E8=A7=92=E3=82=B9?= =?UTF-8?q?=E3=83=9A=E3=83=BC=E3=82=B9=E3=82=92=E3=82=BF=E3=83=96=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index f9057a5e..794abe38 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -9,15 +9,15 @@ BEGIN @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; DROP TEMPORARY TABLE IF EXISTS make_history_tmp; SIGNAL SQLSTATE '45000' - SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; + SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; END; -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = ' UPDATE @@target_table SET - start_datetime = @@target_column - , end_datetime = "9999-12-31 00:00:00" + start_datetime = @@target_column + , end_datetime = "9999-12-31 00:00:00" WHERE start_datetime IS NULL AND end_datetime IS NULL From 0079d1110c54d1da43b12e69b3028840c20b2ff7 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Wed, 20 Jul 2022 13:58:59 +0900 Subject: [PATCH 019/275] =?UTF-8?q?refactor:@@=E3=81=8B=E3=82=89=E5=A7=8B?= =?UTF-8?q?=E3=81=BE=E3=82=8B=E6=96=87=E5=AD=97=E3=81=AE=E8=AA=AC=E6=98=8E?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 794abe38..0777912f 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,4 +1,5 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること +-- @@から始まる文字は後からREPLACEする文字を示す独自ルール CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN -- 例外処理 From e96d8be181dd03e3d36550e6a7fb8a1db38a7d30 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Wed, 20 Jul 2022 15:40:14 +0900 Subject: [PATCH 020/275] =?UTF-8?q?fix:=E4=B8=8D=E8=B6=B3=E9=A0=85?= =?UTF-8?q?=E7=9B=AE=E3=81=A8=E8=A8=AD=E5=AE=9A=E9=A0=85=E7=9B=AE=E3=83=9F?= =?UTF-8?q?=E3=82=B9=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 0777912f..00279493 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -18,7 +18,9 @@ BEGIN UPDATE @@target_table SET start_datetime = @@target_column - , end_datetime = "9999-12-31 00:00:00" + , end_datetime = "9999-12-31 00:00:00" + , upd_user = CURRENT_USER() + , upd_date = CURRENT_TIMESTAMP() WHERE start_datetime IS NULL AND end_datetime IS NULL @@ -56,7 +58,9 @@ BEGIN ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET - start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND + end_datetime = mht.max_start_datetime - INTERVAL 1 SECOND + , upd_user = CURRENT_USER() + , upd_date = CURRENT_TIMESTAMP() WHERE mht.id IS NOT NULL '; From e2609471aed60b30deac3546cb86fd3e38113007 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Wed, 20 Jul 2022 18:36:34 +0900 Subject: [PATCH 021/275] =?UTF-8?q?refactor:@@=E2=86=92$$=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4=E3=80=81=E4=B8=80=E6=99=82=E3=83=86=E3=83=BC?= =?UTF-8?q?=E3=83=96=E3=83=AB=E3=81=AE=E3=83=86=E3=83=BC=E3=83=96=E3=83=AB?= =?UTF-8?q?=E5=90=8D=E4=BF=AE=E6=AD=A3=E3=80=81DEALLOCATE=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 45 ++++++++++++++-------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 00279493..a83cc581 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,5 +1,5 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること --- @@から始まる文字は後からREPLACEする文字を示す独自ルール +-- $$から始まる文字は後からREPLACEする文字を示す独自ルール CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN -- 例外処理 @@ -8,16 +8,25 @@ BEGIN BEGIN GET DIAGNOSTICS CONDITION 1 @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; - DROP TEMPORARY TABLE IF EXISTS make_history_tmp; + EXECUTE drop_tmp_table; + DEALLOCATE PREPARE drop_tmp_table; SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; END; + -- ②-3で利用する一時テーブル削除のSQLを準備 + -- 例外処理でも利用するため先に記述 + SET @drop_tmp_table = ' + DROP TEMPORARY TABLE IF EXISTS make_history_tmp_$$target_table + '; + SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table", target_table); + PREPARE new_history_save_stmt from @drop_tmp_table; + -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = ' - UPDATE @@target_table + UPDATE $$target_table SET - start_datetime = @@target_column + start_datetime = $$target_column , end_datetime = "9999-12-31 00:00:00" , upd_user = CURRENT_USER() , upd_date = CURRENT_TIMESTAMP() @@ -25,20 +34,21 @@ BEGIN start_datetime IS NULL AND end_datetime IS NULL '; - SET @new_history_save = REPLACE(@new_history_save, "@@target_table", target_table); - SET @new_history_save = REPLACE(@new_history_save, "@@target_column", target_column); + SET @new_history_save = REPLACE(@new_history_save, "$$target_table", target_table); + SET @new_history_save = REPLACE(@new_history_save, "$$target_column", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; + DEALLOCATE PREPARE ew_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 SET @make_history_tmp_create = ' - CREATE TEMPORARY TABLE make_history_tmp + CREATE TEMPORARY TABLE make_history_tmp_$$target_table SELECT id - , MIN(@@target_column) AS min_start_datetime + , MIN($$target_column) AS min_start_datetime , MAX(start_datetime) AS max_start_datetime FROM - @@target_table + $$target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY @@ -46,15 +56,16 @@ BEGIN HAVING count(id) = 2 '; - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_table", target_table); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_column", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; EXECUTE make_history_tmp_create_stmt; + DEALLOCATE PREPARE make_history_tmp_create_stmt; -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' - UPDATE @@target_table tt - LEFT JOIN make_history_tmp mht + UPDATE $$target_table tt + LEFT JOIN make_history_tmp_$$target_table mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET @@ -64,12 +75,14 @@ BEGIN WHERE mht.id IS NOT NULL '; - SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; + DEALLOCATE PREPARE update_end_datetime_stmt; -- ②-3 「②-1」で作成した一時テーブルを削除する - DROP TEMPORARY TABLE make_history_tmp; + EXECUTE drop_tmp_table; + DEALLOCATE PREPARE drop_tmp_table; END \ No newline at end of file From 611acb6a3722499a874476bc52e29e67636f5e15 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Thu, 21 Jul 2022 09:31:40 +0900 Subject: [PATCH 022/275] =?UTF-8?q?refactor:=E4=B8=80=E6=99=82=E3=83=86?= =?UTF-8?q?=E3=83=BC=E3=83=96=E3=83=AB=E5=90=8D=E3=81=AE=E5=A4=89=E6=9B=B4?= =?UTF-8?q?=E3=80=81=E7=B4=B0=E3=81=8B=E3=81=84=E3=83=9F=E3=82=B9=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index a83cc581..f361a850 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -8,19 +8,19 @@ BEGIN BEGIN GET DIAGNOSTICS CONDITION 1 @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; - EXECUTE drop_tmp_table; - DEALLOCATE PREPARE drop_tmp_table; + EXECUTE drop_tmp_table_stmt; + DEALLOCATE PREPARE drop_tmp_table_stmt; SIGNAL SQLSTATE '45000' - SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; END; -- ②-3で利用する一時テーブル削除のSQLを準備 -- 例外処理でも利用するため先に記述 SET @drop_tmp_table = ' - DROP TEMPORARY TABLE IF EXISTS make_history_tmp_$$target_table + DROP TEMPORARY TABLE IF EXISTS $$target_table_make_history_tmp '; SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table", target_table); - PREPARE new_history_save_stmt from @drop_tmp_table; + PREPARE drop_tmp_table_stmt from @drop_tmp_table; -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = ' @@ -38,11 +38,11 @@ BEGIN SET @new_history_save = REPLACE(@new_history_save, "$$target_column", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; - DEALLOCATE PREPARE ew_history_save_stmt; + DEALLOCATE PREPARE new_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 SET @make_history_tmp_create = ' - CREATE TEMPORARY TABLE make_history_tmp_$$target_table + CREATE TEMPORARY TABLE $$target_table_make_history_tmp SELECT id , MIN($$target_column) AS min_start_datetime @@ -65,7 +65,7 @@ BEGIN -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' UPDATE $$target_table tt - LEFT JOIN make_history_tmp_$$target_table mht + LEFT JOIN $$target_table_make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET @@ -82,7 +82,7 @@ BEGIN DEALLOCATE PREPARE update_end_datetime_stmt; -- ②-3 「②-1」で作成した一時テーブルを削除する - EXECUTE drop_tmp_table; - DEALLOCATE PREPARE drop_tmp_table; + EXECUTE drop_tmp_table_stmt; + DEALLOCATE PREPARE drop_tmp_table_stmt; END \ No newline at end of file From 3b5b0dc33c1b8351ad6f76b134b6d98da3af4344 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 21 Jul 2022 10:01:24 +0900 Subject: [PATCH 023/275] =?UTF-8?q?feat:=E3=83=AD=E3=82=B0=E5=87=BA?= =?UTF-8?q?=E5=8A=9B=E3=82=84=E3=80=81Salesforce=E3=81=AE=E6=8E=A5?= =?UTF-8?q?=E7=B6=9A=E3=83=AA=E3=83=88=E3=83=A9=E3=82=A4=E3=82=92=E4=B8=BB?= =?UTF-8?q?=E3=81=A8=E3=81=97=E3=81=9F=E6=A9=9F=E8=83=BD=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/launch.json | 19 ++++ ecs/crm-datafetch/src/aws/s3.py | 19 ++-- ecs/crm-datafetch/src/chk.py | 5 +- ecs/crm-datafetch/src/config/__init__.py | 0 ecs/crm-datafetch/src/constants.py | 64 +++++------ ecs/crm-datafetch/src/conv.py | 5 +- ecs/crm-datafetch/src/converter/__init__.py | 0 ecs/crm-datafetch/src/csvbk.py | 8 +- ecs/crm-datafetch/src/date.py | 5 +- ecs/crm-datafetch/src/end.py | 5 +- ecs/crm-datafetch/src/environments.py | 101 ++++++------------ ecs/crm-datafetch/src/error/exceptions.py | 10 -- ecs/crm-datafetch/src/fetch.py | 99 +++++++++++++++-- ecs/crm-datafetch/src/pre.py | 5 +- ecs/crm-datafetch/src/process.py | 26 +++-- ecs/crm-datafetch/src/resbk.py | 5 +- .../src/salesforce/salesforce_api.py | 14 +-- ecs/crm-datafetch/src/upd.py | 5 +- ecs/crm-datafetch/src/upld.py | 5 +- ecs/crm-datafetch/src/util/logger.py | 6 ++ 20 files changed, 214 insertions(+), 192 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 ecs/crm-datafetch/src/config/__init__.py create mode 100644 ecs/crm-datafetch/src/converter/__init__.py diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..d8f8414f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,19 @@ +{ + // IntelliSense を使用して利用可能な属性を学べます。 + // 既存の属性の説明をホバーして表示します。 + // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: CRMデータ取得ローカル実行", + "type": "python", + "request": "launch", + // windowsだと\区切りかも + "program": "ecs\\crm-datafetch\\main.py", + "console": "integratedTerminal", + "justMyCode": true, + "envFile": "${workspaceFolder}/.env", + "args": ["local"] + } + ] + } \ No newline at end of file diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index e5235016..229b8666 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -6,6 +6,7 @@ from src.environments import ( IMPORT_DATA_BUCKET, OBJECT_INFO_FOLDER, OBJECT_INFO_FILENAME, + PROCESS_RESULT_FOLDER, CRM_IMPORT_DATA_FOLDER, CRM_IMPORT_DATA_BACKUP_FOLDER, LAST_FETCH_DATE_FOLDER, @@ -49,7 +50,7 @@ class S3ResourceNonBucket: def __init__(self) -> None: self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) - def copy(self, src_bucket, src_key, dest_bucket, dest_key): + def copy(self, src_bucket, src_key, dest_bucket, dest_key) -> None: copy_source = {'Bucket': src_bucket, 'Key': src_key} self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) @@ -62,13 +63,13 @@ class ConfigBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(CRM_CONFIG_BUCKET) - def get_object_info_file(self): + def get_object_info_file(self) -> str: return self.__s3_resource.get_object(f'{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}') - def get_last_fetch_datetime_file(self, file_name): + def get_last_fetch_datetime_file(self, file_name) -> str: return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_name}') - def put_last_fetch_datetime_file(self, file_name, data): + def put_last_fetch_datetime_file(self, file_name, data) -> None: self.__s3_resource.put_object( f'{LAST_FETCH_DATE_FOLDER}/{file_name}', data) return @@ -80,7 +81,7 @@ class DataBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(IMPORT_DATA_BUCKET) - def put_csv(self, file_name, data): + def put_csv(self, file_name, data) -> None: object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_name}' self.__s3_resource.put_object(object_key, data) return @@ -92,16 +93,16 @@ class BackupBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(CRM_BACKUP_BUCKET) - def put_response_json(self, file_name, data): + def put_response_json(self, file_name, data) -> None: object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_name}' self.__s3_resource.put_object(object_key, json.dumps(data)) return - def put_csv_bk(self, file_name, data): + def put_csv_bk(self, file_name, data) -> None: object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_name}' self.__s3_resource.put_object(object_key, data) return - def put_result_json(self, file_name, data): - object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_name}' + def put_result_json(self, file_name, data) -> None: + object_key = f'{PROCESS_RESULT_FOLDER}/{file_name}' self.__s3_resource.put_object(object_key, json.dumps(data)) diff --git a/ecs/crm-datafetch/src/chk.py b/ecs/crm-datafetch/src/chk.py index 1342243e..077f0ee2 100644 --- a/ecs/crm-datafetch/src/chk.py +++ b/ecs/crm-datafetch/src/chk.py @@ -1,4 +1,4 @@ -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import( CHK_JP_NAME ) @@ -8,9 +8,6 @@ from src.error.exceptions import( from src.config.objects import TargetObject -logger = Logger().get_logger() - - def check_object_info(object_info, execute_datetime): # ① オブジェクト情報形式チェック処理開始ログを出力する logger.info('I-CHK-01 オブジェクト情報形式チェック処理を開始します') diff --git a/ecs/crm-datafetch/src/config/__init__.py b/ecs/crm-datafetch/src/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/constants.py b/ecs/crm-datafetch/src/constants.py index fcd6b4a3..a394df16 100644 --- a/ecs/crm-datafetch/src/constants.py +++ b/ecs/crm-datafetch/src/constants.py @@ -1,38 +1,38 @@ # environments(task settings file) -LOG_LEVEL = "LOG_LEVEL" # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する -CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = 'CRM_AUTH_RETRY_MIN_INTERVAL' # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = 'CRM_AUTH_RETRY_MAX_INTERVAL' # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = 'CRM_GET_RECORD_COUNT_TIMEOUT' # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = 'CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT' # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_INTERVAL' # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL' # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL' # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_TIMEOUT = 'CRM_FETCH_RECORD_TIMEOUT' # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = 'CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT' # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_RETRY_INTERVAL = 'CRM_FETCH_RECORD_RETRY_INTERVAL' # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MIN_INTERVAL' # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MAX_INTERVAL' # CRMのレコード取得処理のリトライ時の最大待ち秒数 +LOG_LEVEL = "LOG_LEVEL" # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 +CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = 'CRM_AUTH_RETRY_MIN_INTERVAL' # CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = 'CRM_AUTH_RETRY_MAX_INTERVAL' # CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = 'CRM_GET_RECORD_COUNT_TIMEOUT' # CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = 'CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT' # CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_INTERVAL' # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL' # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL' # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_TIMEOUT = 'CRM_FETCH_RECORD_TIMEOUT' # CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = 'CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT' # CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_RETRY_INTERVAL = 'CRM_FETCH_RECORD_RETRY_INTERVAL' # CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MIN_INTERVAL' # CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MAX_INTERVAL' # CRMのレコード取得処理のリトライ時の最大待ち秒数 # environments(ECS Task Enviroment) -CRM_AUTH_DOMAIN = 'CRM_AUTH_DOMAIN' # CRMのAPI実行のための認証エンドポイントのドメイン -CRM_USER_NAME = 'CRM_USER_NAME' # CRMのAPI実行用ユーザ名 -CRM_USER_PASSWORD = 'CRM_USER_PASSWORD' # CRMのAPI実行用ユーザパスワード -CRM_USER_SECURITY_TOKEN = 'CRM_USER_SECURITY_TOKEN' # CRMのAPI実行用ユーザのセキュリティトークン -CRM_CONFIG_BUCKET = 'CRM_CONFIG_BUCKET' # CRMデータ取得用の設定ファイルを格納するバケット名 -CRM_BACKUP_BUCKET = 'CRM_BACKUP_BUCKET' # CRMのバックアップデータを格納するバケット名 -IMPORT_DATA_BUCKET = 'IMPORT_DATA_BUCKET' # CRMの取込データを格納するバケット名 -OBJECT_INFO_FOLDER = 'OBJECT_INFO_FOLDER' # CRM取得対象オブジェクトの情報を格納するフォルダパス -OBJECT_INFO_FILENAME = 'OBJECT_INFO_FILENAME' # CRM取得対象オブジェクトの情報のファイル名 -PROCESS_RESULT_FOLDER = 'PROCESS_RESULT_FOLDER' # CRMデータ取得結果を格納するフォルダパス -PROCESS_RESULT_FILENAME = 'PROCESS_RESULT_FILENAME' # CRMデータ取得結果を格納するファイル名 -LAST_FETCH_DATE_FOLDER = 'LAST_FETCH_DATE_FOLDER' # CRMからの最終取得日時ファイルを格納するフォルダパス -CRM_IMPORT_DATA_FOLDER ='CRM_IMPORT_DATA_FOLDER' # CRMから取得し、取込用に変換したデータを格納するフォルダ -LAST_FETCH_DATE_BACKUP_FOLDER = 'LAST_FETCH_DATE_BACKUP_FOLDER' # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -RESPONSE_JSON_BACKUP_FOLDER = 'RESPONSE_JSON_BACKUP_FOLDER' # CRMから取得した生データのバックアップを格納するフォルダパス -CRM_IMPORT_DATA_BACKUP_FOLDER = 'CRM_IMPORT_DATA_BACKUP_FOLDER' # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ +CRM_AUTH_DOMAIN = 'CRM_AUTH_DOMAIN' # CRMのAPI実行のための認証エンドポイントのドメイン +CRM_USER_NAME = 'CRM_USER_NAME' # CRMのAPI実行用ユーザ名 +CRM_USER_PASSWORD = 'CRM_USER_PASSWORD' # CRMのAPI実行用ユーザパスワード +CRM_USER_SECURITY_TOKEN = 'CRM_USER_SECURITY_TOKEN' # CRMのAPI実行用ユーザのセキュリティトークン +CRM_CONFIG_BUCKET = 'CRM_CONFIG_BUCKET' # CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_BACKUP_BUCKET = 'CRM_BACKUP_BUCKET' # CRMのバックアップデータを格納するバケット名 +IMPORT_DATA_BUCKET = 'IMPORT_DATA_BUCKET' # CRMの取込データを格納するバケット名 +OBJECT_INFO_FOLDER = 'OBJECT_INFO_FOLDER' # CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FILENAME = 'OBJECT_INFO_FILENAME' # CRM取得対象オブジェクトの情報のファイル名 +PROCESS_RESULT_FOLDER = 'PROCESS_RESULT_FOLDER' # CRMデータ取得結果を格納するフォルダパス +PROCESS_RESULT_FILENAME = 'PROCESS_RESULT_FILENAME' # CRMデータ取得結果を格納するファイル名 +LAST_FETCH_DATE_FOLDER = 'LAST_FETCH_DATE_FOLDER' # CRMからの最終取得日時ファイルを格納するフォルダパス +CRM_IMPORT_DATA_FOLDER = 'CRM_IMPORT_DATA_FOLDER' # CRMから取得し、取込用に変換したデータを格納するフォルダ +LAST_FETCH_DATE_BACKUP_FOLDER = 'LAST_FETCH_DATE_BACKUP_FOLDER' # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +RESPONSE_JSON_BACKUP_FOLDER = 'RESPONSE_JSON_BACKUP_FOLDER' # CRMから取得した生データのバックアップを格納するフォルダパス +CRM_IMPORT_DATA_BACKUP_FOLDER = 'CRM_IMPORT_DATA_BACKUP_FOLDER' # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ # 時刻フォーマット # .000ZはUTCを表す。ミリ秒までの考慮は不要なので固定で指定 diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/conv.py index 88b86ae8..eb17c4c3 100644 --- a/ecs/crm-datafetch/src/conv.py +++ b/ecs/crm-datafetch/src/conv.py @@ -1,5 +1,5 @@ from datetime import datetime -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import ( CONV_JP_NAME, ) @@ -9,9 +9,6 @@ from src.error.exceptions import( from src.converter.converter import CSVStringConverter -logger = Logger().get_logger() - - def convert_crm_csvdata(target_object, sf_object_jsons): # ① CSV変換処理の開始ログを出力する logger.info(f'I-CONV-01 [{target_object.object_name}] のCSV変換処理を開始します') diff --git a/ecs/crm-datafetch/src/converter/__init__.py b/ecs/crm-datafetch/src/converter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/csvbk.py b/ecs/crm-datafetch/src/csvbk.py index db490c63..75c5f780 100644 --- a/ecs/crm-datafetch/src/csvbk.py +++ b/ecs/crm-datafetch/src/csvbk.py @@ -1,4 +1,4 @@ -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.aws.s3 import BackupBucket from src.constants import ( CSVBK_JP_NAME @@ -8,9 +8,6 @@ from src.error.exceptions import( ) -logger = Logger().get_logger() - - def backup_crm_csvdata(target_object, date_path, csv_object): # ① CSVバックアップ処理の開始ログを出力する logger.info( @@ -22,6 +19,9 @@ def backup_crm_csvdata(target_object, date_path, csv_object): backup_bucket.put_csv_bk( f'{date_path}/{target_object.upload_file_name}.csv', csv_object.csv_buffer) + logger.debug( + f'D-CSVBK-02 [{target_object.object_name}] のCSVデータバックアップ 正常終了') + except Exception as e: raise FileUploadException( 'E-CSVBK-01', CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') diff --git a/ecs/crm-datafetch/src/date.py b/ecs/crm-datafetch/src/date.py index 8fff0d94..adcfdd4c 100644 --- a/ecs/crm-datafetch/src/date.py +++ b/ecs/crm-datafetch/src/date.py @@ -1,4 +1,4 @@ -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import( DATE_JP_NAME ) @@ -15,9 +15,6 @@ from src.config.objects import LastFetchDatetime from src.aws.s3 import ConfigBucket -logger = Logger().get_logger() - - def set_datetime_period(target_object, execute_datetime): # ① データ取得期間設定処理の開始ログを出力する logger.info( diff --git a/ecs/crm-datafetch/src/end.py b/ecs/crm-datafetch/src/end.py index e2b274b3..7604b5b3 100644 --- a/ecs/crm-datafetch/src/end.py +++ b/ecs/crm-datafetch/src/end.py @@ -1,4 +1,4 @@ -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import ( END_JP_NAME ) @@ -11,9 +11,6 @@ from src.error.exceptions import( from src.aws.s3 import BackupBucket -logger = Logger().get_logger() - - def updload_result_data(process_result, date_path): # ① 取得処理実施結果アップロード処理のログを出力する logger.info( diff --git a/ecs/crm-datafetch/src/environments.py b/ecs/crm-datafetch/src/environments.py index 8d63cc64..262a2fbd 100644 --- a/ecs/crm-datafetch/src/environments.py +++ b/ecs/crm-datafetch/src/environments.py @@ -1,73 +1,38 @@ import os - -from src.constants import ( - LOG_LEVEL, - LOG_LEVEL_INFO, - CRM_AUTH_TIMEOUT, - CRM_AUTH_MAX_RETRY_ATTEMPT, - CRM_AUTH_RETRY_INTERVAL, - CRM_AUTH_RETRY_MIN_INTERVAL, - CRM_AUTH_RETRY_MAX_INTERVAL, - CRM_GET_RECORD_COUNT_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, - CRM_AUTH_DOMAIN, - CRM_USER_NAME, - CRM_USER_PASSWORD, - CRM_USER_SECURITY_TOKEN, - CRM_CONFIG_BUCKET, - CRM_BACKUP_BUCKET, - IMPORT_DATA_BUCKET, - OBJECT_INFO_FOLDER, - OBJECT_INFO_FILENAME, - PROCESS_RESULT_FOLDER, - PROCESS_RESULT_FILENAME, - LAST_FETCH_DATE_FOLDER, - CRM_IMPORT_DATA_FOLDER, - LAST_FETCH_DATE_BACKUP_FOLDER, - RESPONSE_JSON_BACKUP_FOLDER, - CRM_IMPORT_DATA_BACKUP_FOLDER -) +import src.constants as constants # environments(task settings file) -LOG_LEVEL = os.environ.get(LOG_LEVEL, LOG_LEVEL_INFO) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する -CRM_AUTH_TIMEOUT = os.environ["CRM_AUTH_TIMEOUT"] # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ["CRM_AUTH_MAX_RETRY_ATTEMPT"] # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_RETRY_INTERVAL = os.environ["CRM_AUTH_RETRY_INTERVAL"] # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ["CRM_AUTH_RETRY_MIN_INTERVAL"] # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ["CRM_AUTH_RETRY_MAX_INTERVAL"] # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ["CRM_GET_RECORD_COUNT_TIMEOUT"] # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ["CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT"] # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_INTERVAL"] # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL"] # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL"] # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ["CRM_FETCH_RECORD_TIMEOUT"] # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ["CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT"] # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_INTERVAL"] # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MIN_INTERVAL"] # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MAX_INTERVAL"] # CRMのレコード取得処理のリトライ時の最大待ち秒数 +LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT] # CRMへの認証処理のタイムアウト秒数 +CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT] # CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL] # CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL] # CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL] # CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT] # CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT] # CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL] # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL] # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL] # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT] # CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT] # CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL] # CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL] # CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL] # CRMのレコード取得処理のリトライ時の最大待ち秒数 # environments(ECS Task Enviroment) -CRM_AUTH_DOMAIN = os.environ["CRM_AUTH_DOMAIN"] # CRMのAPI実行のための認証エンドポイントのドメイン -CRM_USER_NAME = os.environ["CRM_USER_NAME"] # CRMのAPI実行用ユーザ名 -CRM_USER_PASSWORD = os.environ["CRM_USER_PASSWORD"] # CRMのAPI実行用ユーザパスワード -CRM_USER_SECURITY_TOKEN = os.environ["CRM_USER_SECURITY_TOKEN"] # CRMのAPI実行用ユーザのセキュリティトークン -CRM_CONFIG_BUCKET = os.environ["CRM_CONFIG_BUCKET"] # CRMデータ取得用の設定ファイルを格納するバケット名 -CRM_BACKUP_BUCKET = os.environ["CRM_BACKUP_BUCKET"] # CRMのバックアップデータを格納するバケット名 -IMPORT_DATA_BUCKET = os.environ["IMPORT_DATA_BUCKET"] # CRMの取込データを格納するバケット名 -OBJECT_INFO_FOLDER = os.environ["OBJECT_INFO_FOLDER"] # CRM取得対象オブジェクトの情報を格納するフォルダパス -OBJECT_INFO_FILENAME = os.environ["OBJECT_INFO_FILENAME"] # CRM取得対象オブジェクトの情報のファイル名 -PROCESS_RESULT_FOLDER = os.environ["PROCESS_RESULT_FOLDER"] # CRMデータ取得結果を格納するフォルダパス -PROCESS_RESULT_FILENAME = os.environ["PROCESS_RESULT_FILENAME"] # CRMデータ取得結果を格納するファイル名 -LAST_FETCH_DATE_FOLDER = os.environ["LAST_FETCH_DATE_FOLDER"] # CRMからの最終取得日時ファイルを格納するフォルダパス -CRM_IMPORT_DATA_FOLDER = os.environ["CRM_IMPORT_DATA_FOLDER"] # CRMから取得し、取込用に変換したデータを格納するフォルダ -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ["LAST_FETCH_DATE_BACKUP_FOLDER"] # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -RESPONSE_JSON_BACKUP_FOLDER = os.environ["FETCH_DATA_BACKUP_FOLDER"] # CRMから取得した生データのバックアップを格納するフォルダパス -CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ["CRM_IMPORT_DATA_BACKUP_FOLDER"] # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ +CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN] # CRMのAPI実行のための認証エンドポイントのドメイン +CRM_USER_NAME = os.environ[constants.CRM_USER_NAME] # CRMのAPI実行用ユーザ名 +CRM_USER_PASSWORD = os.environ[constants.CRM_USER_PASSWORD] # CRMのAPI実行用ユーザパスワード +CRM_USER_SECURITY_TOKEN = os.environ[constants.CRM_USER_SECURITY_TOKEN] # CRMのAPI実行用ユーザのセキュリティトークン +CRM_CONFIG_BUCKET = os.environ[constants.CRM_CONFIG_BUCKET] # CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET] # CRMのバックアップデータを格納するバケット名 +IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET] # CRMの取込データを格納するバケット名 +OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER] # CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME] # CRM取得対象オブジェクトの情報のファイル名 +PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER] # CRMデータ取得結果を格納するフォルダパス +PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME] # CRMデータ取得結果を格納するファイル名 +LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER] # CRMからの最終取得日時ファイルを格納するフォルダパス +CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER] # CRMから取得し、取込用に変換したデータを格納するフォルダ +LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER] # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER] # CRMから取得した生データのバックアップを格納するフォルダパス +CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER] # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ diff --git a/ecs/crm-datafetch/src/error/exceptions.py b/ecs/crm-datafetch/src/error/exceptions.py index 528c262d..71a174c0 100644 --- a/ecs/crm-datafetch/src/error/exceptions.py +++ b/ecs/crm-datafetch/src/error/exceptions.py @@ -30,16 +30,6 @@ class DataConvertException(MeDaCaCRMDataFetchException): pass -class TimeOutException(MeDaCaCRMDataFetchException): - """タイムアウトが発生した場合の例外""" - pass - - -class RetryExceededException(MeDaCaCRMDataFetchException): - """リトライ処理が発生した場合の例外""" - pass - - class SalesforceAPIException(MeDaCaCRMDataFetchException): """SalseforceのAPI実行失敗が発生した場合の例外""" pass diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch.py index bac313fd..9670767a 100644 --- a/ecs/crm-datafetch/src/fetch.py +++ b/ecs/crm-datafetch/src/fetch.py @@ -1,7 +1,23 @@ -from src.util.logger import Logger +from requests.exceptions import ConnectTimeout, ReadTimeout +from tenacity import retry, stop_after_attempt +from tenacity.wait import wait_exponential +from src.util.logger import logger_instance as logger from src.constants import( FETCH_JP_NAME ) +from src.environments import( + CRM_AUTH_TIMEOUT, + CRM_GET_RECORD_COUNT_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_TIMEOUT, + CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + CRM_FETCH_RECORD_RETRY_MAX_INTERVAL +) from src.error.exceptions import( SalesforceAPIException, DataConvertException @@ -12,9 +28,6 @@ from src.salesforce.salesforce_api import( ) -logger = Logger().get_logger() - - def fetch_crm_data(target_object, last_fetch_datetime): # ① CRMデータ取得処理開始ログを出力する logger.info( @@ -25,31 +38,37 @@ def fetch_crm_data(target_object, last_fetch_datetime): last_fetch_datetime_from = last_fetch_datetime.last_fetch_datetime_from last_fetch_datetime_to = last_fetch_datetime.last_fetch_datetime_to + global count_contime_counter, count_readtime_counter, count_counter, data_contime_counter, data_readtime_counter, data_counter + count_contime_counter = 1 + count_readtime_counter = 1 + count_counter = 1 + data_contime_counter = 1 + data_readtime_counter = 1 + data_counter = 1 + try: # ② 取得対象オブジェクトの取得期間内のレコード件数を取得する logger.info(f'I-FETCH-02 [{object_name}] の件数取得を開始します') - saleforce_count = SalesForceCount() - record_count = saleforce_count.fetch_sf_count_retry( + record_count = fetch_sf_count_retry( object_name, last_fetch_datetime_from, last_fetch_datetime_to) logger.info(f'I-FETCH-03 [{object_name}] の件数：[{record_count}]') except Exception as e: raise SalesforceAPIException( - 'E-FETCH-01', FETCH_JP_NAME, f'[{object_name}] の件数取得に失敗しました [{e}]') + 'E-FETCH-01', FETCH_JP_NAME, f'[{object_name}] の件数取得に失敗しました エラー内容：[{e}]') try: # ③ 取得対象オブジェクトのレコードを取得する logger.info(f'I-FETCH-04 [{object_name}] のレコード取得を開始します') - saleforce_data = SalesForceData() - record_generator = saleforce_data.fetch_sf_data_retry( + record_generator = fetch_sf_data_retry( columns, object_name, last_fetch_datetime_from, last_fetch_datetime_to) except Exception as e: raise SalesforceAPIException( - 'E-FETCH-02', FETCH_JP_NAME, f'[{object_name}] のレコード取得に失敗しました [{e}]') + 'E-FETCH-02', FETCH_JP_NAME, f'[{object_name}] のレコード取得に失敗しました エラー内容：[{e}]') try: # ④ 取得対象オブジェクトをJSONに変換 @@ -62,10 +81,68 @@ def fetch_crm_data(target_object, last_fetch_datetime): except Exception as e: raise DataConvertException( - 'E-FETCH-03', FETCH_JP_NAME, f'[{object_name}] のレコードのJSON変換に失敗しました [{e}]') + 'E-FETCH-03', FETCH_JP_NAME, f'[{object_name}] のレコードのJSON変換に失敗しました エラー内容：[{e}]') # ⑤ CRMデータ取得処理終了ログを出力する logger.info(f'I-FETCH-06 [{object_name}] のレコード取得が成功しました') # ⑥ 次の処理へ移行する return sf_object_jsons + + +@retry(wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) +def fetch_sf_count_retry(object_name, last_update_datetime_from, last_update_datetime_to): + try: + global count_contime_counter, count_readtime_counter, count_counter + + saleforce_count = SalesForceCount() + return saleforce_count.fetch_sf_count(object_name, last_update_datetime_from, last_update_datetime_to) + + except ConnectTimeout as e: + if count_contime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_contime_counter += 1 + logger.warn(f'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') + raise e + + except ReadTimeout as e: + if count_readtime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_readtime_counter += 1 + logger.warn( + f'W-FETCH-02 [{object_name}] の件数取得処理がタイムアウトしたため、リトライします：[{CRM_GET_RECORD_COUNT_TIMEOUT}] エラー内容：[{e}]') + raise e + + except Exception as e: + if count_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_counter += 1 + logger.warn( + f'W-FETCH-03 [{object_name}] の件数取得に失敗したため、リトライします エラー内容：[{e}]') + raise e + + +@retry(wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) +def fetch_sf_data_retry(columns, object_name, last_update_datetime_from, last_update_datetime_to): + try: + global data_contime_counter, data_readtime_counter, data_counter + + saleforce_data = SalesForceData() + return saleforce_data.fetch_sf_data(columns, object_name, last_update_datetime_from, last_update_datetime_to) + + except ConnectTimeout as e: + if data_contime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_contime_counter += 1 + logger.warn(f'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') + raise e + + except ReadTimeout as e: + if data_readtime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_readtime_counter += 1 + logger.warn( + f'W-FETCH-05 [{object_name}] のレコード取得処理がタイムアウトしたため、リトライします：[{CRM_FETCH_RECORD_TIMEOUT}] エラー内容：[{e}]') + raise e + + except Exception as e: + if data_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_counter += 1 + logger.warn( + f'W-FETCH-06 [{object_name}] のレコード取得に失敗したため、リトライします エラー内容：[{e}]') + raise e diff --git a/ecs/crm-datafetch/src/pre.py b/ecs/crm-datafetch/src/pre.py index 50071fe1..c8190f12 100644 --- a/ecs/crm-datafetch/src/pre.py +++ b/ecs/crm-datafetch/src/pre.py @@ -1,5 +1,5 @@ from datetime import datetime -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import( PRE_JP_NAME, YYYYMMDDTHHMMSSTZ @@ -18,9 +18,6 @@ from src.parser.json_parse import JsonParser from src.config.objects import FetchTargetObjects -logger = Logger().get_logger() - - def prepare_get_data(): # ① データ取得準備処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/process.py b/ecs/crm-datafetch/src/process.py index 5ccbd716..09d938af 100644 --- a/ecs/crm-datafetch/src/process.py +++ b/ecs/crm-datafetch/src/process.py @@ -8,13 +8,11 @@ from src.csvbk import backup_crm_csvdata # CS from src.upld import copy_crm_csvdata # CSVアップロード処理 from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 from src.end import updload_result_data # 取得処理実施結果アップロード処理 -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.error.exceptions import( MeDaCaCRMDataFetchException ) -logger = Logger().get_logger() - def main() -> None: try: @@ -108,14 +106,14 @@ def main() -> None: logger.info(f'I-CTRL-16 [{target_object.object_name}] 処理正常終了') except MeDaCaCRMDataFetchException as e: - logger.error(f'{e.error_id} {e}') - logger.exception( - f'I-ERR-03 [{object_info.get("object_name")}] の[{e.func_name}]でエラーが発生しました 次のオブジェクトの処理に移行します') + logger.info(f'{e.error_id} {e}') + logger.info( + f'I-ERR-03 [{object_info.get("object_name")}] の[{e.func_name}]でエラーが発生しました 次のオブジェクトの処理に移行します', exc_info=True) continue except Exception as e: - logger.exception( - f'I-ERR-04 [{object_info.get("object_name")}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e) + logger.info( + f'I-ERR-04 [{object_info.get("object_name")}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e, exc_info=True) continue # ④ すべてのオブジェクトの処理が完了したことと、オブジェクト毎の処理結果をログに出力する @@ -127,22 +125,22 @@ def main() -> None: # ⑥ 最終結果をチェックし、チェック結果をログに出力 if not all([v == 'success' for v in process_result.values()]): - logger.error('E-CTRL-01 一部のデータ取得に失敗しています。') + logger.error('E-CTRL-01 一部のデータ取得に失敗しています。詳細はログをご確認ください。') else: - logger.info('I-CTRL-18 すべてのデータの取得に成功しました。') + logger.info('I-CTRL-20 すべてのデータの取得に成功しました。') # ⑦ CRMデータ取得処理終了ログを出力する - logger.info('I-CTRL-20 CRMデータ取得処理を終了します') + logger.info('I-CTRL-21 CRMデータ取得処理を終了します') return exit(0) except MeDaCaCRMDataFetchException as e: - logger.error(f'{e.error_id} {e}') - logger.exception(f'I-ERR-01 [{e.func_name}]でエラーが発生したため、処理を終了します') + logger.error(f'E-ERR-01 [{e.func_name}]でエラーが発生したため、処理を終了します') + logger.exception(f'{e.error_id} {e}') return exit(0) except Exception as e: - logger.exception('I-ERR-02 予期せぬエラーが発生したため、処理を終了します', e) + logger.exception('E-ERR-02 予期せぬエラーが発生したため、処理を終了します', e) return exit(0) diff --git a/ecs/crm-datafetch/src/resbk.py b/ecs/crm-datafetch/src/resbk.py index 02b3c53b..21d516ef 100644 --- a/ecs/crm-datafetch/src/resbk.py +++ b/ecs/crm-datafetch/src/resbk.py @@ -1,4 +1,4 @@ -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import( RESBK_JP_NAME ) @@ -8,9 +8,6 @@ from src.error.exceptions import( from src.aws.s3 import BackupBucket -logger = Logger().get_logger() - - def backup_crm_data(object_name, sf_object_jsons, date_path): # ① CRM電文データバックアップ処理の開始ログを出力する logger.info(f'I-RESBK-01 [{object_name}] のCRM電文データバックアップ処理を開始します') diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index ad466782..0bce5675 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,7 +1,4 @@ -from tenacity import retry, stop_after_attempt, stop_after_delay -from tenacity.wait import wait_exponential from simple_salesforce import Salesforce -from src.util.async_retry import TimeoutManager, AsyncRetry from src.environments import( CRM_AUTH_DOMAIN, CRM_USER_NAME, @@ -17,13 +14,14 @@ from src.environments import( CRM_GET_RECORD_COUNT_RETRY_INTERVAL, CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, # CRMのレコード取得処理のタイムアウト秒数 + CRM_FETCH_RECORD_TIMEOUT, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, CRM_FETCH_RECORD_RETRY_INTERVAL, CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, CRM_FETCH_RECORD_RETRY_MAX_INTERVAL ) + FETCH_SOQL = """SELECT {column_or_expression} FROM {object_name} WHERE SystemModStamp > {last_update_datetime_from} AND SystemModStamp <= {last_update_datetime_to} @@ -57,10 +55,6 @@ class SalesForceCount(): count_soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) return count_res.get('records')[0].get('expr0') - @retry(wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT) | stop_after_delay(CRM_GET_RECORD_COUNT_TIMEOUT)) - def fetch_sf_count_retry(self, object_name, last_update_datetime_from, last_update_datetime_to): - return self.fetch_sf_count(object_name, last_update_datetime_from, last_update_datetime_to) - class SalesForceData(): def fetch_sf_data(self, columns, object_name, last_update_datetime_from, last_update_datetime_to): @@ -72,7 +66,3 @@ class SalesForceData(): ) self.__sf = SalesfoeceApi() return self.__sf.sf_query_all_iter(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) - - @retry(wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT) | stop_after_delay(CRM_FETCH_RECORD_TIMEOUT)) - def fetch_sf_data_retry(self, columns, object_name, last_update_datetime_from, last_update_datetime_to): - return self.fetch_sf_data(columns, object_name, last_update_datetime_from, last_update_datetime_to) diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upd.py index 4e3cb90d..f8dd4350 100644 --- a/ecs/crm-datafetch/src/upd.py +++ b/ecs/crm-datafetch/src/upd.py @@ -1,5 +1,5 @@ import json -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.aws.s3 import ConfigBucket from src.constants import ( UPD_JP_NAME @@ -9,9 +9,6 @@ from src.error.exceptions import( ) -logger = Logger().get_logger() - - def updload_last_fetch_datetime(target_object, last_fetch_datetime): # ① 前回取得日時ファイル更新処理の開始ログを出力する logger.info( diff --git a/ecs/crm-datafetch/src/upld.py b/ecs/crm-datafetch/src/upld.py index 57e33e25..c3c3140f 100644 --- a/ecs/crm-datafetch/src/upld.py +++ b/ecs/crm-datafetch/src/upld.py @@ -1,4 +1,4 @@ -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import ( UPLD_JP_NAME ) @@ -14,9 +14,6 @@ from src.error.exceptions import( from src.aws.s3 import S3ResourceNonBucket -logger = Logger().get_logger() - - def copy_crm_csvdata(target_object, date_path): # ① CSVデータアップロード処理の開始ログを出力する logger.info( diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index d9892da8..ec0baf55 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -1,6 +1,9 @@ import logging from src.environments import LOG_LEVEL +for name in ["boto3", "botocore", "s3transfer", "urllib3"]: + logging.getLogger(name).setLevel(logging.WARNING) + class Logger(): __logger: logging.Logger @@ -27,3 +30,6 @@ class Logger(): def get_logger(self) -> logging.Logger: return self.__logger + + +logger_instance = Logger().get_logger() From c62db70d9a00ceea5c1b233a1ad1f543290d4a1e Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Thu, 21 Jul 2022 10:14:16 +0900 Subject: [PATCH 024/275] =?UTF-8?q?fix:=E3=83=88=E3=83=A9=E3=83=B3?= =?UTF-8?q?=E3=82=B6=E3=82=AF=E3=82=B7=E3=83=A7=E3=83=B3=E5=87=A6=E7=90=86?= =?UTF-8?q?=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index f361a850..7573bf9b 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -10,10 +10,13 @@ BEGIN @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; EXECUTE drop_tmp_table_stmt; DEALLOCATE PREPARE drop_tmp_table_stmt; + ROLLBACK; SIGNAL SQLSTATE '45000' SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; END; + START TRANSACTION; + -- ②-3で利用する一時テーブル削除のSQLを準備 -- 例外処理でも利用するため先に記述 SET @drop_tmp_table = ' @@ -85,4 +88,6 @@ BEGIN EXECUTE drop_tmp_table_stmt; DEALLOCATE PREPARE drop_tmp_table_stmt; + COMMIT; + END \ No newline at end of file From 805dfb26de33e64c40b5301abbb442b329b961c8 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Thu, 21 Jul 2022 13:30:26 +0900 Subject: [PATCH 025/275] =?UTF-8?q?refactor:=E4=B8=A6=E5=88=97=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=81=A7=E3=81=8D=E3=81=AA=E3=81=84=E3=82=B3=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E8=BF=BD=E5=8A=A0=E3=80=81=E4=BE=8B=E5=A4=96?= =?UTF-8?q?=E3=81=A7=E4=BD=BF=E7=94=A8=E3=81=99=E3=82=8B=E3=83=A6=E3=83=BC?= =?UTF-8?q?=E3=82=B6=E5=A4=89=E6=95=B0=E3=81=AE=E5=88=9D=E6=9C=9F=E5=8C=96?= =?UTF-8?q?=E3=80=81$$=E7=B5=82=E7=AB=AF=E6=96=87=E5=AD=97=E3=81=AE?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 34 ++++++++++++---------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 7573bf9b..b60e2681 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,5 +1,6 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること --- $$から始まる文字は後からREPLACEする文字を示す独自ルール +-- $$から始まり$$で終わる文字は後からREPLACEする文字を示す独自ルール +-- crm_historyストアドプロシージャは、同一セッション内での並列処理を実行することができない CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN -- 例外処理 @@ -15,21 +16,22 @@ BEGIN SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; END; + SET @error_state = NULL, @error_msg = NULL; START TRANSACTION; -- ②-3で利用する一時テーブル削除のSQLを準備 -- 例外処理でも利用するため先に記述 SET @drop_tmp_table = ' - DROP TEMPORARY TABLE IF EXISTS $$target_table_make_history_tmp + DROP TEMPORARY TABLE IF EXISTS $$target_table$$_make_history_tmp '; - SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table", target_table); + SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table$$", target_table); PREPARE drop_tmp_table_stmt from @drop_tmp_table; -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = ' - UPDATE $$target_table + UPDATE $$target_table$$ SET - start_datetime = $$target_column + start_datetime = $$target_column$$ , end_datetime = "9999-12-31 00:00:00" , upd_user = CURRENT_USER() , upd_date = CURRENT_TIMESTAMP() @@ -37,21 +39,21 @@ BEGIN start_datetime IS NULL AND end_datetime IS NULL '; - SET @new_history_save = REPLACE(@new_history_save, "$$target_table", target_table); - SET @new_history_save = REPLACE(@new_history_save, "$$target_column", target_column); + SET @new_history_save = REPLACE(@new_history_save, "$$target_table$$", target_table); + SET @new_history_save = REPLACE(@new_history_save, "$$target_column$$", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; DEALLOCATE PREPARE new_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 SET @make_history_tmp_create = ' - CREATE TEMPORARY TABLE $$target_table_make_history_tmp + CREATE TEMPORARY TABLE $$target_table$$_make_history_tmp SELECT id - , MIN($$target_column) AS min_start_datetime + , MIN($$target_column$$) AS min_start_datetime , MAX(start_datetime) AS max_start_datetime FROM - $$target_table + $$target_table$$ WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY @@ -59,16 +61,16 @@ BEGIN HAVING count(id) = 2 '; - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_table", target_table); - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_column", target_column); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_table$$", target_table); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_column$$", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; EXECUTE make_history_tmp_create_stmt; DEALLOCATE PREPARE make_history_tmp_create_stmt; -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' - UPDATE $$target_table tt - LEFT JOIN $$target_table_make_history_tmp mht + UPDATE $$target_table$$ tt + LEFT JOIN $$target_table$$_make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET @@ -78,8 +80,8 @@ BEGIN WHERE mht.id IS NOT NULL '; - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column", target_column); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; DEALLOCATE PREPARE update_end_datetime_stmt; From 06e8a5769eb955712d83aff6a98d9af5ce9c30e2 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 21 Jul 2022 14:00:25 +0900 Subject: [PATCH 026/275] =?UTF-8?q?refactor:=20launch.json=E3=81=AE?= =?UTF-8?q?=E5=89=8A=E9=99=A4=EF=BC=88=E7=AB=B6=E5=90=88=E3=81=AE=E3=81=9F?= =?UTF-8?q?=E3=82=81=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/launch.json | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index d8f8414f..00000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - // IntelliSense を使用して利用可能な属性を学べます。 - // 既存の属性の説明をホバーして表示します。 - // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Python: CRMデータ取得ローカル実行", - "type": "python", - "request": "launch", - // windowsだと\区切りかも - "program": "ecs\\crm-datafetch\\main.py", - "console": "integratedTerminal", - "justMyCode": true, - "envFile": "${workspaceFolder}/.env", - "args": ["local"] - } - ] - } \ No newline at end of file From df6d75780b62e255adbb319a5215eb91799ecbd5 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 19:20:00 +0900 Subject: [PATCH 027/275] =?UTF-8?q?fix(Pipfile,Pipfile.lock):=20Pipfile?= =?UTF-8?q?=E3=81=AB=E3=82=88=E3=82=8B=E4=BE=9D=E5=AD=98=E9=96=A2=E4=BF=82?= =?UTF-8?q?=E3=81=AE=E7=AE=A1=E7=90=86=E3=81=A8=E3=82=B3=E3=83=B3=E3=83=86?= =?UTF-8?q?=E3=83=8A=E3=81=AB=E3=83=AA=E3=83=AA=E3=83=BC=E3=82=B9=E3=81=99?= =?UTF-8?q?=E3=82=8B=E3=81=9F=E3=82=81=E3=81=AElock=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=A4=E3=83=AB=E3=81=8C=E6=9B=B4=E6=96=B0=E3=81=95=E3=82=8C?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=AA=E3=81=8B=E3=81=A3=E3=81=9F=E3=81=AE?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Pipfile | 21 +- ecs/crm-datafetch/Pipfile.lock | 419 +++++++++++++++++++++++++++++++++ 2 files changed, 437 insertions(+), 3 deletions(-) diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index ec9569f7..c9758d2f 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -1,4 +1,19 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + [packages] -boto3 -simple-salesforce -tenacity \ No newline at end of file +boto3 = "*" +simple-salesforce = "*" +tenacity = "*" + +[dev-packages] +autopep8 = "*" +flake8 = "*" + +[requires] +python_version = "3.8" + +[pipenv] +allow_prereleases = true diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index e69de29b..a2cb4458 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -0,0 +1,419 @@ +{ + "_meta": { + "hash": { + "sha256": "ec1d83143aff859500979be73f67196dcfe2298ad3553a7d81ad0605a277d672" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "attrs": { + "hashes": [ + "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", + "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==21.4.0" + }, + "authlib": { + "hashes": [ + "sha256:1286e2d5ef5bfe5a11cc2d0a0d1031f0393f6ce4d61f5121cfe87fa0054e98bd", + "sha256:6e74a4846ac36dfc882b3cc2fbd3d9eb410a627f2f2dc11771276655345223b1" + ], + "version": "==1.0.1" + }, + "boto3": { + "hashes": [ + "sha256:5c775dcb12ca5d6be3f5aa3c49d77783faa64eb30fd3f4af93ff116bb42f9ffb", + "sha256:5d9bcc355cf6edd7f3849fedac4252e12a0aa2b436cdbc0d4371b16a0f852a30" + ], + "index": "pypi", + "version": "==1.24.34" + }, + "botocore": { + "hashes": [ + "sha256:0d824a5315f5f5c3bea53c14107a69695ef43190edf647f1281bac8f172ca77c", + "sha256:9c695d47f1f1212f3e306e51f7bacdf67e58055194ddcf7d8296660b124cf135" + ], + "markers": "python_version >= '3.7'", + "version": "==1.27.34" + }, + "cached-property": { + "hashes": [ + "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", + "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" + ], + "version": "==1.5.2" + }, + "certifi": { + "hashes": [ + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==2022.6.15" + }, + "cffi": { + "hashes": [ + "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", + "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", + "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", + "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", + "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", + "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", + "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", + "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", + "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", + "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", + "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", + "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", + "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", + "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", + "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", + "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", + "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", + "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", + "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", + "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", + "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", + "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", + "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", + "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", + "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", + "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", + "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", + "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", + "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", + "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", + "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", + "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", + "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", + "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", + "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", + "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", + "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", + "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", + "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", + "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", + "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", + "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", + "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", + "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", + "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", + "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", + "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", + "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", + "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", + "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", + "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", + "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", + "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", + "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", + "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", + "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", + "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", + "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", + "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", + "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", + "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", + "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", + "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", + "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" + ], + "version": "==1.15.1" + }, + "charset-normalizer": { + "hashes": [ + "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", + "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==2.1.0" + }, + "cryptography": { + "hashes": [ + "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", + "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596", + "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", + "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", + "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab", + "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", + "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", + "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b", + "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441", + "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa", + "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d", + "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", + "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a", + "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", + "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", + "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", + "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282", + "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67", + "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", + "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", + "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", + "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==37.0.4" + }, + "idna": { + "hashes": [ + "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + ], + "markers": "python_version >= '3.5'", + "version": "==3.3" + }, + "isodate": { + "hashes": [ + "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96", + "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9" + ], + "version": "==0.6.1" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "lxml": { + "hashes": [ + "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318", + "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c", + "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b", + "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000", + "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73", + "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d", + "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb", + "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8", + "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2", + "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345", + "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94", + "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e", + "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b", + "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc", + "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a", + "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9", + "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc", + "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387", + "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb", + "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7", + "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4", + "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97", + "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67", + "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627", + "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7", + "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd", + "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3", + "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7", + "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130", + "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b", + "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036", + "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785", + "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca", + "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91", + "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc", + "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536", + "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391", + "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3", + "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d", + "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21", + "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3", + "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d", + "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29", + "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715", + "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed", + "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25", + "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c", + "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785", + "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837", + "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4", + "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b", + "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2", + "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067", + "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448", + "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d", + "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2", + "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc", + "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c", + "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5", + "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84", + "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8", + "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf", + "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7", + "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e", + "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb", + "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b", + "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3", + "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad", + "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8", + "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.9.1" + }, + "platformdirs": { + "hashes": [ + "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788", + "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19" + ], + "markers": "python_version >= '3.7'", + "version": "==2.5.2" + }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.8.2" + }, + "pytz": { + "hashes": [ + "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", + "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" + ], + "version": "==2022.1" + }, + "requests": { + "hashes": [ + "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" + ], + "markers": "python_version >= '3.7' and python_version < '4'", + "version": "==2.28.1" + }, + "requests-file": { + "hashes": [ + "sha256:07d74208d3389d01c38ab89ef403af0cfec63957d53a0081d8eca738d0247d8e", + "sha256:dfe5dae75c12481f68ba353183c53a65e6044c923e64c24b2209f6c7570ca953" + ], + "version": "==1.5.1" + }, + "requests-toolbelt": { + "hashes": [ + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" + ], + "version": "==0.9.1" + }, + "s3transfer": { + "hashes": [ + "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", + "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" + ], + "markers": "python_version >= '3.7'", + "version": "==0.6.0" + }, + "simple-salesforce": { + "hashes": [ + "sha256:15d6943e52252c9cc28e1779803354f2a36c88b72056499e07eb06cd652f149c", + "sha256:7931038081c445e9459ddc014aaf7f540b1131a31596956cb5d7c0e7b7e0c4cb" + ], + "index": "pypi", + "version": "==1.12.1" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.16.0" + }, + "tenacity": { + "hashes": [ + "sha256:43242a20e3e73291a28bcbcacfd6e000b02d3857a9a9fff56b297a27afdc932f", + "sha256:f78f4ea81b0fabc06728c11dc2a8c01277bfc5181b321a4770471902e3eb844a" + ], + "index": "pypi", + "version": "==8.0.1" + }, + "urllib3": { + "hashes": [ + "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", + "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.10" + }, + "zeep": { + "hashes": [ + "sha256:5867f2eadd6b028d9751f4155af590d3aaf9280e3a0ed5e15a53343921c956e5", + "sha256:81c491092b71f5b276de8c63dfd452be3f322622c48a54f3a497cf913bdfb2f4" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==4.1.0" + } + }, + "develop": { + "autopep8": { + "hashes": [ + "sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979", + "sha256:ed77137193bbac52d029a52c59bec1b0629b5a186c495f1eb21b126ac466083f" + ], + "index": "pypi", + "version": "==1.6.0" + }, + "flake8": { + "hashes": [ + "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", + "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" + ], + "index": "pypi", + "version": "==4.0.1" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "pycodestyle": { + "hashes": [ + "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", + "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.8.0" + }, + "pyflakes": { + "hashes": [ + "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", + "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "version": "==0.10.2" + } + } +} From 5e88f3b519c5d89ff6d84e3807b1a224351818f6 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 21 Jul 2022 19:32:33 +0900 Subject: [PATCH 028/275] =?UTF-8?q?feat:=20Dokcer=E9=96=A2=E9=80=A3?= =?UTF-8?q?=E3=81=AE=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Dockerfile | 12 +- ecs/crm-datafetch/Pipfile | 16 +- ecs/crm-datafetch/Pipfile.lock | 371 +++++++++++++++++++++++++++++++++ 3 files changed, 390 insertions(+), 9 deletions(-) diff --git a/ecs/crm-datafetch/Dockerfile b/ecs/crm-datafetch/Dockerfile index 88e588e1..56087bf0 100644 --- a/ecs/crm-datafetch/Dockerfile +++ b/ecs/crm-datafetch/Dockerfile @@ -4,16 +4,16 @@ ENV TZ="Asia/Tokyo" WORKDIR /usr/src/app COPY Pipfile Pipfile.lock ./ -RUN pip install --no-cache-dir -r requirements.txt && \ - cp /usr/share/zoneinfo/Asia/Tokyo /etc/localtime - -RUN \ +RUN \ apt update -y && \ # パッケージのセキュリティアップデートのみを適用するコマンド apt install -y unattended-upgrades && \ - unattended-upgrades + unattended-upgrades && \ + pip install pipenv --no-cache-dir && \ + pipenv install --system --deploy && \ + pip uninstall -y pipenv virtualenv-clone virtualenv COPY main.py ./ -COPY crm-datafetch /usr/src +COPY src ./ CMD [ "python", "./main.py" ] diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index ec9569f7..3c0e1a61 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -1,4 +1,14 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + [packages] -boto3 -simple-salesforce -tenacity \ No newline at end of file +boto3 = "*" +simple-salesforce = "*" +tenacity = "*" + +[dev-packages] + +[requires] +python_version = "3.8" diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index e69de29b..b66b415b 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -0,0 +1,371 @@ +{ + "_meta": { + "hash": { + "sha256": "84e1c5fdd458ebb8022dd649f03e59828d8658b62a572f524a60850e2a1d626c" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "attrs": { + "hashes": [ + "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", + "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==21.4.0" + }, + "authlib": { + "hashes": [ + "sha256:1286e2d5ef5bfe5a11cc2d0a0d1031f0393f6ce4d61f5121cfe87fa0054e98bd", + "sha256:6e74a4846ac36dfc882b3cc2fbd3d9eb410a627f2f2dc11771276655345223b1" + ], + "version": "==1.0.1" + }, + "boto3": { + "hashes": [ + "sha256:5c775dcb12ca5d6be3f5aa3c49d77783faa64eb30fd3f4af93ff116bb42f9ffb", + "sha256:5d9bcc355cf6edd7f3849fedac4252e12a0aa2b436cdbc0d4371b16a0f852a30" + ], + "index": "pypi", + "version": "==1.24.34" + }, + "botocore": { + "hashes": [ + "sha256:0d824a5315f5f5c3bea53c14107a69695ef43190edf647f1281bac8f172ca77c", + "sha256:9c695d47f1f1212f3e306e51f7bacdf67e58055194ddcf7d8296660b124cf135" + ], + "markers": "python_version >= '3.7'", + "version": "==1.27.34" + }, + "cached-property": { + "hashes": [ + "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", + "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" + ], + "version": "==1.5.2" + }, + "certifi": { + "hashes": [ + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" + ], + "markers": "python_version >= '3.6'", + "version": "==2022.6.15" + }, + "cffi": { + "hashes": [ + "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", + "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", + "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", + "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", + "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", + "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", + "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", + "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", + "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", + "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", + "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", + "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", + "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", + "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", + "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", + "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", + "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", + "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", + "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", + "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", + "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", + "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", + "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", + "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", + "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", + "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", + "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", + "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", + "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", + "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", + "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", + "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", + "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", + "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", + "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", + "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", + "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", + "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", + "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", + "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", + "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", + "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", + "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", + "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", + "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", + "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", + "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", + "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", + "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", + "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", + "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", + "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", + "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", + "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", + "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", + "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", + "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", + "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", + "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", + "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", + "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", + "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", + "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", + "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" + ], + "version": "==1.15.1" + }, + "charset-normalizer": { + "hashes": [ + "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", + "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" + ], + "markers": "python_version >= '3.6'", + "version": "==2.1.0" + }, + "cryptography": { + "hashes": [ + "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", + "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596", + "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", + "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", + "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab", + "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", + "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", + "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b", + "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441", + "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa", + "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d", + "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", + "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a", + "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", + "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", + "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", + "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282", + "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67", + "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", + "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", + "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", + "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" + ], + "markers": "python_version >= '3.6'", + "version": "==37.0.4" + }, + "idna": { + "hashes": [ + "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + ], + "markers": "python_version >= '3.5'", + "version": "==3.3" + }, + "isodate": { + "hashes": [ + "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96", + "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9" + ], + "version": "==0.6.1" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "lxml": { + "hashes": [ + "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318", + "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c", + "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b", + "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000", + "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73", + "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d", + "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb", + "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8", + "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2", + "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345", + "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94", + "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e", + "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b", + "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc", + "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a", + "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9", + "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc", + "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387", + "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb", + "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7", + "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4", + "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97", + "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67", + "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627", + "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7", + "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd", + "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3", + "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7", + "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130", + "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b", + "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036", + "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785", + "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca", + "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91", + "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc", + "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536", + "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391", + "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3", + "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d", + "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21", + "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3", + "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d", + "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29", + "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715", + "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed", + "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25", + "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c", + "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785", + "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837", + "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4", + "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b", + "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2", + "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067", + "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448", + "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d", + "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2", + "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc", + "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c", + "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5", + "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84", + "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8", + "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf", + "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7", + "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e", + "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb", + "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b", + "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3", + "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad", + "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8", + "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.9.1" + }, + "platformdirs": { + "hashes": [ + "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788", + "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19" + ], + "markers": "python_version >= '3.7'", + "version": "==2.5.2" + }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "pytz": { + "hashes": [ + "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", + "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" + ], + "version": "==2022.1" + }, + "requests": { + "hashes": [ + "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" + ], + "markers": "python_version >= '3.7' and python_version < '4'", + "version": "==2.28.1" + }, + "requests-file": { + "hashes": [ + "sha256:07d74208d3389d01c38ab89ef403af0cfec63957d53a0081d8eca738d0247d8e", + "sha256:dfe5dae75c12481f68ba353183c53a65e6044c923e64c24b2209f6c7570ca953" + ], + "version": "==1.5.1" + }, + "requests-toolbelt": { + "hashes": [ + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" + ], + "version": "==0.9.1" + }, + "s3transfer": { + "hashes": [ + "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", + "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" + ], + "markers": "python_version >= '3.7'", + "version": "==0.6.0" + }, + "simple-salesforce": { + "hashes": [ + "sha256:15d6943e52252c9cc28e1779803354f2a36c88b72056499e07eb06cd652f149c", + "sha256:7931038081c445e9459ddc014aaf7f540b1131a31596956cb5d7c0e7b7e0c4cb" + ], + "index": "pypi", + "version": "==1.12.1" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "tenacity": { + "hashes": [ + "sha256:43242a20e3e73291a28bcbcacfd6e000b02d3857a9a9fff56b297a27afdc932f", + "sha256:f78f4ea81b0fabc06728c11dc2a8c01277bfc5181b321a4770471902e3eb844a" + ], + "index": "pypi", + "version": "==8.0.1" + }, + "urllib3": { + "hashes": [ + "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", + "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.10" + }, + "zeep": { + "hashes": [ + "sha256:5867f2eadd6b028d9751f4155af590d3aaf9280e3a0ed5e15a53343921c956e5", + "sha256:81c491092b71f5b276de8c63dfd452be3f322622c48a54f3a497cf913bdfb2f4" + ], + "markers": "python_version >= '3.6'", + "version": "==4.1.0" + } + }, + "develop": {} +} From 91e70710494f26bd6ef14c01b1fb82e7a38a7fa9 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 19:41:28 +0900 Subject: [PATCH 029/275] =?UTF-8?q?Revert=20"fix(Pipfile,Pipfile.lock):=20?= =?UTF-8?q?Pipfile=E3=81=AB=E3=82=88=E3=82=8B=E4=BE=9D=E5=AD=98=E9=96=A2?= =?UTF-8?q?=E4=BF=82=E3=81=AE=E7=AE=A1=E7=90=86=E3=81=A8=E3=82=B3=E3=83=B3?= =?UTF-8?q?=E3=83=86=E3=83=8A=E3=81=AB=E3=83=AA=E3=83=AA=E3=83=BC=E3=82=B9?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=81=9F=E3=82=81=E3=81=AElock=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=8C=E6=9B=B4=E6=96=B0=E3=81=95?= =?UTF-8?q?=E3=82=8C=E3=81=A6=E3=81=84=E3=81=AA=E3=81=8B=E3=81=A3=E3=81=9F?= =?UTF-8?q?=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit df6d75780b62e255adbb319a5215eb91799ecbd5. --- ecs/crm-datafetch/Pipfile | 21 +- ecs/crm-datafetch/Pipfile.lock | 419 --------------------------------- 2 files changed, 3 insertions(+), 437 deletions(-) diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index c9758d2f..ec9569f7 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -1,19 +1,4 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - [packages] -boto3 = "*" -simple-salesforce = "*" -tenacity = "*" - -[dev-packages] -autopep8 = "*" -flake8 = "*" - -[requires] -python_version = "3.8" - -[pipenv] -allow_prereleases = true +boto3 +simple-salesforce +tenacity \ No newline at end of file diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index a2cb4458..e69de29b 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -1,419 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "ec1d83143aff859500979be73f67196dcfe2298ad3553a7d81ad0605a277d672" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.8" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "attrs": { - "hashes": [ - "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", - "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==21.4.0" - }, - "authlib": { - "hashes": [ - "sha256:1286e2d5ef5bfe5a11cc2d0a0d1031f0393f6ce4d61f5121cfe87fa0054e98bd", - "sha256:6e74a4846ac36dfc882b3cc2fbd3d9eb410a627f2f2dc11771276655345223b1" - ], - "version": "==1.0.1" - }, - "boto3": { - "hashes": [ - "sha256:5c775dcb12ca5d6be3f5aa3c49d77783faa64eb30fd3f4af93ff116bb42f9ffb", - "sha256:5d9bcc355cf6edd7f3849fedac4252e12a0aa2b436cdbc0d4371b16a0f852a30" - ], - "index": "pypi", - "version": "==1.24.34" - }, - "botocore": { - "hashes": [ - "sha256:0d824a5315f5f5c3bea53c14107a69695ef43190edf647f1281bac8f172ca77c", - "sha256:9c695d47f1f1212f3e306e51f7bacdf67e58055194ddcf7d8296660b124cf135" - ], - "markers": "python_version >= '3.7'", - "version": "==1.27.34" - }, - "cached-property": { - "hashes": [ - "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", - "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" - ], - "version": "==1.5.2" - }, - "certifi": { - "hashes": [ - "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", - "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" - ], - "markers": "python_full_version >= '3.6.0'", - "version": "==2022.6.15" - }, - "cffi": { - "hashes": [ - "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", - "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", - "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", - "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", - "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", - "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", - "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", - "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", - "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", - "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", - "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", - "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", - "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", - "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", - "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", - "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", - "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", - "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", - "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", - "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", - "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", - "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", - "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", - "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", - "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", - "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", - "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", - "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", - "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", - "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", - "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", - "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", - "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", - "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", - "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", - "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", - "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", - "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", - "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", - "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", - "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", - "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", - "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", - "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", - "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", - "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", - "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", - "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", - "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", - "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", - "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", - "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", - "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", - "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", - "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", - "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", - "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", - "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", - "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", - "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", - "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", - "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", - "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", - "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" - ], - "version": "==1.15.1" - }, - "charset-normalizer": { - "hashes": [ - "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", - "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" - ], - "markers": "python_full_version >= '3.6.0'", - "version": "==2.1.0" - }, - "cryptography": { - "hashes": [ - "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", - "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596", - "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", - "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", - "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab", - "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", - "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", - "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b", - "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441", - "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa", - "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d", - "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", - "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a", - "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", - "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", - "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", - "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282", - "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67", - "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", - "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", - "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", - "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" - ], - "markers": "python_full_version >= '3.6.0'", - "version": "==37.0.4" - }, - "idna": { - "hashes": [ - "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", - "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" - ], - "markers": "python_version >= '3.5'", - "version": "==3.3" - }, - "isodate": { - "hashes": [ - "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96", - "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9" - ], - "version": "==0.6.1" - }, - "jmespath": { - "hashes": [ - "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", - "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" - ], - "markers": "python_version >= '3.7'", - "version": "==1.0.1" - }, - "lxml": { - "hashes": [ - "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318", - "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c", - "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b", - "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000", - "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73", - "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d", - "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb", - "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8", - "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2", - "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345", - "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94", - "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e", - "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b", - "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc", - "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a", - "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9", - "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc", - "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387", - "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb", - "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7", - "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4", - "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97", - "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67", - "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627", - "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7", - "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd", - "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3", - "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7", - "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130", - "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b", - "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036", - "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785", - "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca", - "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91", - "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc", - "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536", - "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391", - "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3", - "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d", - "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21", - "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3", - "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d", - "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29", - "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715", - "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed", - "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25", - "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c", - "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785", - "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837", - "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4", - "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b", - "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2", - "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067", - "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448", - "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d", - "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2", - "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc", - "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c", - "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5", - "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84", - "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8", - "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf", - "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7", - "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e", - "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb", - "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b", - "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3", - "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad", - "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8", - "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==4.9.1" - }, - "platformdirs": { - "hashes": [ - "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788", - "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19" - ], - "markers": "python_version >= '3.7'", - "version": "==2.5.2" - }, - "pycparser": { - "hashes": [ - "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", - "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" - ], - "version": "==2.21" - }, - "python-dateutil": { - "hashes": [ - "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", - "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", - "version": "==2.8.2" - }, - "pytz": { - "hashes": [ - "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", - "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" - ], - "version": "==2022.1" - }, - "requests": { - "hashes": [ - "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", - "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" - ], - "markers": "python_version >= '3.7' and python_version < '4'", - "version": "==2.28.1" - }, - "requests-file": { - "hashes": [ - "sha256:07d74208d3389d01c38ab89ef403af0cfec63957d53a0081d8eca738d0247d8e", - "sha256:dfe5dae75c12481f68ba353183c53a65e6044c923e64c24b2209f6c7570ca953" - ], - "version": "==1.5.1" - }, - "requests-toolbelt": { - "hashes": [ - "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", - "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" - ], - "version": "==0.9.1" - }, - "s3transfer": { - "hashes": [ - "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", - "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" - ], - "markers": "python_version >= '3.7'", - "version": "==0.6.0" - }, - "simple-salesforce": { - "hashes": [ - "sha256:15d6943e52252c9cc28e1779803354f2a36c88b72056499e07eb06cd652f149c", - "sha256:7931038081c445e9459ddc014aaf7f540b1131a31596956cb5d7c0e7b7e0c4cb" - ], - "index": "pypi", - "version": "==1.12.1" - }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", - "version": "==1.16.0" - }, - "tenacity": { - "hashes": [ - "sha256:43242a20e3e73291a28bcbcacfd6e000b02d3857a9a9fff56b297a27afdc932f", - "sha256:f78f4ea81b0fabc06728c11dc2a8c01277bfc5181b321a4770471902e3eb844a" - ], - "index": "pypi", - "version": "==8.0.1" - }, - "urllib3": { - "hashes": [ - "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", - "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", - "version": "==1.26.10" - }, - "zeep": { - "hashes": [ - "sha256:5867f2eadd6b028d9751f4155af590d3aaf9280e3a0ed5e15a53343921c956e5", - "sha256:81c491092b71f5b276de8c63dfd452be3f322622c48a54f3a497cf913bdfb2f4" - ], - "markers": "python_full_version >= '3.6.0'", - "version": "==4.1.0" - } - }, - "develop": { - "autopep8": { - "hashes": [ - "sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979", - "sha256:ed77137193bbac52d029a52c59bec1b0629b5a186c495f1eb21b126ac466083f" - ], - "index": "pypi", - "version": "==1.6.0" - }, - "flake8": { - "hashes": [ - "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", - "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" - ], - "index": "pypi", - "version": "==4.0.1" - }, - "mccabe": { - "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" - ], - "version": "==0.6.1" - }, - "pycodestyle": { - "hashes": [ - "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", - "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==2.8.0" - }, - "pyflakes": { - "hashes": [ - "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", - "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.4.0" - }, - "toml": { - "hashes": [ - "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", - "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", - "version": "==0.10.2" - } - } -} From c6f15e22b9b4d01b34684171b17056ae47efde29 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 19:46:49 +0900 Subject: [PATCH 030/275] =?UTF-8?q?feat(Pipfile,Pipefile.lock):=20?= =?UTF-8?q?=E9=96=8B=E7=99=BA=E7=94=A8=E3=83=91=E3=83=83=E3=82=B1=E3=83=BC?= =?UTF-8?q?=E3=82=B8=E3=81=A8=E3=81=97=E3=81=A6=E3=80=81=E3=83=AA=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=83=BC(flake8)=E3=81=A8=E3=83=95=E3=82=A9=E3=83=BC?= =?UTF-8?q?=E3=83=9E=E3=83=83=E3=82=BF(autopep8)=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Pipfile | 2 ++ ecs/crm-datafetch/Pipfile.lock | 52 ++++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 3c0e1a61..33501d98 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -9,6 +9,8 @@ simple-salesforce = "*" tenacity = "*" [dev-packages] +autopep8 = "*" +flake8 = "*" [requires] python_version = "3.8" diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index b66b415b..c8c60540 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "84e1c5fdd458ebb8022dd649f03e59828d8658b62a572f524a60850e2a1d626c" + "sha256": "ec1d83143aff859500979be73f67196dcfe2298ad3553a7d81ad0605a277d672" }, "pipfile-spec": 6, "requires": { @@ -367,5 +367,53 @@ "version": "==4.1.0" } }, - "develop": {} + "develop": { + "autopep8": { + "hashes": [ + "sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979", + "sha256:ed77137193bbac52d029a52c59bec1b0629b5a186c495f1eb21b126ac466083f" + ], + "index": "pypi", + "version": "==1.6.0" + }, + "flake8": { + "hashes": [ + "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", + "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" + ], + "index": "pypi", + "version": "==4.0.1" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "pycodestyle": { + "hashes": [ + "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", + "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.8.0" + }, + "pyflakes": { + "hashes": [ + "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", + "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + } + } } From a53e60cbf19934f4a4259452b51e895dd150dbf1 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 19:48:59 +0900 Subject: [PATCH 031/275] =?UTF-8?q?docs:=20=E3=83=AD=E3=83=BC=E3=82=AB?= =?UTF-8?q?=E3=83=AB=E9=96=8B=E7=99=BA=E7=92=B0=E5=A2=83=E3=81=AE=E6=A7=8B?= =?UTF-8?q?=E7=AF=89=E3=82=84=E5=AE=9F=E8=A1=8C=E6=96=B9=E6=B3=95=E3=81=AB?= =?UTF-8?q?=E3=81=A4=E3=81=84=E3=81=A6=E3=80=81README=E3=81=AB=E8=BF=BD?= =?UTF-8?q?=E8=A8=98=E3=81=97=E3=80=81=E5=91=A8=E8=BE=BA=E3=81=AE=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=82=E4=B8=80=E7=B7=92=E3=81=AB?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 5 +- .../.vscode/recommend_settings.json | 20 ++++++ ecs/crm-datafetch/README.md | 69 +++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 ecs/crm-datafetch/.vscode/recommend_settings.json create mode 100644 ecs/crm-datafetch/README.md diff --git a/.gitignore b/.gitignore index e4f6d9fb..6733371f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ lambda/mbj-newdwh2021-staging-NoticeToSlack/package-lock.json lambda/mbj-newdwh2021-staging-NoticeToSlack/node_modules/* lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json -lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* \ No newline at end of file +lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* + +**/.vscode/settings.json +.env \ No newline at end of file diff --git a/ecs/crm-datafetch/.vscode/recommend_settings.json b/ecs/crm-datafetch/.vscode/recommend_settings.json new file mode 100644 index 00000000..a29de1db --- /dev/null +++ b/ecs/crm-datafetch/.vscode/recommend_settings.json @@ -0,0 +1,20 @@ +{ + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": ["--max-line-length=120"], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Args": [ + "--max-line-length", + "120" + ] + } + \ No newline at end of file diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md new file mode 100644 index 00000000..5a23e966 --- /dev/null +++ b/ecs/crm-datafetch/README.md @@ -0,0 +1,69 @@ +# CRMデータ連携 データ取得処理 ECSタスク + +## 前提事項 + +### ツールのバージョン + +- Python 3.8.x +- PipEnv(Pythonの依存関係管理用モジュール) + +### 開発環境 + +- Visual Studio Code + +## 開発環境構築 + +※下記の操作は基本的にVSCode上で行います。 + +- [ファイル]-[フォルダーを開く]から、当フォルダを選択して開く + +- [Wiki | Pythonの環境構築](https://nds-tyo.backlog.com/alias/wiki/1874930)にて、pyenvの導入まで完了させる + - **pyenvの導入はマストではないが、Pythonのバージョンが前提のバージョンと同一であることを確認して開発を進めてください** + - **確認しながら開発するのは煩わしいため、導入を強く推奨します。** + +- ローカルのPythonでPipEnvをインストールする + + ```sh + pip install pipenv + ``` + +- pipenvの仮想環境と依存パッケージをインストール。このとき、初回実行にはpythonの仮想環境のパスがターミナルに表示されるため、控えておく + + ```sh + # 開発用パッケージも含めてインストール + pipenv install --dev + ``` + +- VSCodeのコマンドパレットを[表示]-[コマンドパレット]から開き、`Python: Select interpreter`を選択して実行する +- Pythonの実行環境を聞かれるため、先に控えたパスと一致するものを選択する + - 出てこない場合、一度VSCodeを閉じて再度開き直す + +- 当フォルダ直下の`.vscode`フォルダ内にある`recommend_settings.json`をコピーし、同フォルダ内に`settings.json`を作成する + +## ローカルでの起動方法 + +- 当フォルダ直下の`.vscode`フォルダ内に`launch.json`を作成する +- 以下のJSONを入力して保存する + + ```json + { + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + // エントリーポイントのファイルに変更すること + "program": "<エントリーポイントになるファイル>", + "console": "integratedTerminal", + "justMyCode": true, + // 環境変数が必要な場合に読み込む環境変数ファイル + "envFile": "${workspaceFolder}/.env", + } + ] + } + ``` + +- 環境変数が必要な場合、直接設定するか、上記JSONの`"envFile"`に設定されたパスに`.env`ファイルを作成し、環境変数を入力する +- キーボードの「F5」キーを押して起動する + - デバッグモードで実行されるため、適当なところにブレークポイントを置いてデバッグすることができる From 24149be0b1b00071ac8abbed876dc563ff887247 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 19:55:37 +0900 Subject: [PATCH 032/275] =?UTF-8?q?style:=20=E3=83=95=E3=82=A9=E3=83=BC?= =?UTF-8?q?=E3=83=9E=E3=83=83=E3=82=BF=E3=82=92=E9=81=A9=E7=94=A8=E3=80=82?= =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=81=AE=E6=9B=B8=E3=81=8D?= =?UTF-8?q?=E3=81=A3=E3=81=B7=E3=82=8A=E3=82=92=E3=82=92=E8=80=83=E6=85=AE?= =?UTF-8?q?=E3=81=97=E3=81=A61=E8=A1=8C150=E6=96=87=E5=AD=97=E3=81=BE?= =?UTF-8?q?=E3=81=A7=E3=81=AF=E8=A8=B1=E5=AE=B9=E3=81=99=E3=82=8B=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.vscode/recommend_settings.json | 34 +++---- ecs/crm-datafetch/src/aws/s3.py | 27 ++---- ecs/crm-datafetch/src/chk.py | 10 +- ecs/crm-datafetch/src/config/objects.py | 4 +- ecs/crm-datafetch/src/constants.py | 5 +- ecs/crm-datafetch/src/conv.py | 11 +-- ecs/crm-datafetch/src/converter/converter.py | 13 +-- ecs/crm-datafetch/src/csvbk.py | 13 +-- ecs/crm-datafetch/src/date.py | 20 ++-- ecs/crm-datafetch/src/end.py | 14 +-- ecs/crm-datafetch/src/environments.py | 97 +++++++++++++------ ecs/crm-datafetch/src/fetch.py | 39 +++----- ecs/crm-datafetch/src/parser/json_parse.py | 4 +- ecs/crm-datafetch/src/pre.py | 22 ++--- ecs/crm-datafetch/src/process.py | 24 +++-- ecs/crm-datafetch/src/resbk.py | 10 +- .../src/salesforce/salesforce_api.py | 37 +++---- ecs/crm-datafetch/src/upd.py | 11 +-- ecs/crm-datafetch/src/upld.py | 18 +--- ecs/crm-datafetch/src/util/logger.py | 1 + 20 files changed, 184 insertions(+), 230 deletions(-) diff --git a/ecs/crm-datafetch/.vscode/recommend_settings.json b/ecs/crm-datafetch/.vscode/recommend_settings.json index a29de1db..72dacf55 100644 --- a/ecs/crm-datafetch/.vscode/recommend_settings.json +++ b/ecs/crm-datafetch/.vscode/recommend_settings.json @@ -1,20 +1,16 @@ { - "[python]": { - "editor.defaultFormatter": null, - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.organizeImports": true - } - }, - "python.linting.lintOnSave": true, - "python.linting.enabled": true, - "python.linting.pylintEnabled": false, - "python.linting.flake8Enabled": true, - "python.linting.flake8Args": ["--max-line-length=120"], - "python.formatting.provider": "autopep8", - "python.formatting.autopep8Args": [ - "--max-line-length", - "120" - ] - } - \ No newline at end of file + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": ["--max-line-length=150"], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Args": ["--max-line-length", "150"] +} diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index 229b8666..e8691e16 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -1,22 +1,13 @@ -import boto3 import json -from src.environments import ( - CRM_CONFIG_BUCKET, - CRM_BACKUP_BUCKET, - IMPORT_DATA_BUCKET, - OBJECT_INFO_FOLDER, - OBJECT_INFO_FILENAME, - PROCESS_RESULT_FOLDER, - CRM_IMPORT_DATA_FOLDER, - CRM_IMPORT_DATA_BACKUP_FOLDER, - LAST_FETCH_DATE_FOLDER, - RESPONSE_JSON_BACKUP_FOLDER -) -from src.constants import ( - AWS_RESOURCE_S3, - AWS_CLINET_S3, - S3_RESPONSE_BODY -) + +import boto3 +from src.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY +from src.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, + CRM_IMPORT_DATA_BACKUP_FOLDER, + CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET, + LAST_FETCH_DATE_FOLDER, OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER, PROCESS_RESULT_FOLDER, + RESPONSE_JSON_BACKUP_FOLDER) class S3Clinet: diff --git a/ecs/crm-datafetch/src/chk.py b/ecs/crm-datafetch/src/chk.py index 077f0ee2..b842d19b 100644 --- a/ecs/crm-datafetch/src/chk.py +++ b/ecs/crm-datafetch/src/chk.py @@ -1,11 +1,7 @@ -from src.util.logger import logger_instance as logger -from src.constants import( - CHK_JP_NAME -) -from src.error.exceptions import( - InvalidConfigException -) from src.config.objects import TargetObject +from src.constants import CHK_JP_NAME +from src.error.exceptions import InvalidConfigException +from src.util.logger import logger_instance as logger def check_object_info(object_info, execute_datetime): diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index edce5b33..8d9f6f85 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,6 +1,4 @@ -from src.constants import ( - DATE_PATTERN_YYYYMMDDTHHMMSSTZ -) +from src.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ from src.util.dict_checker import DictCheck diff --git a/ecs/crm-datafetch/src/constants.py b/ecs/crm-datafetch/src/constants.py index a394df16..fde2ec65 100644 --- a/ecs/crm-datafetch/src/constants.py +++ b/ecs/crm-datafetch/src/constants.py @@ -1,5 +1,6 @@ # environments(task settings file) -LOG_LEVEL = "LOG_LEVEL" # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +LOG_LEVEL = "LOG_LEVEL" CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 @@ -70,4 +71,4 @@ END_JP_NAME = '取得処理実施結果アップロード処理' # CSVチェック CSV_TRUE_VALUE = '1' -CSV_FALSE_VALUE = '0' \ No newline at end of file +CSV_FALSE_VALUE = '0' diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/conv.py index eb17c4c3..cfcc6a8a 100644 --- a/ecs/crm-datafetch/src/conv.py +++ b/ecs/crm-datafetch/src/conv.py @@ -1,12 +1,9 @@ from datetime import datetime -from src.util.logger import logger_instance as logger -from src.constants import ( - CONV_JP_NAME, -) -from src.error.exceptions import( - DataConvertException -) + +from src.constants import CONV_JP_NAME from src.converter.converter import CSVStringConverter +from src.error.exceptions import DataConvertException +from src.util.logger import logger_instance as logger def convert_crm_csvdata(target_object, sf_object_jsons): diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 1b785c35..18fc2923 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -1,13 +1,10 @@ -import re -import io import csv +import io +import re from datetime import datetime -from src.constants import( - CSV_TRUE_VALUE, - CSV_FALSE_VALUE, - CRM_DATETIME_FORMAT, - YYYYMMDDHHMMSS -) + +from src.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, + CSV_TRUE_VALUE, YYYYMMDDHHMMSS) class CSVStringConverter: diff --git a/ecs/crm-datafetch/src/csvbk.py b/ecs/crm-datafetch/src/csvbk.py index 75c5f780..375b273e 100644 --- a/ecs/crm-datafetch/src/csvbk.py +++ b/ecs/crm-datafetch/src/csvbk.py @@ -1,11 +1,7 @@ -from src.util.logger import logger_instance as logger from src.aws.s3 import BackupBucket -from src.constants import ( - CSVBK_JP_NAME -) -from src.error.exceptions import( - FileUploadException -) +from src.constants import CSVBK_JP_NAME +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def backup_crm_csvdata(target_object, date_path, csv_object): @@ -24,7 +20,8 @@ def backup_crm_csvdata(target_object, date_path, csv_object): except Exception as e: raise FileUploadException( - 'E-CSVBK-01', CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') + 'E-CSVBK-01', + CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') # ③ CSVバックアップ処理の終了ログを出力する logger.info( diff --git a/ecs/crm-datafetch/src/date.py b/ecs/crm-datafetch/src/date.py index adcfdd4c..2896c366 100644 --- a/ecs/crm-datafetch/src/date.py +++ b/ecs/crm-datafetch/src/date.py @@ -1,18 +1,10 @@ -from src.util.logger import logger_instance as logger -from src.constants import( - DATE_JP_NAME -) -from src.environments import ( - CRM_CONFIG_BUCKET, - LAST_FETCH_DATE_FOLDER -) -from src.error.exceptions import( - FileNotFoundException, - InvalidConfigException -) -from src.parser.json_parse import JsonParser -from src.config.objects import LastFetchDatetime from src.aws.s3 import ConfigBucket +from src.config.objects import LastFetchDatetime +from src.constants import DATE_JP_NAME +from src.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.error.exceptions import FileNotFoundException, InvalidConfigException +from src.parser.json_parse import JsonParser +from src.util.logger import logger_instance as logger def set_datetime_period(target_object, execute_datetime): diff --git a/ecs/crm-datafetch/src/end.py b/ecs/crm-datafetch/src/end.py index 7604b5b3..471effc2 100644 --- a/ecs/crm-datafetch/src/end.py +++ b/ecs/crm-datafetch/src/end.py @@ -1,14 +1,8 @@ -from src.util.logger import logger_instance as logger -from src.constants import ( - END_JP_NAME -) -from src.environments import( - PROCESS_RESULT_FILENAME -) -from src.error.exceptions import( - FileUploadException -) from src.aws.s3 import BackupBucket +from src.constants import END_JP_NAME +from src.environments import PROCESS_RESULT_FILENAME +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def updload_result_data(process_result, date_path): diff --git a/ecs/crm-datafetch/src/environments.py b/ecs/crm-datafetch/src/environments.py index 262a2fbd..4a09b2a2 100644 --- a/ecs/crm-datafetch/src/environments.py +++ b/ecs/crm-datafetch/src/environments.py @@ -1,38 +1,71 @@ import os + import src.constants as constants # environments(task settings file) -LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する -CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT] # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT] # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL] # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL] # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL] # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT] # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT] # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL] # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL] # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL] # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT] # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT] # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL] # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL] # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL] # CRMのレコード取得処理のリトライ時の最大待ち秒数 +# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) +# CRMへの認証処理のタイムアウト秒数 +CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT] +# CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT] +# CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL] +# CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL] +# CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL] +# CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT] +# CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT] +# CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL] +# CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL] +# CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL] +# CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT] +# CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT] +# CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL] +# CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL] +# CRMのレコード取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL] # environments(ECS Task Enviroment) -CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN] # CRMのAPI実行のための認証エンドポイントのドメイン -CRM_USER_NAME = os.environ[constants.CRM_USER_NAME] # CRMのAPI実行用ユーザ名 -CRM_USER_PASSWORD = os.environ[constants.CRM_USER_PASSWORD] # CRMのAPI実行用ユーザパスワード -CRM_USER_SECURITY_TOKEN = os.environ[constants.CRM_USER_SECURITY_TOKEN] # CRMのAPI実行用ユーザのセキュリティトークン -CRM_CONFIG_BUCKET = os.environ[constants.CRM_CONFIG_BUCKET] # CRMデータ取得用の設定ファイルを格納するバケット名 -CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET] # CRMのバックアップデータを格納するバケット名 -IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET] # CRMの取込データを格納するバケット名 -OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER] # CRM取得対象オブジェクトの情報を格納するフォルダパス -OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME] # CRM取得対象オブジェクトの情報のファイル名 -PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER] # CRMデータ取得結果を格納するフォルダパス -PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME] # CRMデータ取得結果を格納するファイル名 -LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER] # CRMからの最終取得日時ファイルを格納するフォルダパス -CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER] # CRMから取得し、取込用に変換したデータを格納するフォルダ -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER] # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER] # CRMから取得した生データのバックアップを格納するフォルダパス -CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER] # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ +# CRMのAPI実行のための認証エンドポイントのドメイン +CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN] +# CRMのAPI実行用ユーザ名 +CRM_USER_NAME = os.environ[constants.CRM_USER_NAME] +# CRMのAPI実行用ユーザパスワード +CRM_USER_PASSWORD = os.environ[constants.CRM_USER_PASSWORD] +# CRMのAPI実行用ユーザのセキュリティトークン +CRM_USER_SECURITY_TOKEN = os.environ[constants.CRM_USER_SECURITY_TOKEN] +# CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_CONFIG_BUCKET = os.environ[constants.CRM_CONFIG_BUCKET] +# CRMのバックアップデータを格納するバケット名 +CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET] +# CRMの取込データを格納するバケット名 +IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET] +# CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER] +# CRM取得対象オブジェクトの情報のファイル名 +OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME] +# CRMデータ取得結果を格納するフォルダパス +PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER] +# CRMデータ取得結果を格納するファイル名 +PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME] +# CRMからの最終取得日時ファイルを格納するフォルダパス +LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER] +# CRMから取得し、取込用に変換したデータを格納するフォルダ +CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER] +# CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER] +# CRMから取得した生データのバックアップを格納するフォルダパス +RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER] +# CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ +CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER] diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch.py index 9670767a..893aaadc 100644 --- a/ecs/crm-datafetch/src/fetch.py +++ b/ecs/crm-datafetch/src/fetch.py @@ -1,31 +1,22 @@ from requests.exceptions import ConnectTimeout, ReadTimeout from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential + +from src.constants import FETCH_JP_NAME +from src.environments import (CRM_AUTH_TIMEOUT, + CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, + CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + CRM_FETCH_RECORD_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, + CRM_GET_RECORD_COUNT_TIMEOUT) +from src.error.exceptions import DataConvertException, SalesforceAPIException +from src.salesforce.salesforce_api import SalesForceCount, SalesForceData from src.util.logger import logger_instance as logger -from src.constants import( - FETCH_JP_NAME -) -from src.environments import( - CRM_AUTH_TIMEOUT, - CRM_GET_RECORD_COUNT_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL -) -from src.error.exceptions import( - SalesforceAPIException, - DataConvertException -) -from src.salesforce.salesforce_api import( - SalesForceCount, - SalesForceData -) def fetch_crm_data(target_object, last_fetch_datetime): diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parse.py index 4273125d..f9cda105 100644 --- a/ecs/crm-datafetch/src/parser/json_parse.py +++ b/ecs/crm-datafetch/src/parser/json_parse.py @@ -1,7 +1,9 @@ -import re import json +import re + from src.constants import EXCLUDE_SYMBOL + class JsonParser(): def __init__(self, json_str) -> None: self.__json_str = json_str diff --git a/ecs/crm-datafetch/src/pre.py b/ecs/crm-datafetch/src/pre.py index c8190f12..adb6f68e 100644 --- a/ecs/crm-datafetch/src/pre.py +++ b/ecs/crm-datafetch/src/pre.py @@ -1,21 +1,13 @@ from datetime import datetime -from src.util.logger import logger_instance as logger -from src.constants import( - PRE_JP_NAME, - YYYYMMDDTHHMMSSTZ -) -from src.environments import( - CRM_CONFIG_BUCKET, - OBJECT_INFO_FOLDER, - OBJECT_INFO_FILENAME -) -from src.error.exceptions import( - FileNotFoundException, - InvalidConfigException -) + from src.aws.s3 import ConfigBucket -from src.parser.json_parse import JsonParser from src.config.objects import FetchTargetObjects +from src.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ +from src.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER) +from src.error.exceptions import FileNotFoundException, InvalidConfigException +from src.parser.json_parse import JsonParser +from src.util.logger import logger_instance as logger def prepare_get_data(): diff --git a/ecs/crm-datafetch/src/process.py b/ecs/crm-datafetch/src/process.py index 09d938af..add1e98d 100644 --- a/ecs/crm-datafetch/src/process.py +++ b/ecs/crm-datafetch/src/process.py @@ -1,17 +1,15 @@ -from src.pre import prepare_get_data # データ取得準備処理 -from src.chk import check_object_info # オブジェクト情報形式チェック処理 -from src.date import set_datetime_period # データ取得期間設定処理 -from src.fetch import fetch_crm_data # CRMデータ取得処理 -from src.resbk import backup_crm_data # CRM電文データバックアップ処理 -from src.conv import convert_crm_csvdata # CSV変換処理 -from src.csvbk import backup_crm_csvdata # CSVバックアップ処理 -from src.upld import copy_crm_csvdata # CSVアップロード処理 -from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 -from src.end import updload_result_data # 取得処理実施結果アップロード処理 +from src.chk import check_object_info # オブジェクト情報形式チェック処理 +from src.conv import convert_crm_csvdata # CSV変換処理 +from src.csvbk import backup_crm_csvdata # CSVバックアップ処理 +from src.date import set_datetime_period # データ取得期間設定処理 +from src.end import updload_result_data # 取得処理実施結果アップロード処理 +from src.error.exceptions import MeDaCaCRMDataFetchException +from src.fetch import fetch_crm_data # CRMデータ取得処理 +from src.pre import prepare_get_data # データ取得準備処理 +from src.resbk import backup_crm_data # CRM電文データバックアップ処理 +from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 +from src.upld import copy_crm_csvdata # CSVアップロード処理 from src.util.logger import logger_instance as logger -from src.error.exceptions import( - MeDaCaCRMDataFetchException -) def main() -> None: diff --git a/ecs/crm-datafetch/src/resbk.py b/ecs/crm-datafetch/src/resbk.py index 21d516ef..61bfe306 100644 --- a/ecs/crm-datafetch/src/resbk.py +++ b/ecs/crm-datafetch/src/resbk.py @@ -1,11 +1,7 @@ -from src.util.logger import logger_instance as logger -from src.constants import( - RESBK_JP_NAME -) -from src.error.exceptions import( - FileUploadException -) from src.aws.s3 import BackupBucket +from src.constants import RESBK_JP_NAME +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def backup_crm_data(object_name, sf_object_jsons, date_path): diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index 0bce5675..ac6151a9 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,26 +1,19 @@ from simple_salesforce import Salesforce -from src.environments import( - CRM_AUTH_DOMAIN, - CRM_USER_NAME, - CRM_USER_PASSWORD, - CRM_USER_SECURITY_TOKEN, - CRM_AUTH_TIMEOUT, - CRM_AUTH_MAX_RETRY_ATTEMPT, - CRM_AUTH_RETRY_INTERVAL, - CRM_AUTH_RETRY_MIN_INTERVAL, - CRM_AUTH_RETRY_MAX_INTERVAL, - CRM_GET_RECORD_COUNT_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL -) - +from src.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT, + CRM_AUTH_RETRY_INTERVAL, + CRM_AUTH_RETRY_MAX_INTERVAL, + CRM_AUTH_RETRY_MIN_INTERVAL, CRM_AUTH_TIMEOUT, + CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, + CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + CRM_FETCH_RECORD_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, + CRM_GET_RECORD_COUNT_TIMEOUT, CRM_USER_NAME, + CRM_USER_PASSWORD, CRM_USER_SECURITY_TOKEN) FETCH_SOQL = """SELECT {column_or_expression} FROM {object_name} WHERE SystemModStamp > {last_update_datetime_from} diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upd.py index f8dd4350..d69bddcc 100644 --- a/ecs/crm-datafetch/src/upd.py +++ b/ecs/crm-datafetch/src/upd.py @@ -1,12 +1,9 @@ import json -from src.util.logger import logger_instance as logger + from src.aws.s3 import ConfigBucket -from src.constants import ( - UPD_JP_NAME -) -from src.error.exceptions import( - FileUploadException -) +from src.constants import UPD_JP_NAME +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def updload_last_fetch_datetime(target_object, last_fetch_datetime): diff --git a/ecs/crm-datafetch/src/upld.py b/ecs/crm-datafetch/src/upld.py index c3c3140f..2384ae57 100644 --- a/ecs/crm-datafetch/src/upld.py +++ b/ecs/crm-datafetch/src/upld.py @@ -1,17 +1,9 @@ -from src.util.logger import logger_instance as logger -from src.constants import ( - UPLD_JP_NAME -) -from src.environments import( - CRM_BACKUP_BUCKET, - IMPORT_DATA_BUCKET, - CRM_IMPORT_DATA_FOLDER, - CRM_IMPORT_DATA_BACKUP_FOLDER, -) -from src.error.exceptions import( - FileUploadException -) from src.aws.s3 import S3ResourceNonBucket +from src.constants import UPLD_JP_NAME +from src.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, + CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def copy_crm_csvdata(target_object, date_path): diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index ec0baf55..cb139c98 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -1,4 +1,5 @@ import logging + from src.environments import LOG_LEVEL for name in ["boto3", "botocore", "s3transfer", "urllib3"]: From e532b395df1f4f0c3e4435fea3f538decad37afb Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 20:00:35 +0900 Subject: [PATCH 033/275] =?UTF-8?q?style:=20Flake8=E3=81=AB=E3=82=88?= =?UTF-8?q?=E3=82=8B=E3=83=AA=E3=83=B3=E3=83=88=E4=B8=8D=E6=AD=A3=E3=81=AE?= =?UTF-8?q?=E8=A7=A3=E6=B6=88=E3=80=82=E3=83=95=E3=82=A9=E3=83=BC=E3=83=9E?= =?UTF-8?q?=E3=83=83=E3=83=88=E6=96=87=E5=AD=97=E5=88=97=E3=81=AF=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E3=81=8C=E5=9F=8B=E3=82=81=E8=BE=BC=E3=81=BE=E3=82=8C?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=AA=E3=81=8F=E3=81=A6=E3=82=82=E3=82=A8?= =?UTF-8?q?=E3=83=A9=E3=83=BC=E3=81=AB=E3=81=97=E3=81=AA=E3=81=84=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=80=81=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF=E3=82=92?= =?UTF-8?q?=E7=84=A1=E8=A6=96=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.vscode/recommend_settings.json | 2 +- ecs/crm-datafetch/src/conv.py | 2 -- ecs/crm-datafetch/src/fetch.py | 10 ++++++++-- ecs/crm-datafetch/src/process.py | 2 +- ecs/crm-datafetch/src/upd.py | 6 ++++-- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ecs/crm-datafetch/.vscode/recommend_settings.json b/ecs/crm-datafetch/.vscode/recommend_settings.json index 72dacf55..66a8f2b5 100644 --- a/ecs/crm-datafetch/.vscode/recommend_settings.json +++ b/ecs/crm-datafetch/.vscode/recommend_settings.json @@ -10,7 +10,7 @@ "python.linting.enabled": true, "python.linting.pylintEnabled": false, "python.linting.flake8Enabled": true, - "python.linting.flake8Args": ["--max-line-length=150"], + "python.linting.flake8Args": ["--max-line-length=150", "--ignore=F541"], "python.formatting.provider": "autopep8", "python.formatting.autopep8Args": ["--max-line-length", "150"] } diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/conv.py index cfcc6a8a..2a4d3be8 100644 --- a/ecs/crm-datafetch/src/conv.py +++ b/ecs/crm-datafetch/src/conv.py @@ -1,5 +1,3 @@ -from datetime import datetime - from src.constants import CONV_JP_NAME from src.converter.converter import CSVStringConverter from src.error.exceptions import DataConvertException diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch.py index 893aaadc..baa0219f 100644 --- a/ecs/crm-datafetch/src/fetch.py +++ b/ecs/crm-datafetch/src/fetch.py @@ -81,7 +81,10 @@ def fetch_crm_data(target_object, last_fetch_datetime): return sf_object_jsons -@retry(wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) +@retry( + wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), + stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) def fetch_sf_count_retry(object_name, last_update_datetime_from, last_update_datetime_to): try: global count_contime_counter, count_readtime_counter, count_counter @@ -110,7 +113,10 @@ def fetch_sf_count_retry(object_name, last_update_datetime_from, last_update_dat raise e -@retry(wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) +@retry( + wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, + min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), + stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) def fetch_sf_data_retry(columns, object_name, last_update_datetime_from, last_update_datetime_to): try: global data_contime_counter, data_readtime_counter, data_counter diff --git a/ecs/crm-datafetch/src/process.py b/ecs/crm-datafetch/src/process.py index add1e98d..e914a4d4 100644 --- a/ecs/crm-datafetch/src/process.py +++ b/ecs/crm-datafetch/src/process.py @@ -80,7 +80,7 @@ def main() -> None: # 9. CSV変換処理を呼び出す logger.info( f'I-CTRL-12 [{target_object.object_name}] CSV変換処理呼び出し') - csv_object = convert_crm_csvdata(target_object, sf_object_jsons) + csv_object = convert_crm_csvdata(target_object, sf_object_jsons) # 10. CSVバックアップ処理を呼び出す logger.info( diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upd.py index d69bddcc..d6a8ef03 100644 --- a/ecs/crm-datafetch/src/upd.py +++ b/ecs/crm-datafetch/src/upd.py @@ -12,7 +12,7 @@ def updload_last_fetch_datetime(target_object, last_fetch_datetime): f'I-UPD-01 [{target_object.object_name}] の前回取得日時ファイルの更新処理を開始します') try: - if target_object.is_update_last_fetch_datetime == False: + if target_object.is_update_last_fetch_datetime is False: # ② オブジェクト情報.is_update_last_fetch_datetimeがfalseの場合、以降の処理をスキップする logger.info( f'I-UPD-02 [{target_object.object_name}] の前回取得日時ファイルの更新処理をスキップします') @@ -33,7 +33,9 @@ def updload_last_fetch_datetime(target_object, last_fetch_datetime): except Exception as e: raise FileUploadException( - 'E-UPD-01', UPD_JP_NAME, f'[{target_object.object_name}] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[{target_object.last_fetch_datetime_file_name}] エラー内容:[{e}]') + 'E-UPD-01', + UPD_JP_NAME, + f'[{target_object.object_name}] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[{target_object.last_fetch_datetime_file_name}] エラー内容:[{e}]') # ④ 前回取得日時ファイル更新処理の終了ログを出力する logger.info( From 9b8076451f14b1ec07d5751d47cb79bbc1b709db Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 22 Jul 2022 15:56:43 +0900 Subject: [PATCH 034/275] =?UTF-8?q?feat:crm=5Fdata=5Fsync=E3=81=AE?= =?UTF-8?q?=E6=96=B0=E8=A6=8F=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 rds_mysql/stored_procedure/crm_data_sync.sql diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql new file mode 100644 index 00000000..c039a18a --- /dev/null +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -0,0 +1,41 @@ +-- A5M2で実行時にSQL区切り文字を「;」以外にすること +-- $$から始まる文字は後からREPLACEする文字を示す独自ルール +-- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない +CREATE PROCEDURE crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) +BEGIN + -- 例外処理 + -- エラーが発生した場合に一時テーブルの削除を実施 + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + ROLLBACK; + SIGNAL SQLSTATE '45000' + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; + END; + + SET @error_state = NULL, @error_msg = NULL; + START TRANSACTION; + + -- ①-1 Salesforce側で物理削除されたデータを検出し更新する + SET @update_end_datetime = ' + UPDATE $$target_table$$ tt + LEFT JOIN $$target_table_all$$ tta + ON tt.id = tta.id + AND tt.$$target_column$$ = tta.$$target_column$$ + SET + tt.end_datetime = CURRENT_TIMESTAMP() + , tt.upd_user = CURRENT_USER() + , tt.upd_date = CURRENT_TIMESTAMP() + WHERE + tta.id IS NULL + AND tt.end_datetime = "9999-12-31 00:00:00" + '; + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); + PREPARE update_end_datetime_stmt from @update_end_datetime; + EXECUTE update_end_datetime_stmt; + + COMMIT; +END \ No newline at end of file From af957662e92dda984f4930715afd0c3989694f18 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 22 Jul 2022 15:58:11 +0900 Subject: [PATCH 035/275] =?UTF-8?q?Revert=20"feat:crm=5Fdata=5Fsync?= =?UTF-8?q?=E3=81=AE=E6=96=B0=E8=A6=8F=E4=BD=9C=E6=88=90"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 9b8076451f14b1ec07d5751d47cb79bbc1b709db. --- rds_mysql/stored_procedure/crm_data_sync.sql | 41 -------------------- 1 file changed, 41 deletions(-) delete mode 100644 rds_mysql/stored_procedure/crm_data_sync.sql diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql deleted file mode 100644 index c039a18a..00000000 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ /dev/null @@ -1,41 +0,0 @@ --- A5M2で実行時にSQL区切り文字を「;」以外にすること --- $$から始まる文字は後からREPLACEする文字を示す独自ルール --- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない -CREATE PROCEDURE crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) -BEGIN - -- 例外処理 - -- エラーが発生した場合に一時テーブルの削除を実施 - DECLARE EXIT HANDLER FOR SQLEXCEPTION - BEGIN - GET DIAGNOSTICS CONDITION 1 - @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; - ROLLBACK; - SIGNAL SQLSTATE '45000' - SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; - END; - - SET @error_state = NULL, @error_msg = NULL; - START TRANSACTION; - - -- ①-1 Salesforce側で物理削除されたデータを検出し更新する - SET @update_end_datetime = ' - UPDATE $$target_table$$ tt - LEFT JOIN $$target_table_all$$ tta - ON tt.id = tta.id - AND tt.$$target_column$$ = tta.$$target_column$$ - SET - tt.end_datetime = CURRENT_TIMESTAMP() - , tt.upd_user = CURRENT_USER() - , tt.upd_date = CURRENT_TIMESTAMP() - WHERE - tta.id IS NULL - AND tt.end_datetime = "9999-12-31 00:00:00" - '; - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); - PREPARE update_end_datetime_stmt from @update_end_datetime; - EXECUTE update_end_datetime_stmt; - - COMMIT; -END \ No newline at end of file From 4a73e14bbfb71f6fa32b8f6bbba8696a8b4363d6 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Fri, 22 Jul 2022 16:00:57 +0900 Subject: [PATCH 036/275] =?UTF-8?q?feat:crm=5Fdata=5Fsync=E3=81=AE?= =?UTF-8?q?=E6=96=B0=E8=A6=8F=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 rds_mysql/stored_procedure/crm_data_sync.sql diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql new file mode 100644 index 00000000..c039a18a --- /dev/null +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -0,0 +1,41 @@ +-- A5M2で実行時にSQL区切り文字を「;」以外にすること +-- $$から始まる文字は後からREPLACEする文字を示す独自ルール +-- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない +CREATE PROCEDURE crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) +BEGIN + -- 例外処理 + -- エラーが発生した場合に一時テーブルの削除を実施 + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + ROLLBACK; + SIGNAL SQLSTATE '45000' + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; + END; + + SET @error_state = NULL, @error_msg = NULL; + START TRANSACTION; + + -- ①-1 Salesforce側で物理削除されたデータを検出し更新する + SET @update_end_datetime = ' + UPDATE $$target_table$$ tt + LEFT JOIN $$target_table_all$$ tta + ON tt.id = tta.id + AND tt.$$target_column$$ = tta.$$target_column$$ + SET + tt.end_datetime = CURRENT_TIMESTAMP() + , tt.upd_user = CURRENT_USER() + , tt.upd_date = CURRENT_TIMESTAMP() + WHERE + tta.id IS NULL + AND tt.end_datetime = "9999-12-31 00:00:00" + '; + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); + PREPARE update_end_datetime_stmt from @update_end_datetime; + EXECUTE update_end_datetime_stmt; + + COMMIT; +END \ No newline at end of file From 8f65e0f87f56643efcf0a027b09f5d22b1ad75c6 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 25 Jul 2022 15:07:41 +0900 Subject: [PATCH 037/275] =?UTF-8?q?refactor:=E2=91=A1-2=E5=86=85=E9=83=A8?= =?UTF-8?q?=E7=B5=90=E5=90=88=E3=81=AB=E5=A4=89=E6=9B=B4=E3=81=97Where?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index b60e2681..366cde70 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -70,15 +70,13 @@ BEGIN -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' UPDATE $$target_table$$ tt - LEFT JOIN $$target_table$$_make_history_tmp mht + JOIN $$target_table$$_make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET end_datetime = mht.max_start_datetime - INTERVAL 1 SECOND , upd_user = CURRENT_USER() , upd_date = CURRENT_TIMESTAMP() - WHERE - mht.id IS NOT NULL '; SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); From 440e6c8fd094f0800d3d8d9b7555361ae45fdf28 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 26 Jul 2022 09:54:16 +0900 Subject: [PATCH 038/275] =?UTF-8?q?fix:=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E5=90=8D=E5=A4=89=E6=9B=B4=E3=80=81=E7=92=B0=E5=A2=83?= =?UTF-8?q?=E5=A4=89=E6=95=B0=E3=81=AE=E3=83=91=E3=82=B9=E3=82=92=E5=A4=89?= =?UTF-8?q?=E6=9B=B4=E3=81=97=E3=81=9F=E3=82=82=E3=81=AE=E3=82=92=E4=B8=80?= =?UTF-8?q?=E5=BA=A6=E3=82=B3=E3=83=9F=E3=83=83=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.vscode/launch.json | 16 ++++++++++++++ ecs/crm-datafetch/main.py | 4 ++-- ecs/crm-datafetch/src/aws/s3.py | 4 ++-- ...csvbk.py => backup_crm_csvdata_process.py} | 2 +- .../{resbk.py => backup_crm_data_process.py} | 2 +- .../{chk.py => check_object_info_process.py} | 2 +- ecs/crm-datafetch/src/config/objects.py | 2 +- .../src/{process.py => controller.py} | 22 +++++++++---------- ...conv.py => convert_crm_csvdata_process.py} | 2 +- ecs/crm-datafetch/src/converter/converter.py | 2 +- .../{upld.py => copy_crm_csvdata_process.py} | 4 ++-- .../{fetch.py => fetch_crm_data_process.py} | 4 ++-- ecs/crm-datafetch/src/parser/json_parse.py | 2 +- .../{pre.py => prepare_get_data_process.py} | 4 ++-- .../src/salesforce/salesforce_api.py | 2 +- ...date.py => set_datetime_period_process.py} | 4 ++-- .../src/{ => system_var}/constants.py | 0 .../src/{ => system_var}/environments.py | 2 +- ...{end.py => updload_result_data_process.py} | 4 ++-- ... => upload_last_fetch_datetime_process.py} | 2 +- ecs/crm-datafetch/src/util/logger.py | 2 +- 21 files changed, 52 insertions(+), 36 deletions(-) create mode 100644 ecs/crm-datafetch/.vscode/launch.json rename ecs/crm-datafetch/src/{csvbk.py => backup_crm_csvdata_process.py} (96%) rename ecs/crm-datafetch/src/{resbk.py => backup_crm_data_process.py} (96%) rename ecs/crm-datafetch/src/{chk.py => check_object_info_process.py} (95%) rename ecs/crm-datafetch/src/{process.py => controller.py} (87%) rename ecs/crm-datafetch/src/{conv.py => convert_crm_csvdata_process.py} (95%) rename ecs/crm-datafetch/src/{upld.py => copy_crm_csvdata_process.py} (92%) rename ecs/crm-datafetch/src/{fetch.py => fetch_crm_data_process.py} (98%) rename ecs/crm-datafetch/src/{pre.py => prepare_get_data_process.py} (95%) rename ecs/crm-datafetch/src/{date.py => set_datetime_period_process.py} (95%) rename ecs/crm-datafetch/src/{ => system_var}/constants.py (100%) rename ecs/crm-datafetch/src/{ => system_var}/environments.py (99%) rename ecs/crm-datafetch/src/{end.py => updload_result_data_process.py} (91%) rename ecs/crm-datafetch/src/{upd.py => upload_last_fetch_datetime_process.py} (97%) diff --git a/ecs/crm-datafetch/.vscode/launch.json b/ecs/crm-datafetch/.vscode/launch.json new file mode 100644 index 00000000..2e39a541 --- /dev/null +++ b/ecs/crm-datafetch/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + // エントリーポイントのファイルに変更すること + "program": "main.py", + "console": "integratedTerminal", + "justMyCode": true, + // 環境変数が必要な場合に読み込む環境変数ファイル + "envFile": "${workspaceFolder}/.env", + } + ] +} \ No newline at end of file diff --git a/ecs/crm-datafetch/main.py b/ecs/crm-datafetch/main.py index b0edf545..6e197d30 100644 --- a/ecs/crm-datafetch/main.py +++ b/ecs/crm-datafetch/main.py @@ -1,4 +1,4 @@ -import src.process as process +import src.controller as controller if __name__ == '__main__': - process.main() + controller.main() diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index e8691e16..dec0ec62 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -1,8 +1,8 @@ import json import boto3 -from src.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY -from src.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, +from src.system_var.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY +from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET, LAST_FETCH_DATE_FOLDER, OBJECT_INFO_FILENAME, diff --git a/ecs/crm-datafetch/src/csvbk.py b/ecs/crm-datafetch/src/backup_crm_csvdata_process.py similarity index 96% rename from ecs/crm-datafetch/src/csvbk.py rename to ecs/crm-datafetch/src/backup_crm_csvdata_process.py index 375b273e..1960e7b0 100644 --- a/ecs/crm-datafetch/src/csvbk.py +++ b/ecs/crm-datafetch/src/backup_crm_csvdata_process.py @@ -1,5 +1,5 @@ from src.aws.s3 import BackupBucket -from src.constants import CSVBK_JP_NAME +from src.system_var.constants import CSVBK_JP_NAME from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/resbk.py b/ecs/crm-datafetch/src/backup_crm_data_process.py similarity index 96% rename from ecs/crm-datafetch/src/resbk.py rename to ecs/crm-datafetch/src/backup_crm_data_process.py index 61bfe306..130da769 100644 --- a/ecs/crm-datafetch/src/resbk.py +++ b/ecs/crm-datafetch/src/backup_crm_data_process.py @@ -1,5 +1,5 @@ from src.aws.s3 import BackupBucket -from src.constants import RESBK_JP_NAME +from src.system_var.constants import RESBK_JP_NAME from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/chk.py b/ecs/crm-datafetch/src/check_object_info_process.py similarity index 95% rename from ecs/crm-datafetch/src/chk.py rename to ecs/crm-datafetch/src/check_object_info_process.py index b842d19b..91551d21 100644 --- a/ecs/crm-datafetch/src/chk.py +++ b/ecs/crm-datafetch/src/check_object_info_process.py @@ -1,5 +1,5 @@ from src.config.objects import TargetObject -from src.constants import CHK_JP_NAME +from src.system_var.constants import CHK_JP_NAME from src.error.exceptions import InvalidConfigException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 8d9f6f85..c00274bf 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,4 +1,4 @@ -from src.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ +from src.system_var.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ from src.util.dict_checker import DictCheck diff --git a/ecs/crm-datafetch/src/process.py b/ecs/crm-datafetch/src/controller.py similarity index 87% rename from ecs/crm-datafetch/src/process.py rename to ecs/crm-datafetch/src/controller.py index e914a4d4..ebd3b3cd 100644 --- a/ecs/crm-datafetch/src/process.py +++ b/ecs/crm-datafetch/src/controller.py @@ -1,14 +1,14 @@ -from src.chk import check_object_info # オブジェクト情報形式チェック処理 -from src.conv import convert_crm_csvdata # CSV変換処理 -from src.csvbk import backup_crm_csvdata # CSVバックアップ処理 -from src.date import set_datetime_period # データ取得期間設定処理 -from src.end import updload_result_data # 取得処理実施結果アップロード処理 +from src.check_object_info_process import check_object_info # オブジェクト情報形式チェック処理 +from src.convert_crm_csvdata_process import convert_crm_csvdata # CSV変換処理 +from src.backup_crm_csvdata_process import backup_crm_csvdata # CSVバックアップ処理 +from src.set_datetime_period_process import set_datetime_period # データ取得期間設定処理 +from src.updload_result_data_process import updload_result_data # 取得処理実施結果アップロード処理 from src.error.exceptions import MeDaCaCRMDataFetchException -from src.fetch import fetch_crm_data # CRMデータ取得処理 -from src.pre import prepare_get_data # データ取得準備処理 -from src.resbk import backup_crm_data # CRM電文データバックアップ処理 -from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 -from src.upld import copy_crm_csvdata # CSVアップロード処理 +from src.fetch_crm_data_process import fetch_crm_data # CRMデータ取得処理 +from src.prepare_get_data_process import prepare_get_data # データ取得準備処理 +from src.backup_crm_data_process import backup_crm_data # CRM電文データバックアップ処理 +from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime # 前回取得日時ファイル更新 +from src.copy_crm_csvdata_process import copy_crm_csvdata # CSVアップロード処理 from src.util.logger import logger_instance as logger @@ -95,7 +95,7 @@ def main() -> None: # 12. 前回取得日時ファイル更新処理を呼びだす logger.info( f'I-CTRL-15 [{target_object.object_name}] 前回取得日時ファイル更新処理呼び出し') - updload_last_fetch_datetime(target_object, last_fetch_datetime) + upload_last_fetch_datetime(target_object, last_fetch_datetime) # 13. オブジェクト処理結果の更新 process_result[target_object.object_name] = 'success' diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/convert_crm_csvdata_process.py similarity index 95% rename from ecs/crm-datafetch/src/conv.py rename to ecs/crm-datafetch/src/convert_crm_csvdata_process.py index 2a4d3be8..4ef6dd11 100644 --- a/ecs/crm-datafetch/src/conv.py +++ b/ecs/crm-datafetch/src/convert_crm_csvdata_process.py @@ -1,4 +1,4 @@ -from src.constants import CONV_JP_NAME +from src.system_var.constants import CONV_JP_NAME from src.converter.converter import CSVStringConverter from src.error.exceptions import DataConvertException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 18fc2923..7b4adb1e 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -3,7 +3,7 @@ import io import re from datetime import datetime -from src.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, +from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, CSV_TRUE_VALUE, YYYYMMDDHHMMSS) diff --git a/ecs/crm-datafetch/src/upld.py b/ecs/crm-datafetch/src/copy_crm_csvdata_process.py similarity index 92% rename from ecs/crm-datafetch/src/upld.py rename to ecs/crm-datafetch/src/copy_crm_csvdata_process.py index 2384ae57..ecebe9ea 100644 --- a/ecs/crm-datafetch/src/upld.py +++ b/ecs/crm-datafetch/src/copy_crm_csvdata_process.py @@ -1,6 +1,6 @@ from src.aws.s3 import S3ResourceNonBucket -from src.constants import UPLD_JP_NAME -from src.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, +from src.system_var.constants import UPLD_JP_NAME +from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py similarity index 98% rename from ecs/crm-datafetch/src/fetch.py rename to ecs/crm-datafetch/src/fetch_crm_data_process.py index baa0219f..eac5478b 100644 --- a/ecs/crm-datafetch/src/fetch.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -2,8 +2,8 @@ from requests.exceptions import ConnectTimeout, ReadTimeout from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential -from src.constants import FETCH_JP_NAME -from src.environments import (CRM_AUTH_TIMEOUT, +from src.system_var.constants import FETCH_JP_NAME +from src.system_var.environments import (CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, CRM_FETCH_RECORD_RETRY_INTERVAL, CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parse.py index f9cda105..5e7249b9 100644 --- a/ecs/crm-datafetch/src/parser/json_parse.py +++ b/ecs/crm-datafetch/src/parser/json_parse.py @@ -1,7 +1,7 @@ import json import re -from src.constants import EXCLUDE_SYMBOL +from src.system_var.constants import EXCLUDE_SYMBOL class JsonParser(): diff --git a/ecs/crm-datafetch/src/pre.py b/ecs/crm-datafetch/src/prepare_get_data_process.py similarity index 95% rename from ecs/crm-datafetch/src/pre.py rename to ecs/crm-datafetch/src/prepare_get_data_process.py index adb6f68e..e96de09c 100644 --- a/ecs/crm-datafetch/src/pre.py +++ b/ecs/crm-datafetch/src/prepare_get_data_process.py @@ -2,8 +2,8 @@ from datetime import datetime from src.aws.s3 import ConfigBucket from src.config.objects import FetchTargetObjects -from src.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ -from src.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, +from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ +from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, OBJECT_INFO_FOLDER) from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index ac6151a9..a6b9c458 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,5 +1,5 @@ from simple_salesforce import Salesforce -from src.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT, +from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT, CRM_AUTH_RETRY_INTERVAL, CRM_AUTH_RETRY_MAX_INTERVAL, CRM_AUTH_RETRY_MIN_INTERVAL, CRM_AUTH_TIMEOUT, diff --git a/ecs/crm-datafetch/src/date.py b/ecs/crm-datafetch/src/set_datetime_period_process.py similarity index 95% rename from ecs/crm-datafetch/src/date.py rename to ecs/crm-datafetch/src/set_datetime_period_process.py index 2896c366..ea281fba 100644 --- a/ecs/crm-datafetch/src/date.py +++ b/ecs/crm-datafetch/src/set_datetime_period_process.py @@ -1,7 +1,7 @@ from src.aws.s3 import ConfigBucket from src.config.objects import LastFetchDatetime -from src.constants import DATE_JP_NAME -from src.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.system_var.constants import DATE_JP_NAME +from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/constants.py b/ecs/crm-datafetch/src/system_var/constants.py similarity index 100% rename from ecs/crm-datafetch/src/constants.py rename to ecs/crm-datafetch/src/system_var/constants.py diff --git a/ecs/crm-datafetch/src/environments.py b/ecs/crm-datafetch/src/system_var/environments.py similarity index 99% rename from ecs/crm-datafetch/src/environments.py rename to ecs/crm-datafetch/src/system_var/environments.py index 4a09b2a2..dfa87ae3 100644 --- a/ecs/crm-datafetch/src/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -1,6 +1,6 @@ import os -import src.constants as constants +import src.system_var.constants as constants # environments(task settings file) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する diff --git a/ecs/crm-datafetch/src/end.py b/ecs/crm-datafetch/src/updload_result_data_process.py similarity index 91% rename from ecs/crm-datafetch/src/end.py rename to ecs/crm-datafetch/src/updload_result_data_process.py index 471effc2..e70f4679 100644 --- a/ecs/crm-datafetch/src/end.py +++ b/ecs/crm-datafetch/src/updload_result_data_process.py @@ -1,6 +1,6 @@ from src.aws.s3 import BackupBucket -from src.constants import END_JP_NAME -from src.environments import PROCESS_RESULT_FILENAME +from src.system_var.constants import END_JP_NAME +from src.system_var.environments import PROCESS_RESULT_FILENAME from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py similarity index 97% rename from ecs/crm-datafetch/src/upd.py rename to ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py index d6a8ef03..f606c9ad 100644 --- a/ecs/crm-datafetch/src/upd.py +++ b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py @@ -1,7 +1,7 @@ import json from src.aws.s3 import ConfigBucket -from src.constants import UPD_JP_NAME +from src.system_var.constants import UPD_JP_NAME from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index cb139c98..689d9c36 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -1,6 +1,6 @@ import logging -from src.environments import LOG_LEVEL +from src.system_var.environments import LOG_LEVEL for name in ["boto3", "botocore", "s3transfer", "urllib3"]: logging.getLogger(name).setLevel(logging.WARNING) From 016e4465e5ac3576aa023962205d055464bb13eb Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 26 Jul 2022 10:32:23 +0900 Subject: [PATCH 039/275] =?UTF-8?q?refactor:EXISTS=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 21 ++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql index c039a18a..01d27b9a 100644 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -20,16 +20,21 @@ BEGIN -- ①-1 Salesforce側で物理削除されたデータを検出し更新する SET @update_end_datetime = ' UPDATE $$target_table$$ tt - LEFT JOIN $$target_table_all$$ tta - ON tt.id = tta.id - AND tt.$$target_column$$ = tta.$$target_column$$ SET - tt.end_datetime = CURRENT_TIMESTAMP() - , tt.upd_user = CURRENT_USER() - , tt.upd_date = CURRENT_TIMESTAMP() + tt.end_datetime = CURRENT_TIMESTAMP () + , tt.upd_user = CURRENT_USER () + , tt.upd_date = CURRENT_TIMESTAMP () WHERE - tta.id IS NULL - AND tt.end_datetime = "9999-12-31 00:00:00" + tt.end_datetime = "9999-12-31 00:00:00" + AND NOT EXISTS ( + SELECT + * + FROM + $$target_table_all$$ tta + WHERE + tt.id = tta.id + AND tt.$$target_column$$ = tta.$$target_column$$ + ) '; SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); From d04af5c6443d8a1b61168f700e202ea4fbbb6f74 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 26 Jul 2022 11:29:00 +0900 Subject: [PATCH 040/275] =?UTF-8?q?refactor:INNER=E3=82=92=E6=98=8E?= =?UTF-8?q?=E7=A4=BA=E7=9A=84=E3=81=AB=E8=A8=98=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 366cde70..10bb0095 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -70,7 +70,7 @@ BEGIN -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' UPDATE $$target_table$$ tt - JOIN $$target_table$$_make_history_tmp mht + INNER JOIN $$target_table$$_make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET From 76750b86b3e8f76ef340f2814cf5240c58bab7bf Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 26 Jul 2022 11:33:32 +0900 Subject: [PATCH 041/275] =?UTF-8?q?refactor:EXISTS=E3=81=AE=E4=B8=AD?= =?UTF-8?q?=E8=BA=AB=E3=81=AESELECT=E5=8F=A5=E3=82=92ID=E3=81=AE=E3=81=BF?= =?UTF-8?q?=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql index 01d27b9a..155bfb99 100644 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -28,7 +28,7 @@ BEGIN tt.end_datetime = "9999-12-31 00:00:00" AND NOT EXISTS ( SELECT - * + tt.id FROM $$target_table_all$$ tta WHERE @@ -37,8 +37,8 @@ BEGIN ) '; SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; From 66b79a3115b35a84a6aaf9867567233985d3559d Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 26 Jul 2022 11:43:03 +0900 Subject: [PATCH 042/275] =?UTF-8?q?refactor:=E5=8F=82=E7=85=A7ID=E3=82=92t?= =?UTF-8?q?ta=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql index 155bfb99..e9986c0f 100644 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -28,7 +28,7 @@ BEGIN tt.end_datetime = "9999-12-31 00:00:00" AND NOT EXISTS ( SELECT - tt.id + tta.id FROM $$target_table_all$$ tta WHERE From a41a40004b4f9ed97304b5d77537f0994be52da7 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Wed, 27 Jul 2022 23:28:28 +0900 Subject: [PATCH 043/275] =?UTF-8?q?fix:=20=E3=82=BD=E3=83=BC=E3=82=B9?= =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=89=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E3=81=AE=E6=8C=87=E6=91=98=E5=8F=8D=E6=98=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/main.py | 5 +- ecs/crm-datafetch/src/aws/s3.py | 76 +- .../src/backup_crm_csv_data_process.py | 58 + .../src/backup_crm_csvdata_process.py | 31 - .../src/backup_crm_data_process.py | 31 +- .../src/check_object_info_process.py | 26 +- ecs/crm-datafetch/src/config/objects.py | 243 +- ecs/crm-datafetch/src/controller.py | 257 +- .../src/convert_crm_csv_data_process.py | 51 + .../src/convert_crm_csvdata_process.py | 25 - .../src/converter/convert_factory.py | 55 + ecs/crm-datafetch/src/converter/converter.py | 99 +- .../src/copy_crm_csv_data_process.py | 57 + .../src/copy_crm_csvdata_process.py | 32 - ecs/crm-datafetch/src/error/exceptions.py | 2 +- .../src/fetch_crm_data_process.py | 64 +- ...ocess.py => prepare_data_fetch_process.py} | 46 +- .../src/salesforce/salesforce_api.py | 53 +- .../src/salesforce/soql_builder.py | 34 + .../src/set_datetime_period_process.py | 41 +- ecs/crm-datafetch/src/system_var/constants.py | 33 +- .../src/system_var/environments.py | 48 +- .../src/upload_last_fetch_datetime_process.py | 50 +- ...ocess.py => upload_result_data_process.py} | 27 +- ecs/crm-datafetch/src/util/dict_checker.py | 75 +- .../src/util/execute_datetime.py | 17 + ecs/crm-datafetch/src/util/logger.py | 1 + .../crm/object_info/crm_object_list_all.json | 47 + .../crm/object_info/crm_object_list_diff.json | 3043 ++++++++++++++++- 29 files changed, 3947 insertions(+), 680 deletions(-) create mode 100644 ecs/crm-datafetch/src/backup_crm_csv_data_process.py delete mode 100644 ecs/crm-datafetch/src/backup_crm_csvdata_process.py create mode 100644 ecs/crm-datafetch/src/convert_crm_csv_data_process.py delete mode 100644 ecs/crm-datafetch/src/convert_crm_csvdata_process.py create mode 100644 ecs/crm-datafetch/src/converter/convert_factory.py create mode 100644 ecs/crm-datafetch/src/copy_crm_csv_data_process.py delete mode 100644 ecs/crm-datafetch/src/copy_crm_csvdata_process.py rename ecs/crm-datafetch/src/{prepare_get_data_process.py => prepare_data_fetch_process.py} (76%) create mode 100644 ecs/crm-datafetch/src/salesforce/soql_builder.py rename ecs/crm-datafetch/src/{updload_result_data_process.py => upload_result_data_process.py} (64%) create mode 100644 ecs/crm-datafetch/src/util/execute_datetime.py create mode 100644 s3/config/crm/object_info/crm_object_list_all.json diff --git a/ecs/crm-datafetch/main.py b/ecs/crm-datafetch/main.py index 6e197d30..dcae9825 100644 --- a/ecs/crm-datafetch/main.py +++ b/ecs/crm-datafetch/main.py @@ -1,4 +1,5 @@ -import src.controller as controller +from src.controller import controller +"""CRMデータ取得処理のエントリーポイント""" if __name__ == '__main__': - controller.main() + controller() diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index dec0ec62..e1d0dc70 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -1,7 +1,7 @@ import json import boto3 -from src.system_var.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY +from src.system_var.constants import AWS_RESOURCE_S3, S3_RESPONSE_BODY, S3_CHAR_CODE from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET, @@ -10,16 +10,6 @@ from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, RESPONSE_JSON_BACKUP_FOLDER) -class S3Clinet: - def __init__(self) -> None: - self.__s3_client = boto3.client(AWS_CLINET_S3) - - def copy_object(self, src_bucket, src_key, dest_bucket, dest_key): - self.__s3_client.copy( - {"Bucket": src_bucket, "key": src_key}, dest_bucket, dest_key) - return - - class S3Resource: def __init__(self, bucket_name: str) -> None: self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) @@ -28,41 +18,47 @@ class S3Resource: def get_object(self, object_key: str) -> str: response = self.__s3_bucket.Object(object_key).get() body = response[S3_RESPONSE_BODY].read() + return body.decode(S3_CHAR_CODE) - return body.decode('utf-8') - - def put_object(self, object_key: str, data) -> str: + def put_object(self, object_key: str, data: str) -> None: s3_object = self.__s3_bucket.Object(object_key) - s3_object.put(Body=data.encode('utf-8'), ContentEncoding='utf-8') + s3_object.put(Body=data.encode(S3_CHAR_CODE), ContentEncoding=S3_CHAR_CODE) return - -class S3ResourceNonBucket: - def __init__(self) -> None: - self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) - - def copy(self, src_bucket, src_key, dest_bucket, dest_key) -> None: + def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: copy_source = {'Bucket': src_bucket, 'Key': src_key} self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) - return +#class S3ResourceNonBucket: +# def __init__(self) -> None: +# self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) +# +# def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: +# copy_source = {'Bucket': src_bucket, 'Key': src_key} +# self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) +# return + + class ConfigBucket: __s3_resource: S3Resource = None def __init__(self) -> None: self.__s3_resource = S3Resource(CRM_CONFIG_BUCKET) + def __str__(self) -> str: + return CRM_CONFIG_BUCKET + def get_object_info_file(self) -> str: return self.__s3_resource.get_object(f'{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}') - def get_last_fetch_datetime_file(self, file_name) -> str: - return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_name}') + def get_last_fetch_datetime_file(self, file_path: str) -> str: + return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_path}') - def put_last_fetch_datetime_file(self, file_name, data) -> None: + def put_last_fetch_datetime_file(self, file_path: str, data: str) -> None: self.__s3_resource.put_object( - f'{LAST_FETCH_DATE_FOLDER}/{file_name}', data) + f'{LAST_FETCH_DATE_FOLDER}/{file_path}', data) return @@ -72,11 +68,19 @@ class DataBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(IMPORT_DATA_BUCKET) - def put_csv(self, file_name, data) -> None: - object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_name}' + def __str__(self) -> str: + return IMPORT_DATA_BUCKET + + def put_csv(self, file_path: str, data: str) -> None: + object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, data) return + def put_csv_from(self, src_bucket: str, src_key: str): + self.__s3_resource.copy(src_bucket, src_key, str(self), CRM_IMPORT_DATA_FOLDER) + return + + class BackupBucket: __s3_resource: S3Resource = None @@ -84,16 +88,20 @@ class BackupBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(CRM_BACKUP_BUCKET) - def put_response_json(self, file_name, data) -> None: - object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_name}' + def __str__(self) -> str: + return CRM_BACKUP_BUCKET + + def put_response_json(self, file_path: str, data: dict) -> None: + object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, json.dumps(data)) return - def put_csv_bk(self, file_name, data) -> None: - object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_name}' + def put_csv(self, file_path: str, data: str) -> None: + object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, data) return - def put_result_json(self, file_name, data) -> None: - object_key = f'{PROCESS_RESULT_FOLDER}/{file_name}' + def put_result_json(self, file_path: str, data: dict) -> None: + object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, json.dumps(data)) + return \ No newline at end of file diff --git a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py new file mode 100644 index 00000000..f14490cf --- /dev/null +++ b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py @@ -0,0 +1,58 @@ +from src.aws.s3 import BackupBucket +from src.config.objects import TargetObject +from src.system_var.constants import CSVBK_JP_NAME +from src.error.exceptions import FileUploadException +from src.util.execute_datetime import ExecuteDateTime +from src.util.logger import logger_instance as logger + + +def backup_crm_csv_data_process(target_object: TargetObject, exetute_datetime: ExecuteDateTime, csv_string: str): + """ + CSVバックアップ処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + csv_string : str + csvデータ + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + + # ① CSVバックアップ処理の開始ログを出力する + target_object_name = target_object.object_name + upload_file_name = target_object.upload_file_name + + logger.info( + f'I-CSVBK-01 [{target_object_name}] のCSVデータのバックアップ処理を開始します ファイル名：[{upload_file_name}.csv]') + + try: + # ② CRMバックアップ保管用バケットに、変換後のCSVデータのバックアップを保管する + backup_bucket = BackupBucket() + backup_bucket.put_csv( + f'{exetute_datetime.to_path()}/{upload_file_name}.csv', csv_string) + + logger.debug( + f'D-CSVBK-02 [{target_object_name}] のCSVデータバックアップ 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-CSVBK-01', + CSVBK_JP_NAME, f'[{target_object_name}] CSVデータのバックアップに失敗しました ファイル名：[{upload_file_name}.csv] エラー内容:[{e}]') + + # ③ CSVバックアップ処理の終了ログを出力する + logger.info( + f'I-CSVBK-03 [{target_object_name}] のCSVデータのバックアップ処理を終了します') + + # ④ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/backup_crm_csvdata_process.py b/ecs/crm-datafetch/src/backup_crm_csvdata_process.py deleted file mode 100644 index 1960e7b0..00000000 --- a/ecs/crm-datafetch/src/backup_crm_csvdata_process.py +++ /dev/null @@ -1,31 +0,0 @@ -from src.aws.s3 import BackupBucket -from src.system_var.constants import CSVBK_JP_NAME -from src.error.exceptions import FileUploadException -from src.util.logger import logger_instance as logger - - -def backup_crm_csvdata(target_object, date_path, csv_object): - # ① CSVバックアップ処理の開始ログを出力する - logger.info( - f'I-CSVBK-01 [{target_object.object_name}] のCSVデータのバックアップ処理を開始します ファイル名：[{target_object.upload_file_name}.csv]') - - try: - # ② CRMバックアップ保管用バケットに、変換後のCSVデータのバックアップを保管する - backup_bucket = BackupBucket() - backup_bucket.put_csv_bk( - f'{date_path}/{target_object.upload_file_name}.csv', csv_object.csv_buffer) - - logger.debug( - f'D-CSVBK-02 [{target_object.object_name}] のCSVデータバックアップ 正常終了') - - except Exception as e: - raise FileUploadException( - 'E-CSVBK-01', - CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') - - # ③ CSVバックアップ処理の終了ログを出力する - logger.info( - f'I-CSVBK-03 [{target_object.object_name}] のCSVデータのバックアップ処理を終了します') - - # ④ 次の処理へ移行する - return diff --git a/ecs/crm-datafetch/src/backup_crm_data_process.py b/ecs/crm-datafetch/src/backup_crm_data_process.py index 130da769..a37eb583 100644 --- a/ecs/crm-datafetch/src/backup_crm_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_data_process.py @@ -1,19 +1,42 @@ from src.aws.s3 import BackupBucket -from src.system_var.constants import RESBK_JP_NAME from src.error.exceptions import FileUploadException +from src.system_var.constants import RESBK_JP_NAME +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def backup_crm_data(object_name, sf_object_jsons, date_path): +def backup_crm_data_process(object_name: str, sf_object_dict: dict, execute_datetime: ExecuteDateTime): + """ + CRM電文データバックアップ処理 + + Parameters + ---------- + object_name : str + 取得対象オブジェクト情報インスタンス + sf_object_dict : dict + Salesforceオブジェクトデータ + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + # ① CRM電文データバックアップ処理の開始ログを出力する logger.info(f'I-RESBK-01 [{object_name}] のCRM電文データバックアップ処理を開始します') try: # ② CRMバックアップ保管用バケットに、CRMから取得したJSONの電文データのバックアップを保管する - file_name = f'{date_path}/{object_name}.json' + file_name = f'{execute_datetime.to_path()}/{object_name}.json' backup_bucket = BackupBucket() - backup_bucket.put_response_json(file_name, sf_object_jsons) + backup_bucket.put_response_json(file_name, sf_object_dict) logger.debug(f'D-RESBK-02 [{object_name}] のJSONデータバックアップ 正常終了') diff --git a/ecs/crm-datafetch/src/check_object_info_process.py b/ecs/crm-datafetch/src/check_object_info_process.py index 91551d21..ed2a34da 100644 --- a/ecs/crm-datafetch/src/check_object_info_process.py +++ b/ecs/crm-datafetch/src/check_object_info_process.py @@ -1,10 +1,32 @@ from src.config.objects import TargetObject -from src.system_var.constants import CHK_JP_NAME from src.error.exceptions import InvalidConfigException +from src.util.execute_datetime import ExecuteDateTime +from src.system_var.constants import CHK_JP_NAME from src.util.logger import logger_instance as logger -def check_object_info(object_info, execute_datetime): +def check_object_info_process(object_info: dict, execute_datetime: ExecuteDateTime): + """ + オブジェクト情報形式チェック処理 + + Parameters + ---------- + object_info : dict + 取得対象オブジェクト情報 + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + + Returns + ------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + + Raises + ------ + InvalidConfigException + オブジェクト情報定義が不正だった場合 + """ + # ① オブジェクト情報形式チェック処理開始ログを出力する logger.info('I-CHK-01 オブジェクト情報形式チェック処理を開始します') diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index c00274bf..464f7b0e 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,193 +1,138 @@ -from src.system_var.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ -from src.util.dict_checker import DictCheck +from src.system_var.constants import ( DATE_PATTERN_YYYYMMDDTHHMMSSTZ, + OBJECTS_KEY, + OBJECTS_TYPE, + OBJECT_NAME_KEY, + OBJECT_NAME_TYPE, + COLUMNS_KEY, + COLUMNS_TYPE, + IS_SKIP_KEY, + IS_SKIP_TYPE, + IS_UPDATE_LAST_FETCH_DATETIME_KEY, + IS_UPDATE_LAST_FETCH_DATETIME_TYPE, + LAST_FETCH_DATETIME_FILE_NAME_KEY, + LAST_FETCH_DATETIME_FILE_NAME_TYPE, + UPLOAD_FILE_NAME_KEY, + UPLOAD_FILE_NAME_TYPE, + DATETIME_COLUMN_KEY, + DATETIME_COLUMN_TYPE, + LAST_FETCH_DATETIME_FROM_KEY, + LAST_FETCH_DATETIME_TO_KEY, + DATETIME_COLUMN_DEFAULT_VALUE +) + +from src.util.dict_checker import DictChecker class FetchTargetObjects(): def __init__(self, object_info_file_dict) -> None: - self.__dict_check = DictCheck() self.__objects = object_info_file_dict - self.__key = 'objects' - self.check_key_objects() + self.__dict_checker = DictChecker(self.__objects) + self.validate() self.__i = 0 def __iter__(self): return self def __next__(self): - if self.__i == len(self.__objects[self.__key]): + if self.__i == len(self.__objects[OBJECTS_KEY]): raise StopIteration() - value = self.__objects['objects'][self.__i] + value = self.__objects[OBJECTS_KEY][self.__i] self.__i += 1 return value - def check_key_objects(self) -> None: - __check_key = self.__key - __check_type = list - self.__dict_check.check_key_exist_and_value_type( - self.__objects, __check_key, __check_type) + def validate(self) -> None: + self.__dict_checker.assert_key_exist(OBJECTS_KEY) + self.__dict_checker.assert_data_type(OBJECTS_KEY, OBJECTS_TYPE) class TargetObject(): def __init__(self, object_info, execute_datetime) -> None: - self.__dict_check = DictCheck() + self.__dict_checker = DictChecker(object_info) self.__object_info = object_info - self.execute_datetime = execute_datetime - self.check_key_object_name() - self.check_key_columns() - self.check_key_is_skip() - self.check_key_is_update_last_fetch_datetime() - self.check_key_last_fetch_datetime_file_name() - self.check_key_upload_file_name() - self.object_name = self.__object_info['object_name'] - self.columns = self.__object_info['columns'] - self.is_skip = self.set_is_skip() - self.is_update_last_fetch_datetime = self.set_is_update_last_fetch_datetime() - self.last_fetch_datetime_file_name = self.set_fetch_datetime_file_name() - self.upload_file_name = self.set_upload_file_name() + self.__execute_datetime = execute_datetime + self.__validate() - def check_key_object_name(self) -> None: - ''' - オブジェクト名チェック - ''' - __check_key = 'object_name' - __check_type = str - self.__dict_check.check_key_exist_and_value_type( - self.__object_info, __check_key, __check_type) + def __validate(self) -> None: + self.__validate_required_properties() + self.__validate_optional_properties() return - def check_key_columns(self) -> None: - ''' - カラム情報チェック - ''' - __key = 'columns' - __type = list - self.__dict_check.check_key_exist_and_value_type( - self.__object_info, __key, __type) + def __validate_required_properties(self) -> None: + self.__dict_checker.assert_key_exist(OBJECT_NAME_KEY) + self.__dict_checker.assert_data_type(OBJECT_NAME_KEY, OBJECT_NAME_TYPE) + self.__dict_checker.assert_key_exist(COLUMNS_KEY) + self.__dict_checker.assert_data_type(COLUMNS_KEY, COLUMNS_TYPE) return - def check_key_is_skip(self,) -> None: - ''' - スキップフラグ型チェック - ''' - __check_key = 'is_skip' - __check_type = bool - self.__dict_check.check_key_exist_case_value_type( - self.__object_info, __check_key, __check_type) + def __validate_optional_properties(self) -> None: + if self.__dict_checker.check_key_exist(IS_SKIP_KEY): + self.__dict_checker.assert_data_type(IS_SKIP_KEY, IS_SKIP_TYPE) + + if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY): + self.__dict_checker.assert_data_type(IS_UPDATE_LAST_FETCH_DATETIME_KEY, IS_UPDATE_LAST_FETCH_DATETIME_TYPE) + + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY): + self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_FILE_NAME_KEY, LAST_FETCH_DATETIME_FILE_NAME_TYPE) + + if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): + self.__dict_checker.assert_data_type(UPLOAD_FILE_NAME_KEY, UPLOAD_FILE_NAME_TYPE) + + if self.__dict_checker.check_key_exist(DATETIME_COLUMN_KEY): + self.__dict_checker.assert_data_type(DATETIME_COLUMN_KEY, DATETIME_COLUMN_TYPE) return - def check_key_is_update_last_fetch_datetime(self) -> None: - ''' - 前回取得日時ファイル更新フラグチェック - ''' - __check_key = 'is_update_last_fetch_datetime' - __check_type = bool - self.__dict_check.check_key_exist_case_value_type( - self.__object_info, __check_key, __check_type) + @property + def object_name(self) -> str: + return self.__object_info[OBJECT_NAME_KEY] - return + @property + def columns(self) -> list: + return self.__object_info[COLUMNS_KEY] - def check_key_last_fetch_datetime_file_name(self) -> None: - ''' - 前回取得日時ファイル名型チェック - ''' - __check_key = 'last_fetch_datetime_file_name' - __check_type = str - self.__dict_check.check_key_exist_case_value_type( - self.__object_info, __check_key, __check_type) + @property + def is_skip(self) -> bool: + return self.__object_info[IS_SKIP_KEY] if self.__dict_checker.check_key_exist(IS_SKIP_KEY) else False - return + @property + def is_update_last_fetch_datetime(self) -> bool: + return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY) else False - def check_key_upload_file_name(self) -> None: - ''' - アップロードファイル名称型チェック - ''' - __check_key = 'upload_file_name' - __check_type = str - self.__dict_check.check_key_exist_case_value_type( - self.__object_info, __check_key, __check_type) + @property + def last_fetch_datetime_file_name(self) -> str: + return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY] if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}.json' - return - def set_is_skip(self) -> bool: - ''' - スキップフラグ設定 - ''' - __check_key = 'is_skip' - if self.__dict_check.check_key_exist(self.__object_info, __check_key): - return self.__object_info[__check_key] - else: - return False + @property + def upload_file_name(self) -> str: + return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' - def set_is_update_last_fetch_datetime(self) -> bool: - ''' - 前回取得日時ファイル更新フラグ設定 - ''' - __check_key = 'is_update_last_fetch_datetime' - if self.__dict_check.check_key_exist(self.__object_info, __check_key): - return self.__object_info[__check_key] - else: - return False - - def set_fetch_datetime_file_name(self) -> str: - ''' - 前回取得日時ファイル名設定 - ''' - __check_key = 'last_fetch_datetime_file_name' - if self.__dict_check.check_key_exist(self.__object_info, __check_key): - return self.__object_info[__check_key] - else: - return self.__object_info['object_name'] + '.json' - - def set_upload_file_name(self) -> str: - ''' - アップロードファイル名称設定 - ''' - __check_key = 'upload_file_name' - if self.__dict_check.check_key_exist(self.__object_info, __check_key): - return self.__object_info[__check_key].format(execute_datetime=self.execute_datetime) - else: - return 'CRM_' + self.__object_info['object_name'] + '_' + self.execute_datetime + @property + def datetime_column(self) -> str: + return self.__object_info[DATETIME_COLUMN_KEY] if self.__dict_checker.check_key_exist(DATETIME_COLUMN_KEY) else DATETIME_COLUMN_DEFAULT_VALUE class LastFetchDatetime(): - def __init__(self, last_fetch_datetime_file_name, last_fetch_datetime_file_dict, execute_datetime) -> None: - self.__dict_check = DictCheck() - self.execute_datetime = execute_datetime + def __init__(self, last_fetch_datetime_file_dict, execute_datetime) -> None: + self.__dict_checker = DictChecker(last_fetch_datetime_file_dict) + self.__execute_datetime = execute_datetime self.__last_fetch_datetime_file_dict = last_fetch_datetime_file_dict - self.last_fetch_datetime_file_name = last_fetch_datetime_file_name - self.check_key_last_fetch_datetime_from - self.check_key_last_fetch_datetime_to - self.last_fetch_datetime_from = self.__last_fetch_datetime_file_dict[ - 'last_fetch_datetime_from'] - self.last_fetch_datetime_to = self.set_last_fetch_datetime_to() + self.__validate() - def check_key_last_fetch_datetime_from(self) -> None: - ''' - データ取得開始日時チェック - ''' - __check_key = 'last_fetch_datetime_from' - __regex_str = DATE_PATTERN_YYYYMMDDTHHMMSSTZ - self.__dict_check.check_key_exsit_and_regex( - self.__last_fetch_datetime_file_dict, __check_key, __regex_str) + def __validate(self) -> None: + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FROM_KEY): + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) - def check_key_last_fetch_datetime_to(self) -> None: - ''' - データ取得終了日時チェック - ''' - __check_key = 'last_fetch_datetime_to' - __regex_str = DATE_PATTERN_YYYYMMDDTHHMMSSTZ - self.__dict_check.check_key_exsit_case_regex( - self.__last_fetch_datetime_file_dict, __check_key, __regex_str) + return - def set_last_fetch_datetime_to(self) -> str: - ''' - データ取得終了日時設定 - ''' - __check_key = 'last_fetch_datetime_to' - if self.__dict_check.check_key_exist(self.__last_fetch_datetime_file_dict, __check_key): - return self.__last_fetch_datetime_file_dict[__check_key].format(execute_datetime=self.execute_datetime) - else: - return self.execute_datetime + @property + def last_fetch_datetime_from(self) -> str: + return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_FROM_KEY] + + @property + def last_fetch_datetime_to(self) -> str: + return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FROM_KEY) else self.__execute_datetime diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index ebd3b3cd..acbd2383 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -1,125 +1,45 @@ -from src.check_object_info_process import check_object_info # オブジェクト情報形式チェック処理 -from src.convert_crm_csvdata_process import convert_crm_csvdata # CSV変換処理 -from src.backup_crm_csvdata_process import backup_crm_csvdata # CSVバックアップ処理 -from src.set_datetime_period_process import set_datetime_period # データ取得期間設定処理 -from src.updload_result_data_process import updload_result_data # 取得処理実施結果アップロード処理 +import gc + +from src.backup_crm_csv_data_process import backup_crm_csv_data_process +from src.backup_crm_data_process import backup_crm_data_process +from src.check_object_info_process import check_object_info_process +from src.config.objects import FetchTargetObjects +from src.convert_crm_csv_data_process import convert_crm_csv_data_process +from src.copy_crm_csv_data_process import copy_crm_csv_data_process from src.error.exceptions import MeDaCaCRMDataFetchException -from src.fetch_crm_data_process import fetch_crm_data # CRMデータ取得処理 -from src.prepare_get_data_process import prepare_get_data # データ取得準備処理 -from src.backup_crm_data_process import backup_crm_data # CRM電文データバックアップ処理 -from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime # 前回取得日時ファイル更新 -from src.copy_crm_csvdata_process import copy_crm_csvdata # CSVアップロード処理 +from src.util.execute_datetime import ExecuteDateTime +from src.fetch_crm_data_process import fetch_crm_data_process +from src.prepare_data_fetch_process import prepare_data_fetch_process +from src.set_datetime_period_process import set_datetime_period_process +from src.system_var.constants import OBJECT_NAME_KEY +from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime_process +from src.upload_result_data_process import upload_result_data_process from src.util.logger import logger_instance as logger -def main() -> None: +def controller() -> None: + """コントロール処理""" + try: # ① CRMデータ取得処理開始ログを出力する logger.info('I-CTRL-01 CRMデータ取得処理を開始します') - fetch_target_objects = None # オブジェクト情報ファイル用オブジェクト - execute_datetime = None # 実行日次文字列 - date_path = None # 実行日次のパス文字列 - process_result = None # オブジェクトごとの実行結果JSON - # ② データ取得準備処理を呼び出す logger.info('I-CTRL-02 データ取得準備処理呼び出し') - fetch_target_objects, execute_datetime, date_path, process_result = prepare_get_data() + fetch_target_objects, execute_datetime, process_result = prepare_data_fetch_process() # ③ object_infoのobjectsキーの値の件数分ループする logger.info('I-CTRL-03 取得対象オブジェクトのループ処理開始') - for object_info in fetch_target_objects: - try: - # 1. オブジェクト処理結果の初期化 - target_object = None # オブジェクトごとの情報格納用オブジェクト - last_fetch_datetime = None # オブジェクトごとの取得日付格納用オブジェクト - sf_object_jsons = None # オブジェクトごとのSalesforce取得変数JSON - csv_object = None # CSVオブジェクト - process_result[object_info.get('object_name')] = 'fail' # オブジェクト処理結果 - - logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：{object_info}') - - # 2. オブジェクト情報形式チェック処理を呼び出す - logger.info('I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') - - target_object = check_object_info(object_info, execute_datetime) - - # 3. 処理対象のオブジェクト名をログ出力する - logger.info( - f'I-CTRL-06 [{target_object.object_name}]のデータ取得を開始します') - - # 4. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する - if target_object.is_skip is True: - logger.info( - f'I-CTRL-07 [{target_object.object_name}]のデータ取得処理をスキップします') - continue - - # 5. データ取得期間設定処理を呼び出す - logger.info( - f'I-CTRL-08 [{target_object.object_name}]のデータ取得期間設定処理呼び出し') - - last_fetch_datetime = set_datetime_period(target_object, execute_datetime) - - # 6. CRMデータ取得処理を呼び出す - logger.info( - f'I-CTRL-09 [{target_object.object_name}]のデータ取得処理呼び出し') - - sf_object_jsons = fetch_crm_data(target_object, last_fetch_datetime) - - # 7. 出力ファイル名をログ出力する - logger.info( - f'I-CTRL-10 [{target_object.object_name}] の出力ファイル名は [{target_object.upload_file_name}]となります') - - # 8. CRM電文データバックアップ処理を呼び出す - logger.info( - f'I-CTRL-11 [{target_object.object_name}] CRM電文データバックアップ処理呼び出し') - backup_crm_data(target_object.object_name, sf_object_jsons, date_path) - - # 9. CSV変換処理を呼び出す - logger.info( - f'I-CTRL-12 [{target_object.object_name}] CSV変換処理呼び出し') - csv_object = convert_crm_csvdata(target_object, sf_object_jsons) - - # 10. CSVバックアップ処理を呼び出す - logger.info( - f'I-CTRL-13 [{target_object.object_name}] CSVデータバックアップ処理呼び出し') - backup_crm_csvdata(target_object, date_path, csv_object) - - # 11. CSVアップロード処理を呼び出す - logger.info( - f'I-CTRL-14 [{target_object.object_name}] CSVデータアップロード処理呼び出し') - copy_crm_csvdata(target_object, date_path) - - # 12. 前回取得日時ファイル更新処理を呼びだす - logger.info( - f'I-CTRL-15 [{target_object.object_name}] 前回取得日時ファイル更新処理呼び出し') - upload_last_fetch_datetime(target_object, last_fetch_datetime) - - # 13. オブジェクト処理結果の更新 - process_result[target_object.object_name] = 'success' - - # 14. オブジェクトのアップロードが完了した旨をログに出力する - logger.info(f'I-CTRL-16 [{target_object.object_name}] 処理正常終了') - - except MeDaCaCRMDataFetchException as e: - logger.info(f'{e.error_id} {e}') - logger.info( - f'I-ERR-03 [{object_info.get("object_name")}] の[{e.func_name}]でエラーが発生しました 次のオブジェクトの処理に移行します', exc_info=True) - continue - - except Exception as e: - logger.info( - f'I-ERR-04 [{object_info.get("object_name")}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e, exc_info=True) - continue + process_result = fetch_crm_data(fetch_target_objects, execute_datetime, process_result) # ④ すべてのオブジェクトの処理が完了したことと、オブジェクト毎の処理結果をログに出力する logger.info(f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result}]') # ⑤ 取得処理実施結果アップロード処理を呼び出す logger.info('I-CTRL-19 CRM_取得処理実施結果ファイルアップロード処理開始') - updload_result_data(process_result, date_path) + upload_result_data_process(process_result, execute_datetime) # ⑥ 最終結果をチェックし、チェック結果をログに出力 if not all([v == 'success' for v in process_result.values()]): @@ -142,5 +62,136 @@ def main() -> None: return exit(0) -if __name__ == '__main__': - main() +def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: ExecuteDateTime, process_result: dict): + """ + 取得対象オブジェクト情報をループし、１オブジェクトごとのデータを取得する + + Parameters + ---------- + fetch_target_objects : FetchTargetObjects + CRMオブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + process_result : dict + 取得処理実行結果辞書オブジェクト + + Returns + ------- + process_result : dict + 取得処理実行結果辞書オブジェクト + + """ + for object_info in fetch_target_objects: + try: + process_result[object_info.get(OBJECT_NAME_KEY)] = 'fail' + + fetch_crm_data_per_object(object_info, execute_datetime, process_result) + + process_result[object_info.get(OBJECT_NAME_KEY)] = 'success' + + except MeDaCaCRMDataFetchException as e: + logger.info(f'{e.error_id} {e}') + logger.info( + f'I-ERR-03 [{object_info.get(OBJECT_NAME_KEY)}] の[{e.func_name}]でエラーが発生しました 次のオブジェクトの処理に移行します', exc_info=True) + continue + + except Exception as e: + logger.info( + f'I-ERR-04 [{object_info.get(OBJECT_NAME_KEY)}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e, exc_info=True) + continue + + return process_result + +def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTime) -> None: + """ + オブジェクトごとにCRMのデータを取得し、取込フォルダにアップロードする + + Parameters + ---------- + object_info : dict + 取得対象オブジェクト情報 + execute_datetime : object + 実行日次取得インスタンス + + Returns + ------- + なし + + Raises + ------ + FileNotFoundException + S3上のファイルが存在しない場合 + InvalidConfigException + オブジェクト情報定義が不正だった場合 + + """ + + # 1. オブジェクト処理結果の初期化 + logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：{object_info}') + + # 2. オブジェクト情報形式チェック処理を呼び出す + logger.info('I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') + + target_object = check_object_info_process(object_info, execute_datetime) + target_object_name = target_object.object_name + + # 3. 処理対象のオブジェクト名をログ出力する + logger.info( + f'I-CTRL-06 [{target_object_name}]のデータ取得を開始します') + + # 4. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する + if target_object.is_skip is True: + logger.info( + f'I-CTRL-07 [{target_object_name}]のデータ取得処理をスキップします') + return + + # 5. データ取得期間設定処理を呼び出す + logger.info( + f'I-CTRL-08 [{target_object_name}]のデータ取得期間設定処理呼び出し') + + last_fetch_datetime = set_datetime_period_process(target_object, execute_datetime) + + # 6. CRMデータ取得処理を呼び出す + logger.info( + f'I-CTRL-09 [{target_object_name}]のデータ取得処理呼び出し') + + crm_data_response = fetch_crm_data_process(target_object, last_fetch_datetime) + + # 7. 出力ファイル名をログ出力する + logger.info( + f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{target_object.upload_file_name}]となります') + + # 8. CRM電文データバックアップ処理を呼び出す + logger.info( + f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し') + backup_crm_data_process(target_object_name, crm_data_response, execute_datetime) + + # 9. CSV変換処理を呼び出す + logger.info( + f'I-CTRL-12 [{target_object.object_name}] CSV変換処理呼び出し') + csv_string = convert_crm_csv_data_process(target_object, crm_data_response) + + # 10. CSVバックアップ処理を呼び出す + logger.info( + f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し') + backup_crm_csv_data_process(target_object, execute_datetime, csv_string) + + # 11. CSVアップロード処理を呼び出す + logger.info( + f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し') + copy_crm_csv_data_process(target_object, execute_datetime) + + # 12. メモリ解放 + del crm_data_response + del csv_string + gc.collect() + + # 13. 前回取得日時ファイル更新処理を呼びだす + logger.info( + f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し') + upload_last_fetch_datetime_process(target_object, last_fetch_datetime) + + # 14. オブジェクトのアップロードが完了した旨をログに出力する + logger.info(f'I-CTRL-16 [{target_object_name}] 処理正常終了') + + return \ No newline at end of file diff --git a/ecs/crm-datafetch/src/convert_crm_csv_data_process.py b/ecs/crm-datafetch/src/convert_crm_csv_data_process.py new file mode 100644 index 00000000..1ed05c3a --- /dev/null +++ b/ecs/crm-datafetch/src/convert_crm_csv_data_process.py @@ -0,0 +1,51 @@ +from src.config.objects import TargetObject +from src.converter.converter import CSVStringConverter +from src.error.exceptions import DataConvertException +from src.system_var.constants import CONV_JP_NAME +from src.util.logger import logger_instance as logger + + +def convert_crm_csv_data_process(target_object: TargetObject, crm_data_response: dict): + """ + CSV変換処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + crm_data_response : dict + Salesforceオブジェクトデータ + + Returns + ------- + csv_string : str + csvデータ + + Raises + ------ + DataConvertException + データ変換が失敗した場合 + """ + + # ① CSV変換処理の開始ログを出力する + target_object_name = target_object.object_name + + logger.info(f'I-CONV-01 [{target_object_name}] のCSV変換処理を開始します') + + try: + # ② CSV変換 + CSVStringConverter(target_object, crm_data_response) + csv_string = CSVStringConverter.convert() + + + logger.debug(f'D-CONV-02 [{target_object_name}] のCSV変換処理 正常終了') + + except Exception as e: + raise DataConvertException( + 'E-CONV-01', CONV_JP_NAME, f'[{target_object_name}] CSV変換に失敗しました エラー内容:[{e}]') + + # ③ CSV変換処理の終了ログを出力する + logger.info(f'I-CONV-03 [{target_object_name}] のCSV変換処理を終了します') + + # ④ 次の処理へ移行する + return csv_string diff --git a/ecs/crm-datafetch/src/convert_crm_csvdata_process.py b/ecs/crm-datafetch/src/convert_crm_csvdata_process.py deleted file mode 100644 index 4ef6dd11..00000000 --- a/ecs/crm-datafetch/src/convert_crm_csvdata_process.py +++ /dev/null @@ -1,25 +0,0 @@ -from src.system_var.constants import CONV_JP_NAME -from src.converter.converter import CSVStringConverter -from src.error.exceptions import DataConvertException -from src.util.logger import logger_instance as logger - - -def convert_crm_csvdata(target_object, sf_object_jsons): - # ① CSV変換処理の開始ログを出力する - logger.info(f'I-CONV-01 [{target_object.object_name}] のCSV変換処理を開始します') - - try: - # ② CSV変換 - csv_object = CSVStringConverter(target_object, sf_object_jsons) - - logger.debug(f'D-CONV-02 [{target_object.object_name}] のCSV変換処理 正常終了') - - except Exception as e: - raise DataConvertException( - 'E-CONV-01', CONV_JP_NAME, f'[{target_object.object_name}] CSV変換に失敗しました エラー内容:[{e}]') - - # ③ CSV変換処理の終了ログを出力する - logger.info(f'I-CONV-03 [{target_object.object_name}] のCSV変換処理を終了します') - - # ④ 次の処理へ移行する - return csv_object diff --git a/ecs/crm-datafetch/src/converter/convert_factory.py b/ecs/crm-datafetch/src/converter/convert_factory.py new file mode 100644 index 00000000..48673f7e --- /dev/null +++ b/ecs/crm-datafetch/src/converter/convert_factory.py @@ -0,0 +1,55 @@ +import re +from datetime import datetime + +from src.config.objects import TargetObject +from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, + CSV_TRUE_VALUE, YYYYMMDDHHMMSS, DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC) + + +class ConvertStrategyFactory: + def __init__(self) -> None: + self.__none_value_convert_strategy = NoneValueConvertStrategy() + self.__float_convert_strategy = FloatConvertStrategy() + self.__boolean_convert_strategy = BooleanConvertStrategy() + self.__datetime_convert_strategy = DatatimeConvertStrategy() + + def create(self, value): + + converted_value = value + + if value is None: + converted_value = self.__none_value_convert_strategy.convert_value() + + # 指数表記で取得できるパターン。指数表記を整数表記に変換する。 + elif type(value) == float: + converted_value = self.__float_convert_strategy.convert_value(value) + + # SQLの真偽値に対応するために変換する + elif type(value) == bool: + converted_value = self.__boolean_convert_strategy.convert_value(value) + + elif type(value) == str and re.fullmatch(DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, value): + converted_value = self.__datetime_convert_strategy.convert_value(value) + + return converted_value + + +class NoneValueConvertStrategy: + def convert_value(self) -> str: + return '' + + +class BooleanConvertStrategy: + def convert_value(self, convert_value: str) -> bool: + return CSV_TRUE_VALUE if convert_value is True else CSV_FALSE_VALUE + + +class DatatimeConvertStrategy: + def convert_value(self, convert_value: str) -> str: + return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).strftime(YYYYMMDDHHMMSS) + + +class FloatConvertStrategy: + def convert_value(self, convert_value: str) -> int: + return int(convert_value) + diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 7b4adb1e..bf7faaee 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -1,33 +1,35 @@ import csv import io -import re -from datetime import datetime -from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, - CSV_TRUE_VALUE, YYYYMMDDHHMMSS) +from src.config.objects import TargetObject +from src.converter.convert_factory import ConvertStrategyFactory class CSVStringConverter: - def __init__(self, target_object, sf_object_jsons) -> None: + def __init__(self, target_object: TargetObject, sf_object_jsons: dict) -> None: self.__target_object = target_object self.__sf_object_jsons = sf_object_jsons - self.__extracted_sf_object_jsons = self.extract_sf_object_jsons() - self.csv_data = self.convert_to_csv() - self.csv_buffer = self.write_csv() + self.__convert_strategy_factory = ConvertStrategyFactory() - def extract_sf_object_jsons(self) -> list: + def convert(self) -> str: + extracted_sf_object_jsons = self.__extract_sf_object_jsons() + csv_data = self.__convert_to_csv(extracted_sf_object_jsons) + csv_string = self.__write_csv_string(csv_data) + return csv_string + + def __extract_sf_object_jsons(self) -> list: try: extracted_sf_object_jsons = [] for sf_object_json in self.__sf_object_jsons: extracted_sf_object_jsons.append( - self.extract_necessary_props_from(sf_object_json)) + self.__extract_necessary_props_from(sf_object_json)) return extracted_sf_object_jsons except Exception as e: - raise Exception('必要なjsonのデータ抽出に失敗しました') + raise Exception('必要なjsonのデータ抽出に失敗しました', e) - def extract_necessary_props_from(self, sf_object_json) -> dict: + def __extract_necessary_props_from(self, sf_object_json) -> dict: try: clone_sf_object = {**sf_object_json} @@ -39,19 +41,18 @@ class CSVStringConverter: return uppercase_key_sf_object except Exception as e: - raise Exception('必要なjsonのデータ成形に失敗しました') + raise Exception('必要なjsonのデータ成形に失敗しました', e) - def convert_to_csv(self) -> list: + def __convert_to_csv(self, extracted_sf_object_jsons) -> list: try: columns = self.__target_object.columns csv_data = [] - for i, json_object in enumerate(self.__extracted_sf_object_jsons, 1): + for i, json_object in enumerate(extracted_sf_object_jsons, 1): csv_row = [] for column in columns: v = json_object[column.upper()] - converted_value = CSVStringConverterFactory( - v).value_convert() + converted_value = self.__convert_strategy_factory.create(v) csv_row.append(converted_value) @@ -62,74 +63,16 @@ class CSVStringConverter: raise Exception( f'CSV変換に失敗しました カラム名：[{column}] 行番号: [{i}] エラー内容:[{e}]') - def write_csv(self) -> str: + def __write_csv_string(self, csv_data) -> str: try: with io.StringIO(newline='') as string_stream: writer = csv.writer(string_stream, delimiter=',', lineterminator='\r\n', doublequote=True, quotechar='"', quoting=csv.QUOTE_ALL, strict=True) writer.writerow(self.__target_object.columns) - writer.writerows(self.csv_data) + writer.writerows(csv_data) csv_value = string_stream.getvalue() return csv_value except Exception as e: - raise Exception('csvデータの取得に失敗しました') - - -class NoneValueConverter: - def __init__(self, convert_value) -> None: - self.__convert_value = convert_value - self.value = self.convert_value() - - def convert_value(self) -> str: - return '' - - -class BooleanConverter: - def __init__(self, convert_value) -> None: - self.__convert_value = convert_value - self.value = self.convert_value() - - def convert_value(self) -> bool: - return CSV_TRUE_VALUE if self.__convert_value is True else CSV_FALSE_VALUE - - -class DatatimeConverter: - def __init__(self, convert_value) -> None: - self.__convert_value = convert_value - self.value = self.convert_value() - - def convert_value(self) -> str: - return datetime.strptime(self.__convert_value, CRM_DATETIME_FORMAT).strftime(YYYYMMDDHHMMSS) - - -class FloatConverter: - def __init__(self, convert_value) -> None: - self.__convert_value = convert_value - self.value = self.convert_value() - - def convert_value(self) -> int: - return int(self.__convert_value) - - -class CSVStringConverterFactory: - def __init__(self, v) -> None: - self.__value = v - - def value_convert(self): - - converted_value = self.__value - - if self.__value is None: - converted_value = NoneValueConverter(self.__value).value - # 指数表記で取得できるパターン。指数表記を整数表記に変換する。 - elif type(self.__value) == float: - converted_value = FloatConverter(self.__value).value - # SQLの真偽値に対応するために変換する - elif type(self.__value) == bool: - converted_value = BooleanConverter(self.__value).value - elif type(self.__value) == str and re.fullmatch(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.000\+0000', self.__value): - converted_value = DatatimeConverter(self.__value).value - - return converted_value + raise Exception('csvデータの取得に失敗しました', e) diff --git a/ecs/crm-datafetch/src/copy_crm_csv_data_process.py b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py new file mode 100644 index 00000000..57a3659f --- /dev/null +++ b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py @@ -0,0 +1,57 @@ +from src.aws.s3 import BackupBucket, DataBucket +from src.config.objects import TargetObject +from src.error.exceptions import FileUploadException +from src.system_var.constants import UPLD_JP_NAME +from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, + CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) +from src.util.execute_datetime import ExecuteDateTime +from src.util.logger import logger_instance as logger + + +def copy_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime): + """ + CSVアップロード処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + + # ① CSVデータアップロード処理の開始ログを出力する + target_object_name = target_object.object_name + upload_file_name = target_object.upload_file_name + + logger.info( + f'I-UPLD-01 [{target_object_name}] のCSVデータアップロード処理を開始します ファイル名：[{upload_file_name}.csv]') + + try: + # ② CRMバックアップ保管用バケットに保管した変換後のCSVデータをデータ取込バケットにコピーする + data_bucket = DataBucket() + backup_bucket = BackupBucket() + data_bucket.put_csv_from(str(backup_bucket), f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{execute_datetime.to_path()}/{upload_file_name}.csv') + + logger.debug( + f'D-UPLD-02 [{target_object_name}] のCSVデータアップロード 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-UPLD-01', UPLD_JP_NAME, f'[{target_object_name}] CSVデータのアップロードに失敗しました ファイル名：[{upload_file_name}.csv] エラー内容:[{e}]') + + # ③ CSVデータアップロード処理の終了ログを出力する + logger.info( + f'I-UPLD-03 [{target_object_name}] のCSVデータのアップロード処理を終了します') + + # ④ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/copy_crm_csvdata_process.py b/ecs/crm-datafetch/src/copy_crm_csvdata_process.py deleted file mode 100644 index ecebe9ea..00000000 --- a/ecs/crm-datafetch/src/copy_crm_csvdata_process.py +++ /dev/null @@ -1,32 +0,0 @@ -from src.aws.s3 import S3ResourceNonBucket -from src.system_var.constants import UPLD_JP_NAME -from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, - CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) -from src.error.exceptions import FileUploadException -from src.util.logger import logger_instance as logger - - -def copy_crm_csvdata(target_object, date_path): - # ① CSVデータアップロード処理の開始ログを出力する - logger.info( - f'I-UPLD-01 [{target_object.object_name}] のCSVデータアップロード処理を開始します ファイル名：[{target_object.upload_file_name}.csv]') - - try: - # ② CRMバックアップ保管用バケットに保管した変換後のCSVデータをデータ取込バケットにコピーする - s3_resource_non_bucket = S3ResourceNonBucket() - s3_resource_non_bucket.copy(CRM_BACKUP_BUCKET, f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{date_path}/{target_object.upload_file_name}.csv', - IMPORT_DATA_BUCKET, f'{CRM_IMPORT_DATA_FOLDER}/{target_object.upload_file_name}.csv') - - logger.debug( - f'D-UPLD-02 [{target_object.object_name}] のCSVデータアップロード 正常終了') - - except Exception as e: - raise FileUploadException( - 'E-UPLD-01', UPLD_JP_NAME, f'[{target_object.object_name}] CSVデータのアップロードに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') - - # ③ CSVデータアップロード処理の終了ログを出力する - logger.info( - f'I-UPLD-03 [{target_object.object_name}] のCSVデータのアップロード処理を終了します') - - # ④ 次の処理へ移行する - return diff --git a/ecs/crm-datafetch/src/error/exceptions.py b/ecs/crm-datafetch/src/error/exceptions.py index 71a174c0..f231804b 100644 --- a/ecs/crm-datafetch/src/error/exceptions.py +++ b/ecs/crm-datafetch/src/error/exceptions.py @@ -4,7 +4,7 @@ from abc import ABCMeta class MeDaCaCRMDataFetchException(Exception, metaclass=ABCMeta): """MeDaCaシステム固有のカスタムエラークラス""" - def __init__(self, error_id: str, func_name, message) -> None: + def __init__(self, error_id: str, func_name: str, message: str) -> None: super().__init__(message) self.func_name = func_name self.error_id = error_id diff --git a/ecs/crm-datafetch/src/fetch_crm_data_process.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py index eac5478b..3a89da1e 100644 --- a/ecs/crm-datafetch/src/fetch_crm_data_process.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -2,6 +2,10 @@ from requests.exceptions import ConnectTimeout, ReadTimeout from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential +from src.config.objects import TargetObject, LastFetchDatetime +from src.error.exceptions import DataConvertException, SalesforceAPIException +from src.salesforce.salesforce_api import SalesforceApiClient +from src.salesforce.soql_builder import SOQLBuilder from src.system_var.constants import FETCH_JP_NAME from src.system_var.environments import (CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, @@ -14,20 +18,38 @@ from src.system_var.environments import (CRM_AUTH_TIMEOUT, CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_TIMEOUT) -from src.error.exceptions import DataConvertException, SalesforceAPIException -from src.salesforce.salesforce_api import SalesForceCount, SalesForceData from src.util.logger import logger_instance as logger -def fetch_crm_data(target_object, last_fetch_datetime): +def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): + """ + CRMデータ取得処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + last_fetch_datetime : LastFetchDatetime + 取得対象オブジェクト情報インスタンス + + Returns + ------- + sf_object_dict : dict + Salesforceオブジェクトデータ + + Raises + ------ + SalesforceAPIException + SalseforceのAPI実行失敗が発生した場合 + DataConvertException + データ変換が失敗した場合 + """ + # ① CRMデータ取得処理開始ログを出力する logger.info( f'I-FETCH-01 [{target_object.object_name}] のCRMからのデータ取得処理を開始します') object_name = target_object.object_name - columns = ','.join(target_object.columns) - last_fetch_datetime_from = last_fetch_datetime.last_fetch_datetime_from - last_fetch_datetime_to = last_fetch_datetime.last_fetch_datetime_to global count_contime_counter, count_readtime_counter, count_counter, data_contime_counter, data_readtime_counter, data_counter count_contime_counter = 1 @@ -41,8 +63,10 @@ def fetch_crm_data(target_object, last_fetch_datetime): # ② 取得対象オブジェクトの取得期間内のレコード件数を取得する logger.info(f'I-FETCH-02 [{object_name}] の件数取得を開始します') - record_count = fetch_sf_count_retry( - object_name, last_fetch_datetime_from, last_fetch_datetime_to) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + count_soql = soql_builder.create_count_soql() + + record_count = fetch_record_count_retry(count_soql, object_name) logger.info(f'I-FETCH-03 [{object_name}] の件数：[{record_count}]') @@ -54,8 +78,9 @@ def fetch_crm_data(target_object, last_fetch_datetime): # ③ 取得対象オブジェクトのレコードを取得する logger.info(f'I-FETCH-04 [{object_name}] のレコード取得を開始します') - record_generator = fetch_sf_data_retry( - columns, object_name, last_fetch_datetime_from, last_fetch_datetime_to) + fetch_soql = soql_builder.create_fetch_soql() + + record_all = fetch_sf_data_retry(fetch_soql, object_name) except Exception as e: raise SalesforceAPIException( @@ -65,10 +90,7 @@ def fetch_crm_data(target_object, last_fetch_datetime): # ④ 取得対象オブジェクトをJSONに変換 logger.info(f'I-FETCH-05 [{object_name}] のレコードをJSONに変換します') - sf_object_jsons = [] - - for record in record_generator: - sf_object_jsons.append(record) + crm_data_response = [record for record in record_all] except Exception as e: raise DataConvertException( @@ -78,19 +100,19 @@ def fetch_crm_data(target_object, last_fetch_datetime): logger.info(f'I-FETCH-06 [{object_name}] のレコード取得が成功しました') # ⑥ 次の処理へ移行する - return sf_object_jsons + return crm_data_response @retry( wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) -def fetch_sf_count_retry(object_name, last_update_datetime_from, last_update_datetime_to): +def fetch_record_count_retry(soql: str, object_name: str): try: global count_contime_counter, count_readtime_counter, count_counter - saleforce_count = SalesForceCount() - return saleforce_count.fetch_sf_count(object_name, last_update_datetime_from, last_update_datetime_to) + salesforce_api_client = SalesforceApiClient() + return salesforce_api_client.fetch_sf_count(soql) except ConnectTimeout as e: if count_contime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: @@ -117,12 +139,12 @@ def fetch_sf_count_retry(object_name, last_update_datetime_from, last_update_dat wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) -def fetch_sf_data_retry(columns, object_name, last_update_datetime_from, last_update_datetime_to): +def fetch_sf_data_retry(soql: str, object_name: str): try: global data_contime_counter, data_readtime_counter, data_counter - saleforce_data = SalesForceData() - return saleforce_data.fetch_sf_data(columns, object_name, last_update_datetime_from, last_update_datetime_to) + salesforce_api_client = SalesforceApiClient() + return salesforce_api_client.fetch_sf_data(soql) except ConnectTimeout as e: if data_contime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: diff --git a/ecs/crm-datafetch/src/prepare_get_data_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py similarity index 76% rename from ecs/crm-datafetch/src/prepare_get_data_process.py rename to ecs/crm-datafetch/src/prepare_data_fetch_process.py index e96de09c..409651ad 100644 --- a/ecs/crm-datafetch/src/prepare_get_data_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -1,24 +1,46 @@ -from datetime import datetime - from src.aws.s3 import ConfigBucket from src.config.objects import FetchTargetObjects -from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ -from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, - OBJECT_INFO_FOLDER) from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser +from src.system_var.constants import PRE_JP_NAME +from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER) +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def prepare_get_data(): +def prepare_data_fetch_process(): + """ + データ取得準備処理 + + Parameters + ---------- + なし + + Returns + ------- + fetch_target_objects : FetchTargetObjects + CRMオブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + process_result : dict + 取得処理実行結果辞書オブジェクト + + Raises + ------ + FileNotFoundException + S3上のファイルが存在しない場合 + InvalidConfigException + オブジェクト情報定義が不正だった場合 + + """ # ① データ取得準備処理の開始ログを出力する logger.info('I-PRE-01 データ取得準備処理を開始します') # ② 取得処理開始年月日時分秒を控える - execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) - date_path = execute_datetime.rstrip('000Z').translate( - str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) + + execute_datetime = ExecuteDateTime() logger.info(f'I-PRE-02 データ取得処理開始日時：{execute_datetime}') @@ -29,7 +51,7 @@ def prepare_get_data(): f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') config_bucket = ConfigBucket() - object_info_file_json = config_bucket.get_object_info_file() + object_info_file_str = config_bucket.get_object_info_file() logger.debug('D-PRE-04 CRM_取得オブジェクト情報ファイルの取得成功しました') @@ -41,7 +63,7 @@ def prepare_get_data(): # ④ CRM_取得オブジェクト情報ファイルをパースし、メモリ上に展開する logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします') - json_parser = JsonParser(object_info_file_json) + json_parser = JsonParser(object_info_file_str) object_info_file_dict = json_parser.json_parser() logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました') @@ -69,4 +91,4 @@ def prepare_get_data(): logger.info('I-PRE-09 データ取得準備処理を終了します') # ⑧ 次の処理へ移行する - return(fetch_target_objects, execute_datetime, date_path, process_result) + return(fetch_target_objects, execute_datetime, process_result) diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index a6b9c458..0c073b9a 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,61 +1,28 @@ from simple_salesforce import Salesforce -from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT, - CRM_AUTH_RETRY_INTERVAL, - CRM_AUTH_RETRY_MAX_INTERVAL, - CRM_AUTH_RETRY_MIN_INTERVAL, CRM_AUTH_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + +from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_TIMEOUT, CRM_USER_NAME, CRM_USER_PASSWORD, CRM_USER_SECURITY_TOKEN) -FETCH_SOQL = """SELECT {column_or_expression} FROM {object_name} - WHERE SystemModStamp > {last_update_datetime_from} - AND SystemModStamp <= {last_update_datetime_to} -""" -class SalesfoeceApi(): +class SalesforceApiClient(): def __init__(self) -> None: self.__sf = Salesforce(username=CRM_USER_NAME, password=CRM_USER_PASSWORD, security_token=CRM_USER_SECURITY_TOKEN, domain=CRM_AUTH_DOMAIN ) - def sf_query(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + def query(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): return self.__sf.query(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) - def sf_query_all_iter(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): - return self.__sf.query_all_iter(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) + def query_all(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + return self.__sf.query_all(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) - -class SalesForceCount(): - def fetch_sf_count(self, object_name, last_update_datetime_from, last_update_datetime_to): - count_soql = FETCH_SOQL.format( - column_or_expression='COUNT(Id)', - object_name=object_name, - last_update_datetime_from=last_update_datetime_from, - last_update_datetime_to=last_update_datetime_to - ) - self.__sf = SalesfoeceApi() - count_res = self.__sf.sf_query( - count_soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) + def fetch_sf_count(self, soql: str): + count_res = self.query(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) return count_res.get('records')[0].get('expr0') - -class SalesForceData(): - def fetch_sf_data(self, columns, object_name, last_update_datetime_from, last_update_datetime_to): - soql = FETCH_SOQL.format( - column_or_expression=columns, - object_name=object_name, - last_update_datetime_from=last_update_datetime_from, - last_update_datetime_to=last_update_datetime_to - ) - self.__sf = SalesfoeceApi() - return self.__sf.sf_query_all_iter(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + def fetch_sf_data(self, soql: str): + return self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) diff --git a/ecs/crm-datafetch/src/salesforce/soql_builder.py b/ecs/crm-datafetch/src/salesforce/soql_builder.py new file mode 100644 index 00000000..a84744a8 --- /dev/null +++ b/ecs/crm-datafetch/src/salesforce/soql_builder.py @@ -0,0 +1,34 @@ +from src.config.objects import TargetObject, LastFetchDatetime + + +class SOQLBuilder: + def __init__(self, target_object: TargetObject, last_fetch_datetime: LastFetchDatetime) -> None: + self.__SELECT_SOQL = """SELECT {column_or_expression} FROM {object_name} + WHERE {datetime_column} > {last_fetch_datetime_from} + AND {datetime_column} <= {last_fetch_datetime_to} + """ + self.__target_object = target_object + self.__last_fetch_datetime = last_fetch_datetime + + def create_count_soql(self): + count_soql = self.__SELECT_SOQL.format( + column_or_expression='COUNT(Id)', + object_name=self.__target_object.object_name, + last_fetch_datetime_from=self.__last_fetch_datetime.last_fetch_datetime_from, + last_fetch_datetime_to=self.__last_fetch_datetime.last_fetch_datetime_to, + datetime_column=self.__target_object.datetime_column + ) + + return count_soql + + def create_fetch_soql(self): + columns = ','.join(self.__target_object.columns) + fetch_soql = self.__SELECT_SOQL.format( + column_or_expression=columns, + object_name=self.__target_object.object_name, + last_fetch_datetime_from=self.__last_fetch_datetime.last_fetch_datetime_from, + last_fetch_datetime_to=self.__last_fetch_datetime.last_fetch_datetime_to, + datetime_column=self.__target_object.datetime_column + ) + + return fetch_soql diff --git a/ecs/crm-datafetch/src/set_datetime_period_process.py b/ecs/crm-datafetch/src/set_datetime_period_process.py index ea281fba..9795ab2e 100644 --- a/ecs/crm-datafetch/src/set_datetime_period_process.py +++ b/ecs/crm-datafetch/src/set_datetime_period_process.py @@ -1,13 +1,39 @@ from src.aws.s3 import ConfigBucket -from src.config.objects import LastFetchDatetime -from src.system_var.constants import DATE_JP_NAME -from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.config.objects import TargetObject, LastFetchDatetime from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser +from src.system_var.constants import DATE_JP_NAME +from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def set_datetime_period(target_object, execute_datetime): + + +def set_datetime_period_process(target_object: TargetObject, execute_datetime: ExecuteDateTime): + """ + データ取得期間設定処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + + Returns + ------- + last_fetch_datetime : LastFetchDatetime + 取得対象オブジェクト情報インスタンス + + Raises + ------ + FileNotFoundException + S3上のファイルが存在しない場合 + InvalidConfigException + オブジェクト情報定義が不正だった場合 + """ + # ① データ取得期間設定処理の開始ログを出力する logger.info( f'I-DATE-01 [{target_object.object_name}] のデータ取得期間設定処理を開始します') @@ -18,7 +44,7 @@ def set_datetime_period(target_object, execute_datetime): f'I-DATE-02 前回取得日時ファイルの取得開始します ファイルパス：[s3://{CRM_CONFIG_BUCKET}/{LAST_FETCH_DATE_FOLDER}/{target_object.last_fetch_datetime_file_name}]') s3_config_bucket = ConfigBucket() - last_fetch_datetime_file_json = s3_config_bucket.get_last_fetch_datetime_file( + last_fetch_datetime_file_str = s3_config_bucket.get_last_fetch_datetime_file( target_object.last_fetch_datetime_file_name) logger.info(f'I-DATE-03 前回取得日時ファイルの取得成功しました') @@ -32,11 +58,10 @@ def set_datetime_period(target_object, execute_datetime): # ④ データの取得期間を設定する logger.debug(f'D-DATE-04 前回取得日時ファイルの形式チェックを開始します') - json_parser = JsonParser(last_fetch_datetime_file_json) + json_parser = JsonParser(last_fetch_datetime_file_str) last_fetch_datetime_file_dict = json_parser.json_parser() - last_fetch_datetime = LastFetchDatetime(target_object.last_fetch_datetime_file_name, - last_fetch_datetime_file_dict, execute_datetime) + last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_file_dict, execute_datetime) logger.debug(f'D-DATE-05 前回取得日時ファイルの形式チェック 正常終了') logger.info( diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index fde2ec65..b84d8f1d 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -1,6 +1,5 @@ # environments(task settings file) -# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する -LOG_LEVEL = "LOG_LEVEL" +LOG_LEVEL = 'LOG_LEVEL' # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 @@ -17,7 +16,7 @@ CRM_FETCH_RECORD_RETRY_INTERVAL = 'CRM_FETCH_RECORD_RETRY_INTERVAL' CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MIN_INTERVAL' # CRMのレコード取得処理のリトライ時の最小待ち秒数 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MAX_INTERVAL' # CRMのレコード取得処理のリトライ時の最大待ち秒数 -# environments(ECS Task Enviroment) +# environments(ECS Task Environment) CRM_AUTH_DOMAIN = 'CRM_AUTH_DOMAIN' # CRMのAPI実行のための認証エンドポイントのドメイン CRM_USER_NAME = 'CRM_USER_NAME' # CRMのAPI実行用ユーザ名 CRM_USER_PASSWORD = 'CRM_USER_PASSWORD' # CRMのAPI実行用ユーザパスワード @@ -40,16 +39,17 @@ CRM_IMPORT_DATA_BACKUP_FOLDER = 'CRM_IMPORT_DATA_BACKUP_FOLDER' YYYYMMDDTHHMMSSTZ = '%Y-%m-%dT%H:%M:%S.000Z' CRM_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.000%z' YYYYMMDDHHMMSS = '%Y-%m-%d %H:%M:%S' +MILLISEC_FORMAT = '000Z' # aws AWS_RESOURCE_S3 = 's3' -AWS_CLINET_S3 = 's3' S3_RESPONSE_BODY = 'Body' - +S3_CHAR_CODE = 'utf-8' # 正規表現チェック EXCLUDE_SYMBOL = ['#', '/'] -DATE_PATTERN_YYYYMMDDTHHMMSSTZ = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9].000Z' +DATE_PATTERN_YYYYMMDDTHHMMSSTZ = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' +DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.000\+0000' # logger LOG_FORMAT = '[%(levelname)s]\t%(asctime)s\t%(message)s\n' @@ -72,3 +72,24 @@ END_JP_NAME = '取得処理実施結果アップロード処理' # CSVチェック CSV_TRUE_VALUE = '1' CSV_FALSE_VALUE = '0' + +# オブジェクト変数 +OBJECTS_KEY = 'objects' +OBJECTS_TYPE = list +OBJECT_NAME_KEY = 'object_name' +OBJECT_NAME_TYPE = str +COLUMNS_KEY = 'columns' +COLUMNS_TYPE = list +IS_SKIP_KEY = 'is_skip' +IS_SKIP_TYPE = bool +IS_UPDATE_LAST_FETCH_DATETIME_KEY = 'is_update_last_fetch_datetime' +IS_UPDATE_LAST_FETCH_DATETIME_TYPE = bool +LAST_FETCH_DATETIME_FILE_NAME_KEY = 'last_fetch_datetime_file_name' +LAST_FETCH_DATETIME_FILE_NAME_TYPE = str +UPLOAD_FILE_NAME_KEY = 'upload_file_name' +UPLOAD_FILE_NAME_TYPE = str +DATETIME_COLUMN_KEY = 'datetime_column' +DATETIME_COLUMN_TYPE = str +DATETIME_COLUMN_DEFAULT_VALUE = 'SystemModstamp' +LAST_FETCH_DATETIME_TO_KEY = 'last_fetch_datetime_to' +LAST_FETCH_DATETIME_FROM_KEY = 'last_fetch_datetime_from' diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index dfa87ae3..5ff41929 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -6,37 +6,37 @@ import src.system_var.constants as constants # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT] +CRM_AUTH_TIMEOUT = os.environ.get(constants.CRM_AUTH_TIMEOUT, 100) # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT] +CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3) # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL] +CRM_AUTH_RETRY_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5) # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL] +CRM_AUTH_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5) # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL] +CRM_AUTH_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50) # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT] +CRM_GET_RECORD_COUNT_TIMEOUT = os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300) # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT] +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3) # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL] +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5) # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL] +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, 5) # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL] +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50) # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT] +CRM_FETCH_RECORD_TIMEOUT = os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300) # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT] +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3) # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL] +CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5) # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL] +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5) # CRMのレコード取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL] +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50) -# environments(ECS Task Enviroment) +# environments(ECS Task Environment) # CRMのAPI実行のための認証エンドポイントのドメイン CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN] # CRMのAPI実行用ユーザ名 @@ -52,20 +52,20 @@ CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET] # CRMの取込データを格納するバケット名 IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET] # CRM取得対象オブジェクトの情報を格納するフォルダパス -OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER] +OBJECT_INFO_FOLDER = os.environ.get(constants.OBJECT_INFO_FOLDER, 'crm/object_info') # CRM取得対象オブジェクトの情報のファイル名 OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME] # CRMデータ取得結果を格納するフォルダパス -PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER] +PROCESS_RESULT_FOLDER = os.environ.get(constants.PROCESS_RESULT_FOLDER, 'data_import') # CRMデータ取得結果を格納するファイル名 -PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME] +PROCESS_RESULT_FILENAME = os.environ.get(constants.PROCESS_RESULT_FILENAME, 'process_result.json') # CRMからの最終取得日時ファイルを格納するフォルダパス -LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER] +LAST_FETCH_DATE_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_FOLDER, 'crm/last_fetch_datetime') # CRMから取得し、取込用に変換したデータを格納するフォルダ -CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER] +CRM_IMPORT_DATA_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_FOLDER, 'crm/target') # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER] +LAST_FETCH_DATE_BACKUP_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_BACKUP_FOLDER, 'last_fetch_datetime') # CRMから取得した生データのバックアップを格納するフォルダパス -RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER] +RESPONSE_JSON_BACKUP_FOLDER = os.environ.get(constants.RESPONSE_JSON_BACKUP_FOLDER, 'response_json') # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ -CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER] +CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_BACKUP_FOLDER, 'data_import') diff --git a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py index f606c9ad..e83e58ab 100644 --- a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py +++ b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py @@ -1,12 +1,33 @@ import json from src.aws.s3 import ConfigBucket -from src.system_var.constants import UPD_JP_NAME +from src.config.objects import TargetObject, LastFetchDatetime from src.error.exceptions import FileUploadException +from src.system_var.constants import UPD_JP_NAME from src.util.logger import logger_instance as logger -def updload_last_fetch_datetime(target_object, last_fetch_datetime): +def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): + """ + 前回取得日時ファイル更新 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + last_fetch_datetime : LastFetchDatetime + 取得対象オブジェクト情報インスタンス + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + # ① 前回取得日時ファイル更新処理の開始ログを出力する logger.info( f'I-UPD-01 [{target_object.object_name}] の前回取得日時ファイルの更新処理を開始します') @@ -16,20 +37,21 @@ def updload_last_fetch_datetime(target_object, last_fetch_datetime): # ② オブジェクト情報.is_update_last_fetch_datetimeがfalseの場合、以降の処理をスキップする logger.info( f'I-UPD-02 [{target_object.object_name}] の前回取得日時ファイルの更新処理をスキップします') - else: - # ③ 前回取得日時ファイル.last_fetch_datetime_fromに取得処理開始年月日時分秒を設定する - # 前回取得日時ファイル.last_fetch_datetime_toに空文字を設定する - last_fetch_datetime_dict = { - 'last_fetch_datetime_from': last_fetch_datetime.last_fetch_datetime_to, - 'last_fetch_datetime_to': '' - } + return - config_bucket = ConfigBucket() - config_bucket.put_last_fetch_datetime_file( - target_object.last_fetch_datetime_file_name, json.dumps(last_fetch_datetime_dict)) + # ③ 前回取得日時ファイル.last_fetch_datetime_fromに取得処理開始年月日時分秒を設定する + # 前回取得日時ファイル.last_fetch_datetime_toに空文字を設定する + last_fetch_datetime_dict = { + 'last_fetch_datetime_from': last_fetch_datetime.last_fetch_datetime_to, + 'last_fetch_datetime_to': '' + } - logger.info( - f'D-UPD-03 [{target_object.object_name}] の前回取得日時ファイル更新処理 正常終了') + config_bucket = ConfigBucket() + config_bucket.put_last_fetch_datetime_file( + target_object.last_fetch_datetime_file_name, json.dumps(last_fetch_datetime_dict)) + + logger.info( + f'D-UPD-03 [{target_object.object_name}] の前回取得日時ファイル更新処理 正常終了') except Exception as e: raise FileUploadException( diff --git a/ecs/crm-datafetch/src/updload_result_data_process.py b/ecs/crm-datafetch/src/upload_result_data_process.py similarity index 64% rename from ecs/crm-datafetch/src/updload_result_data_process.py rename to ecs/crm-datafetch/src/upload_result_data_process.py index e70f4679..855ec900 100644 --- a/ecs/crm-datafetch/src/updload_result_data_process.py +++ b/ecs/crm-datafetch/src/upload_result_data_process.py @@ -1,11 +1,32 @@ from src.aws.s3 import BackupBucket +from src.error.exceptions import FileUploadException from src.system_var.constants import END_JP_NAME from src.system_var.environments import PROCESS_RESULT_FILENAME -from src.error.exceptions import FileUploadException +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def updload_result_data(process_result, date_path): +def upload_result_data_process(process_result: dict, execute_datetime: ExecuteDateTime): + """ + 取得処理実施結果アップロード処理 + + Parameters + ---------- + process_result : dict + 取得処理実行結果辞書オブジェクト + last_fetch_datetime : LastFetchDatetime + 取得対象オブジェクト情報インスタンス + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + # ① 取得処理実施結果アップロード処理のログを出力する logger.info( f'I-END-01 取得処理実施結果アップロード処理を開始します') @@ -14,7 +35,7 @@ def updload_result_data(process_result, date_path): # ② CRMバックアップ保管用バケットに、取得処理実施結果のJSONデータを保管する backup_bucket = BackupBucket() backup_bucket.put_result_json( - f'{date_path}/{PROCESS_RESULT_FILENAME}', process_result) + f'{execute_datetime.to_path()}/{PROCESS_RESULT_FILENAME}', process_result) logger.debug(f'D-END-02 取得処理実施結果アップロード 正常終了') diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py index a8bfed62..9d498b5e 100644 --- a/ecs/crm-datafetch/src/util/dict_checker.py +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -1,64 +1,39 @@ import re -class DictCheck: - def __init__(self) -> None: - pass +class DictChecker: + def __init__(self, object_dict: dict) -> None: + self.__object_dict = object_dict - def check_key_exist(self, object_dict: dict, check_key: str) -> bool: - ''' - 辞書型キー存在チェック - ''' - return True if check_key in object_dict and object_dict[check_key] != '' else False + def check_key_exist(self, check_key: str) -> bool: + """辞書型キー存在チェック""" + return check_key in self.__object_dict and self.__object_dict[check_key] != '' - def check_value_type(self, object_dict: dict, check_key: str, check_type: type) -> bool: - ''' - 辞書型バリュー型チェック - ''' - return True if isinstance(object_dict[check_key], check_type) else False + def check_data_type(self, check_key: str, check_type: type) -> bool: + """辞書型バリュー型チェック""" + return isinstance(self.__object_dict[check_key], check_type) - def check_regex(self, regex_str: str, object_dict: dict, check_key: str) -> bool: - ''' - 辞書型バリュー正規表現チェック - ''' - return True if re.fullmatch(regex_str, object_dict[check_key]) else False + def check_match_pattern(self, regex_str: str, check_key: str) -> bool: + """辞書型バリュー正規表現チェック""" + return True if re.fullmatch(regex_str, self.__object_dict[check_key]) else False - def check_key_exist_and_value_type(self, object_dict: dict, check_key: str, check_type: type) -> None: - ''' - 辞書型キー存在チェック＆バリュー型チェック - ''' - if not self.check_key_exist(object_dict, check_key): + def assert_key_exist(self, check_key: str, check_type: type) -> None: + """辞書型キー存在検査""" + if not self.check_key_exist(check_key): raise Exception(f'「{check_key}」キーは必須です') - elif not self.check_value_type(object_dict, check_key, check_type): + return + + def assert_data_type(self, check_key: str, check_type: type) -> None: + """バリュー型検査""" + if not self.check_data_type(check_key, check_type): raise Exception(f'「{check_key}」キーの値は「{check_type}」でなければなりません') - def check_key_exist_case_value_type(self, object_dict: dict, check_key: str, check_type: type): - ''' - 辞書型キー存在した場合のバリュー型チェック - ''' - if not self.check_key_exist(object_dict, check_key): - pass + return - elif not self.check_value_type(object_dict, check_key, check_type): - raise Exception(f'「{check_key}」キーの値は「{check_type}」でなければなりません') - - def check_key_exsit_and_regex(self, object_dict: dict, check_key: str, regex_str: str): - ''' - 辞書型キー存在チェック＆バリュー正規表現チェック - ''' - if not self.check_key_exist(object_dict, check_key): - raise Exception(f'「{check_key}」キーは必須です') - - elif not self.check_regex(regex_str, object_dict, check_key): + def assert_match_pattern(self, check_key: str, regex_str: str): + """正規表現検査""" + if not self.check_match_pattern(regex_str, check_key): raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') - def check_key_exsit_case_regex(self, object_dict: dict, check_key: str, regex_str: str): - ''' - 辞書型キー存在した場合のバリュー正規表現チェック - ''' - if not self.check_key_exist(object_dict, check_key): - pass - - elif not self.check_regex(regex_str, object_dict, check_key): - raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') + return diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py new file mode 100644 index 00000000..2f07736a --- /dev/null +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -0,0 +1,17 @@ +from datetime import datetime + +from src.system_var.constants import( + YYYYMMDDTHHMMSSTZ, + MILLISEC_FORMAT +) + + +class ExecuteDateTime: + def __init__(self): + self.__execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) + + def __str__(self) -> str: + return self.__execute_datetime + + def to_path(self) -> str: + return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index 689d9c36..ec1df277 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -2,6 +2,7 @@ import logging from src.system_var.environments import LOG_LEVEL +"""boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する""" for name in ["boto3", "botocore", "s3transfer", "urllib3"]: logging.getLogger(name).setLevel(logging.WARNING) diff --git a/s3/config/crm/object_info/crm_object_list_all.json b/s3/config/crm/object_info/crm_object_list_all.json new file mode 100644 index 00000000..a65cf1bb --- /dev/null +++ b/s3/config/crm/object_info/crm_object_list_all.json @@ -0,0 +1,47 @@ +{ + "objects": [ + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": false, + "last_fetch_datetime_file_name": "Territory2_ALL.json", + "upload_file_name": "CRM_Territory2_ALL_{execute_datetime}.csv" + }, + { + "object_name": "UserTerritory2Association", + "columns": [ + "Id", + "UserId", + "Territory2Id", + "IsActive", + "RoleInTerritory2", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": false, + "last_fetch_datetime_file_name": "UserTerritory2Association_ALL.json", + "upload_file_name": "CRM_UserTerritory2Association_ALL_{execute_datetime}.csv" + } + ] +} diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index 78ec31a7..00f7fe89 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -1,41 +1,3008 @@ { "objects": [ - { - # これはコメント行です - "object_name": "Account", - "columns": [ - "Id", - "Name", - "SystemModStamp", - "NumberOfEmployees", - "IsDeleted" - ] - }, - { - "object_name": "Territory2", - "is_skip": false, - "can_update_last_update": false, - "upload_file_name": "CRM_Territory2_All_{execute_datetime}_{split_csv_suffix_number:03}", - "last_fetch_datetime_file_name": "Territory2.json", - "columns": [ - "Id", - "Name", - "Territory2TypeId", - "SystemModStamp" - ] - }, - { - "object_name": "AccountShare", - "is_skip": false, - "datetime_column": "LastModifiedDate", - "columns": [ - "Id", - "Name", - "Territory2TypeId", - "LastModifiedDate" - ] - } + { + "object_name": "Clm_Presentation_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Mobile_ID_vod__c", + "Presentation_Id_vod__c", + "Product_vod__c", + "Default_Presentation_vod__c", + "Training_vod__c", + "ParentId_vod__c", + "Hidden_vod__c", + "Type_vod__c", + "Approved_vod__c", + "Copied_From_vod__c", + "Copy_Date_vod__c", + "Survey_vod__c", + "Original_Record_ID_vod__c", + "Directory_vod__c", + "End_Date_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "VExternal_Id_vod__c", + "Vault_DNS_vod__c", + "Vault_Doc_Id_vod__c", + "Vault_External_Id_vod__c", + "Vault_GUID_vod__c", + "Vault_Last_Modified_Date_Time_vod__c", + "Version_vod__c", + "Enable_Survey_Overlay_vod__c", + "Description_vod__c", + "Keywords_vod__c", + "Content_Channel_vod__c", + "original_material_approved_in_veritas__c", + "keywords__c", + "trade_team__c", + "ewizard_link__c", + "business_function__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Clm_Presentation_Slide_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Clm_Presentation_vod__c", + "Key_Message_vod__c", + "Display_Order_vod__c", + "Sub_Presentation_vod__c", + "Mobile_ID_vod__c", + "External_ID_vod__c", + "VExternal_Id_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Insight_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Clinical_Trial_vod__c", + "Date_vod__c", + "Description_vod__c", + "Entity_Reference_Id_vod__c", + "Interaction_vod__c", + "Medical_Event_vod__c", + "Mobile_ID_vod__c", + "Other_Source_vod__c", + "Override_Lock_vod__c", + "Publication_vod__c", + "Status_vod__c", + "Summary_vod__c", + "Unlock_vod__c", + "Commercial_Medical__c", + "MSJ_Level_1A__c", + "MSJ_Level_1B__c", + "MSJ_Level_2A__c", + "MSJ_Level_2B__c", + "MSJ_Level_3A__c", + "MSJ_Level_3B__c", + "MSJ_Level_4A__c", + "MSJ_Level_4B__c", + "MSJ_SubStatus__c", + "MSJ_Type_A__c", + "MSJ_Type_B__c", + "MSJ_Description_Backup__c", + "MSJ_Country__c", + "MSJ_Received_at_Boomi__c", + "MSJ_Level_1A_Value__c", + "MSJ_Level_1B_Value__c", + "MSJ_Level_2A_Value__c", + "MSJ_Level_2B_Value__c", + "MSJ_Level_3A_Value__c", + "MSJ_Level_3B_Value__c", + "MSJ_Level_4A_Value__c", + "MSJ_Level_4B_Value__c", + "MSJ_Hospital_ID__c", + "MSJ_Hospital_Name__c", + "MSJ_Hospital__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_MR_Weekly_Report__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Account_Name__c", + "MSJ_Activity_Results_Summary__c", + "MSJ_MUID__c", + "MSJ_Next_Week_Action_What__c", + "MSJ_Next_Week_Action_When__c", + "MSJ_Next_Week_Action_Where__c", + "MSJ_Next_Week_Action_Who__c", + "MSJ_Report_Week__c", + "MSJ_Target_Patient_Count__c", + "Mobile_ID_vod__c", + "MSJ_Activity_Results_Summary_HN__c", + "MSJ_Next_Week_Action_Where_HN__c", + "MSJ_Next_Week_Action_Who_HN__c", + "MSJ_Next_Week_Action_What_HN__c", + "MSJ_Next_Week_Action_When_HN__c", + "MSJ_Target_Patient_Count_HN__c", + "MSJ_Activity_Results_Summary_MCC__c", + "MSJ_Next_Week_Action_Where_MCC__c", + "MSJ_Next_Week_Action_Who_MCC__c", + "MSJ_Next_Week_Action_What_MCC__c", + "MSJ_Next_Week_Action_When_MCC__c", + "MSJ_Target_Patient_Count_MCC__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Account_Territory_Loader_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "External_ID_vod__c", + "Territory_vod__c", + "Mobile_ID_vod__c", + "Territory_To_Add_vod__c", + "Territory_to_Drop_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Event_Attendee_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Attendee_vod__c", + "User_vod__c", + "Medical_Event_vod__c", + "Attendee_Type_vod__c", + "Status_vod__c", + "Contact_vod__c", + "Attendee_Name_vod__c", + "Account_vod__c", + "Start_Date_vod__c", + "Signature_vod__c", + "Signature_Datetime_vod__c", + "MSJ_Copy_Account_Type__c", + "MSJ_Evaluation__c", + "MSJ_Hospital__c", + "MSJ_Role__c", + "Mobile_ID_vod__c", + "MSJ_Evaluation_Comment__c", + "Position_vod__c", + "Talk_Title_vod__c", + "MSJ_Attendee_Reaction__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "ObjectTerritory2Association", + "columns": [ + "Id", + "ObjectId", + "Territory2Id", + "AssociationCause", + "SobjectType", + "IsDeleted", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Key_Message_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Description_vod__c", + "Product_vod__c", + "Product_Strategy_vod__c", + "Display_Order_vod__c", + "Active_vod__c", + "Category_vod__c", + "Vehicle_vod__c", + "CLM_ID_vod__c", + "Custom_Reaction_vod__c", + "Slide_Version_vod__c", + "Language_vod__c", + "Media_File_CRC_vod__c", + "Media_File_Name_vod__c", + "Media_File_Size_vod__c", + "Segment_vod__c", + "Disable_Actions_vod__c", + "VExternal_Id_vod__c", + "CDN_Path_vod__c", + "Status_vod__c", + "Vault_DNS_vod__c", + "Vault_Doc_Id_vod__c", + "Vault_External_Id_vod__c", + "Vault_GUID_vod__c", + "Vault_Last_Modified_Date_Time_vod__c", + "Is_Shared_Resource_vod__c", + "iOS_Viewer_vod__c", + "iOS_Resolution_vod__c", + "approved_for_distribution_date__c", + "approved_for_use_date__c", + "ewizard_link__c", + "expiration_date__c", + "keywords__c", + "trade_team__c", + "business_function__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Group", + "columns": [ + "Id", + "Name", + "DeveloperName", + "RelatedId", + "Type", + "Email", + "OwnerId", + "DoesSendEmailToMembers", + "DoesIncludeBosses", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Event_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Primary_Product__c", + "Description_vod__c", + "Start_Date_vod__c", + "Location__c", + "End_Date_vod__c", + "Secondary_Product__c", + "Website__c", + "Active_vod__c", + "Event_Type__c", + "MSJ_Area__c", + "MSJ_Business_Unit__c", + "MSJ_Comment__c", + "MSJ_Company__c", + "MSJ_Expense_App_No__c", + "MSJ_Form__c", + "MSJ_HQ_Area__c", + "MSJ_Location__c", + "MSJ_MR__c", + "MSJ_Number_of_Attendee__c", + "MSJ_Product__c", + "MSJ_Site__c", + "MSJ_Type__c", + "MSJ_Number_of_Attendee_Auto_Calc__c", + "MSJ_Number_of_Attendee_Invited__c", + "Account_vod__c", + "MSJ_MUID__c", + "Country_Name_vod__c", + "MSJ_CE_SIPAGL_Updater__c", + "MSJ_CE_SIPAGL_1A__c", + "MSJ_CE_SIPAGL_1B__c", + "MSJ_CE_SIPAGL_2__c", + "MSJ_CE_SIPAGL_3__c", + "MSJ_CE_SIPAGL_4__c", + "MSJ_CE_SIPAGL_5__c", + "MSJ_CE_SIPAGL_Comment__c", + "MSJ_CE_SIPAGL_1A_date__c", + "MSJ_CE_SIPAGL_6__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Medical_Event_Evaluation__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Medical_Event__c", + "MSJ_Evaluation_Comment__c", + "MSJ_Evaluation__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Coaching_Report_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Mobile_ID_vod__c", + "Manager_vod__c", + "Employee_vod__c", + "Review_Date__c", + "Review_Period__c", + "Status__c", + "Comments__c", + "Strategic_Planning__c", + "Customer_Focus__c", + "Knowledge_Expertise__c", + "Business_Account_Planning__c", + "Call_Productivity__c", + "Overall_Rating__c", + "MSJ_A01__c", + "MSJ_A02__c", + "MSJ_A03__c", + "MSJ_AM_Memo__c", + "MSJ_Aid_Total__c", + "MSJ_C0_GC__c", + "MSJ_C1_GC__c", + "MSJ_C2_GC__c", + "MSJ_Countermeasure__c", + "MSJ_Deadline__c", + "MSJ_Double_Visit_Time__c", + "MSJ_Hospital__c", + "MSJ_K01_FE__c", + "MSJ_K01_ONC__c", + "MSJ_K02_FE__c", + "MSJ_K02_ONC__c", + "MSJ_K03_FE__c", + "MSJ_K03_ONC__c", + "MSJ_K04_FE__c", + "MSJ_K04_ONC__c", + "MSJ_K05_FE__c", + "MSJ_K05_ONC__c", + "MSJ_K06_FE__c", + "MSJ_K06_ONC__c", + "MSJ_K0_GC__c", + "MSJ_K1_GC__c", + "MSJ_K2_GC__c", + "MSJ_Knowledge_Total__c", + "MSJ_L0_GC__c", + "MSJ_L1_GC__c", + "MSJ_L2_GC__c", + "MSJ_MR_GC__c", + "MSJ_MR_Problems__c", + "MSJ_N0_GC__c", + "MSJ_N1_GC__c", + "MSJ_N2_GC__c", + "MSJ_Num_of_DTL__c", + "MSJ_P01__c", + "MSJ_P02__c", + "MSJ_P03__c", + "MSJ_P04__c", + "MSJ_P05__c", + "MSJ_P0_GC__c", + "MSJ_P1_GC__c", + "MSJ_P2_GC__c", + "MSJ_PlanningTotal__c", + "MSJ_R0_GC__c", + "MSJ_R1_GC__c", + "MSJ_R2_GC__c", + "MSJ_S01__c", + "MSJ_S02__c", + "MSJ_S03__c", + "MSJ_S04__c", + "MSJ_S05__c", + "MSJ_S06__c", + "MSJ_S07__c", + "MSJ_S08__c", + "MSJ_S09__c", + "MSJ_S10__c", + "MSJ_S11__c", + "MSJ_S12__c", + "MSJ_Skill_Total__c", + "MSJ_After_Call_01__c", + "MSJ_After_Call_02__c", + "MSJ_After_Call_03__c", + "MSJ_After_Call_04__c", + "MSJ_Closing__c", + "MSJ_Comment_by_MR__c", + "MSJ_Confirmed_by_MR__c", + "MSJ_Createdby__c", + "MSJ_FT_AM_Name__c", + "MSJ_Interview_Preparation__c", + "MSJ_Interview_Reflection__c", + "MSJ_Notify_To_MR__c", + "MSJ_Opening__c", + "MSJ_Others_01_Result__c", + "MSJ_Others_01__c", + "MSJ_Others_02_Result__c", + "MSJ_Others_02__c", + "MSJ_Patient_Thinking__c", + "MSJ_Probing__c", + "MSJ_Supporting__c", + "MSJ_Patient_Thinking_for_FE__c", + "MSJ_After_Call_05__c", + "MSJ_After_Call_06__c", + "MSJ_After_Call_07__c", + "MSJ_After_Call_08__c", + "MSJ_Createdby_FE__c", + "MSJ_Createdby_ONC__c", + "MSJ_Development_Level__c", + "MSJ_Interview_Prep_01__c", + "MSJ_Interview_Prep_02__c", + "MSJ_Leadership_Style__c", + "MSJ_Overcome_01__c", + "MSJ_Overcome_02__c", + "MSJ_Overcome_03__c", + "MSJ_Overcome_04__c", + "MSJ_Review_01__c", + "MSJ_Review_02__c", + "MSJ_SK_01__c", + "MSJ_SK_02__c", + "MSJ_SK_03__c", + "MSJ_SK_04__c", + "MSJ_SK_05__c", + "MSJ_SK_06__c", + "MSJ_SK_07__c", + "MSJ_SK_08__c", + "MSJ_SK_09__c", + "MSJ_SK_10__c", + "MSJ_Specific_Action__c", + "MSJ_Training_Point__c", + "MSJ_Efforts_of_Year__c", + "MSJ_Efforts_of_Month__c", + "MSJ_Skill_Task__c", + "MSJ_Action_of_This_Month__c", + "MSJ_Achievement_of_This_Month__c", + "MSJ_Comment_from_AM__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Call_Comments_vod__c", + "Sample_Card_vod__c", + "Add_Detail_vod__c", + "Property_vod__c", + "Account_vod__c", + "zvod_Product_Discussion_vod__c", + "Status_vod__c", + "Parent_Address_vod__c", + "Account_Plan_vod__c", + "zvod_SaveNew_vod__c", + "Next_Call_Notes_vod__c", + "Pre_Call_Notes_vod__c", + "Mobile_ID_vod__c", + "zvod_Account_Credentials_vod_c_vod__c", + "zvod_Account_Preferred_Name_vod_c_vod__c", + "zvod_Account_Sample_Status_vod_c_vod__c", + "zvod_Attendees_vod__c", + "zvod_Key_Messages_vod__c", + "zvod_Detailing_vod__c", + "zvod_Expenses_vod__c", + "zvod_Followup_vod__c", + "zvod_Samples_vod__c", + "zvod_Save_vod__c", + "zvod_Submit_vod__c", + "zvod_Delete_vod__c", + "Activity_Type__c", + "Significant_Event__c", + "Location_vod__c", + "Subject_vod__c", + "Unlock_vod__c", + "Call_Datetime_vod__c", + "Disbursed_To_vod__c", + "Disclaimer_vod__c", + "Request_Receipt_vod__c", + "Signature_Date_vod__c", + "Signature_vod__c", + "Territory_vod__c", + "Submitted_By_Mobile_vod__c", + "Call_Type_vod__c", + "Add_Key_Message_vod__c", + "Address_vod__c", + "Attendees_vod__c", + "Attendee_Type_vod__c", + "Call_Date_vod__c", + "Detailed_Products_vod__c", + "No_Disbursement_vod__c", + "Parent_Call_vod__c", + "User_vod__c", + "Contact_vod__c", + "zvod_Entity_vod__c", + "Medical_Event_vod__c", + "Mobile_Created_Datetime_vod__c", + "Mobile_Last_Modified_Datetime_vod__c", + "License_vod__c", + "Is_Parent_Call_vod__c", + "Entity_Display_Name_vod__c", + "Override_Lock_vod__c", + "Last_Device_vod__c", + "Ship_Address_Line_1_vod__c", + "Ship_Address_Line_2_vod__c", + "Ship_City_vod__c", + "Ship_Country_vod__c", + "Ship_License_Expiration_Date_vod__c", + "Ship_License_Status_vod__c", + "Ship_License_vod__c", + "Ship_State_vod__c", + "Ship_To_Address_vod__c", + "Ship_Zip_vod__c", + "Ship_To_Address_Text_vod__c", + "CLM_vod__c", + "zvod_CLMDetails_vod__c", + "Is_Sampled_Call_vod__c", + "zvod_Surveys_vod__c", + "Presentations_vod__c", + "Entity_Reference_Id_vod__c", + "Error_Reference_Call_vod__c", + "Duration_vod__c", + "Color_vod__c", + "Allowed_Products_vod__c", + "zvod_Attachments_vod__c", + "Sample_Card_Reason_vod__c", + "ASSMCA_vod__c", + "Address_Line_1_vod__c", + "Address_Line_2_vod__c", + "City_vod__c", + "DEA_Address_Line_1_vod__c", + "DEA_Address_Line_2_vod__c", + "DEA_Address_vod__c", + "DEA_City_vod__c", + "DEA_Expiration_Date_vod__c", + "DEA_State_vod__c", + "DEA_Zip_4_vod__c", + "DEA_Zip_vod__c", + "DEA_vod__c", + "Ship_Zip_4_vod__c", + "State_vod__c", + "Zip_4_vod__c", + "Zip_vod__c", + "Sample_Send_Card_vod__c", + "zvod_Address_vod_c_DEA_Status_vod_c_vod__c", + "Signature_Page_Image_vod__c", + "Credentials_vod__c", + "Salutation_vod__c", + "zvod_Account_Call_Reminder_vod_c_vod__c", + "MSJ_Meeting_Duration__c", + "MSJ_Double_Visit_AM__c", + "zvod_Business_Account_vod__c", + "Product_Priority_1_vod__c", + "Product_Priority_2_vod__c", + "Product_Priority_3_vod__c", + "Product_Priority_4_vod__c", + "Product_Priority_5_vod__c", + "zvod_More_Actions_vod__c", + "zvod_Call_Conflict_Status_vod__c", + "Signature_Timestamp_vod__c", + "Expense_Amount_vod__c", + "Total_Expense_Attendees_Count_vod__c", + "Attendee_list_vod__c", + "Expense_Post_Status_vod__c", + "Attendee_Post_Status_vod__c", + "Expense_System_External_ID_vod__c", + "Incurred_Expense_vod__c", + "Assigner_vod__c", + "Assignment_Datetime_vod__c", + "zvod_Call_Objective_vod__c", + "Signature_Location_Longitude_vod__c", + "Signature_Location_Latitude_vod__c", + "Location_Services_Status_vod__c", + "MSJ_Double_Visit_Other__c", + "MSJ_Comment__c", + "MSJ_For_Reporting__c", + "MSJ_Number_of_Attendees__c", + "MSJ_Main_Dept__c", + "Planned_Type_vjh__c", + "Cobrowse_URL_Participant_vod__c", + "MSJ_Activity_Method_Text__c", + "MSJ_Activity_Method__c", + "MSJ_Classification__c", + "MSJ_Double_Visit_MSL__c", + "MSJ_MSL_Comment_for_MR__c", + "MSJ_APD__c", + "Medical_Inquiry_vod__c", + "MSJ_Call_Type_MSJ__c", + "MSJ_Prescription_Request__c", + "MSJ_Patient_Follow__c", + "Child_Account_Id_vod__c", + "Child_Account_vod__c", + "Location_Id_vod__c", + "Location_Name_vod__c", + "MSJ_Comments_about_technology__c", + "Remote_Meeting_vod__c", + "Veeva_Remote_Meeting_Id_vod__c", + "MSJ_Activity_Type_Report__c", + "MSJ_Activity_Type__c", + "MSJ_Activity__c", + "MSJ_Comments__c", + "MSJ_Therapy__c", + "MSJ_Time_Hrs__c", + "EMDS_CO_Reference__c", + "EMDS_Call_Sub_Type__c", + "EMDS_Call_Type__c", + "EMDS_Call_Unsuccessful__c", + "EMDS_Congress_Type__c", + "EMDS_Date_of_Service__c", + "EMDS_Fertility_DisInterest__c", + "EMDS_Fertility_Interest__c", + "EMDS_Installed_Equipment__c", + "EMDS_Pipeline_Stage_Value__c", + "EMDS_Pipeline_Stage__c", + "EMDS_Pipeline__c", + "EMDS_Reason_for_Call__c", + "EMDS_Training_Completed__c", + "MSJ_BrainStorming__c", + "MSJ_SIPAGL_1A__c", + "MSJ_SIPAGL_1B__c", + "MSJ_SIPAGL_2__c", + "MSJ_SIPAGL_3__c", + "MSJ_SIPAGL_4A__c", + "MSJ_SIPAGL_5A__c", + "MSJ_SIPAGL_comment__c", + "MSJ_SIPAGL_4B__c", + "MSJ_SIPAGL_5B__c", + "Location_Text_vod__c", + "Call_Channel_vod__c", + "MSJ_Scientific_Interaction__c", + "MSJ_Activity_Email_Reply__c", + "MSJ_Interaction_Duration__c", + "MSJ_SIPAGL_1A_date__c", + "MSJ_CoPromotion__c", + "Call_Channel_Formula_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Detail_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Is_Parent_Call_vod__c", + "Call2_vod__c", + "Product_vod__c", + "Detail_Priority_vod__c", + "Mobile_ID_vod__c", + "Override_Lock_vod__c", + "Type_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Key_Message_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Call2_vod__c", + "Reaction_vod__c", + "Product_vod__c", + "Key_Message_vod__c", + "Mobile_ID_vod__c", + "Contact_vod__c", + "Call_Date_vod__c", + "User_vod__c", + "Category_vod__c", + "Vehicle_vod__c", + "Is_Parent_Call_vod__c", + "Override_Lock_vod__c", + "CLM_ID_vod__c", + "Slide_Version_vod__c", + "Duration_vod__c", + "Presentation_ID_vod__c", + "Start_Time_vod__c", + "Attendee_Type_vod__c", + "Entity_Reference_Id_vod__c", + "Segment_vod__c", + "Display_Order_vod__c", + "Clm_Presentation_Name_vod__c", + "Clm_Presentation_Version_vod__c", + "Clm_Presentation_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call_Clickstream_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Answer_vod__c", + "Call_vod__c", + "Key_Message_vod__c", + "Mobile_ID_vod__c", + "Popup_Opened_vod__c", + "Possible_Answers_vod__c", + "Presentation_ID_vod__c", + "Product_vod__c", + "Range_Value_vod__c", + "Rollover_Entered_vod__c", + "Selected_Items_vod__c", + "CLM_ID_vod__c", + "Question_vod__c", + "Survey_Type_vod__c", + "Text_Entered_vod__c", + "Toggle_Button_On_vod__c", + "Track_Element_Description_vod__c", + "Track_Element_Id_vod__c", + "Track_Element_Type_vod__c", + "Usage_Duration_vod__c", + "Usage_Start_Time_vod__c", + "AuxillaryId_vod__c", + "ParentId_vod__c", + "Revision_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Discussion_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Call2_vod__c", + "Activity__c", + "Comments__c", + "Contact_vod__c", + "Call_Date_vod__c", + "Product_Strategy_vod__c", + "Product_Tactic_vod__c", + "Restricted_Comments__c", + "Product_vod__c", + "Presentation__c", + "Discussion_Topics__c", + "Slides__c", + "User_vod__c", + "Indication__c", + "Mobile_ID_vod__c", + "Medical_Event_vod__c", + "Is_Parent_Call_vod__c", + "Override_Lock_vod__c", + "zvod_Product_Map_vod__c", + "Attendee_Type_vod__c", + "Entity_Reference_Id_vod__c", + "Account_Tactic_vod__c", + "MSJ_Material_Type__c", + "MSJ_Discussion_Contents__c", + "MSJ_IST_Minutes__c", + "MSJ_Off_Label_Minutes__c", + "MSJ_Discussion_Objectives__c", + "MSJ_Insight__c", + "EMDS_Materials__c", + "EMDS_Topic__c", + "MSJ_Visit_Purpose__c", + "MSJ_Insight_Count__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Time_Off_Territory_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Reason_vod__c", + "Territory_vod__c", + "Date_vod__c", + "Status_vod__c", + "Time_vod__c", + "Hours_vod__c", + "Mobile_ID_vod__c", + "Hours_off_vod__c", + "Start_Time_vod__c", + "MSJ_Day__c", + "MSJ_Comment__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Dynamic_Attribute_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Active_vod__c", + "Dynamic_Attribute_Configuration_vod__c", + "Dynamic_Attribute_Description_vod__c", + "Dynamic_Attribute_Help_Text_vod__c", + "Dynamic_Attribute_Label_vod__c", + "Dynamic_Attribute_Name_vod__c", + "Dynamic_Attribute_Record_Type_vod__c", + "Dynamic_Attribute_Value_Checkbox_vod__c", + "Dynamic_Attribute_Value_Date_Time_vod__c", + "Dynamic_Attribute_Value_Date_vod__c", + "Dynamic_Attribute_Value_Number_vod__c", + "Dynamic_Attribute_Value_Text_Area_vod__c", + "Dynamic_Attribute_Value_Text_vod__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Dynamic_Attribute_Configuration_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Applies_To_vod__c", + "Attribute_Label_vod__c", + "Attribute_Name_vod__c", + "Available_Values_vod__c", + "Description_vod__c", + "Detail_Group_vod__c", + "Display_Order_vod__c", + "External_ID_vod__c", + "Help_Text_vod__c", + "Product_vod__c", + "Read_Only_vod__c", + "Section_Name_vod__c", + "Sharing_Group_vod__c", + "Status_vod__c", + "Track_Changes_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Profile", + "columns": [ + "Id", + "Name", + "PermissionsEmailSingle", + "PermissionsEmailMass", + "PermissionsEditTask", + "PermissionsEditEvent", + "PermissionsExportReport", + "PermissionsImportPersonal", + "PermissionsDataExport", + "PermissionsManageUsers", + "PermissionsEditPublicFilters", + "PermissionsEditPublicTemplates", + "PermissionsModifyAllData", + "PermissionsManageCases", + "PermissionsManageSolutions", + "PermissionsCustomizeApplication", + "PermissionsEditReadonlyFields", + "PermissionsRunReports", + "PermissionsViewSetup", + "PermissionsTransferAnyEntity", + "PermissionsNewReportBuilder", + "PermissionsManageSelfService", + "PermissionsManageCssUsers", + "PermissionsActivateContract", + "PermissionsApproveContract", + "PermissionsImportLeads", + "PermissionsManageLeads", + "PermissionsTransferAnyLead", + "PermissionsViewAllData", + "PermissionsEditPublicDocuments", + "PermissionsViewEncryptedData", + "PermissionsEditBrandTemplates", + "PermissionsEditHtmlTemplates", + "PermissionsManageTranslation", + "PermissionsDeleteActivatedContract", + "PermissionsSendSitRequests", + "PermissionsApiUserOnly", + "PermissionsManageRemoteAccess", + "PermissionsCanUseNewDashboardBuilder", + "PermissionsManageCategories", + "PermissionsConvertLeads", + "PermissionsTestInstanceCreate", + "PermissionsPasswordNeverExpires", + "PermissionsUseTeamReassignWizards", + "PermissionsInstallMultiforce", + "PermissionsPublishMultiforce", + "PermissionsEditOppLineItemUnitPrice", + "PermissionsManageTerritories", + "PermissionsCreateMultiforce", + "PermissionsBulkApiHardDelete", + "PermissionsInboundMigrationToolsUser", + "PermissionsSolutionImport", + "PermissionsManageCallCenters", + "PermissionsManageSynonyms", + "PermissionsOutboundMigrationToolsUser", + "PermissionsViewContent", + "PermissionsManageEmailClientConfig", + "PermissionsEnableNotifications", + "PermissionsManageDataIntegrations", + "PermissionsDistributeFromPersWksp", + "PermissionsViewDataCategories", + "PermissionsManageDataCategories", + "PermissionsAuthorApex", + "PermissionsManageMobile", + "PermissionsApiEnabled", + "PermissionsManageCustomReportTypes", + "PermissionsEditCaseComments", + "PermissionsTransferAnyCase", + "PermissionsContentAdministrator", + "PermissionsCreateWorkspaces", + "PermissionsManageContentPermissions", + "PermissionsManageContentProperties", + "PermissionsManageContentTypes", + "PermissionsScheduleJob", + "PermissionsManageExchangeConfig", + "PermissionsManageAnalyticSnapshots", + "PermissionsScheduleReports", + "PermissionsManageBusinessHourHolidays", + "PermissionsManageDynamicDashboards", + "PermissionsManageInteraction", + "PermissionsViewMyTeamsDashboards", + "PermissionsResetPasswords", + "PermissionsFlowUFLRequired", + "PermissionsActivitiesAccess", + "PermissionsEmailTemplateManagement", + "PermissionsEmailAdministration", + "PermissionsChatterFileLink", + "PermissionsForceTwoFactor", + "PermissionsViewEventLogFiles", + "PermissionsManageNetworks", + "PermissionsManageAuthProviders", + "PermissionsRunFlow", + "PermissionsCreateCustomizeDashboards", + "PermissionsCreateDashboardFolders", + "PermissionsViewPublicDashboards", + "PermissionsManageDashbdsInPubFolders", + "PermissionsCreateCustomizeReports", + "PermissionsCreateReportFolders", + "PermissionsViewPublicReports", + "PermissionsManageReportsInPubFolders", + "PermissionsEditMyDashboards", + "PermissionsEditMyReports", + "PermissionsViewAllUsers", + "PermissionsConnectOrgToEnvironmentHub", + "PermissionsCreateCustomizeFilters", + "PermissionsContentHubUser", + "PermissionsGovernNetworks", + "PermissionsSalesConsole", + "PermissionsTwoFactorApi", + "PermissionsDeleteTopics", + "PermissionsEditTopics", + "PermissionsCreateTopics", + "PermissionsAssignTopics", + "PermissionsIdentityEnabled", + "PermissionsIdentityConnect", + "PermissionsContentWorkspaces", + "PermissionsCustomMobileAppsAccess", + "PermissionsViewHelpLink", + "PermissionsManageProfilesPermissionsets", + "PermissionsAssignPermissionSets", + "PermissionsManageRoles", + "PermissionsManageIpAddresses", + "PermissionsManageSharing", + "PermissionsManageInternalUsers", + "PermissionsManagePasswordPolicies", + "PermissionsManageLoginAccessPolicies", + "PermissionsManageCustomPermissions", + "PermissionsStdAutomaticActivityCapture", + "PermissionsManageTwoFactor", + "PermissionsDebugApex", + "PermissionsLightningExperienceUser", + "PermissionsConfigCustomRecs", + "PermissionsSubmitMacrosAllowed", + "PermissionsBulkMacrosAllowed", + "PermissionsManageSessionPermissionSets", + "PermissionsCreateAuditFields", + "PermissionsUpdateWithInactiveOwner", + "PermissionsManageSandboxes", + "PermissionsAutomaticActivityCapture", + "PermissionsImportCustomObjects", + "PermissionsDelegatedTwoFactor", + "PermissionsSelectFilesFromSalesforce", + "PermissionsModerateNetworkUsers", + "PermissionsMergeTopics", + "PermissionsSubscribeToLightningReports", + "PermissionsManagePvtRptsAndDashbds", + "PermissionsAllowLightningLogin", + "PermissionsCampaignInfluence2", + "PermissionsViewDataAssessment", + "PermissionsCanApproveFeedPost", + "PermissionsAllowViewEditConvertedLeads", + "PermissionsShowCompanyNameAsUserBadge", + "PermissionsAccessCMC", + "PermissionsViewHealthCheck", + "PermissionsManageHealthCheck", + "PermissionsPackaging2", + "PermissionsManageCertificates", + "PermissionsCreateReportInLightning", + "PermissionsPreventClassicExperience", + "PermissionsListEmailSend", + "PermissionsChangeDashboardColors", + "PermissionsManageRecommendationStrategies", + "PermissionsManagePropositions", + "PermissionsSubscribeReportRolesGrps", + "PermissionsSubscribeDashboardRolesGrps", + "PermissionsUseWebLink", + "PermissionsHasUnlimitedNBAExecutions", + "PermissionsViewOnlyEmbeddedAppUser", + "PermissionsViewAllActivities", + "PermissionsSubscribeReportToOtherUsers", + "PermissionsLightningConsoleAllowedForUser", + "PermissionsSubscribeReportsRunAsUser", + "PermissionsSubscribeToLightningDashboards", + "PermissionsSubscribeDashboardToOtherUsers", + "PermissionsCreateLtngTempInPub", + "PermissionsTransactionalEmailSend", + "PermissionsViewPrivateStaticResources", + "PermissionsCreateLtngTempFolder", + "PermissionsApexRestServices", + "PermissionsEnableCommunityAppLauncher", + "PermissionsGiveRecognitionBadge", + "PermissionsUseMySearch", + "PermissionsLtngPromoReserved01UserPerm", + "PermissionsManageSubscriptions", + "PermissionsManageSurveys", + "PermissionsUseAssistantDialog", + "PermissionsUseQuerySuggestions", + "PermissionsViewRoles", + "PermissionsLMOutboundMessagingUserPerm", + "PermissionsModifyDataClassification", + "PermissionsPrivacyDataAccess", + "PermissionsQueryAllFiles", + "PermissionsModifyMetadata", + "PermissionsManageCMS", + "PermissionsSandboxTestingInCommunityApp", + "PermissionsCanEditPrompts", + "PermissionsViewUserPII", + "PermissionsManageHubConnections", + "PermissionsB2BMarketingAnalyticsUser", + "PermissionsTraceXdsQueries", + "PermissionsViewAllCustomSettings", + "PermissionsViewAllForeignKeyNames", + "PermissionsHeadlessCMSAccess", + "PermissionsLMEndMessagingSessionUserPerm", + "PermissionsConsentApiUpdate", + "PermissionsAccessContentBuilder", + "PermissionsAccountSwitcherUser", + "PermissionsManageC360AConnections", + "PermissionsManageReleaseUpdates", + "PermissionsViewAllProfiles", + "PermissionsSkipIdentityConfirmation", + "PermissionsSendCustomNotifications", + "PermissionsPackaging2Delete", + "PermissionsFSCComprehensiveUserAccess", + "PermissionsManageTrustMeasures", + "PermissionsViewTrustMeasures", + "PermissionsIsotopeCToCUser", + "PermissionsIsotopeAccess", + "PermissionsIsotopeLEX", + "PermissionsQuipMetricsAccess", + "PermissionsQuipUserEngagementMetrics", + "PermissionsManageExternalConnections", + "PermissionsAIViewInsightObjects", + "PermissionsAICreateInsightObjects", + "PermissionsNativeWebviewScrolling", + "PermissionsViewDeveloperName", + "Type", + "UserLicenseId", + "UserType", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "Description", + "LastViewedDate", + "LastReferencedDate" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "My_Setup_Products_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Product_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Activity_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_External_ID_Map_vod__c", + "Account_vod__c", + "Call_vod__c", + "City_vod__c", + "Client_Name_vod__c", + "Client_OS_vod__c", + "Client_Type_vod__c", + "Country_vod__c", + "Debug_vod__c", + "Device_vod__c", + "IP_Address_vod__c", + "Multichannel_Activity_vod__c", + "Referring_Site_vod__c", + "Region_vod__c", + "Sent_Email_vod__c", + "Session_Id_vod__c", + "Site_vod__c", + "Start_DateTime_vod__c", + "Total_Duration_vod__c", + "URL_vod__c", + "User_Agent_vod__c", + "VExternal_Id_vod__c", + "Viewport_Height_vod__c", + "Viewport_Width_vod__c", + "Color_vod__c", + "Icon_vod__c", + "MCD_Primary_Key_vod__c", + "Record_Type_Name_vod__c", + "MSJ_Date_Opened__c", + "MSJ_Sent_Date__c", + "MSJ_Email_Subject__c", + "MSJ_Opens__c", + "MSJ_Email_Status__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Activity_Line_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Multichannel_Activity_vod__c", + "Call_vod__c", + "Custom_vod__c", + "DateTime_vod__c", + "Debug_vod__c", + "Detail_Group_VExternal_Id_vod__c", + "Detail_Group_vod__c", + "Duration_vod__c", + "Event_Subtype_vod__c", + "Event_Type_vod__c", + "Key_Message_VExternal_Id_vod__c", + "Key_Message_vod__c", + "Multichannel_Content_Asset_Id_vod__c", + "Multichannel_Content_Asset_Version_vod__c", + "Multichannel_Content_Asset_vod__c", + "Multichannel_Content_vod__c", + "Product_VExternal_Id_vod__c", + "Product_vod__c", + "Sent_Email_vod__c", + "VExternal_Id_vod__c", + "Video_Last_Viewed_Time_vod__c", + "Video_Length_vod__c", + "Video_Total_Time_Spent_vod__c", + "View_Order_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Consent_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Capture_Datetime_vod__c", + "Channel_Value_vod__c", + "Detail_Group_vod__c", + "External_ID_vod__c", + "Last_Device_vod__c", + "Mobile_ID_vod__c", + "Opt_Expiration_Date_vod__c", + "Opt_Type_vod__c", + "Optout_Event_Type_vod__c", + "Product_vod__c", + "Signature_Datetime_vod__c", + "Signature_ID_vod__c", + "Signature_vod__c", + "Sample_Consent_Template_Data_vod__c", + "Sample_Consent_Template_vod__c", + "Consent_Line_vod__c", + "Consent_Type_vod__c", + "Default_Consent_Text_vod__c", + "Disclaimer_Text_vod__c", + "Sub_Channel_Key_vod__c", + "Consent_Confirm_Datetime_vod__c", + "Related_Transaction_Id_vod__c", + "Sent_Email_vod__c", + "Content_Type_vod__c", + "Receipt_Email_vod__c", + "Receipt_Sent_Email_Transaction_Id_vod__c", + "Receipt_Sent_Email_vod__c", + "Captured_By_vod__c", + "Opt_Out_Disclaimer_Text_vod__c", + "Channel_Source_vod__c", + "Union_Id_vod__c", + "User_Last_Notified_vod__c", + "Sub_Channel_Display_Name__c", + "MSJ_Consent_Source__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Inquiry_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Address_Line_1_vod__c", + "Address_Line_2_vod__c", + "City_vod__c", + "Delivery_Method_vod__c", + "Email_vod__c", + "Fax_Number_vod__c", + "Inquiry_Text__c", + "Lock_vod__c", + "Mobile_ID_vod__c", + "Phone_Number_vod__c", + "Product__c", + "Rush_Delivery__c", + "Signature_Date_vod__c", + "Signature_vod__c", + "State_vod__c", + "Status_vod__c", + "Zip_vod__c", + "zvod_Delivery_Method_vod__c", + "zvod_Disclaimer_vod__c", + "Submitted_By_Mobile_vod__c", + "Disclaimer_vod__c", + "Entity_Reference_Id_vod__c", + "Call2_vod__c", + "Country_vod__c", + "Override_Lock_vod__c", + "MSJ_Department__c", + "MSJ_Doctor_Name__c", + "MSJ_Hospital_Name__c", + "MSJ_Indication__c", + "MSJ_Inquiry_Assignment__c", + "MSJ_Inquiry_Date__c", + "MSJ_Inquiry_Input_Manager__c", + "MSJ_Inquiry_Input_User__c", + "MSJ_MSL_Manager__c", + "MSJ_Notice_to_MR__c", + "MSJ_Person_in_charge_1__c", + "MSJ_Person_in_charge_2__c", + "MSJ_Product_for_MEC__c", + "MSJ_Product_for_MR__c", + "MSJ_Reply_Date__c", + "MSJ_Reply_User__c", + "MSJ_Reply__c", + "MSJ_Title__c", + "MSJ_AE_Infomation__c", + "MSJ_FAQ_Number_Report__c", + "MSJ_Return_Call_Report__c", + "MSJ_Inquiry_Origin_Report__c", + "MSJ_AE_Report__c", + "MSJ_Background__c", + "MSJ_MSL_Support__c", + "MSJ_Material_Requirement__c", + "MSJ_Hospital_Name_Disp__c", + "MSJ_Hospital__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Email_Activity_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Sent_Email_vod__c", + "Activity_DateTime_vod__c", + "City_vod__c", + "Click_URL_vod__c", + "Client_Name_vod__c", + "Client_OS_vod__c", + "Client_Type_vod__c", + "Country_vod__c", + "Device_Type_vod__c", + "Event_Msg_vod__c", + "Event_type_vod__c", + "IP_Address_vod__c", + "Region_vod__c", + "User_Agent_vod__c", + "Vault_Doc_ID_vod__c", + "Vault_Doc_Name_vod__c", + "Vault_Document_Major_Version_vod__c", + "Vault_Document_Minor_Version_vod__c", + "Vault_Document_Number_vod__c", + "Vault_Document_Title_vod__c", + "Vault_Instance_ID_vod__c", + "Preference_Modification_vod__c", + "Approved_Document_vod__c", + "Link_Name_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "User", + "columns": [ + "Id", + "Username", + "LastName", + "FirstName", + "Name", + "CompanyName", + "Division", + "Department", + "Title", + "Street", + "City", + "State", + "PostalCode", + "Country", + "Latitude", + "Longitude", + "GeocodeAccuracy", + "Address", + "Email", + "EmailPreferencesAutoBcc", + "EmailPreferencesAutoBccStayInTouch", + "EmailPreferencesStayInTouchReminder", + "SenderEmail", + "SenderName", + "Signature", + "StayInTouchSubject", + "StayInTouchSignature", + "StayInTouchNote", + "Phone", + "Fax", + "MobilePhone", + "Alias", + "CommunityNickname", + "BadgeText", + "IsActive", + "TimeZoneSidKey", + "UserRoleId", + "LocaleSidKey", + "ReceivesInfoEmails", + "ReceivesAdminInfoEmails", + "EmailEncodingKey", + "ProfileId", + "UserType", + "LanguageLocaleKey", + "EmployeeNumber", + "DelegatedApproverId", + "ManagerId", + "LastLoginDate", + "LastPasswordChangeDate", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "NumberOfFailedLogins", + "OfflineTrialExpirationDate", + "OfflinePdaTrialExpirationDate", + "UserPermissionsMarketingUser", + "UserPermissionsOfflineUser", + "UserPermissionsWirelessUser", + "UserPermissionsAvantgoUser", + "UserPermissionsCallCenterAutoLogin", + "UserPermissionsSFContentUser", + "UserPermissionsInteractionUser", + "UserPermissionsSupportUser", + "UserPermissionsChatterAnswersUser", + "ForecastEnabled", + "UserPreferencesActivityRemindersPopup", + "UserPreferencesEventRemindersCheckboxDefault", + "UserPreferencesTaskRemindersCheckboxDefault", + "UserPreferencesReminderSoundOff", + "UserPreferencesDisableAllFeedsEmail", + "UserPreferencesApexPagesDeveloperMode", + "UserPreferencesReceiveNoNotificationsAsApprover", + "UserPreferencesReceiveNotificationsAsDelegatedApprover", + "UserPreferencesHideCSNGetChatterMobileTask", + "UserPreferencesHideCSNDesktopTask", + "UserPreferencesHideChatterOnboardingSplash", + "UserPreferencesHideSecondChatterOnboardingSplash", + "UserPreferencesShowTitleToExternalUsers", + "UserPreferencesShowManagerToExternalUsers", + "UserPreferencesShowEmailToExternalUsers", + "UserPreferencesShowWorkPhoneToExternalUsers", + "UserPreferencesShowMobilePhoneToExternalUsers", + "UserPreferencesShowFaxToExternalUsers", + "UserPreferencesShowStreetAddressToExternalUsers", + "UserPreferencesShowCityToExternalUsers", + "UserPreferencesShowStateToExternalUsers", + "UserPreferencesShowPostalCodeToExternalUsers", + "UserPreferencesShowCountryToExternalUsers", + "UserPreferencesShowProfilePicToGuestUsers", + "UserPreferencesShowTitleToGuestUsers", + "UserPreferencesShowCityToGuestUsers", + "UserPreferencesShowStateToGuestUsers", + "UserPreferencesShowPostalCodeToGuestUsers", + "UserPreferencesShowCountryToGuestUsers", + "UserPreferencesHideInvoicesRedirectConfirmation", + "UserPreferencesHideStatementsRedirectConfirmation", + "UserPreferencesPathAssistantCollapsed", + "UserPreferencesCacheDiagnostics", + "UserPreferencesShowEmailToGuestUsers", + "UserPreferencesShowManagerToGuestUsers", + "UserPreferencesShowWorkPhoneToGuestUsers", + "UserPreferencesShowMobilePhoneToGuestUsers", + "UserPreferencesShowFaxToGuestUsers", + "UserPreferencesShowStreetAddressToGuestUsers", + "UserPreferencesLightningExperiencePreferred", + "UserPreferencesPreviewLightning", + "UserPreferencesHideEndUserOnboardingAssistantModal", + "UserPreferencesHideLightningMigrationModal", + "UserPreferencesHideSfxWelcomeMat", + "UserPreferencesHideBiggerPhotoCallout", + "UserPreferencesGlobalNavBarWTShown", + "UserPreferencesGlobalNavGridMenuWTShown", + "UserPreferencesCreateLEXAppsWTShown", + "UserPreferencesFavoritesWTShown", + "UserPreferencesRecordHomeSectionCollapseWTShown", + "UserPreferencesRecordHomeReservedWTShown", + "UserPreferencesFavoritesShowTopFavorites", + "UserPreferencesExcludeMailAppAttachments", + "UserPreferencesSuppressTaskSFXReminders", + "UserPreferencesSuppressEventSFXReminders", + "UserPreferencesPreviewCustomTheme", + "UserPreferencesHasCelebrationBadge", + "UserPreferencesUserDebugModePref", + "UserPreferencesSRHOverrideActivities", + "UserPreferencesNewLightningReportRunPageEnabled", + "UserPreferencesReverseOpenActivitiesView", + "UserPreferencesNativeEmailClient", + "UserPreferencesHideBrowseProductRedirectConfirmation", + "UserPreferencesHideOnlineSalesAppWelcomeMat", + "ContactId", + "AccountId", + "CallCenterId", + "Extension", + "FederationIdentifier", + "AboutMe", + "FullPhotoUrl", + "SmallPhotoUrl", + "IsExtIndicatorVisible", + "OutOfOfficeMessage", + "MediumPhotoUrl", + "DigestFrequency", + "DefaultGroupNotificationFrequency", + "LastViewedDate", + "LastReferencedDate", + "BannerPhotoUrl", + "SmallBannerPhotoUrl", + "MediumBannerPhotoUrl", + "IsProfilePhotoActive", + "IndividualId", + "Last_Mobile_Connect_vod__c", + "Last_Tablet_Connect_vod__c", + "Last_Mobile_Connect_Version_vod__c", + "Last_Tablet_Connect_Version_vod__c", + "Last_Mobile_Sync_vod__c", + "Last_Tablet_Sync_vod__c", + "RaiseLoggingLevel_vod__c", + "SendDetailedLog_vod__c", + "Last_Blackberry_Connect_vod__c", + "Last_Blackberry_Connect_Version_vod__c", + "Last_Blackberry_Sync_vod__c", + "Force_Full_Refresh_vod__c", + "Override_SystemModstamp_Timestamp_vod__c", + "Facetime_Email_vod__c", + "Facetime_Phone_vod__c", + "Product_Expertise_vod__c", + "Available_vod__c", + "Available_Last_Update_vod__c", + "Last_iPad_Connect_Version_vod__c", + "Last_iPad_Connect_vod__c", + "Last_iPad_Sync_vod__c", + "Inventory_Order_Allocation_Group_vod__c", + "Concur_User_Id_vod__c", + "Last_iPad_iOS_Version_vod__c", + "Approved_Email_Admin_vod__c", + "Last_WinModern_Connect_Version_vod__c", + "Last_WinModern_Connect_vod__c", + "Last_WinModern_Sync_vod__c", + "Primary_Territory_vod__c", + "Analytics_Admin_vod__c", + "Content_Admin_vod__c", + "Last_WinModern_Windows_Version_vod__c", + "Upload_VTrans_vod__c", + "Sync_Frequency_vjh__c", + "Clear_Client_Sync_Errors_vod__c", + "Remote_Meeting_Host_Id_vod__c", + "Remote_Meeting_Host_Token_vod__c", + "Last_iPhone_Connect_Version_vod__c", + "Last_iPhone_Connect_vod__c", + "Last_iPhone_Sync_vod__c", + "Last_iPhone_iOS_Version_vod__c", + "Remote_Meeting_Start_From_CRM_Online_vod__c", + "Country_Code_vod__c", + "User_Type_vod__c", + "Engage_Group_Provisioning_Status_vod__c", + "Engage_Group_Request_vod__c", + "Engage_Group_vod__c", + "Last_CRMDesktop_Mac_Sync_vod__c", + "Last_CRMDesktop_Mac_Version_vod__c", + "Last_CRMDesktop_Windows_Sync_vod__c", + "Last_CRMDesktop_Windows_Version_vod__c", + "MSJ_Test_User__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "UserTerritory2Association", + "columns": [ + "Id", + "UserId", + "Territory2Id", + "IsActive", + "RoleInTerritory2", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Remote_Meeting_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Meeting_Id_vod__c", + "Meeting_Name_vod__c", + "Mobile_ID_vod__c", + "Scheduled_DateTime_vod__c", + "Scheduled_vod__c", + "Attendance_Report_Process_Status_vod__c", + "Latest_Meeting_Start_Datetime_vod__c", + "Meeting_Password_vod__c", + "Meeting_Outcome_Status_vod__c", + "Allow_for_Joining_via_Zoom_vod__c", + "Zoom_Join_Token_vod__c", + "VExternal_Id_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "RecordType", + "columns": [ + "Id", + "Name", + "DeveloperName", + "NamespacePrefix", + "Description", + "BusinessProcessId", + "SobjectType", + "IsActive", + "IsPersonType", + "CreatedById", + "CreatedDate", + "LastModifiedById", + "LastModifiedDate", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "UserRole", + "columns": [ + "Id", + "Name", + "ParentRoleId", + "RollupDescription", + "OpportunityAccessForAccountOwner", + "CaseAccessForAccountOwner", + "ContactAccessForAccountOwner", + "ForecastUserId", + "MayForecastManagerShare", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "PortalAccountId", + "PortalType", + "PortalAccountOwnerId" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Account", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "Name", + "LastName", + "FirstName", + "Salutation", + "RecordTypeId", + "Phone", + "Fax", + "Website", + "PhotoUrl", + "NumberOfEmployees", + "Ownership", + "OwnerId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "IsExcludedFromRealign", + "PersonContactId", + "IsPersonAccount", + "PersonMailingStreet", + "PersonMailingCity", + "PersonMailingState", + "PersonMailingPostalCode", + "PersonMailingCountry", + "PersonMailingLatitude", + "PersonMailingLongitude", + "PersonMailingGeocodeAccuracy", + "PersonMailingAddress", + "PersonOtherStreet", + "PersonOtherCity", + "PersonOtherState", + "PersonOtherPostalCode", + "PersonOtherCountry", + "PersonOtherLatitude", + "PersonOtherLongitude", + "PersonOtherGeocodeAccuracy", + "PersonOtherAddress", + "PersonMobilePhone", + "PersonHomePhone", + "PersonOtherPhone", + "PersonAssistantPhone", + "PersonEmail", + "PersonTitle", + "PersonDepartment", + "PersonAssistantName", + "PersonBirthdate", + "PersonHasOptedOutOfEmail", + "PersonHasOptedOutOfFax", + "PersonDoNotCall", + "PersonLastCURequestDate", + "PersonLastCUUpdateDate", + "PersonEmailBouncedReason", + "PersonEmailBouncedDate", + "PersonIndividualId", + "Jigsaw", + "JigsawCompanyId", + "AccountSource", + "SicDesc", + "External_ID_vod__c", + "Credentials_vod__c", + "Territory_vod__c", + "Exclude_from_Zip_to_Terr_Processing_vod__c", + "Group_Specialty_1_vod__c", + "Group_Specialty_2_vod__c", + "Specialty_1_vod__c", + "Specialty_2_vod__c", + "Formatted_Name_vod__c", + "Territory_Test_vod__c", + "Mobile_ID_vod__c", + "Gender_vod__c", + "ID_vod__c", + "Do_Not_Sync_Sales_Data_vod__c", + "ID2_vod__c", + "Preferred_Name_vod__c", + "Sample_Default_vod__c", + "Segmentations_vod__c", + "Restricted_Products_vod__c", + "Payer_Id_vod__c", + "Alternate_Name_vod__c", + "Do_Not_Call_vod__c", + "MSJ_Beds__c", + "Spend_Amount__c", + "PDRP_Opt_Out_vod__c", + "Spend_Status_Value_vod__c", + "PDRP_Opt_Out_Date_vod__c", + "Spend_Status_vod__c", + "Enable_Restricted_Products_vod__c", + "Call_Reminder_vod__c", + "Account_Group_vod__c", + "Primary_Parent_vod__c", + "Color_vod__c", + "Middle_vod__c", + "Suffix_vod__c", + "MSJ_Type__c", + "No_Orders_vod__c", + "MSJ_BU_ONC__c", + "MSJ_BU_FE__c", + "Account_Search_FirstLast_vod__c", + "Account_Search_LastFirst_vod__c", + "MSJ_Operation__c", + "Practice_at_Hospital_vod__c", + "Practice_Near_Hospital_vod__c", + "Do_Not_Create_Child_Account_vod__c", + "Total_MDs_DOs__c", + "AHA__c", + "Order_Type_vod__c", + "NPI_vod__c", + "ME__c", + "Speaker__c", + "Investigator_vod__c", + "Default_Order_Type_vod__c", + "Tax_Status__c", + "Model__c", + "Offerings__c", + "Departments__c", + "Account_Type__c", + "MSJ_ONC_Tier__c", + "Account_Search_Business_vod__c", + "Business_Professional_Person_vod__c", + "Hospital_Type_vod__c", + "Account_Class_vod__c", + "Furigana_vod__c", + "MSJ_JISART__c", + "Total_Revenue_000__c", + "Net_Income_Loss_000__c", + "PMPM_Income_Loss_000__c", + "Commercial_Premiums_PMPM__c", + "Medical_Loss_Ratio__c", + "Medical_Expenses_PMPM__c", + "Commercial_Patient_Days_1000__c", + "HMO_Market_Shr__c", + "HMO__c", + "HMO_POS__c", + "PPO__c", + "PPO_POS__c", + "Medicare__c", + "Medicaid__c", + "MSJ_HP_Name_E__c", + "MSJ_Department__c", + "MSJ_Date_Of_Birth__c", + "MSJ_FE_GF_Potential__c", + "MSJ_FE_SZ_Potential__c", + "MSJ_EB_CRC_Ladder__c", + "MSJ_EB_CRC_Segment__c", + "MSJ_EB_HN_Segment__c", + "Business_Description__c", + "Regional_Strategy__c", + "Contracts_Process__c", + "MSJ_GF_segment__c", + "MSJ_DCF_DR_Code__c", + "MSJ_SZ_Segment__c", + "MSJ_Remark__c", + "MSJ_Title__c", + "MSJ_Role__c", + "MSJ_Kana__c", + "MSJ_Specialism__c", + "MSJ_Graduated_from__c", + "MSJ_Year_Graduation__c", + "Target__c", + "KOL_vod__c", + "MSJ_EPPV_Code__c", + "MSJ_DCF_HP_Code__c", + "Total_Lives__c", + "Total_Physicians_Enrolled__c", + "MSJ_Delete__c", + "MSJ_KOL_LOL__c", + "MSJ_ONC_Status__c", + "Account_Identifier_vod__c", + "Approved_Email_Opt_Type_vod__c", + "Language_vod__c", + "MSJ_KRAS_Routine_Date__c", + "MSJ_KRAS_Routine__c", + "MSJ_DRP_Target__c", + "MSJ_Fertility_Evaluation_Score__c", + "MSJ_Fertility_Tracking_Last_Modify_Date__c", + "Total_Pharmacists__c", + "MSJ_Number_of_Gonadotropin__c", + "MSJ_Number_of_IUI_cycle__c", + "MSJ_Number_of_OI_monthly_cycle__c", + "MSJ_OI_Protocol_learning_level__c", + "MSJ_H_N_Tier__c", + "MSJ_XLK_Segment__c", + "MSJ_XLK_Tier__c", + "Career_Status_vod__c", + "Photo_vod__c", + "MSJ_EB_H_N_LA_Segment__c", + "MSJ_EB_H_N_RM_Segment__c", + "MSJ_FE_CE_Potential__c", + "MSJ_FE_1C_potential__c", + "MSJ_FE_OV_potential__c", + "MSJ_FE_Tech_potential__c", + "MSJ_CE_segment__c", + "MSJ_1C_segment__c", + "MSJ_OV_segment__c", + "MSJ_Tech_segment__c", + "MSJ_Target_Call_Num__c", + "MSJ_DR_Change_Log__c", + "MSJ_Global_scientific_exposure__c", + "MSJ_H_index__c", + "MSJ_Num_of_Article_3Y__c", + "MSJ_Num_of_Article__c", + "MSJ_Num_of_Article_as_1st_Author_3Y__c", + "MSJ_Num_of_article_growth_rate_3Y__c", + "MSJ_Num_of_cited_3Y__c", + "MSJ_Num_of_impact_factor_3Y__c", + "MSJ_impact_factor_as_1st_Author_3Y__c", + "EMDS_Has_Pipeline_Opportunity__c", + "EMDS_Pipeline_Count__c", + "MSJ_BVC_Segment__c", + "MSJ_BVC_Tier__c", + "MSJ_BVC_AcctOpen__c", + "MSJ_BVC_MCC_Patients__c", + "MSJ_ONC_HP_Segment__c", + "MSJ_AE_Department__c", + "MSJ_AE_Facility__c", + "MSJ_AE_Name__c", + "MSJ_AE_Title__c", + "MSJ_Email__c", + "MSJ_FE_GF2_Potential__c", + "MSJ_FE_Location_potential__c", + "MSJ_GF2_segment__c", + "MSJ_OPTIN_target__c", + "MSJ_Merck_Specialty1__c", + "MSJ_Merck_Specialty2__c", + "MSJ_Marketing_Cloud_Integration__c", + "MSJ_Marketing_Cloud1__c", + "MSJ_Marketing_Cloud2__c", + "MSJ_Marketing_Cloud3__c", + "MSJ_Marketing_Cloud4__c", + "MSJ_Medical_Department__c", + "MSJ_Marketing_Cloud0__c", + "Mobile_ID_vod__pc", + "H1Insights__H1_NPI_Value_for_Testing__pc", + "H1Insights__H1_Person_ID__pc", + "H1Insights__H1_Request_Status__pc", + "H1Insights__H1_URL__pc", + "H1Insights__NPI_Number__pc", + "H1Insights__NPI_Number_for_H1_Insights__pc", + "MSJ_Marketing_Cloud_Integration__pc" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Type_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Header_vod__c", + "Channel_Label_vod__c", + "Channel_Source_vod__c", + "Consent_Expires_In_vod__c", + "Default_Consent_Type_vod__c", + "Disclaimer_Text_vod__c", + "Display_Order_vod__c", + "Product_Preference_vod__c", + "zvod_Consent_Default_Consent_Text_vod__c", + "zvod_Consent_Line_vod__c", + "zvod_Signature_Capture_vod__c", + "Double_Opt_In_vod__c", + "zvod_Consent_Activity_Tracking_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Header_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Header_Help_Text_vod__c", + "Country_vod__c", + "Inactive_Datetime_vod__c", + "Language_vod__c", + "Status_vod__c", + "Signature_Required_On_Opt_Out_vod__c", + "Request_Receipt_vod__c", + "Subscription_Option_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Line_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Type_vod__c", + "Detail_Group_Display_Name_vod__c", + "Detail_Group_vod__c", + "Display_Order_vod__c", + "End_Date_vod__c", + "Group_By_vod__c", + "Product_Display_Name_vod__c", + "Product_vod__c", + "Start_Date_vod__c", + "Sub_Channel_Description_vod__c", + "Sub_Channel_Display_Name_vod__c", + "Sub_Channel_Key_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Inquiry_Assignment__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Medical_Inquiry__c", + "MSJ_Close__c", + "MSJ_Doctor_Name__c", + "MSJ_Hospital_Name__c", + "MSJ_Indication__c", + "MSJ_Inquiry_Text__c", + "MSJ_MEC_User__c", + "MSJ_MSL_Manager__c", + "MSJ_MSL_User__c", + "MSJ_Notice_to_MR__c", + "MSJ_Product_for_MEC__c", + "MSJ_Product_for_MR__c", + "MSJ_Reply_Date__c", + "MSJ_Reply__c", + "MSJ_AE_Infomation__c", + "MSJ_Cancel__c", + "MSJ_FAQ_number_c__c", + "MSJ_Return_Call__c", + "MSJ_Inquiry_Origin__c", + "First_Response__c", + "Inquiry_Created_Date__c", + "Inquiry_Type_1__c", + "Inquiry_Type_2__c", + "MSJ_First_User__c", + "MSJ_MEC_Comment__c", + "MSJ_Send_Email__c", + "MSJ_Temp_Aggregated_Info__c", + "MSJ_AE_Report__c", + "MSJ_Background__c", + "MSJ_Inquiry_Date__c", + "MSJ_MSL_Support__c", + "MSJ_Handover_Comment__c", + "MSJ_Handover_Email__c", + "MSJ_Material_Requirement__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Approved_Document_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Detail_Group_vod__c", + "Document_Description_vod__c", + "Document_Host_URL_vod__c", + "Document_ID_vod__c", + "Document_Last_Mod_DateTime_vod__c", + "Email_Allows_Documents_vod__c", + "Email_Domain_vod__c", + "Email_Fragment_HTML_vod__c", + "Email_From_Address_vod__c", + "Email_From_Name_vod__c", + "Email_HTML_1_vod__c", + "Email_HTML_2_vod__c", + "Email_ReplyTo_Address_vod__c", + "Email_ReplyTo_Name_vod__c", + "Email_Subject_vod__c", + "Email_Template_Fragment_Document_ID_vod__c", + "Email_Template_Fragment_HTML_vod__c", + "ISI_Document_ID_vod__c", + "Language_vod__c", + "Other_Document_ID_List_vod__c", + "PI_Document_ID_vod__c", + "Piece_Document_ID_vod__c", + "Product_vod__c", + "Status_vod__c", + "Territory_vod__c", + "Vault_Instance_ID_vod__c", + "Allow_Any_Product_Fragment_vod__c", + "Allowed_Document_IDs_vod__c", + "Engage_Document_Id_vod__c", + "Vault_Document_ID_vod__c", + "Key_Message_vod__c", + "Events_Management_Subtype_vod__c", + "Survey_vod__c", + "Content_Type_vod__c", + "Bcc_vod__c", + "Audience_vod__c", + "WeChat_Template_ID_vod__c", + "Check_Consent_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Child_Account_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Parent_Account_vod__c", + "Child_Account_vod__c", + "External_ID_vod__c", + "Mobile_ID_vod__c", + "Primary_vod__c", + "Copy_Address_vod__c", + "Child_Name_vod__c", + "Parent_Name_vod__c", + "Parent_Child_Name_vod__c", + "Account_Code__c", + "Child_Department__c", + "Child_Role__c", + "Child_Title__c", + "Child_Remark__c", + "MSJ_1C_segment__c", + "MSJ_BU_FE__c", + "MSJ_BU_ONC__c", + "MSJ_BVC_Segment__c", + "MSJ_CE_segment__c", + "MSJ_Child_Account_Link__c", + "MSJ_DCF_DR_Code__c", + "MSJ_DCF_HP_Code__c", + "MSJ_DR_Change_Log__c", + "MSJ_Delete__c", + "MSJ_Department__c", + "MSJ_EB_CRC_Segment__c", + "MSJ_EB_HN_Segment__c", + "MSJ_EB_H_N_LA_Segment__c", + "MSJ_EB_H_N_RM_Segment__c", + "MSJ_External_ID__c", + "MSJ_Fax__c", + "MSJ_GF2_segment__c", + "MSJ_GF_segment__c", + "MSJ_KOL_LOL__c", + "MSJ_KOL__c", + "MSJ_ONC_HP_Segment__c", + "MSJ_OPTIN_target__c", + "MSJ_OV_segment__c", + "MSJ_Parent_Child_Name__c", + "MSJ_Phone__c", + "MSJ_Remark__c", + "MSJ_Target_Call_Num__c", + "MSJ_Tech_segment__c", + "MSJ_Title__c", + "MSJ_XLK_Segment__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Hospital_Medical_Regimen__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Account_Name__c", + "MSJ_Delete_Date__c", + "MSJ_Delete_Flag__c", + "MSJ_Indication__c", + "MSJ_Line__c", + "MSJ_Medical_Regimen__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Medical_Regimen__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Delete_Date__c", + "MSJ_Delete_Flag__c", + "MSJ_Indication__c", + "MSJ_Remark__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Patient__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Account_Name__c", + "MSJ_CRC_Group__c", + "MSJ_Casus_or_Transfer_Point__c", + "MSJ_Entry_Date__c", + "MSJ_IST_Name__c", + "MSJ_Indication__c", + "MSJ_Line__c", + "MSJ_MR_Comments__c", + "MSJ_MUID__c", + "MSJ_Medical_Regimen__c", + "MSJ_Month__c", + "MSJ_Report_Comments__c", + "MSJ_Start_Date_Of_Administration__c", + "MSJ_Year__c", + "Mobile_ID_vod__c", + "MSJ_CRC_RAS_KRAS__c", + "MSJ_End_Date_Of_Administration__c", + "MSJ_End_Date_of_Stop_Administration__c", + "MSJ_HN_Hospitalized_Type__c", + "MSJ_Start_Date_of_Stop_Administration__c", + "MSJ_Patient_Status__c", + "MSJ_Patient_TA__c", + "MSJ_Child_Account_Name__c", + "MSJ_Child_Account__c", + "MSJ_Parent_Account_Name__c", + "MSJ_Parent_Child_Name__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consumer_site__c", + "Product_info__c", + "Therapeutic_Class_vod__c", + "Parent_Product_vod__c", + "Therapeutic_Area_vod__c", + "Product_Type_vod__c", + "Require_Key_Message_vod__c", + "Cost_vod__c", + "External_ID_vod__c", + "Manufacturer_vod__c", + "Company_Product_vod__c", + "Controlled_Substance_vod__c", + "Description_vod__c", + "Sample_Quantity_Picklist_vod__c", + "Display_Order_vod__c", + "No_Metrics_vod__c", + "Distributor_vod__c", + "Sample_Quantity_Bound_vod__c", + "Sample_U_M_vod__c", + "No_Details_vod__c", + "Quantity_Per_Case_vod__c", + "Schedule_vod__c", + "Restricted_vod__c", + "Pricing_Rule_Quantity_Bound_vod__c", + "No_Promo_Items_vod__c", + "User_Aligned_vod__c", + "Restricted_States_vod__c", + "Sort_Code_vod__c", + "No_Cycle_Plans_vod__c", + "Inventory_Order_UOM_vod__c", + "Inventory_Quantity_Per_Case_vod__c", + "VExternal_Id_vod__c", + "Country__c", + "MSJ_Product_Classification__c", + "MSJ_Indication__c", + "MSJ_Therapeutic_Area__c", + "MSJ_Global_Brand__c", + "MSJ_Global_Business_Unit__c", + "MSJ_Molecules__c", + "MSJ_SBU__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_Group_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Description_vod__c", + "Product_vod__c", + "Product_Catalog_vod__c", + "Start_Date_vod__c", + "End_Date_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_Metrics_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Awareness__c", + "Selling_Stage__c", + "Formulary_Status__c", + "Movement__c", + "Products_vod__c", + "Segment__c", + "X12_mo_trx_chg__c", + "Speaker_Skills__c", + "Investigator_Readiness__c", + "Engagements__c", + "Mobile_ID_vod__c", + "External_ID_vod__c", + "MSJ_Patient__c", + "Detail_Group_vod__c", + "MSJ_EB_1st_Line_Liver_Meta__c", + "MSJ_EB_1st_Line_Multi_Meta__c", + "MSJ_EB_2nd_Line_Mono__c", + "MSJ_EB_2nd_Line_Combination__c", + "MSJ_EB_3rd_Line_Mono__c", + "MSJ_EB_3rd_Line_Combination__c", + "EMDS_Ability__c", + "EMDS_Brand_Loyalty__c", + "EMDS_Decision_Maker__c", + "EMDS_Early_Tech_Adopter__c", + "EMDS_Influence__c", + "EMDS_Main_Driver__c", + "EMDS_Priority__c", + "EMDS_Willingness__c", + "MSJ_KTL_Type__c", + "MSJ_KTL_Tier__c", + "MSJ_Publications__c", + "MSJ_Clinical_Trials__c", + "MSJ_Speaker_for_Medical_Events__c", + "MSJ_Advisor_to_Medical_Affairs__c", + "MSJ_Guidelines_Treatment_Standards__c", + "MSJ_Therapeutic_Area_Expertise__c", + "MSJ_MAP_GAP__c", + "MSJ_Associations__c", + "MSJ_Tier_Score__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Assignment_Type_vod__c", + "Channels_vod__c", + "End_Date_vod__c", + "Expired_vod__c", + "External_ID_vod__c", + "Language_vod__c", + "Lock_vod__c", + "Open_vod__c", + "Product_vod__c", + "Region_vod__c", + "Segment_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "Territory_vod__c", + "zvod_Questions_vod__c", + "zvod_Segments_vod__c", + "zvod_Targets_vod__c", + "Max_Score_vod__c", + "Min_Score_vod__c", + "Autotarget_vod__c", + "Territories_vod__c", + "Target_Type_vod__c", + "MSJ_External_ID__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_Target_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_Display_Name_vod__c", + "Account_vod__c", + "Channels_vod__c", + "End_Date_vod__c", + "Entity_Reference_Id_vod__c", + "External_ID_vod__c", + "Language_vod__c", + "Lock_vod__c", + "Mobile_ID_vod__c", + "No_Autoassign_vod__c", + "Not_Completed_vod__c", + "Region_vod__c", + "Segment_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "Survey_vod__c", + "Territory_vod__c", + "zvod_Address_vod__c", + "zvod_Specialty_vod__c", + "Score_vod__c", + "User_vod__c", + "Child_Account_vod__c", + "Location_Entity_Reference_Id_vod__c", + "Location_vod__c", + "Target_Type_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_Question_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Survey_vod__c", + "Answer_Choice_vod__c", + "External_ID_vod__c", + "Max_Score_vod__c", + "Min_Score_vod__c", + "Order_vod__c", + "Question_vod__c", + "Required_vod__c", + "Text_vod__c", + "Condition_vod__c", + "Source_ID_vod__c", + "MSJ_External_ID__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Question_Response_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Survey_Target_vod__c", + "Answer_Choice_vod__c", + "Date_vod__c", + "Datetime_vod__c", + "External_ID_vod__c", + "Mobile_ID_vod__c", + "Number_vod__c", + "Order_vod__c", + "Question_Text_vod__c", + "Required_vod__c", + "Response_Hash_vod__c", + "Response_vod__c", + "Score_vod__c", + "Survey_Question_vod__c", + "Text_vod__c", + "Type_vod__c", + "Condition_vod__c", + "Inactive_Condition_vod__c", + "Source_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Sent_Fragment_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Sent_Email_vod__c", + "Account_vod__c", + "Email_Template_vod__c", + "Sent_Fragment_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Sent_Email_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Account_Email_vod__c", + "Account_vod__c", + "Approved_Email_Template_vod__c", + "Capture_Datetime_vod__c", + "Detail_Group_vod__c", + "Email_Config_Values_vod__c", + "Email_Content2_vod__c", + "Email_Content_vod__c", + "Email_Fragments_vod__c", + "Email_Sent_Date_vod__c", + "Failure_Msg_vod__c", + "Last_Activity_Date_vod__c", + "Last_Device_vod__c", + "MC_Capture_Datetime_vod__c", + "Mobile_ID_vod__c", + "Opened_vod__c", + "Product_Display_vod__c", + "Product_vod__c", + "Sender_Email_vod__c", + "Status_vod__c", + "Valid_Consent_Exists_vod__c", + "Approved_Document_Views_vod__c", + "Click_Count_vod__c", + "Last_Click_Date_vod__c", + "Last_Open_Date_vod__c", + "Open_Count_vod__c", + "Receipt_Entity_Type_vod__c", + "Receipt_Record_Id_vod__c", + "Territory_vod__c", + "Call2_vod__c", + "Medical_Inquiry_vod__c", + "Parent_Email_vod__c", + "Related_Transaction_ID_vod__c", + "Case_vod__c", + "Key_Message_vod__c", + "Suggestion_vod__c", + "EM_Attendee_vod__c", + "EM_Event_Speaker_vod__c", + "EM_Event_Team_Member_vod__c", + "Event_Attendee_vod__c", + "Event_vod__c", + "Medical_Event_vod__c", + "Scheduled_Send_Datetime_vod__c", + "User_vod__c", + "Content_Type_vod__c", + "Bcc_vod__c", + "Event_Attendee_Mobile_Id_vod__c", + "Event_Mobile_Id_vod__c", + "Activity_Tracking_Mode_vod__c", + "Email_Source_vod__c", + "Subject_vod__c", + "User_Input_Text_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + } ] - } - - \ No newline at end of file +} \ No newline at end of file From 49a589478ac61b318035d7628cf48683bdef2e49 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Thu, 28 Jul 2022 11:55:36 +0900 Subject: [PATCH 044/275] =?UTF-8?q?feat:CRM=E3=81=AE=E6=8B=A1=E5=BC=B5SQL?= =?UTF-8?q?=E3=80=81=E8=A8=AD=E5=AE=9A=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/crm/settings/CRM_Account.txt | 13 +++++ s3/data/crm/settings/CRM_AccountShare.txt | 13 +++++ s3/data/crm/settings/CRM_AccountShare_ex.sql | 1 + .../CRM_Account_Territory_Loader_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Account_ex.sql | 1 + .../settings/CRM_Approved_Document_vod__c.txt | 13 +++++ .../crm/settings/CRM_Call2_Detail_vod__c.txt | 13 +++++ .../settings/CRM_Call2_Discussion_vod__c.txt | 13 +++++ .../settings/CRM_Call2_Key_Message_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Call2_vod__c.txt | 13 +++++ .../settings/CRM_Call_Clickstream_vod__c.txt | 13 +++++ .../crm/settings/CRM_Child_Account_vod__c.txt | 13 +++++ .../settings/CRM_Child_Account_vod__c_ex.sql | 1 + .../CRM_Clm_Presentation_Slide_vod__c.txt | 13 +++++ .../settings/CRM_Clm_Presentation_vod__c.txt | 13 +++++ .../settings/CRM_Coaching_Report_vod__c.txt | 13 +++++ .../settings/CRM_Consent_Header_vod__c.txt | 13 +++++ .../crm/settings/CRM_Consent_Line_vod__c.txt | 13 +++++ .../crm/settings/CRM_Consent_Type_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Contact.txt | 13 +++++ s3/data/crm/settings/CRM_Contact_ex.sql | 1 + ...Dynamic_Attribute_Configuration_vod__c.txt | 13 +++++ .../settings/CRM_Dynamic_Attribute_vod__c.txt | 13 +++++ .../settings/CRM_Email_Activity_vod__c.txt | 13 +++++ .../settings/CRM_Event_Attendee_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Group.txt | 13 +++++ s3/data/crm/settings/CRM_Group_ex.sql | 1 + .../crm/settings/CRM_Key_Message_vod__c.txt | 13 +++++ .../CRM_MSJ_Hospital_Medical_Regimen__c.txt | 13 +++++ .../CRM_MSJ_Inquiry_Assignment__c.txt | 13 +++++ .../settings/CRM_MSJ_MR_Weekly_Report__c.txt | 13 +++++ .../CRM_MSJ_Medical_Event_Evaluation__c.txt | 13 +++++ .../settings/CRM_MSJ_Medical_Regimen__c.txt | 13 +++++ s3/data/crm/settings/CRM_MSJ_Patient__c.txt | 13 +++++ .../crm/settings/CRM_Medical_Event_vod__c.txt | 13 +++++ .../settings/CRM_Medical_Inquiry_vod__c.txt | 13 +++++ .../settings/CRM_Medical_Insight_vod__c.txt | 13 +++++ .../CRM_Multichannel_Activity_Line_vod__c.txt | 13 +++++ .../CRM_Multichannel_Activity_vod__c.txt | 13 +++++ .../CRM_Multichannel_Consent_vod__c.txt | 13 +++++ .../settings/CRM_My_Setup_Products_vod__c.txt | 13 +++++ .../CRM_ObjectTerritory2Association.txt | 13 +++++ .../CRM_ObjectTerritory2Association_ex.sql | 1 + .../crm/settings/CRM_Product_Group_vod__c.txt | 13 +++++ .../settings/CRM_Product_Metrics_vod__c.txt | 13 +++++ .../CRM_Product_Metrics_vod__c_ex.sql | 1 + s3/data/crm/settings/CRM_Product_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Profile.txt | 13 +++++ s3/data/crm/settings/CRM_Profile_ex.sql | 1 + .../settings/CRM_Question_Response_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_RecordType.txt | 13 +++++ .../settings/CRM_Remote_Meeting_vod__c.txt | 13 +++++ .../crm/settings/CRM_Sent_Email_vod__c.txt | 13 +++++ .../crm/settings/CRM_Sent_Fragment_vod__c.txt | 13 +++++ .../settings/CRM_Survey_Question_vod__c.txt | 13 +++++ .../crm/settings/CRM_Survey_Target_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Survey_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Territory2.txt | 13 +++++ s3/data/crm/settings/CRM_Territory2_ALL.txt | 14 +++++ s3/data/crm/settings/CRM_Territory2_ex.sql | 2 + .../CRM_Time_Off_Territory_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_User.txt | 13 +++++ s3/data/crm/settings/CRM_UserRole.txt | 13 +++++ .../CRM_UserTerritory2Association.txt | 13 +++++ .../CRM_UserTerritory2Association_ALL.txt | 14 +++++ .../CRM_UserTerritory2Association_ex.sql | 2 + s3/data/crm/settings/configmap.config | 58 +++++++++++++++++++ 67 files changed, 800 insertions(+) create mode 100644 s3/data/crm/settings/CRM_Account.txt create mode 100644 s3/data/crm/settings/CRM_AccountShare.txt create mode 100644 s3/data/crm/settings/CRM_AccountShare_ex.sql create mode 100644 s3/data/crm/settings/CRM_Account_Territory_Loader_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Account_ex.sql create mode 100644 s3/data/crm/settings/CRM_Approved_Document_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call2_Detail_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call2_Discussion_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call2_Key_Message_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call2_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call_Clickstream_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Child_Account_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql create mode 100644 s3/data/crm/settings/CRM_Clm_Presentation_Slide_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Clm_Presentation_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Coaching_Report_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Consent_Header_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Consent_Line_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Consent_Type_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Contact.txt create mode 100644 s3/data/crm/settings/CRM_Contact_ex.sql create mode 100644 s3/data/crm/settings/CRM_Dynamic_Attribute_Configuration_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Dynamic_Attribute_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Email_Activity_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Group.txt create mode 100644 s3/data/crm/settings/CRM_Group_ex.sql create mode 100644 s3/data/crm/settings/CRM_Key_Message_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Hospital_Medical_Regimen__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Inquiry_Assignment__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_MR_Weekly_Report__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Medical_Event_Evaluation__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Medical_Regimen__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Patient__c.txt create mode 100644 s3/data/crm/settings/CRM_Medical_Event_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Medical_Inquiry_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Medical_Insight_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Multichannel_Activity_Line_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Multichannel_Activity_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Multichannel_Consent_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_My_Setup_Products_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_ObjectTerritory2Association.txt create mode 100644 s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql create mode 100644 s3/data/crm/settings/CRM_Product_Group_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Product_Metrics_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql create mode 100644 s3/data/crm/settings/CRM_Product_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Profile.txt create mode 100644 s3/data/crm/settings/CRM_Profile_ex.sql create mode 100644 s3/data/crm/settings/CRM_Question_Response_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_RecordType.txt create mode 100644 s3/data/crm/settings/CRM_Remote_Meeting_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Sent_Email_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Sent_Fragment_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Survey_Question_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Survey_Target_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Survey_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Territory2.txt create mode 100644 s3/data/crm/settings/CRM_Territory2_ALL.txt create mode 100644 s3/data/crm/settings/CRM_Territory2_ex.sql create mode 100644 s3/data/crm/settings/CRM_Time_Off_Territory_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_User.txt create mode 100644 s3/data/crm/settings/CRM_UserRole.txt create mode 100644 s3/data/crm/settings/CRM_UserTerritory2Association.txt create mode 100644 s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt create mode 100644 s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql create mode 100644 s3/data/crm/settings/configmap.config diff --git a/s3/data/crm/settings/CRM_Account.txt b/s3/data/crm/settings/CRM_Account.txt new file mode 100644 index 00000000..bee3992c --- /dev/null +++ b/s3/data/crm/settings/CRM_Account.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +248 +Id,IsDeleted,MasterRecordId,Name,LastName,FirstName,Salutation,RecordTypeId,Phone,Fax,Website,PhotoUrl,NumberOfEmployees,Ownership,OwnerId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,IsExcludedFromRealign,PersonContactId,IsPersonAccount,PersonMailingStreet,PersonMailingCity,PersonMailingState,PersonMailingPostalCode,PersonMailingCountry,PersonMailingLatitude,PersonMailingLongitude,PersonMailingGeocodeAccuracy,PersonMailingAddress,PersonOtherStreet,PersonOtherCity,PersonOtherState,PersonOtherPostalCode,PersonOtherCountry,PersonOtherLatitude,PersonOtherLongitude,PersonOtherGeocodeAccuracy,PersonOtherAddress,PersonMobilePhone,PersonHomePhone,PersonOtherPhone,PersonAssistantPhone,PersonEmail,PersonTitle,PersonDepartment,PersonAssistantName,PersonBirthdate,PersonHasOptedOutOfEmail,PersonHasOptedOutOfFax,PersonDoNotCall,PersonLastCURequestDate,PersonLastCUUpdateDate,PersonEmailBouncedReason,PersonEmailBouncedDate,PersonIndividualId,Jigsaw,JigsawCompanyId,AccountSource,SicDesc,External_ID_vod__c,Credentials_vod__c,Territory_vod__c,Exclude_from_Zip_to_Terr_Processing_vod__c,Group_Specialty_1_vod__c,Group_Specialty_2_vod__c,Specialty_1_vod__c,Specialty_2_vod__c,Formatted_Name_vod__c,Territory_Test_vod__c,Mobile_ID_vod__c,Gender_vod__c,ID_vod__c,Do_Not_Sync_Sales_Data_vod__c,ID2_vod__c,Preferred_Name_vod__c,Sample_Default_vod__c,Segmentations_vod__c,Restricted_Products_vod__c,Payer_Id_vod__c,Alternate_Name_vod__c,Do_Not_Call_vod__c,MSJ_Beds__c,Spend_Amount__c,PDRP_Opt_Out_vod__c,Spend_Status_Value_vod__c,PDRP_Opt_Out_Date_vod__c,Spend_Status_vod__c,Enable_Restricted_Products_vod__c,Call_Reminder_vod__c,Account_Group_vod__c,Primary_Parent_vod__c,Color_vod__c,Middle_vod__c,Suffix_vod__c,MSJ_Type__c,No_Orders_vod__c,MSJ_BU_ONC__c,MSJ_BU_FE__c,Account_Search_FirstLast_vod__c,Account_Search_LastFirst_vod__c,MSJ_Operation__c,Practice_at_Hospital_vod__c,Practice_Near_Hospital_vod__c,Do_Not_Create_Child_Account_vod__c,Total_MDs_DOs__c,AHA__c,Order_Type_vod__c,NPI_vod__c,ME__c,Speaker__c,Investigator_vod__c,Default_Order_Type_vod__c,Tax_Status__c,Model__c,Offerings__c,Departments__c,Account_Type__c,MSJ_ONC_Tier__c,Account_Search_Business_vod__c,Business_Professional_Person_vod__c,Hospital_Type_vod__c,Account_Class_vod__c,Furigana_vod__c,MSJ_JISART__c,Total_Revenue_000__c,Net_Income_Loss_000__c,PMPM_Income_Loss_000__c,Commercial_Premiums_PMPM__c,Medical_Loss_Ratio__c,Medical_Expenses_PMPM__c,Commercial_Patient_Days_1000__c,HMO_Market_Shr__c,HMO__c,HMO_POS__c,PPO__c,PPO_POS__c,Medicare__c,Medicaid__c,MSJ_HP_Name_E__c,MSJ_Department__c,MSJ_Date_Of_Birth__c,MSJ_FE_GF_Potential__c,MSJ_FE_SZ_Potential__c,MSJ_EB_CRC_Ladder__c,MSJ_EB_CRC_Segment__c,MSJ_EB_HN_Segment__c,Business_Description__c,Regional_Strategy__c,Contracts_Process__c,MSJ_GF_segment__c,MSJ_DCF_DR_Code__c,MSJ_SZ_Segment__c,MSJ_Remark__c,MSJ_Title__c,MSJ_Role__c,MSJ_Kana__c,MSJ_Specialism__c,MSJ_Graduated_from__c,MSJ_Year_Graduation__c,Target__c,KOL_vod__c,MSJ_EPPV_Code__c,MSJ_DCF_HP_Code__c,Total_Lives__c,Total_Physicians_Enrolled__c,MSJ_Delete__c,MSJ_KOL_LOL__c,MSJ_ONC_Status__c,Account_Identifier_vod__c,Approved_Email_Opt_Type_vod__c,Language_vod__c,MSJ_KRAS_Routine_Date__c,MSJ_KRAS_Routine__c,MSJ_DRP_Target__c,MSJ_Fertility_Evaluation_Score__c,MSJ_Fertility_Tracking_Last_Modify_Date__c,Total_Pharmacists__c,MSJ_Number_of_Gonadotropin__c,MSJ_Number_of_IUI_cycle__c,MSJ_Number_of_OI_monthly_cycle__c,MSJ_OI_Protocol_learning_level__c,MSJ_H_N_Tier__c,MSJ_XLK_Segment__c,MSJ_XLK_Tier__c,Career_Status_vod__c,Photo_vod__c,MSJ_EB_H_N_LA_Segment__c,MSJ_EB_H_N_RM_Segment__c,MSJ_FE_CE_Potential__c,MSJ_FE_1C_potential__c,MSJ_FE_OV_potential__c,MSJ_FE_Tech_potential__c,MSJ_CE_segment__c,MSJ_1C_segment__c,MSJ_OV_segment__c,MSJ_Tech_segment__c,MSJ_Target_Call_Num__c,MSJ_DR_Change_Log__c,MSJ_Global_scientific_exposure__c,MSJ_H_index__c,MSJ_Num_of_Article_3Y__c,MSJ_Num_of_Article__c,MSJ_Num_of_Article_as_1st_Author_3Y__c,MSJ_Num_of_article_growth_rate_3Y__c,MSJ_Num_of_cited_3Y__c,MSJ_Num_of_impact_factor_3Y__c,MSJ_impact_factor_as_1st_Author_3Y__c,EMDS_Has_Pipeline_Opportunity__c,EMDS_Pipeline_Count__c,MSJ_BVC_Segment__c,MSJ_BVC_Tier__c,MSJ_BVC_AcctOpen__c,MSJ_BVC_MCC_Patients__c,MSJ_ONC_HP_Segment__c,MSJ_AE_Department__c,MSJ_AE_Facility__c,MSJ_AE_Name__c,MSJ_AE_Title__c,MSJ_Email__c,MSJ_FE_GF2_Potential__c,MSJ_FE_Location_potential__c,MSJ_GF2_segment__c,MSJ_OPTIN_target__c,MSJ_Merck_Specialty1__c,MSJ_Merck_Specialty2__c,MSJ_Marketing_Cloud_Integration__c,MSJ_Marketing_Cloud1__c,MSJ_Marketing_Cloud2__c,MSJ_Marketing_Cloud3__c,MSJ_Marketing_Cloud4__c,MSJ_Medical_Department__c,MSJ_Marketing_Cloud0__c,Mobile_ID_vod__pc,H1Insights__H1_NPI_Value_for_Testing__pc,H1Insights__H1_Person_ID__pc,H1Insights__H1_Request_Status__pc,H1Insights__H1_URL__pc,H1Insights__NPI_Number__pc,H1Insights__NPI_Number_for_H1_Insights__pc,MSJ_Marketing_Cloud_Integration__pc +id,is_deleted,master_record_id,name,last_name,first_name,salutation,record_type_id,phone,fax,website,photo_url,number_of_employees,ownership,owner_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,is_excluded_from_realign,person_contact_id,is_person_account,person_mailing_street,person_mailing_city,person_mailing_state,person_mailing_postal_code,person_mailing_country,person_mailing_latitude,person_mailing_longitude,person_mailing_geocode_accuracy,person_mailing_address,person_other_street,person_other_city,person_other_state,person_other_postal_code,person_other_country,person_other_latitude,person_other_longitude,person_other_geocode_accuracy,person_other_address,person_mobile_phone,person_home_phone,person_other_phone,person_assistant_phone,person_email,person_title,person_department,person_assistant_name,person_birthdate,person_has_opted_out_of_email,person_has_opted_out_of_fax,person_do_not_call,person_last_curequest_date,person_last_cuupdate_date,person_email_bounced_reason,person_email_bounced_date,person_individual_id,jigsaw,jigsaw_company_id,account_source,sic_desc,external_id_vod__c,credentials_vod__c,territory_vod__c,exclude_from_zip_to_terr_processing_vod__c,group_specialty_1_vod__c,group_specialty_2_vod__c,specialty_1_vod__c,specialty_2_vod__c,formatted_name_vod__c,territory_test_vod__c,mobile_id_vod__c,gender_vod__c,id_vod__c,do_not_sync_sales_data_vod__c,id2_vod__c,preferred_name_vod__c,sample_default_vod__c,segmentations_vod__c,restricted_products_vod__c,payer_id_vod__c,alternate_name_vod__c,do_not_call_vod__c,msj_beds__c,spend_amount__c,pdrp_opt_out_vod__c,spend_status_value_vod__c,pdrp_opt_out_date_vod__c,spend_status_vod__c,enable_restricted_products_vod__c,call_reminder_vod__c,account_group_vod__c,primary_parent_vod__c,color_vod__c,middle_vod__c,suffix_vod__c,msj_type__c,no_orders_vod__c,msj_bu_onc__c,msj_bu_fe__c,account_search_first_last_vod__c,account_search_last_first_vod__c,msj_operation__c,practice_at_hospital_vod__c,practice_near_hospital_vod__c,do_not_create_child_account_vod__c,total_mds_dos__c,aha__c,order_type_vod__c,npi_vod__c,me__c,speaker__c,investigator_vod__c,default_order_type_vod__c,tax_status__c,model__c,offerings__c,departments__c,account_type__c,msj_onc_tier__c,account_search_business_vod__c,business_professional_person_vod__c,hospital_type_vod__c,account_class_vod__c,furigana_vod__c,msj_jisart__c,total_revenue_000__c,net_income_loss_000__c,pmpm_income_loss_000__c,commercial_premiums_pmpm__c,medical_loss_ratio__c,medical_expenses_pmpm__c,commercial_patient_days_1000__c,hmo_market_shr__c,hmo__c,hmo_pos__c,ppo__c,ppo_pos__c,medicare__c,medicaid__c,msj_hp_name_e__c,msj_department__c,msj_date_of_birth__c,msj_fe_gf_potential__c,msj_fe_sz_potential__c,msj_eb_crc_ladder__c,msj_eb_crc_segment__c,msj_eb_hn_segment__c,business_description__c,regional_strategy__c,contracts_process__c,msj_gf_segment__c,msj_dcf_dr_code__c,msj_sz_segment__c,msj_remark__c,msj_title__c,msj_role__c,msj_kana__c,msj_specialism__c,msj_graduated_from__c,msj_year_graduation__c,target__c,kol_vod__c,msj_eppv_code__c,msj_dcf_hp_code__c,total_lives__c,total_physicians_enrolled__c,msj_delete__c,msj_kol_lol__c,msj_onc_status__c,account_identifier_vod__c,approved_email_opt_type_vod__c,language_vod__c,msj_kras_routine_date__c,msj_kras_routine__c,msj_drp_target__c,msj_fertility_evaluation_score__c,msj_fertility_tracking_last_modify_date__c,total_pharmacists__c,msj_number_of_gonadotropin__c,msj_number_of_iui_cycle__c,msj_number_of_oi_monthly_cycle__c,msj_oi_protocol_learning_level__c,msj_h_n_tier__c,msj_xlk_segment__c,msj_xlk_tier__c,career_status_vod__c,photo_vod__c,msj_eb_h_n_la_segment__c,msj_eb_h_n_rm_segment__c,msj_fe_ce_potential__c,msj_fe_1_c_potential__c,msj_fe_ov_potential__c,msj_fe_tech_potential__c,msj_ce_segment__c,msj_1_c_segment__c,msj_ov_segment__c,msj_tech_segment__c,msj_target_call_num__c,msj_dr_change_log__c,msj_global_scientific_exposure__c,msj_h_index__c,msj_num_of_article_3_y__c,msj_num_of_article__c,msj_num_of_article_as_1st_author_3_y__c,msj_num_of_article_growth_rate_3_y__c,msj_num_of_cited_3_y__c,msj_num_of_impact_factor_3_y__c,msj_impact_factor_as_1st_author_3_y__c,emds_has_pipeline_opportunity__c,emds_pipeline_count__c,msj_bvc_segment__c,msj_bvc_tier__c,msj_bvc_acct_open__c,msj_bvc_mcc_patients__c,msj_onc_hp_segment__c,msj_ae_department__c,msj_ae_facility__c,msj_ae_name__c,msj_ae_title__c,msj_email__c,msj_fe_gf2_potential__c,msj_fe_location_potential__c,msj_gf2_segment__c,msj_optin_target__c,msj_merck_specialty1__c,msj_merck_specialty2__c,msj_marketing_cloud_integration__c,msj_marketing_cloud1__c,msj_marketing_cloud2__c,msj_marketing_cloud3__c,msj_marketing_cloud4__c,msj_medical_department__c,msj_marketing_cloud0__c,mobile_id_vod__pc,h1_insights__h1_npi_value_for_testing__pc,h1_insights__h1_person_id__pc,h1_insights__h1_request_status__pc,h1_insights__h1_url__pc,h1_insights__npi_number__pc,h1_insights__npi_number_for_h1_insights__pc,msj_marketing_cloud_integration__pc +src02.crm_account +org02.crm_account +CRM_Account_ex.sql + diff --git a/s3/data/crm/settings/CRM_AccountShare.txt b/s3/data/crm/settings/CRM_AccountShare.txt new file mode 100644 index 00000000..8176af1e --- /dev/null +++ b/s3/data/crm/settings/CRM_AccountShare.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +11 +Id,AccountId,UserOrGroupId,AccountAccessLevel,OpportunityAccessLevel,CaseAccessLevel,ContactAccessLevel,RowCause,LastModifiedDate,LastModifiedById,IsDeleted +id,account_id,user_or_group_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,row_cause,last_modified_date,last_modified_by_id,is_deleted +src02.crm_account_share +org02.crm_account_share +CRM_AccountShare_ex.sql + diff --git a/s3/data/crm/settings/CRM_AccountShare_ex.sql b/s3/data/crm/settings/CRM_AccountShare_ex.sql new file mode 100644 index 00000000..d18e68db --- /dev/null +++ b/s3/data/crm/settings/CRM_AccountShare_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_account_share', 'last_modified_date'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Account_Territory_Loader_vod__c.txt b/s3/data/crm/settings/CRM_Account_Territory_Loader_vod__c.txt new file mode 100644 index 00000000..676e8b39 --- /dev/null +++ b/s3/data/crm/settings/CRM_Account_Territory_Loader_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +19 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,External_ID_vod__c,Territory_vod__c,Mobile_ID_vod__c,Territory_To_Add_vod__c,Territory_to_Drop_vod__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,external_id_vod__c,territory_vod__c,mobile_id_vod__c,territory_to_add_vod__c,territory_to_drop_vod__c +src02.crm_account_territory_loader_vod__c +org02.crm_account_territory_loader_vod__c + + diff --git a/s3/data/crm/settings/CRM_Account_ex.sql b/s3/data/crm/settings/CRM_Account_ex.sql new file mode 100644 index 00000000..792d7a28 --- /dev/null +++ b/s3/data/crm/settings/CRM_Account_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_account', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Approved_Document_vod__c.txt b/s3/data/crm/settings/CRM_Approved_Document_vod__c.txt new file mode 100644 index 00000000..23683a8c --- /dev/null +++ b/s3/data/crm/settings/CRM_Approved_Document_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +53 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Detail_Group_vod__c,Document_Description_vod__c,Document_Host_URL_vod__c,Document_ID_vod__c,Document_Last_Mod_DateTime_vod__c,Email_Allows_Documents_vod__c,Email_Domain_vod__c,Email_Fragment_HTML_vod__c,Email_From_Address_vod__c,Email_From_Name_vod__c,Email_HTML_1_vod__c,Email_HTML_2_vod__c,Email_ReplyTo_Address_vod__c,Email_ReplyTo_Name_vod__c,Email_Subject_vod__c,Email_Template_Fragment_Document_ID_vod__c,Email_Template_Fragment_HTML_vod__c,ISI_Document_ID_vod__c,Language_vod__c,Other_Document_ID_List_vod__c,PI_Document_ID_vod__c,Piece_Document_ID_vod__c,Product_vod__c,Status_vod__c,Territory_vod__c,Vault_Instance_ID_vod__c,Allow_Any_Product_Fragment_vod__c,Allowed_Document_IDs_vod__c,Engage_Document_Id_vod__c,Vault_Document_ID_vod__c,Key_Message_vod__c,Events_Management_Subtype_vod__c,Survey_vod__c,Content_Type_vod__c,Bcc_vod__c,Audience_vod__c,WeChat_Template_ID_vod__c,Check_Consent_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,detail_group_vod__c,document_description_vod__c,document_host_url_vod__c,document_id_vod__c,document_last_mod_date_time_vod__c,email_allows_documents_vod__c,email_domain_vod__c,email_fragment_html_vod__c,email_from_address_vod__c,email_from_name_vod__c,email_html_1_vod__c,email_html_2_vod__c,email_reply_to_address_vod__c,email_reply_to_name_vod__c,email_subject_vod__c,email_template_fragment_document_id_vod__c,email_template_fragment_html_vod__c,isi_document_id_vod__c,language_vod__c,other_document_id_list_vod__c,pi_document_id_vod__c,piece_document_id_vod__c,product_vod__c,status_vod__c,territory_vod__c,vault_instance_id_vod__c,allow_any_product_fragment_vod__c,allowed_document_ids_vod__c,engage_document_id_vod__c,vault_document_id_vod__c,key_message_vod__c,events_management_subtype_vod__c,survey_vod__c,content_type_vod__c,bcc_vod__c,audience_vod__c,we_chat_template_id_vod__c,check_consent_vod__c +src02.crm_approved_document_vod__c +org02.crm_approved_document_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call2_Detail_vod__c.txt b/s3/data/crm/settings/CRM_Call2_Detail_vod__c.txt new file mode 100644 index 00000000..67a53029 --- /dev/null +++ b/s3/data/crm/settings/CRM_Call2_Detail_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +17 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Is_Parent_Call_vod__c,Call2_vod__c,Product_vod__c,Detail_Priority_vod__c,Mobile_ID_vod__c,Override_Lock_vod__c,Type_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,is_parent_call_vod__c,call2_vod__c,product_vod__c,detail_priority_vod__c,mobile_id_vod__c,override_lock_vod__c,type_vod__c +src02.crm_call2_detail_vod__c +org02.crm_call2_detail_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call2_Discussion_vod__c.txt b/s3/data/crm/settings/CRM_Call2_Discussion_vod__c.txt new file mode 100644 index 00000000..c86c840c --- /dev/null +++ b/s3/data/crm/settings/CRM_Call2_Discussion_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +44 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Account_vod__c,Call2_vod__c,Activity__c,Comments__c,Contact_vod__c,Call_Date_vod__c,Product_Strategy_vod__c,Product_Tactic_vod__c,Restricted_Comments__c,Product_vod__c,Presentation__c,Discussion_Topics__c,Slides__c,User_vod__c,Indication__c,Mobile_ID_vod__c,Medical_Event_vod__c,Is_Parent_Call_vod__c,Override_Lock_vod__c,zvod_Product_Map_vod__c,Attendee_Type_vod__c,Entity_Reference_Id_vod__c,Account_Tactic_vod__c,MSJ_Material_Type__c,MSJ_Discussion_Contents__c,MSJ_IST_Minutes__c,MSJ_Off_Label_Minutes__c,MSJ_Discussion_Objectives__c,MSJ_Insight__c,EMDS_Materials__c,EMDS_Topic__c,MSJ_Visit_Purpose__c,MSJ_Insight_Count__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,account_vod__c,call2_vod__c,activity__c,comments__c,contact_vod__c,call_date_vod__c,product_strategy_vod__c,product_tactic_vod__c,restricted_comments__c,product_vod__c,presentation__c,discussion_topics__c,slides__c,user_vod__c,indication__c,mobile_id_vod__c,medical_event_vod__c,is_parent_call_vod__c,override_lock_vod__c,zvod_product_map_vod__c,attendee_type_vod__c,entity_reference_id_vod__c,account_tactic_vod__c,msj_material_type__c,msj_discussion_contents__c,msj_ist_minutes__c,msj_off_label_minutes__c,msj_discussion_objectives__c,msj_insight__c,emds_materials__c,emds_topic__c,msj_visit_purpose__c,msj_insight_count__c +src02.crm_call2_discussion_vod__c +org02.crm_call2_discussion_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call2_Key_Message_vod__c.txt b/s3/data/crm/settings/CRM_Call2_Key_Message_vod__c.txt new file mode 100644 index 00000000..40267bd0 --- /dev/null +++ b/s3/data/crm/settings/CRM_Call2_Key_Message_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +35 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Account_vod__c,Call2_vod__c,Reaction_vod__c,Product_vod__c,Key_Message_vod__c,Mobile_ID_vod__c,Contact_vod__c,Call_Date_vod__c,User_vod__c,Category_vod__c,Vehicle_vod__c,Is_Parent_Call_vod__c,Override_Lock_vod__c,CLM_ID_vod__c,Slide_Version_vod__c,Duration_vod__c,Presentation_ID_vod__c,Start_Time_vod__c,Attendee_Type_vod__c,Entity_Reference_Id_vod__c,Segment_vod__c,Display_Order_vod__c,Clm_Presentation_Name_vod__c,Clm_Presentation_Version_vod__c,Clm_Presentation_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,account_vod__c,call2_vod__c,reaction_vod__c,product_vod__c,key_message_vod__c,mobile_id_vod__c,contact_vod__c,call_date_vod__c,user_vod__c,category_vod__c,vehicle_vod__c,is_parent_call_vod__c,override_lock_vod__c,clm_id_vod__c,slide_version_vod__c,duration_vod__c,presentation_id_vod__c,start_time_vod__c,attendee_type_vod__c,entity_reference_id_vod__c,segment_vod__c,display_order_vod__c,clm_presentation_name_vod__c,clm_presentation_version_vod__c,clm_presentation_vod__c +src02.crm_call2_key_message_vod__c +org02.crm_call2_key_message_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call2_vod__c.txt b/s3/data/crm/settings/CRM_Call2_vod__c.txt new file mode 100644 index 00000000..6b2c330e --- /dev/null +++ b/s3/data/crm/settings/CRM_Call2_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +205 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Call_Comments_vod__c,Sample_Card_vod__c,Add_Detail_vod__c,Property_vod__c,Account_vod__c,zvod_Product_Discussion_vod__c,Status_vod__c,Parent_Address_vod__c,Account_Plan_vod__c,zvod_SaveNew_vod__c,Next_Call_Notes_vod__c,Pre_Call_Notes_vod__c,Mobile_ID_vod__c,zvod_Account_Credentials_vod_c_vod__c,zvod_Account_Preferred_Name_vod_c_vod__c,zvod_Account_Sample_Status_vod_c_vod__c,zvod_Attendees_vod__c,zvod_Key_Messages_vod__c,zvod_Detailing_vod__c,zvod_Expenses_vod__c,zvod_Followup_vod__c,zvod_Samples_vod__c,zvod_Save_vod__c,zvod_Submit_vod__c,zvod_Delete_vod__c,Activity_Type__c,Significant_Event__c,Location_vod__c,Subject_vod__c,Unlock_vod__c,Call_Datetime_vod__c,Disbursed_To_vod__c,Disclaimer_vod__c,Request_Receipt_vod__c,Signature_Date_vod__c,Signature_vod__c,Territory_vod__c,Submitted_By_Mobile_vod__c,Call_Type_vod__c,Add_Key_Message_vod__c,Address_vod__c,Attendees_vod__c,Attendee_Type_vod__c,Call_Date_vod__c,Detailed_Products_vod__c,No_Disbursement_vod__c,Parent_Call_vod__c,User_vod__c,Contact_vod__c,zvod_Entity_vod__c,Medical_Event_vod__c,Mobile_Created_Datetime_vod__c,Mobile_Last_Modified_Datetime_vod__c,License_vod__c,Is_Parent_Call_vod__c,Entity_Display_Name_vod__c,Override_Lock_vod__c,Last_Device_vod__c,Ship_Address_Line_1_vod__c,Ship_Address_Line_2_vod__c,Ship_City_vod__c,Ship_Country_vod__c,Ship_License_Expiration_Date_vod__c,Ship_License_Status_vod__c,Ship_License_vod__c,Ship_State_vod__c,Ship_To_Address_vod__c,Ship_Zip_vod__c,Ship_To_Address_Text_vod__c,CLM_vod__c,zvod_CLMDetails_vod__c,Is_Sampled_Call_vod__c,zvod_Surveys_vod__c,Presentations_vod__c,Entity_Reference_Id_vod__c,Error_Reference_Call_vod__c,Duration_vod__c,Color_vod__c,Allowed_Products_vod__c,zvod_Attachments_vod__c,Sample_Card_Reason_vod__c,ASSMCA_vod__c,Address_Line_1_vod__c,Address_Line_2_vod__c,City_vod__c,DEA_Address_Line_1_vod__c,DEA_Address_Line_2_vod__c,DEA_Address_vod__c,DEA_City_vod__c,DEA_Expiration_Date_vod__c,DEA_State_vod__c,DEA_Zip_4_vod__c,DEA_Zip_vod__c,DEA_vod__c,Ship_Zip_4_vod__c,State_vod__c,Zip_4_vod__c,Zip_vod__c,Sample_Send_Card_vod__c,zvod_Address_vod_c_DEA_Status_vod_c_vod__c,Signature_Page_Image_vod__c,Credentials_vod__c,Salutation_vod__c,zvod_Account_Call_Reminder_vod_c_vod__c,MSJ_Meeting_Duration__c,MSJ_Double_Visit_AM__c,zvod_Business_Account_vod__c,Product_Priority_1_vod__c,Product_Priority_2_vod__c,Product_Priority_3_vod__c,Product_Priority_4_vod__c,Product_Priority_5_vod__c,zvod_More_Actions_vod__c,zvod_Call_Conflict_Status_vod__c,Signature_Timestamp_vod__c,Expense_Amount_vod__c,Total_Expense_Attendees_Count_vod__c,Attendee_list_vod__c,Expense_Post_Status_vod__c,Attendee_Post_Status_vod__c,Expense_System_External_ID_vod__c,Incurred_Expense_vod__c,Assigner_vod__c,Assignment_Datetime_vod__c,zvod_Call_Objective_vod__c,Signature_Location_Longitude_vod__c,Signature_Location_Latitude_vod__c,Location_Services_Status_vod__c,MSJ_Double_Visit_Other__c,MSJ_Comment__c,MSJ_For_Reporting__c,MSJ_Number_of_Attendees__c,MSJ_Main_Dept__c,Planned_Type_vjh__c,Cobrowse_URL_Participant_vod__c,MSJ_Activity_Method_Text__c,MSJ_Activity_Method__c,MSJ_Classification__c,MSJ_Double_Visit_MSL__c,MSJ_MSL_Comment_for_MR__c,MSJ_APD__c,Medical_Inquiry_vod__c,MSJ_Call_Type_MSJ__c,MSJ_Prescription_Request__c,MSJ_Patient_Follow__c,Child_Account_Id_vod__c,Child_Account_vod__c,Location_Id_vod__c,Location_Name_vod__c,MSJ_Comments_about_technology__c,Remote_Meeting_vod__c,Veeva_Remote_Meeting_Id_vod__c,MSJ_Activity_Type_Report__c,MSJ_Activity_Type__c,MSJ_Activity__c,MSJ_Comments__c,MSJ_Therapy__c,MSJ_Time_Hrs__c,EMDS_CO_Reference__c,EMDS_Call_Sub_Type__c,EMDS_Call_Type__c,EMDS_Call_Unsuccessful__c,EMDS_Congress_Type__c,EMDS_Date_of_Service__c,EMDS_Fertility_DisInterest__c,EMDS_Fertility_Interest__c,EMDS_Installed_Equipment__c,EMDS_Pipeline_Stage_Value__c,EMDS_Pipeline_Stage__c,EMDS_Pipeline__c,EMDS_Reason_for_Call__c,EMDS_Training_Completed__c,MSJ_BrainStorming__c,MSJ_SIPAGL_1A__c,MSJ_SIPAGL_1B__c,MSJ_SIPAGL_2__c,MSJ_SIPAGL_3__c,MSJ_SIPAGL_4A__c,MSJ_SIPAGL_5A__c,MSJ_SIPAGL_comment__c,MSJ_SIPAGL_4B__c,MSJ_SIPAGL_5B__c,Location_Text_vod__c,Call_Channel_vod__c,MSJ_Scientific_Interaction__c,MSJ_Activity_Email_Reply__c,MSJ_Interaction_Duration__c,MSJ_SIPAGL_1A_date__c,MSJ_CoPromotion__c,Call_Channel_Formula_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,call_comments_vod__c,sample_card_vod__c,add_detail_vod__c,property_vod__c,account_vod__c,zvod_product_discussion_vod__c,status_vod__c,parent_address_vod__c,account_plan_vod__c,zvod_save_new_vod__c,next_call_notes_vod__c,pre_call_notes_vod__c,mobile_id_vod__c,zvod_account_credentials_vod_c_vod__c,zvod_account_preferred_name_vod_c_vod__c,zvod_account_sample_status_vod_c_vod__c,zvod_attendees_vod__c,zvod_key_messages_vod__c,zvod_detailing_vod__c,zvod_expenses_vod__c,zvod_followup_vod__c,zvod_samples_vod__c,zvod_save_vod__c,zvod_submit_vod__c,zvod_delete_vod__c,activity_type__c,significant_event__c,location_vod__c,subject_vod__c,unlock_vod__c,call_datetime_vod__c,disbursed_to_vod__c,disclaimer_vod__c,request_receipt_vod__c,signature_date_vod__c,signature_vod__c,territory_vod__c,submitted_by_mobile_vod__c,call_type_vod__c,add_key_message_vod__c,address_vod__c,attendees_vod__c,attendee_type_vod__c,call_date_vod__c,detailed_products_vod__c,no_disbursement_vod__c,parent_call_vod__c,user_vod__c,contact_vod__c,zvod_entity_vod__c,medical_event_vod__c,mobile_created_datetime_vod__c,mobile_last_modified_datetime_vod__c,license_vod__c,is_parent_call_vod__c,entity_display_name_vod__c,override_lock_vod__c,last_device_vod__c,ship_address_line_1_vod__c,ship_address_line_2_vod__c,ship_city_vod__c,ship_country_vod__c,ship_license_expiration_date_vod__c,ship_license_status_vod__c,ship_license_vod__c,ship_state_vod__c,ship_to_address_vod__c,ship_zip_vod__c,ship_to_address_text_vod__c,clm_vod__c,zvod_clmdetails_vod__c,is_sampled_call_vod__c,zvod_surveys_vod__c,presentations_vod__c,entity_reference_id_vod__c,error_reference_call_vod__c,duration_vod__c,color_vod__c,allowed_products_vod__c,zvod_attachments_vod__c,sample_card_reason_vod__c,assmca_vod__c,address_line_1_vod__c,address_line_2_vod__c,city_vod__c,dea_address_line_1_vod__c,dea_address_line_2_vod__c,dea_address_vod__c,dea_city_vod__c,dea_expiration_date_vod__c,dea_state_vod__c,dea_zip_4_vod__c,dea_zip_vod__c,dea_vod__c,ship_zip_4_vod__c,state_vod__c,zip_4_vod__c,zip_vod__c,sample_send_card_vod__c,zvod_address_vod_c_dea_status_vod_c_vod__c,signature_page_image_vod__c,credentials_vod__c,salutation_vod__c,zvod_account_call_reminder_vod_c_vod__c,msj_meeting_duration__c,msj_double_visit_am__c,zvod_business_account_vod__c,product_priority_1_vod__c,product_priority_2_vod__c,product_priority_3_vod__c,product_priority_4_vod__c,product_priority_5_vod__c,zvod_more_actions_vod__c,zvod_call_conflict_status_vod__c,signature_timestamp_vod__c,expense_amount_vod__c,total_expense_attendees_count_vod__c,attendee_list_vod__c,expense_post_status_vod__c,attendee_post_status_vod__c,expense_system_external_id_vod__c,incurred_expense_vod__c,assigner_vod__c,assignment_datetime_vod__c,zvod_call_objective_vod__c,signature_location_longitude_vod__c,signature_location_latitude_vod__c,location_services_status_vod__c,msj_double_visit_other__c,msj_comment__c,msj_for_reporting__c,msj_number_of_attendees__c,msj_main_dept__c,planned_type_vjh__c,cobrowse_url_participant_vod__c,msj_activity_method_text__c,msj_activity_method__c,msj_classification__c,msj_double_visit_msl__c,msj_msl_comment_for_mr__c,msj_apd__c,medical_inquiry_vod__c,msj_call_type_msj__c,msj_prescription_request__c,msj_patient_follow__c,child_account_id_vod__c,child_account_vod__c,location_id_vod__c,location_name_vod__c,msj_comments_about_technology__c,remote_meeting_vod__c,veeva_remote_meeting_id_vod__c,msj_activity_type_report__c,msj_activity_type__c,msj_activity__c,msj_comments__c,msj_therapy__c,msj_time_hrs__c,emds_co_reference__c,emds_call_sub_type__c,emds_call_type__c,emds_call_unsuccessful__c,emds_congress_type__c,emds_date_of_service__c,emds_fertility_dis_interest__c,emds_fertility_interest__c,emds_installed_equipment__c,emds_pipeline_stage_value__c,emds_pipeline_stage__c,emds_pipeline__c,emds_reason_for_call__c,emds_training_completed__c,msj_brain_storming__c,msj_sipagl_1_a__c,msj_sipagl_1_b__c,msj_sipagl_2__c,msj_sipagl_3__c,msj_sipagl_4_a__c,msj_sipagl_5_a__c,msj_sipagl_comment__c,msj_sipagl_4_b__c,msj_sipagl_5_b__c,location_text_vod__c,call_channel_vod__c,msj_scientific_interaction__c,msj_activity_email_reply__c,msj_interaction_duration__c,msj_sipagl_1_a_date__c,msj_co_promotion__c,call_channel_formula_vod__c +src02.crm_call2_vod__c +org02.crm_call2_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call_Clickstream_vod__c.txt b/s3/data/crm/settings/CRM_Call_Clickstream_vod__c.txt new file mode 100644 index 00000000..e2b56535 --- /dev/null +++ b/s3/data/crm/settings/CRM_Call_Clickstream_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +34 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Answer_vod__c,Call_vod__c,Key_Message_vod__c,Mobile_ID_vod__c,Popup_Opened_vod__c,Possible_Answers_vod__c,Presentation_ID_vod__c,Product_vod__c,Range_Value_vod__c,Rollover_Entered_vod__c,Selected_Items_vod__c,CLM_ID_vod__c,Question_vod__c,Survey_Type_vod__c,Text_Entered_vod__c,Toggle_Button_On_vod__c,Track_Element_Description_vod__c,Track_Element_Id_vod__c,Track_Element_Type_vod__c,Usage_Duration_vod__c,Usage_Start_Time_vod__c,AuxillaryId_vod__c,ParentId_vod__c,Revision_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,answer_vod__c,call_vod__c,key_message_vod__c,mobile_id_vod__c,popup_opened_vod__c,possible_answers_vod__c,presentation_id_vod__c,product_vod__c,range_value_vod__c,rollover_entered_vod__c,selected_items_vod__c,clm_id_vod__c,question_vod__c,survey_type_vod__c,text_entered_vod__c,toggle_button_on_vod__c,track_element_description_vod__c,track_element_id_vod__c,track_element_type_vod__c,usage_duration_vod__c,usage_start_time_vod__c,auxillary_id_vod__c,parent_id_vod__c,revision_vod__c +src02.crm_call_clickstream_vod__c +org02.crm_call_clickstream_vod__c + + diff --git a/s3/data/crm/settings/CRM_Child_Account_vod__c.txt b/s3/data/crm/settings/CRM_Child_Account_vod__c.txt new file mode 100644 index 00000000..876bef05 --- /dev/null +++ b/s3/data/crm/settings/CRM_Child_Account_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +57 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Parent_Account_vod__c,Child_Account_vod__c,External_ID_vod__c,Mobile_ID_vod__c,Primary_vod__c,Copy_Address_vod__c,Child_Name_vod__c,Parent_Name_vod__c,Parent_Child_Name_vod__c,Account_Code__c,Child_Department__c,Child_Role__c,Child_Title__c,Child_Remark__c,MSJ_1C_segment__c,MSJ_BU_FE__c,MSJ_BU_ONC__c,MSJ_BVC_Segment__c,MSJ_CE_segment__c,MSJ_Child_Account_Link__c,MSJ_DCF_DR_Code__c,MSJ_DCF_HP_Code__c,MSJ_DR_Change_Log__c,MSJ_Delete__c,MSJ_Department__c,MSJ_EB_CRC_Segment__c,MSJ_EB_HN_Segment__c,MSJ_EB_H_N_LA_Segment__c,MSJ_EB_H_N_RM_Segment__c,MSJ_External_ID__c,MSJ_Fax__c,MSJ_GF2_segment__c,MSJ_GF_segment__c,MSJ_KOL_LOL__c,MSJ_KOL__c,MSJ_ONC_HP_Segment__c,MSJ_OPTIN_target__c,MSJ_OV_segment__c,MSJ_Parent_Child_Name__c,MSJ_Phone__c,MSJ_Remark__c,MSJ_Target_Call_Num__c,MSJ_Tech_segment__c,MSJ_Title__c,MSJ_XLK_Segment__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,parent_account_vod__c,child_account_vod__c,external_id_vod__c,mobile_id_vod__c,primary_vod__c,copy_address_vod__c,child_name_vod__c,parent_name_vod__c,parent_child_name_vod__c,account_code__c,child_department__c,child_role__c,child_title__c,child_remark__c,msj_1_c_segment__c,msj_bu_fe__c,msj_bu_onc__c,msj_bvc_segment__c,msj_ce_segment__c,msj_child_account_link__c,msj_dcf_dr_code__c,msj_dcf_hp_code__c,msj_dr_change_log__c,msj_delete__c,msj_department__c,msj_eb_crc_segment__c,msj_eb_hn_segment__c,msj_eb_h_n_la_segment__c,msj_eb_h_n_rm_segment__c,msj_external_id__c,msj_fax__c,msj_gf2_segment__c,msj_gf_segment__c,msj_kol_lol__c,msj_kol__c,msj_onc_hp_segment__c,msj_optin_target__c,msj_ov_segment__c,msj_parent_child_name__c,msj_phone__c,msj_remark__c,msj_target_call_num__c,msj_tech_segment__c,msj_title__c,msj_xlk_segment__c +src02.crm_child_account_vod__c +org02.crm_child_account_vod__c +CRM_Child_Account_vod__c_ex.sql + diff --git a/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql b/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql new file mode 100644 index 00000000..cac33f47 --- /dev/null +++ b/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_child_account_vod__c', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Clm_Presentation_Slide_vod__c.txt b/s3/data/crm/settings/CRM_Clm_Presentation_Slide_vod__c.txt new file mode 100644 index 00000000..422a6926 --- /dev/null +++ b/s3/data/crm/settings/CRM_Clm_Presentation_Slide_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +17 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Clm_Presentation_vod__c,Key_Message_vod__c,Display_Order_vod__c,Sub_Presentation_vod__c,Mobile_ID_vod__c,External_ID_vod__c,VExternal_Id_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,clm_presentation_vod__c,key_message_vod__c,display_order_vod__c,sub_presentation_vod__c,mobile_id_vod__c,external_id_vod__c,vexternal_id_vod__c +src02.crm_clm_presentation_slide_vod__c +org02.crm_clm_presentation_slide_vod__c + + diff --git a/s3/data/crm/settings/CRM_Clm_Presentation_vod__c.txt b/s3/data/crm/settings/CRM_Clm_Presentation_vod__c.txt new file mode 100644 index 00000000..364a308b --- /dev/null +++ b/s3/data/crm/settings/CRM_Clm_Presentation_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +46 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Mobile_ID_vod__c,Presentation_Id_vod__c,Product_vod__c,Default_Presentation_vod__c,Training_vod__c,ParentId_vod__c,Hidden_vod__c,Type_vod__c,Approved_vod__c,Copied_From_vod__c,Copy_Date_vod__c,Survey_vod__c,Original_Record_ID_vod__c,Directory_vod__c,End_Date_vod__c,Start_Date_vod__c,Status_vod__c,VExternal_Id_vod__c,Vault_DNS_vod__c,Vault_Doc_Id_vod__c,Vault_External_Id_vod__c,Vault_GUID_vod__c,Vault_Last_Modified_Date_Time_vod__c,Version_vod__c,Enable_Survey_Overlay_vod__c,Description_vod__c,Keywords_vod__c,Content_Channel_vod__c,original_material_approved_in_veritas__c,keywords__c,trade_team__c,ewizard_link__c,business_function__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,mobile_id_vod__c,presentation_id_vod__c,product_vod__c,default_presentation_vod__c,training_vod__c,parent_id_vod__c,hidden_vod__c,type_vod__c,approved_vod__c,copied_from_vod__c,copy_date_vod__c,survey_vod__c,original_record_id_vod__c,directory_vod__c,end_date_vod__c,start_date_vod__c,status_vod__c,vexternal_id_vod__c,vault_dns_vod__c,vault_doc_id_vod__c,vault_external_id_vod__c,vault_guid_vod__c,vault_last_modified_date_time_vod__c,version_vod__c,enable_survey_overlay_vod__c,description_vod__c,keywords_vod__c,content_channel_vod__c,original_material_approved_in_veritas__c,keywords__c,trade_team__c,ewizard_link__c,business_function__c +src02.crm_clm_presentation_vod__c +org02.crm_clm_presentation_vod__c + + diff --git a/s3/data/crm/settings/CRM_Coaching_Report_vod__c.txt b/s3/data/crm/settings/CRM_Coaching_Report_vod__c.txt new file mode 100644 index 00000000..fd64a473 --- /dev/null +++ b/s3/data/crm/settings/CRM_Coaching_Report_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +144 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Mobile_ID_vod__c,Manager_vod__c,Employee_vod__c,Review_Date__c,Review_Period__c,Status__c,Comments__c,Strategic_Planning__c,Customer_Focus__c,Knowledge_Expertise__c,Business_Account_Planning__c,Call_Productivity__c,Overall_Rating__c,MSJ_A01__c,MSJ_A02__c,MSJ_A03__c,MSJ_AM_Memo__c,MSJ_Aid_Total__c,MSJ_C0_GC__c,MSJ_C1_GC__c,MSJ_C2_GC__c,MSJ_Countermeasure__c,MSJ_Deadline__c,MSJ_Double_Visit_Time__c,MSJ_Hospital__c,MSJ_K01_FE__c,MSJ_K01_ONC__c,MSJ_K02_FE__c,MSJ_K02_ONC__c,MSJ_K03_FE__c,MSJ_K03_ONC__c,MSJ_K04_FE__c,MSJ_K04_ONC__c,MSJ_K05_FE__c,MSJ_K05_ONC__c,MSJ_K06_FE__c,MSJ_K06_ONC__c,MSJ_K0_GC__c,MSJ_K1_GC__c,MSJ_K2_GC__c,MSJ_Knowledge_Total__c,MSJ_L0_GC__c,MSJ_L1_GC__c,MSJ_L2_GC__c,MSJ_MR_GC__c,MSJ_MR_Problems__c,MSJ_N0_GC__c,MSJ_N1_GC__c,MSJ_N2_GC__c,MSJ_Num_of_DTL__c,MSJ_P01__c,MSJ_P02__c,MSJ_P03__c,MSJ_P04__c,MSJ_P05__c,MSJ_P0_GC__c,MSJ_P1_GC__c,MSJ_P2_GC__c,MSJ_PlanningTotal__c,MSJ_R0_GC__c,MSJ_R1_GC__c,MSJ_R2_GC__c,MSJ_S01__c,MSJ_S02__c,MSJ_S03__c,MSJ_S04__c,MSJ_S05__c,MSJ_S06__c,MSJ_S07__c,MSJ_S08__c,MSJ_S09__c,MSJ_S10__c,MSJ_S11__c,MSJ_S12__c,MSJ_Skill_Total__c,MSJ_After_Call_01__c,MSJ_After_Call_02__c,MSJ_After_Call_03__c,MSJ_After_Call_04__c,MSJ_Closing__c,MSJ_Comment_by_MR__c,MSJ_Confirmed_by_MR__c,MSJ_Createdby__c,MSJ_FT_AM_Name__c,MSJ_Interview_Preparation__c,MSJ_Interview_Reflection__c,MSJ_Notify_To_MR__c,MSJ_Opening__c,MSJ_Others_01_Result__c,MSJ_Others_01__c,MSJ_Others_02_Result__c,MSJ_Others_02__c,MSJ_Patient_Thinking__c,MSJ_Probing__c,MSJ_Supporting__c,MSJ_Patient_Thinking_for_FE__c,MSJ_After_Call_05__c,MSJ_After_Call_06__c,MSJ_After_Call_07__c,MSJ_After_Call_08__c,MSJ_Createdby_FE__c,MSJ_Createdby_ONC__c,MSJ_Development_Level__c,MSJ_Interview_Prep_01__c,MSJ_Interview_Prep_02__c,MSJ_Leadership_Style__c,MSJ_Overcome_01__c,MSJ_Overcome_02__c,MSJ_Overcome_03__c,MSJ_Overcome_04__c,MSJ_Review_01__c,MSJ_Review_02__c,MSJ_SK_01__c,MSJ_SK_02__c,MSJ_SK_03__c,MSJ_SK_04__c,MSJ_SK_05__c,MSJ_SK_06__c,MSJ_SK_07__c,MSJ_SK_08__c,MSJ_SK_09__c,MSJ_SK_10__c,MSJ_Specific_Action__c,MSJ_Training_Point__c,MSJ_Efforts_of_Year__c,MSJ_Efforts_of_Month__c,MSJ_Skill_Task__c,MSJ_Action_of_This_Month__c,MSJ_Achievement_of_This_Month__c,MSJ_Comment_from_AM__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,mobile_id_vod__c,manager_vod__c,employee_vod__c,review_date__c,review_period__c,status__c,comments__c,strategic_planning__c,customer_focus__c,knowledge_expertise__c,business_account_planning__c,call_productivity__c,overall_rating__c,msj_a01__c,msj_a02__c,msj_a03__c,msj_am_memo__c,msj_aid_total__c,msj_c0_gc__c,msj_c1_gc__c,msj_c2_gc__c,msj_countermeasure__c,msj_deadline__c,msj_double_visit_time__c,msj_hospital__c,msj_k01_fe__c,msj_k01_onc__c,msj_k02_fe__c,msj_k02_onc__c,msj_k03_fe__c,msj_k03_onc__c,msj_k04_fe__c,msj_k04_onc__c,msj_k05_fe__c,msj_k05_onc__c,msj_k06_fe__c,msj_k06_onc__c,msj_k0_gc__c,msj_k1_gc__c,msj_k2_gc__c,msj_knowledge_total__c,msj_l0_gc__c,msj_l1_gc__c,msj_l2_gc__c,msj_mr_gc__c,msj_mr_problems__c,msj_n0_gc__c,msj_n1_gc__c,msj_n2_gc__c,msj_num_of_dtl__c,msj_p01__c,msj_p02__c,msj_p03__c,msj_p04__c,msj_p05__c,msj_p0_gc__c,msj_p1_gc__c,msj_p2_gc__c,msj_planning_total__c,msj_r0_gc__c,msj_r1_gc__c,msj_r2_gc__c,msj_s01__c,msj_s02__c,msj_s03__c,msj_s04__c,msj_s05__c,msj_s06__c,msj_s07__c,msj_s08__c,msj_s09__c,msj_s10__c,msj_s11__c,msj_s12__c,msj_skill_total__c,msj_after_call_01__c,msj_after_call_02__c,msj_after_call_03__c,msj_after_call_04__c,msj_closing__c,msj_comment_by_mr__c,msj_confirmed_by_mr__c,msj_createdby__c,msj_ft_am_name__c,msj_interview_preparation__c,msj_interview_reflection__c,msj_notify_to_mr__c,msj_opening__c,msj_others_01_result__c,msj_others_01__c,msj_others_02_result__c,msj_others_02__c,msj_patient_thinking__c,msj_probing__c,msj_supporting__c,msj_patient_thinking_for_fe__c,msj_after_call_05__c,msj_after_call_06__c,msj_after_call_07__c,msj_after_call_08__c,msj_createdby_fe__c,msj_createdby_onc__c,msj_development_level__c,msj_interview_prep_01__c,msj_interview_prep_02__c,msj_leadership_style__c,msj_overcome_01__c,msj_overcome_02__c,msj_overcome_03__c,msj_overcome_04__c,msj_review_01__c,msj_review_02__c,msj_sk_01__c,msj_sk_02__c,msj_sk_03__c,msj_sk_04__c,msj_sk_05__c,msj_sk_06__c,msj_sk_07__c,msj_sk_08__c,msj_sk_09__c,msj_sk_10__c,msj_specific_action__c,msj_training_point__c,msj_efforts_of_year__c,msj_efforts_of_month__c,msj_skill_task__c,msj_action_of_this_month__c,msj_achievement_of_this_month__c,msj_comment_from_am__c +src02.crm_coaching_report_vod__c +org02.crm_coaching_report_vod__c + + diff --git a/s3/data/crm/settings/CRM_Consent_Header_vod__c.txt b/s3/data/crm/settings/CRM_Consent_Header_vod__c.txt new file mode 100644 index 00000000..729c3452 --- /dev/null +++ b/s3/data/crm/settings/CRM_Consent_Header_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +22 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consent_Header_Help_Text_vod__c,Country_vod__c,Inactive_Datetime_vod__c,Language_vod__c,Status_vod__c,Signature_Required_On_Opt_Out_vod__c,Request_Receipt_vod__c,Subscription_Option_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consent_header_help_text_vod__c,country_vod__c,inactive_datetime_vod__c,language_vod__c,status_vod__c,signature_required_on_opt_out_vod__c,request_receipt_vod__c,subscription_option_vod__c +src02.crm_consent_header_vod__c +org02.crm_consent_header_vod__c + + diff --git a/s3/data/crm/settings/CRM_Consent_Line_vod__c.txt b/s3/data/crm/settings/CRM_Consent_Line_vod__c.txt new file mode 100644 index 00000000..3cc95e31 --- /dev/null +++ b/s3/data/crm/settings/CRM_Consent_Line_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +25 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consent_Type_vod__c,Detail_Group_Display_Name_vod__c,Detail_Group_vod__c,Display_Order_vod__c,End_Date_vod__c,Group_By_vod__c,Product_Display_Name_vod__c,Product_vod__c,Start_Date_vod__c,Sub_Channel_Description_vod__c,Sub_Channel_Display_Name_vod__c,Sub_Channel_Key_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consent_type_vod__c,detail_group_display_name_vod__c,detail_group_vod__c,display_order_vod__c,end_date_vod__c,group_by_vod__c,product_display_name_vod__c,product_vod__c,start_date_vod__c,sub_channel_description_vod__c,sub_channel_display_name_vod__c,sub_channel_key_vod__c +src02.crm_consent_line_vod__c +org02.crm_consent_line_vod__c + + diff --git a/s3/data/crm/settings/CRM_Consent_Type_vod__c.txt b/s3/data/crm/settings/CRM_Consent_Type_vod__c.txt new file mode 100644 index 00000000..2a3451a0 --- /dev/null +++ b/s3/data/crm/settings/CRM_Consent_Type_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +26 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consent_Header_vod__c,Channel_Label_vod__c,Channel_Source_vod__c,Consent_Expires_In_vod__c,Default_Consent_Type_vod__c,Disclaimer_Text_vod__c,Display_Order_vod__c,Product_Preference_vod__c,zvod_Consent_Default_Consent_Text_vod__c,zvod_Consent_Line_vod__c,zvod_Signature_Capture_vod__c,Double_Opt_In_vod__c,zvod_Consent_Activity_Tracking_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consent_header_vod__c,channel_label_vod__c,channel_source_vod__c,consent_expires_in_vod__c,default_consent_type_vod__c,disclaimer_text_vod__c,display_order_vod__c,product_preference_vod__c,zvod_consent_default_consent_text_vod__c,zvod_consent_line_vod__c,zvod_signature_capture_vod__c,double_opt_in_vod__c,zvod_consent_activity_tracking_vod__c +src02.crm_consent_type_vod__c +org02.crm_consent_type_vod__c + + diff --git a/s3/data/crm/settings/CRM_Contact.txt b/s3/data/crm/settings/CRM_Contact.txt new file mode 100644 index 00000000..21a9c5dd --- /dev/null +++ b/s3/data/crm/settings/CRM_Contact.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +71 +Id,IsDeleted,MasterRecordId,AccountId,IsPersonAccount,LastName,FirstName,Salutation,Name,OtherStreet,OtherCity,OtherState,OtherPostalCode,OtherCountry,OtherLatitude,OtherLongitude,OtherGeocodeAccuracy,OtherAddress,MailingStreet,MailingCity,MailingState,MailingPostalCode,MailingCountry,MailingLatitude,MailingLongitude,MailingGeocodeAccuracy,MailingAddress,Phone,Fax,MobilePhone,HomePhone,OtherPhone,AssistantPhone,ReportsToId,Email,Title,Department,AssistantName,Birthdate,Description,OwnerId,HasOptedOutOfEmail,HasOptedOutOfFax,DoNotCall,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,LastCURequestDate,LastCUUpdateDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,EmailBouncedReason,EmailBouncedDate,IsEmailBounced,PhotoUrl,Jigsaw,JigsawContactId,IndividualId,Mobile_ID_vod__c,H1Insights__H1_NPI_Value_for_Testing__c,H1Insights__H1_Person_ID__c,H1Insights__H1_Request_Status__c,H1Insights__H1_URL__c,H1Insights__NPI_Number__c,H1Insights__NPI_Number_for_H1_Insights__c,MSJ_Marketing_Cloud_Integration__c +id,is_deleted,master_record_id,account_id,is_person_account,last_name,first_name,salutation,name,other_street,other_city,other_state,other_postal_code,other_country,other_latitude,other_longitude,other_geocode_accuracy,other_address,mailing_street,mailing_city,mailing_state,mailing_postal_code,mailing_country,mailing_latitude,mailing_longitude,mailing_geocode_accuracy,mailing_address,phone,fax,mobile_phone,home_phone,other_phone,assistant_phone,reports_to_id,email,title,department,assistant_name,birthdate,description,owner_id,has_opted_out_of_email,has_opted_out_of_fax,do_not_call,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,last_curequest_date,last_cuupdate_date,may_edit,is_locked,last_viewed_date,last_referenced_date,email_bounced_reason,email_bounced_date,is_email_bounced,photo_url,jigsaw,jigsaw_contact_id,individual_id,mobile_id_vod__c,h1_insights__h1_npi_value_for_testing__c,h1_insights__h1_person_id__c,h1_insights__h1_request_status__c,h1_insights__h1_url__c,h1_insights__npi_number__c,h1_insights__npi_number_for_h1_insights__c,msj_marketing_cloud_integration__c +src02.crm_contact +org02.crm_contact +CRM_Contact_ex.sql + diff --git a/s3/data/crm/settings/CRM_Contact_ex.sql b/s3/data/crm/settings/CRM_Contact_ex.sql new file mode 100644 index 00000000..4b916e23 --- /dev/null +++ b/s3/data/crm/settings/CRM_Contact_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_contact', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Dynamic_Attribute_Configuration_vod__c.txt b/s3/data/crm/settings/CRM_Dynamic_Attribute_Configuration_vod__c.txt new file mode 100644 index 00000000..d48828a7 --- /dev/null +++ b/s3/data/crm/settings/CRM_Dynamic_Attribute_Configuration_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +29 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Applies_To_vod__c,Attribute_Label_vod__c,Attribute_Name_vod__c,Available_Values_vod__c,Description_vod__c,Detail_Group_vod__c,Display_Order_vod__c,External_ID_vod__c,Help_Text_vod__c,Product_vod__c,Read_Only_vod__c,Section_Name_vod__c,Sharing_Group_vod__c,Status_vod__c,Track_Changes_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,applies_to_vod__c,attribute_label_vod__c,attribute_name_vod__c,available_values_vod__c,description_vod__c,detail_group_vod__c,display_order_vod__c,external_id_vod__c,help_text_vod__c,product_vod__c,read_only_vod__c,section_name_vod__c,sharing_group_vod__c,status_vod__c,track_changes_vod__c +src02.crm_dynamic_attribute_configuration_vod__c +org02.crm_dynamic_attribute_configuration_vod__c + + diff --git a/s3/data/crm/settings/CRM_Dynamic_Attribute_vod__c.txt b/s3/data/crm/settings/CRM_Dynamic_Attribute_vod__c.txt new file mode 100644 index 00000000..adea9267 --- /dev/null +++ b/s3/data/crm/settings/CRM_Dynamic_Attribute_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +26 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Account_vod__c,Active_vod__c,Dynamic_Attribute_Configuration_vod__c,Dynamic_Attribute_Description_vod__c,Dynamic_Attribute_Help_Text_vod__c,Dynamic_Attribute_Label_vod__c,Dynamic_Attribute_Name_vod__c,Dynamic_Attribute_Record_Type_vod__c,Dynamic_Attribute_Value_Checkbox_vod__c,Dynamic_Attribute_Value_Date_Time_vod__c,Dynamic_Attribute_Value_Date_vod__c,Dynamic_Attribute_Value_Number_vod__c,Dynamic_Attribute_Value_Text_Area_vod__c,Dynamic_Attribute_Value_Text_vod__c,Mobile_ID_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,account_vod__c,active_vod__c,dynamic_attribute_configuration_vod__c,dynamic_attribute_description_vod__c,dynamic_attribute_help_text_vod__c,dynamic_attribute_label_vod__c,dynamic_attribute_name_vod__c,dynamic_attribute_record_type_vod__c,dynamic_attribute_value_checkbox_vod__c,dynamic_attribute_value_date_time_vod__c,dynamic_attribute_value_date_vod__c,dynamic_attribute_value_number_vod__c,dynamic_attribute_value_text_area_vod__c,dynamic_attribute_value_text_vod__c,mobile_id_vod__c +src02.crm_dynamic_attribute_vod__c +org02.crm_dynamic_attribute_vod__c + + diff --git a/s3/data/crm/settings/CRM_Email_Activity_vod__c.txt b/s3/data/crm/settings/CRM_Email_Activity_vod__c.txt new file mode 100644 index 00000000..2c801e06 --- /dev/null +++ b/s3/data/crm/settings/CRM_Email_Activity_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +36 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,Sent_Email_vod__c,Activity_DateTime_vod__c,City_vod__c,Click_URL_vod__c,Client_Name_vod__c,Client_OS_vod__c,Client_Type_vod__c,Country_vod__c,Device_Type_vod__c,Event_Msg_vod__c,Event_type_vod__c,IP_Address_vod__c,Region_vod__c,User_Agent_vod__c,Vault_Doc_ID_vod__c,Vault_Doc_Name_vod__c,Vault_Document_Major_Version_vod__c,Vault_Document_Minor_Version_vod__c,Vault_Document_Number_vod__c,Vault_Document_Title_vod__c,Vault_Instance_ID_vod__c,Preference_Modification_vod__c,Approved_Document_vod__c,Link_Name_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,sent_email_vod__c,activity_date_time_vod__c,city_vod__c,click_url_vod__c,client_name_vod__c,client_os_vod__c,client_type_vod__c,country_vod__c,device_type_vod__c,event_msg_vod__c,event_type_vod__c,ip_address_vod__c,region_vod__c,user_agent_vod__c,vault_doc_id_vod__c,vault_doc_name_vod__c,vault_document_major_version_vod__c,vault_document_minor_version_vod__c,vault_document_number_vod__c,vault_document_title_vod__c,vault_instance_id_vod__c,preference_modification_vod__c,approved_document_vod__c,link_name_vod__c +src02.crm_email_activity_vod__c +org02.crm_email_activity_vod__c + + diff --git a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt new file mode 100644 index 00000000..d5d6c0dc --- /dev/null +++ b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +31 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c +src02.crm_event_attendee_vod__c +org02.crm_event_attendee_vod__c + + diff --git a/s3/data/crm/settings/CRM_Group.txt b/s3/data/crm/settings/CRM_Group.txt new file mode 100644 index 00000000..94f4d0ca --- /dev/null +++ b/s3/data/crm/settings/CRM_Group.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +14 +Id,Name,DeveloperName,RelatedId,Type,Email,OwnerId,DoesSendEmailToMembers,DoesIncludeBosses,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp +id,name,developer_name,related_id,type,email,owner_id,does_send_email_to_members,does_include_bosses,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp +src02.crm_group +org02.crm_group +CRM_Group_ex.sql + diff --git a/s3/data/crm/settings/CRM_Group_ex.sql b/s3/data/crm/settings/CRM_Group_ex.sql new file mode 100644 index 00000000..5e50dab6 --- /dev/null +++ b/s3/data/crm/settings/CRM_Group_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_group', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Key_Message_vod__c.txt b/s3/data/crm/settings/CRM_Key_Message_vod__c.txt new file mode 100644 index 00000000..1c1f1df0 --- /dev/null +++ b/s3/data/crm/settings/CRM_Key_Message_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +48 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Description_vod__c,Product_vod__c,Product_Strategy_vod__c,Display_Order_vod__c,Active_vod__c,Category_vod__c,Vehicle_vod__c,CLM_ID_vod__c,Custom_Reaction_vod__c,Slide_Version_vod__c,Language_vod__c,Media_File_CRC_vod__c,Media_File_Name_vod__c,Media_File_Size_vod__c,Segment_vod__c,Disable_Actions_vod__c,VExternal_Id_vod__c,CDN_Path_vod__c,Status_vod__c,Vault_DNS_vod__c,Vault_Doc_Id_vod__c,Vault_External_Id_vod__c,Vault_GUID_vod__c,Vault_Last_Modified_Date_Time_vod__c,Is_Shared_Resource_vod__c,iOS_Viewer_vod__c,iOS_Resolution_vod__c,approved_for_distribution_date__c,approved_for_use_date__c,ewizard_link__c,expiration_date__c,keywords__c,trade_team__c,business_function__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,description_vod__c,product_vod__c,product_strategy_vod__c,display_order_vod__c,active_vod__c,category_vod__c,vehicle_vod__c,clm_id_vod__c,custom_reaction_vod__c,slide_version_vod__c,language_vod__c,media_file_crc_vod__c,media_file_name_vod__c,media_file_size_vod__c,segment_vod__c,disable_actions_vod__c,vexternal_id_vod__c,cdn_path_vod__c,status_vod__c,vault_dns_vod__c,vault_doc_id_vod__c,vault_external_id_vod__c,vault_guid_vod__c,vault_last_modified_date_time_vod__c,is_shared_resource_vod__c,i_os_viewer_vod__c,i_os_resolution_vod__c,approved_for_distribution_date__c,approved_for_use_date__c,ewizard_link__c,expiration_date__c,keywords__c,trade_team__c,business_function__c +src02.crm_key_message_vod__c +org02.crm_key_message_vod__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Hospital_Medical_Regimen__c.txt b/s3/data/crm/settings/CRM_MSJ_Hospital_Medical_Regimen__c.txt new file mode 100644 index 00000000..c295fc5e --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Hospital_Medical_Regimen__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +18 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,MSJ_Account_Name__c,MSJ_Delete_Date__c,MSJ_Delete_Flag__c,MSJ_Indication__c,MSJ_Line__c,MSJ_Medical_Regimen__c,Mobile_ID_vod__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,msj_account_name__c,msj_delete_date__c,msj_delete_flag__c,msj_indication__c,msj_line__c,msj_medical_regimen__c,mobile_id_vod__c +src02.crm_msj_hospital_medical_regimen__c +org02.crm_msj_hospital_medical_regimen__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Inquiry_Assignment__c.txt b/s3/data/crm/settings/CRM_MSJ_Inquiry_Assignment__c.txt new file mode 100644 index 00000000..a50bf1f3 --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Inquiry_Assignment__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +47 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,MSJ_Medical_Inquiry__c,MSJ_Close__c,MSJ_Doctor_Name__c,MSJ_Hospital_Name__c,MSJ_Indication__c,MSJ_Inquiry_Text__c,MSJ_MEC_User__c,MSJ_MSL_Manager__c,MSJ_MSL_User__c,MSJ_Notice_to_MR__c,MSJ_Product_for_MEC__c,MSJ_Product_for_MR__c,MSJ_Reply_Date__c,MSJ_Reply__c,MSJ_AE_Infomation__c,MSJ_Cancel__c,MSJ_FAQ_number_c__c,MSJ_Return_Call__c,MSJ_Inquiry_Origin__c,First_Response__c,Inquiry_Created_Date__c,Inquiry_Type_1__c,Inquiry_Type_2__c,MSJ_First_User__c,MSJ_MEC_Comment__c,MSJ_Send_Email__c,MSJ_Temp_Aggregated_Info__c,MSJ_AE_Report__c,MSJ_Background__c,MSJ_Inquiry_Date__c,MSJ_MSL_Support__c,MSJ_Handover_Comment__c,MSJ_Handover_Email__c,MSJ_Material_Requirement__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,msj_medical_inquiry__c,msj_close__c,msj_doctor_name__c,msj_hospital_name__c,msj_indication__c,msj_inquiry_text__c,msj_mec_user__c,msj_msl_manager__c,msj_msl_user__c,msj_notice_to_mr__c,msj_product_for_mec__c,msj_product_for_mr__c,msj_reply_date__c,msj_reply__c,msj_ae_infomation__c,msj_cancel__c,msj_faq_number_c__c,msj_return_call__c,msj_inquiry_origin__c,first_response__c,inquiry_created_date__c,inquiry_type_1__c,inquiry_type_2__c,msj_first_user__c,msj_mec_comment__c,msj_send_email__c,msj_temp_aggregated_info__c,msj_ae_report__c,msj_background__c,msj_inquiry_date__c,msj_msl_support__c,msj_handover_comment__c,msj_handover_email__c,msj_material_requirement__c +src02.crm_msj_inquiry_assignment__c +org02.crm_msj_inquiry_assignment__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_MR_Weekly_Report__c.txt b/s3/data/crm/settings/CRM_MSJ_MR_Weekly_Report__c.txt new file mode 100644 index 00000000..89df0b85 --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_MR_Weekly_Report__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +35 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,MSJ_Account_Name__c,MSJ_Activity_Results_Summary__c,MSJ_MUID__c,MSJ_Next_Week_Action_What__c,MSJ_Next_Week_Action_When__c,MSJ_Next_Week_Action_Where__c,MSJ_Next_Week_Action_Who__c,MSJ_Report_Week__c,MSJ_Target_Patient_Count__c,Mobile_ID_vod__c,MSJ_Activity_Results_Summary_HN__c,MSJ_Next_Week_Action_Where_HN__c,MSJ_Next_Week_Action_Who_HN__c,MSJ_Next_Week_Action_What_HN__c,MSJ_Next_Week_Action_When_HN__c,MSJ_Target_Patient_Count_HN__c,MSJ_Activity_Results_Summary_MCC__c,MSJ_Next_Week_Action_Where_MCC__c,MSJ_Next_Week_Action_Who_MCC__c,MSJ_Next_Week_Action_What_MCC__c,MSJ_Next_Week_Action_When_MCC__c,MSJ_Target_Patient_Count_MCC__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,msj_account_name__c,msj_activity_results_summary__c,msj_muid__c,msj_next_week_action_what__c,msj_next_week_action_when__c,msj_next_week_action_where__c,msj_next_week_action_who__c,msj_report_week__c,msj_target_patient_count__c,mobile_id_vod__c,msj_activity_results_summary_hn__c,msj_next_week_action_where_hn__c,msj_next_week_action_who_hn__c,msj_next_week_action_what_hn__c,msj_next_week_action_when_hn__c,msj_target_patient_count_hn__c,msj_activity_results_summary_mcc__c,msj_next_week_action_where_mcc__c,msj_next_week_action_who_mcc__c,msj_next_week_action_what_mcc__c,msj_next_week_action_when_mcc__c,msj_target_patient_count_mcc__c +src02.crm_msj_mr_weekly_report__c +org02.crm_msj_mr_weekly_report__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Medical_Event_Evaluation__c.txt b/s3/data/crm/settings/CRM_MSJ_Medical_Event_Evaluation__c.txt new file mode 100644 index 00000000..5f1546f2 --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Medical_Event_Evaluation__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +14 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,MSJ_Medical_Event__c,MSJ_Evaluation_Comment__c,MSJ_Evaluation__c,Mobile_ID_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,msj_medical_event__c,msj_evaluation_comment__c,msj_evaluation__c,mobile_id_vod__c +src02.crm_msj_medical_event_evaluation__c +org02.crm_msj_medical_event_evaluation__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Medical_Regimen__c.txt b/s3/data/crm/settings/CRM_MSJ_Medical_Regimen__c.txt new file mode 100644 index 00000000..321b8a5e --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Medical_Regimen__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +17 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,MSJ_Delete_Date__c,MSJ_Delete_Flag__c,MSJ_Indication__c,MSJ_Remark__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,msj_delete_date__c,msj_delete_flag__c,msj_indication__c,msj_remark__c +src02.crm_msj_medical_regimen__c +org02.crm_msj_medical_regimen__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Patient__c.txt b/s3/data/crm/settings/CRM_MSJ_Patient__c.txt new file mode 100644 index 00000000..8b1bac96 --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Patient__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +37 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,MSJ_Account_Name__c,MSJ_CRC_Group__c,MSJ_Casus_or_Transfer_Point__c,MSJ_Entry_Date__c,MSJ_IST_Name__c,MSJ_Indication__c,MSJ_Line__c,MSJ_MR_Comments__c,MSJ_MUID__c,MSJ_Medical_Regimen__c,MSJ_Month__c,MSJ_Report_Comments__c,MSJ_Start_Date_Of_Administration__c,MSJ_Year__c,Mobile_ID_vod__c,MSJ_CRC_RAS_KRAS__c,MSJ_End_Date_Of_Administration__c,MSJ_End_Date_of_Stop_Administration__c,MSJ_HN_Hospitalized_Type__c,MSJ_Start_Date_of_Stop_Administration__c,MSJ_Patient_Status__c,MSJ_Patient_TA__c,MSJ_Child_Account_Name__c,MSJ_Child_Account__c,MSJ_Parent_Account_Name__c,MSJ_Parent_Child_Name__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,msj_account_name__c,msj_crc_group__c,msj_casus_or_transfer_point__c,msj_entry_date__c,msj_ist_name__c,msj_indication__c,msj_line__c,msj_mr_comments__c,msj_muid__c,msj_medical_regimen__c,msj_month__c,msj_report_comments__c,msj_start_date_of_administration__c,msj_year__c,mobile_id_vod__c,msj_crc_ras_kras__c,msj_end_date_of_administration__c,msj_end_date_of_stop_administration__c,msj_hn_hospitalized_type__c,msj_start_date_of_stop_administration__c,msj_patient_status__c,msj_patient_ta__c,msj_child_account_name__c,msj_child_account__c,msj_parent_account_name__c,msj_parent_child_name__c +src02.crm_msj_patient__c +org02.crm_msj_patient__c + + diff --git a/s3/data/crm/settings/CRM_Medical_Event_vod__c.txt b/s3/data/crm/settings/CRM_Medical_Event_vod__c.txt new file mode 100644 index 00000000..159a475b --- /dev/null +++ b/s3/data/crm/settings/CRM_Medical_Event_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +52 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Primary_Product__c,Description_vod__c,Start_Date_vod__c,Location__c,End_Date_vod__c,Secondary_Product__c,Website__c,Active_vod__c,Event_Type__c,MSJ_Area__c,MSJ_Business_Unit__c,MSJ_Comment__c,MSJ_Company__c,MSJ_Expense_App_No__c,MSJ_Form__c,MSJ_HQ_Area__c,MSJ_Location__c,MSJ_MR__c,MSJ_Number_of_Attendee__c,MSJ_Product__c,MSJ_Site__c,MSJ_Type__c,MSJ_Number_of_Attendee_Auto_Calc__c,MSJ_Number_of_Attendee_Invited__c,Account_vod__c,MSJ_MUID__c,Country_Name_vod__c,MSJ_CE_SIPAGL_Updater__c,MSJ_CE_SIPAGL_1A__c,MSJ_CE_SIPAGL_1B__c,MSJ_CE_SIPAGL_2__c,MSJ_CE_SIPAGL_3__c,MSJ_CE_SIPAGL_4__c,MSJ_CE_SIPAGL_5__c,MSJ_CE_SIPAGL_Comment__c,MSJ_CE_SIPAGL_1A_date__c,MSJ_CE_SIPAGL_6__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,primary_product__c,description_vod__c,start_date_vod__c,location__c,end_date_vod__c,secondary_product__c,website__c,active_vod__c,event_type__c,msj_area__c,msj_business_unit__c,msj_comment__c,msj_company__c,msj_expense_app_no__c,msj_form__c,msj_hq_area__c,msj_location__c,msj_mr__c,msj_number_of_attendee__c,msj_product__c,msj_site__c,msj_type__c,msj_number_of_attendee_auto_calc__c,msj_number_of_attendee_invited__c,account_vod__c,msj_muid__c,country_name_vod__c,msj_ce_sipagl_updater__c,msj_ce_sipagl_1_a__c,msj_ce_sipagl_1_b__c,msj_ce_sipagl_2__c,msj_ce_sipagl_3__c,msj_ce_sipagl_4__c,msj_ce_sipagl_5__c,msj_ce_sipagl_comment__c,msj_ce_sipagl_1_a_date__c,msj_ce_sipagl_6__c +src02.crm_medical_event_vod__c +org02.crm_medical_event_vod__c + + diff --git a/s3/data/crm/settings/CRM_Medical_Inquiry_vod__c.txt b/s3/data/crm/settings/CRM_Medical_Inquiry_vod__c.txt new file mode 100644 index 00000000..cafca6d9 --- /dev/null +++ b/s3/data/crm/settings/CRM_Medical_Inquiry_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +67 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,Address_Line_1_vod__c,Address_Line_2_vod__c,City_vod__c,Delivery_Method_vod__c,Email_vod__c,Fax_Number_vod__c,Inquiry_Text__c,Lock_vod__c,Mobile_ID_vod__c,Phone_Number_vod__c,Product__c,Rush_Delivery__c,Signature_Date_vod__c,Signature_vod__c,State_vod__c,Status_vod__c,Zip_vod__c,zvod_Delivery_Method_vod__c,zvod_Disclaimer_vod__c,Submitted_By_Mobile_vod__c,Disclaimer_vod__c,Entity_Reference_Id_vod__c,Call2_vod__c,Country_vod__c,Override_Lock_vod__c,MSJ_Department__c,MSJ_Doctor_Name__c,MSJ_Hospital_Name__c,MSJ_Indication__c,MSJ_Inquiry_Assignment__c,MSJ_Inquiry_Date__c,MSJ_Inquiry_Input_Manager__c,MSJ_Inquiry_Input_User__c,MSJ_MSL_Manager__c,MSJ_Notice_to_MR__c,MSJ_Person_in_charge_1__c,MSJ_Person_in_charge_2__c,MSJ_Product_for_MEC__c,MSJ_Product_for_MR__c,MSJ_Reply_Date__c,MSJ_Reply_User__c,MSJ_Reply__c,MSJ_Title__c,MSJ_AE_Infomation__c,MSJ_FAQ_Number_Report__c,MSJ_Return_Call_Report__c,MSJ_Inquiry_Origin_Report__c,MSJ_AE_Report__c,MSJ_Background__c,MSJ_MSL_Support__c,MSJ_Material_Requirement__c,MSJ_Hospital_Name_Disp__c,MSJ_Hospital__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,address_line_1_vod__c,address_line_2_vod__c,city_vod__c,delivery_method_vod__c,email_vod__c,fax_number_vod__c,inquiry_text__c,lock_vod__c,mobile_id_vod__c,phone_number_vod__c,product__c,rush_delivery__c,signature_date_vod__c,signature_vod__c,state_vod__c,status_vod__c,zip_vod__c,zvod_delivery_method_vod__c,zvod_disclaimer_vod__c,submitted_by_mobile_vod__c,disclaimer_vod__c,entity_reference_id_vod__c,call2_vod__c,country_vod__c,override_lock_vod__c,msj_department__c,msj_doctor_name__c,msj_hospital_name__c,msj_indication__c,msj_inquiry_assignment__c,msj_inquiry_date__c,msj_inquiry_input_manager__c,msj_inquiry_input_user__c,msj_msl_manager__c,msj_notice_to_mr__c,msj_person_in_charge_1__c,msj_person_in_charge_2__c,msj_product_for_mec__c,msj_product_for_mr__c,msj_reply_date__c,msj_reply_user__c,msj_reply__c,msj_title__c,msj_ae_infomation__c,msj_faq_number_report__c,msj_return_call_report__c,msj_inquiry_origin_report__c,msj_ae_report__c,msj_background__c,msj_msl_support__c,msj_material_requirement__c,msj_hospital_name_disp__c,msj_hospital__c +src02.crm_medical_inquiry_vod__c +org02.crm_medical_inquiry_vod__c + + diff --git a/s3/data/crm/settings/CRM_Medical_Insight_vod__c.txt b/s3/data/crm/settings/CRM_Medical_Insight_vod__c.txt new file mode 100644 index 00000000..0c477d5d --- /dev/null +++ b/s3/data/crm/settings/CRM_Medical_Insight_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +54 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,Clinical_Trial_vod__c,Date_vod__c,Description_vod__c,Entity_Reference_Id_vod__c,Interaction_vod__c,Medical_Event_vod__c,Mobile_ID_vod__c,Other_Source_vod__c,Override_Lock_vod__c,Publication_vod__c,Status_vod__c,Summary_vod__c,Unlock_vod__c,Commercial_Medical__c,MSJ_Level_1A__c,MSJ_Level_1B__c,MSJ_Level_2A__c,MSJ_Level_2B__c,MSJ_Level_3A__c,MSJ_Level_3B__c,MSJ_Level_4A__c,MSJ_Level_4B__c,MSJ_SubStatus__c,MSJ_Type_A__c,MSJ_Type_B__c,MSJ_Description_Backup__c,MSJ_Country__c,MSJ_Received_at_Boomi__c,MSJ_Level_1A_Value__c,MSJ_Level_1B_Value__c,MSJ_Level_2A_Value__c,MSJ_Level_2B_Value__c,MSJ_Level_3A_Value__c,MSJ_Level_3B_Value__c,MSJ_Level_4A_Value__c,MSJ_Level_4B_Value__c,MSJ_Hospital_ID__c,MSJ_Hospital_Name__c,MSJ_Hospital__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,clinical_trial_vod__c,date_vod__c,description_vod__c,entity_reference_id_vod__c,interaction_vod__c,medical_event_vod__c,mobile_id_vod__c,other_source_vod__c,override_lock_vod__c,publication_vod__c,status_vod__c,summary_vod__c,unlock_vod__c,commercial_medical__c,msj_level_1_a__c,msj_level_1_b__c,msj_level_2_a__c,msj_level_2_b__c,msj_level_3_a__c,msj_level_3_b__c,msj_level_4_a__c,msj_level_4_b__c,msj_sub_status__c,msj_type_a__c,msj_type_b__c,msj_description_backup__c,msj_country__c,msj_received_at_boomi__c,msj_level_1_a_value__c,msj_level_1_b_value__c,msj_level_2_a_value__c,msj_level_2_b_value__c,msj_level_3_a_value__c,msj_level_3_b_value__c,msj_level_4_a_value__c,msj_level_4_b_value__c,msj_hospital_id__c,msj_hospital_name__c,msj_hospital__c +src02.crm_medical_insight_vod__c +org02.crm_medical_insight_vod__c + + diff --git a/s3/data/crm/settings/CRM_Multichannel_Activity_Line_vod__c.txt b/s3/data/crm/settings/CRM_Multichannel_Activity_Line_vod__c.txt new file mode 100644 index 00000000..1b4ed66a --- /dev/null +++ b/s3/data/crm/settings/CRM_Multichannel_Activity_Line_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +35 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Multichannel_Activity_vod__c,Call_vod__c,Custom_vod__c,DateTime_vod__c,Debug_vod__c,Detail_Group_VExternal_Id_vod__c,Detail_Group_vod__c,Duration_vod__c,Event_Subtype_vod__c,Event_Type_vod__c,Key_Message_VExternal_Id_vod__c,Key_Message_vod__c,Multichannel_Content_Asset_Id_vod__c,Multichannel_Content_Asset_Version_vod__c,Multichannel_Content_Asset_vod__c,Multichannel_Content_vod__c,Product_VExternal_Id_vod__c,Product_vod__c,Sent_Email_vod__c,VExternal_Id_vod__c,Video_Last_Viewed_Time_vod__c,Video_Length_vod__c,Video_Total_Time_Spent_vod__c,View_Order_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,multichannel_activity_vod__c,call_vod__c,custom_vod__c,date_time_vod__c,debug_vod__c,detail_group_vexternal_id_vod__c,detail_group_vod__c,duration_vod__c,event_subtype_vod__c,event_type_vod__c,key_message_vexternal_id_vod__c,key_message_vod__c,multichannel_content_asset_id_vod__c,multichannel_content_asset_version_vod__c,multichannel_content_asset_vod__c,multichannel_content_vod__c,product_vexternal_id_vod__c,product_vod__c,sent_email_vod__c,vexternal_id_vod__c,video_last_viewed_time_vod__c,video_length_vod__c,video_total_time_spent_vod__c,view_order_vod__c +src02.crm_multichannel_activity_line_vod__c +org02.crm_multichannel_activity_line_vod__c + + diff --git a/s3/data/crm/settings/CRM_Multichannel_Activity_vod__c.txt b/s3/data/crm/settings/CRM_Multichannel_Activity_vod__c.txt new file mode 100644 index 00000000..9072d9e4 --- /dev/null +++ b/s3/data/crm/settings/CRM_Multichannel_Activity_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +47 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_External_ID_Map_vod__c,Account_vod__c,Call_vod__c,City_vod__c,Client_Name_vod__c,Client_OS_vod__c,Client_Type_vod__c,Country_vod__c,Debug_vod__c,Device_vod__c,IP_Address_vod__c,Multichannel_Activity_vod__c,Referring_Site_vod__c,Region_vod__c,Sent_Email_vod__c,Session_Id_vod__c,Site_vod__c,Start_DateTime_vod__c,Total_Duration_vod__c,URL_vod__c,User_Agent_vod__c,VExternal_Id_vod__c,Viewport_Height_vod__c,Viewport_Width_vod__c,Color_vod__c,Icon_vod__c,MCD_Primary_Key_vod__c,Record_Type_Name_vod__c,MSJ_Date_Opened__c,MSJ_Sent_Date__c,MSJ_Email_Subject__c,MSJ_Opens__c,MSJ_Email_Status__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_external_id_map_vod__c,account_vod__c,call_vod__c,city_vod__c,client_name_vod__c,client_os_vod__c,client_type_vod__c,country_vod__c,debug_vod__c,device_vod__c,ip_address_vod__c,multichannel_activity_vod__c,referring_site_vod__c,region_vod__c,sent_email_vod__c,session_id_vod__c,site_vod__c,start_date_time_vod__c,total_duration_vod__c,url_vod__c,user_agent_vod__c,vexternal_id_vod__c,viewport_height_vod__c,viewport_width_vod__c,color_vod__c,icon_vod__c,mcd_primary_key_vod__c,record_type_name_vod__c,msj_date_opened__c,msj_sent_date__c,msj_email_subject__c,msj_opens__c,msj_email_status__c +src02.crm_multichannel_activity_vod__c +org02.crm_multichannel_activity_vod__c + + diff --git a/s3/data/crm/settings/CRM_Multichannel_Consent_vod__c.txt b/s3/data/crm/settings/CRM_Multichannel_Consent_vod__c.txt new file mode 100644 index 00000000..be8d8b57 --- /dev/null +++ b/s3/data/crm/settings/CRM_Multichannel_Consent_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +48 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,Capture_Datetime_vod__c,Channel_Value_vod__c,Detail_Group_vod__c,External_ID_vod__c,Last_Device_vod__c,Mobile_ID_vod__c,Opt_Expiration_Date_vod__c,Opt_Type_vod__c,Optout_Event_Type_vod__c,Product_vod__c,Signature_Datetime_vod__c,Signature_ID_vod__c,Signature_vod__c,Sample_Consent_Template_Data_vod__c,Sample_Consent_Template_vod__c,Consent_Line_vod__c,Consent_Type_vod__c,Default_Consent_Text_vod__c,Disclaimer_Text_vod__c,Sub_Channel_Key_vod__c,Consent_Confirm_Datetime_vod__c,Related_Transaction_Id_vod__c,Sent_Email_vod__c,Content_Type_vod__c,Receipt_Email_vod__c,Receipt_Sent_Email_Transaction_Id_vod__c,Receipt_Sent_Email_vod__c,Captured_By_vod__c,Opt_Out_Disclaimer_Text_vod__c,Channel_Source_vod__c,Union_Id_vod__c,User_Last_Notified_vod__c,Sub_Channel_Display_Name__c,MSJ_Consent_Source__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,capture_datetime_vod__c,channel_value_vod__c,detail_group_vod__c,external_id_vod__c,last_device_vod__c,mobile_id_vod__c,opt_expiration_date_vod__c,opt_type_vod__c,optout_event_type_vod__c,product_vod__c,signature_datetime_vod__c,signature_id_vod__c,signature_vod__c,sample_consent_template_data_vod__c,sample_consent_template_vod__c,consent_line_vod__c,consent_type_vod__c,default_consent_text_vod__c,disclaimer_text_vod__c,sub_channel_key_vod__c,consent_confirm_datetime_vod__c,related_transaction_id_vod__c,sent_email_vod__c,content_type_vod__c,receipt_email_vod__c,receipt_sent_email_transaction_id_vod__c,receipt_sent_email_vod__c,captured_by_vod__c,opt_out_disclaimer_text_vod__c,channel_source_vod__c,union_id_vod__c,user_last_notified_vod__c,sub_channel_display_name__c,msj_consent_source__c +src02.crm_multichannel_consent_vod__c +org02.crm_multichannel_consent_vod__c + + diff --git a/s3/data/crm/settings/CRM_My_Setup_Products_vod__c.txt b/s3/data/crm/settings/CRM_My_Setup_Products_vod__c.txt new file mode 100644 index 00000000..47c71f53 --- /dev/null +++ b/s3/data/crm/settings/CRM_My_Setup_Products_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +12 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Product_vod__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,product_vod__c +src02.crm_my_setup_products_vod__c +org02.crm_my_setup_products_vod__c + + diff --git a/s3/data/crm/settings/CRM_ObjectTerritory2Association.txt b/s3/data/crm/settings/CRM_ObjectTerritory2Association.txt new file mode 100644 index 00000000..fff7b033 --- /dev/null +++ b/s3/data/crm/settings/CRM_ObjectTerritory2Association.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +9 +Id,ObjectId,Territory2Id,AssociationCause,SobjectType,IsDeleted,LastModifiedDate,LastModifiedById,SystemModstamp +id,object_id,territory2_id,association_cause,sobject_type,is_deleted,last_modified_date,last_modified_by_id,system_modstamp +src02.crm_object_territory2_association +org02.crm_object_territory2_association +CRM_ObjectTerritory2Association_ex.sql + diff --git a/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql new file mode 100644 index 00000000..d142c071 --- /dev/null +++ b/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_object_territory2_association', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Product_Group_vod__c.txt b/s3/data/crm/settings/CRM_Product_Group_vod__c.txt new file mode 100644 index 00000000..7b6c6cff --- /dev/null +++ b/s3/data/crm/settings/CRM_Product_Group_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +17 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Description_vod__c,Product_vod__c,Product_Catalog_vod__c,Start_Date_vod__c,End_Date_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,description_vod__c,product_vod__c,product_catalog_vod__c,start_date_vod__c,end_date_vod__c +src02.crm_product_group_vod__c +org02.crm_product_group_vod__c + + diff --git a/s3/data/crm/settings/CRM_Product_Metrics_vod__c.txt b/s3/data/crm/settings/CRM_Product_Metrics_vod__c.txt new file mode 100644 index 00000000..4629f0eb --- /dev/null +++ b/s3/data/crm/settings/CRM_Product_Metrics_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +52 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,Awareness__c,Selling_Stage__c,Formulary_Status__c,Movement__c,Products_vod__c,Segment__c,X12_mo_trx_chg__c,Speaker_Skills__c,Investigator_Readiness__c,Engagements__c,Mobile_ID_vod__c,External_ID_vod__c,MSJ_Patient__c,Detail_Group_vod__c,MSJ_EB_1st_Line_Liver_Meta__c,MSJ_EB_1st_Line_Multi_Meta__c,MSJ_EB_2nd_Line_Mono__c,MSJ_EB_2nd_Line_Combination__c,MSJ_EB_3rd_Line_Mono__c,MSJ_EB_3rd_Line_Combination__c,EMDS_Ability__c,EMDS_Brand_Loyalty__c,EMDS_Decision_Maker__c,EMDS_Early_Tech_Adopter__c,EMDS_Influence__c,EMDS_Main_Driver__c,EMDS_Priority__c,EMDS_Willingness__c,MSJ_KTL_Type__c,MSJ_KTL_Tier__c,MSJ_Publications__c,MSJ_Clinical_Trials__c,MSJ_Speaker_for_Medical_Events__c,MSJ_Advisor_to_Medical_Affairs__c,MSJ_Guidelines_Treatment_Standards__c,MSJ_Therapeutic_Area_Expertise__c,MSJ_MAP_GAP__c,MSJ_Associations__c,MSJ_Tier_Score__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,awareness__c,selling_stage__c,formulary_status__c,movement__c,products_vod__c,segment__c,x12_mo_trx_chg__c,speaker_skills__c,investigator_readiness__c,engagements__c,mobile_id_vod__c,external_id_vod__c,msj_patient__c,detail_group_vod__c,msj_eb_1st_line_liver_meta__c,msj_eb_1st_line_multi_meta__c,msj_eb_2nd_line_mono__c,msj_eb_2nd_line_combination__c,msj_eb_3rd_line_mono__c,msj_eb_3rd_line_combination__c,emds_ability__c,emds_brand_loyalty__c,emds_decision_maker__c,emds_early_tech_adopter__c,emds_influence__c,emds_main_driver__c,emds_priority__c,emds_willingness__c,msj_ktl_type__c,msj_ktl_tier__c,msj_publications__c,msj_clinical_trials__c,msj_speaker_for_medical_events__c,msj_advisor_to_medical_affairs__c,msj_guidelines_treatment_standards__c,msj_therapeutic_area_expertise__c,msj_map_gap__c,msj_associations__c,msj_tier_score__c +src02.crm_product_metrics_vod__c +org02.crm_product_metrics_vod__c +CRM_Product_Metrics_vod__c_ex.sql + diff --git a/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql b/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql new file mode 100644 index 00000000..af2e678e --- /dev/null +++ b/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_product_metrics_vod__c', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Product_vod__c.txt b/s3/data/crm/settings/CRM_Product_vod__c.txt new file mode 100644 index 00000000..843cec84 --- /dev/null +++ b/s3/data/crm/settings/CRM_Product_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +46 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consumer_site__c,Product_info__c,Therapeutic_Class_vod__c,Parent_Product_vod__c,Therapeutic_Area_vod__c,Product_Type_vod__c,Require_Key_Message_vod__c,Cost_vod__c,External_ID_vod__c,Manufacturer_vod__c,Company_Product_vod__c,Controlled_Substance_vod__c,Description_vod__c,Sample_Quantity_Picklist_vod__c,Display_Order_vod__c,No_Metrics_vod__c,Distributor_vod__c,Sample_Quantity_Bound_vod__c,Sample_U_M_vod__c,No_Details_vod__c,Quantity_Per_Case_vod__c,Schedule_vod__c,Restricted_vod__c,Pricing_Rule_Quantity_Bound_vod__c,No_Promo_Items_vod__c,User_Aligned_vod__c,Restricted_States_vod__c,Sort_Code_vod__c,No_Cycle_Plans_vod__c,Inventory_Order_UOM_vod__c,Inventory_Quantity_Per_Case_vod__c,VExternal_Id_vod__c,Country__c,MSJ_Product_Classification__c,MSJ_Indication__c,MSJ_Therapeutic_Area__c,MSJ_Global_Brand__c,MSJ_Global_Business_Unit__c,MSJ_Molecules__c,MSJ_SBU__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consumer_site__c,product_info__c,therapeutic_class_vod__c,parent_product_vod__c,therapeutic_area_vod__c,product_type_vod__c,require_key_message_vod__c,cost_vod__c,external_id_vod__c,manufacturer_vod__c,company_product_vod__c,controlled_substance_vod__c,description_vod__c,sample_quantity_picklist_vod__c,display_order_vod__c,no_metrics_vod__c,distributor_vod__c,sample_quantity_bound_vod__c,sample_u_m_vod__c,no_details_vod__c,quantity_per_case_vod__c,schedule_vod__c,restricted_vod__c,pricing_rule_quantity_bound_vod__c,no_promo_items_vod__c,user_aligned_vod__c,restricted_states_vod__c,sort_code_vod__c,no_cycle_plans_vod__c,inventory_order_uom_vod__c,inventory_quantity_per_case_vod__c,vexternal_id_vod__c,country__c,msj_product_classification__c,msj_indication__c,msj_therapeutic_area__c,msj_global_brand__c,msj_global_business_unit__c,msj_molecules__c,msj_sbu__c +src02.crm_product_vod__c +org02.crm_product_vod__c + + diff --git a/s3/data/crm/settings/CRM_Profile.txt b/s3/data/crm/settings/CRM_Profile.txt new file mode 100644 index 00000000..558ce629 --- /dev/null +++ b/s3/data/crm/settings/CRM_Profile.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +237 +Id,Name,PermissionsEmailSingle,PermissionsEmailMass,PermissionsEditTask,PermissionsEditEvent,PermissionsExportReport,PermissionsImportPersonal,PermissionsDataExport,PermissionsManageUsers,PermissionsEditPublicFilters,PermissionsEditPublicTemplates,PermissionsModifyAllData,PermissionsManageCases,PermissionsManageSolutions,PermissionsCustomizeApplication,PermissionsEditReadonlyFields,PermissionsRunReports,PermissionsViewSetup,PermissionsTransferAnyEntity,PermissionsNewReportBuilder,PermissionsManageSelfService,PermissionsManageCssUsers,PermissionsActivateContract,PermissionsApproveContract,PermissionsImportLeads,PermissionsManageLeads,PermissionsTransferAnyLead,PermissionsViewAllData,PermissionsEditPublicDocuments,PermissionsViewEncryptedData,PermissionsEditBrandTemplates,PermissionsEditHtmlTemplates,PermissionsManageTranslation,PermissionsDeleteActivatedContract,PermissionsSendSitRequests,PermissionsApiUserOnly,PermissionsManageRemoteAccess,PermissionsCanUseNewDashboardBuilder,PermissionsManageCategories,PermissionsConvertLeads,PermissionsTestInstanceCreate,PermissionsPasswordNeverExpires,PermissionsUseTeamReassignWizards,PermissionsInstallMultiforce,PermissionsPublishMultiforce,PermissionsEditOppLineItemUnitPrice,PermissionsManageTerritories,PermissionsCreateMultiforce,PermissionsBulkApiHardDelete,PermissionsInboundMigrationToolsUser,PermissionsSolutionImport,PermissionsManageCallCenters,PermissionsManageSynonyms,PermissionsOutboundMigrationToolsUser,PermissionsViewContent,PermissionsManageEmailClientConfig,PermissionsEnableNotifications,PermissionsManageDataIntegrations,PermissionsDistributeFromPersWksp,PermissionsViewDataCategories,PermissionsManageDataCategories,PermissionsAuthorApex,PermissionsManageMobile,PermissionsApiEnabled,PermissionsManageCustomReportTypes,PermissionsEditCaseComments,PermissionsTransferAnyCase,PermissionsContentAdministrator,PermissionsCreateWorkspaces,PermissionsManageContentPermissions,PermissionsManageContentProperties,PermissionsManageContentTypes,PermissionsScheduleJob,PermissionsManageExchangeConfig,PermissionsManageAnalyticSnapshots,PermissionsScheduleReports,PermissionsManageBusinessHourHolidays,PermissionsManageDynamicDashboards,PermissionsManageInteraction,PermissionsViewMyTeamsDashboards,PermissionsResetPasswords,PermissionsFlowUFLRequired,PermissionsActivitiesAccess,PermissionsEmailTemplateManagement,PermissionsEmailAdministration,PermissionsChatterFileLink,PermissionsForceTwoFactor,PermissionsViewEventLogFiles,PermissionsManageNetworks,PermissionsManageAuthProviders,PermissionsRunFlow,PermissionsCreateCustomizeDashboards,PermissionsCreateDashboardFolders,PermissionsViewPublicDashboards,PermissionsManageDashbdsInPubFolders,PermissionsCreateCustomizeReports,PermissionsCreateReportFolders,PermissionsViewPublicReports,PermissionsManageReportsInPubFolders,PermissionsEditMyDashboards,PermissionsEditMyReports,PermissionsViewAllUsers,PermissionsConnectOrgToEnvironmentHub,PermissionsCreateCustomizeFilters,PermissionsContentHubUser,PermissionsGovernNetworks,PermissionsSalesConsole,PermissionsTwoFactorApi,PermissionsDeleteTopics,PermissionsEditTopics,PermissionsCreateTopics,PermissionsAssignTopics,PermissionsIdentityEnabled,PermissionsIdentityConnect,PermissionsContentWorkspaces,PermissionsCustomMobileAppsAccess,PermissionsViewHelpLink,PermissionsManageProfilesPermissionsets,PermissionsAssignPermissionSets,PermissionsManageRoles,PermissionsManageIpAddresses,PermissionsManageSharing,PermissionsManageInternalUsers,PermissionsManagePasswordPolicies,PermissionsManageLoginAccessPolicies,PermissionsManageCustomPermissions,PermissionsStdAutomaticActivityCapture,PermissionsManageTwoFactor,PermissionsDebugApex,PermissionsLightningExperienceUser,PermissionsConfigCustomRecs,PermissionsSubmitMacrosAllowed,PermissionsBulkMacrosAllowed,PermissionsManageSessionPermissionSets,PermissionsCreateAuditFields,PermissionsUpdateWithInactiveOwner,PermissionsManageSandboxes,PermissionsAutomaticActivityCapture,PermissionsImportCustomObjects,PermissionsDelegatedTwoFactor,PermissionsSelectFilesFromSalesforce,PermissionsModerateNetworkUsers,PermissionsMergeTopics,PermissionsSubscribeToLightningReports,PermissionsManagePvtRptsAndDashbds,PermissionsAllowLightningLogin,PermissionsCampaignInfluence2,PermissionsViewDataAssessment,PermissionsCanApproveFeedPost,PermissionsAllowViewEditConvertedLeads,PermissionsShowCompanyNameAsUserBadge,PermissionsAccessCMC,PermissionsViewHealthCheck,PermissionsManageHealthCheck,PermissionsPackaging2,PermissionsManageCertificates,PermissionsCreateReportInLightning,PermissionsPreventClassicExperience,PermissionsListEmailSend,PermissionsChangeDashboardColors,PermissionsManageRecommendationStrategies,PermissionsManagePropositions,PermissionsSubscribeReportRolesGrps,PermissionsSubscribeDashboardRolesGrps,PermissionsUseWebLink,PermissionsHasUnlimitedNBAExecutions,PermissionsViewOnlyEmbeddedAppUser,PermissionsViewAllActivities,PermissionsSubscribeReportToOtherUsers,PermissionsLightningConsoleAllowedForUser,PermissionsSubscribeReportsRunAsUser,PermissionsSubscribeToLightningDashboards,PermissionsSubscribeDashboardToOtherUsers,PermissionsCreateLtngTempInPub,PermissionsTransactionalEmailSend,PermissionsViewPrivateStaticResources,PermissionsCreateLtngTempFolder,PermissionsApexRestServices,PermissionsEnableCommunityAppLauncher,PermissionsGiveRecognitionBadge,PermissionsUseMySearch,PermissionsLtngPromoReserved01UserPerm,PermissionsManageSubscriptions,PermissionsManageSurveys,PermissionsUseAssistantDialog,PermissionsUseQuerySuggestions,PermissionsViewRoles,PermissionsLMOutboundMessagingUserPerm,PermissionsModifyDataClassification,PermissionsPrivacyDataAccess,PermissionsQueryAllFiles,PermissionsModifyMetadata,PermissionsManageCMS,PermissionsSandboxTestingInCommunityApp,PermissionsCanEditPrompts,PermissionsViewUserPII,PermissionsManageHubConnections,PermissionsB2BMarketingAnalyticsUser,PermissionsTraceXdsQueries,PermissionsViewAllCustomSettings,PermissionsViewAllForeignKeyNames,PermissionsHeadlessCMSAccess,PermissionsLMEndMessagingSessionUserPerm,PermissionsConsentApiUpdate,PermissionsAccessContentBuilder,PermissionsAccountSwitcherUser,PermissionsManageC360AConnections,PermissionsManageReleaseUpdates,PermissionsViewAllProfiles,PermissionsSkipIdentityConfirmation,PermissionsSendCustomNotifications,PermissionsPackaging2Delete,PermissionsFSCComprehensiveUserAccess,PermissionsManageTrustMeasures,PermissionsViewTrustMeasures,PermissionsIsotopeCToCUser,PermissionsIsotopeAccess,PermissionsIsotopeLEX,PermissionsQuipMetricsAccess,PermissionsQuipUserEngagementMetrics,PermissionsManageExternalConnections,PermissionsAIViewInsightObjects,PermissionsAICreateInsightObjects,PermissionsNativeWebviewScrolling,PermissionsViewDeveloperName,Type,UserLicenseId,UserType,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,Description,LastViewedDate,LastReferencedDate +id,name,permissions_email_single,permissions_email_mass,permissions_edit_task,permissions_edit_event,permissions_export_report,permissions_import_personal,permissions_data_export,permissions_manage_users,permissions_edit_public_filters,permissions_edit_public_templates,permissions_modify_all_data,permissions_manage_cases,permissions_manage_solutions,permissions_customize_application,permissions_edit_readonly_fields,permissions_run_reports,permissions_view_setup,permissions_transfer_any_entity,permissions_new_report_builder,permissions_manage_self_service,permissions_manage_css_users,permissions_activate_contract,permissions_approve_contract,permissions_import_leads,permissions_manage_leads,permissions_transfer_any_lead,permissions_view_all_data,permissions_edit_public_documents,permissions_view_encrypted_data,permissions_edit_brand_templates,permissions_edit_html_templates,permissions_manage_translation,permissions_delete_activated_contract,permissions_send_sit_requests,permissions_api_user_only,permissions_manage_remote_access,permissions_can_use_new_dashboard_builder,permissions_manage_categories,permissions_convert_leads,permissions_test_instance_create,permissions_password_never_expires,permissions_use_team_reassign_wizards,permissions_install_multiforce,permissions_publish_multiforce,permissions_edit_opp_line_item_unit_price,permissions_manage_territories,permissions_create_multiforce,permissions_bulk_api_hard_delete,permissions_inbound_migration_tools_user,permissions_solution_import,permissions_manage_call_centers,permissions_manage_synonyms,permissions_outbound_migration_tools_user,permissions_view_content,permissions_manage_email_client_config,permissions_enable_notifications,permissions_manage_data_integrations,permissions_distribute_from_pers_wksp,permissions_view_data_categories,permissions_manage_data_categories,permissions_author_apex,permissions_manage_mobile,permissions_api_enabled,permissions_manage_custom_report_types,permissions_edit_case_comments,permissions_transfer_any_case,permissions_content_administrator,permissions_create_workspaces,permissions_manage_content_permissions,permissions_manage_content_properties,permissions_manage_content_types,permissions_schedule_job,permissions_manage_exchange_config,permissions_manage_analytic_snapshots,permissions_schedule_reports,permissions_manage_business_hour_holidays,permissions_manage_dynamic_dashboards,permissions_manage_interaction,permissions_view_my_teams_dashboards,permissions_reset_passwords,permissions_flow_uflrequired,permissions_activities_access,permissions_email_template_management,permissions_email_administration,permissions_chatter_file_link,permissions_force_two_factor,permissions_view_event_log_files,permissions_manage_networks,permissions_manage_auth_providers,permissions_run_flow,permissions_create_customize_dashboards,permissions_create_dashboard_folders,permissions_view_public_dashboards,permissions_manage_dashbds_in_pub_folders,permissions_create_customize_reports,permissions_create_report_folders,permissions_view_public_reports,permissions_manage_reports_in_pub_folders,permissions_edit_my_dashboards,permissions_edit_my_reports,permissions_view_all_users,permissions_connect_org_to_environment_hub,permissions_create_customize_filters,permissions_content_hub_user,permissions_govern_networks,permissions_sales_console,permissions_two_factor_api,permissions_delete_topics,permissions_edit_topics,permissions_create_topics,permissions_assign_topics,permissions_identity_enabled,permissions_identity_connect,permissions_content_workspaces,permissions_custom_mobile_apps_access,permissions_view_help_link,permissions_manage_profiles_permissionsets,permissions_assign_permission_sets,permissions_manage_roles,permissions_manage_ip_addresses,permissions_manage_sharing,permissions_manage_internal_users,permissions_manage_password_policies,permissions_manage_login_access_policies,permissions_manage_custom_permissions,permissions_std_automatic_activity_capture,permissions_manage_two_factor,permissions_debug_apex,permissions_lightning_experience_user,permissions_config_custom_recs,permissions_submit_macros_allowed,permissions_bulk_macros_allowed,permissions_manage_session_permission_sets,permissions_create_audit_fields,permissions_update_with_inactive_owner,permissions_manage_sandboxes,permissions_automatic_activity_capture,permissions_import_custom_objects,permissions_delegated_two_factor,permissions_select_files_from_salesforce,permissions_moderate_network_users,permissions_merge_topics,permissions_subscribe_to_lightning_reports,permissions_manage_pvt_rpts_and_dashbds,permissions_allow_lightning_login,permissions_campaign_influence2,permissions_view_data_assessment,permissions_can_approve_feed_post,permissions_allow_view_edit_converted_leads,permissions_show_company_name_as_user_badge,permissions_access_cmc,permissions_view_health_check,permissions_manage_health_check,permissions_packaging2,permissions_manage_certificates,permissions_create_report_in_lightning,permissions_prevent_classic_experience,permissions_list_email_send,permissions_change_dashboard_colors,permissions_manage_recommendation_strategies,permissions_manage_propositions,permissions_subscribe_report_roles_grps,permissions_subscribe_dashboard_roles_grps,permissions_use_web_link,permissions_has_unlimited_nbaexecutions,permissions_view_only_embedded_app_user,permissions_view_all_activities,permissions_subscribe_report_to_other_users,permissions_lightning_console_allowed_for_user,permissions_subscribe_reports_run_as_user,permissions_subscribe_to_lightning_dashboards,permissions_subscribe_dashboard_to_other_users,permissions_create_ltng_temp_in_pub,permissions_transactional_email_send,permissions_view_private_static_resources,permissions_create_ltng_temp_folder,permissions_apex_rest_services,permissions_enable_community_app_launcher,permissions_give_recognition_badge,permissions_use_my_search,permissions_ltng_promo_reserved01_user_perm,permissions_manage_subscriptions,permissions_manage_surveys,permissions_use_assistant_dialog,permissions_use_query_suggestions,permissions_view_roles,permissions_lmoutbound_messaging_user_perm,permissions_modify_data_classification,permissions_privacy_data_access,permissions_query_all_files,permissions_modify_metadata,permissions_manage_cms,permissions_sandbox_testing_in_community_app,permissions_can_edit_prompts,permissions_view_user_pii,permissions_manage_hub_connections,permissions_b2_bmarketing_analytics_user,permissions_trace_xds_queries,permissions_view_all_custom_settings,permissions_view_all_foreign_key_names,permissions_headless_cmsaccess,permissions_lmend_messaging_session_user_perm,permissions_consent_api_update,permissions_access_content_builder,permissions_account_switcher_user,permissions_manage_c360_aconnections,permissions_manage_release_updates,permissions_view_all_profiles,permissions_skip_identity_confirmation,permissions_send_custom_notifications,permissions_packaging2_delete,permissions_fsccomprehensive_user_access,permissions_manage_trust_measures,permissions_view_trust_measures,permissions_isotope_cto_cuser,permissions_isotope_access,permissions_isotope_lex,permissions_quip_metrics_access,permissions_quip_user_engagement_metrics,permissions_manage_external_connections,permissions_aiview_insight_objects,permissions_aicreate_insight_objects,permissions_native_webview_scrolling,permissions_view_developer_name,type,user_license_id,user_type,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,description,last_viewed_date,last_referenced_date +src02.crm_profile +org02.crm_profile +CRM_Profile_ex.sql + diff --git a/s3/data/crm/settings/CRM_Profile_ex.sql b/s3/data/crm/settings/CRM_Profile_ex.sql new file mode 100644 index 00000000..b858ab02 --- /dev/null +++ b/s3/data/crm/settings/CRM_Profile_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_profile', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Question_Response_vod__c.txt b/s3/data/crm/settings/CRM_Question_Response_vod__c.txt new file mode 100644 index 00000000..a994ab4a --- /dev/null +++ b/s3/data/crm/settings/CRM_Question_Response_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +30 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Survey_Target_vod__c,Answer_Choice_vod__c,Date_vod__c,Datetime_vod__c,External_ID_vod__c,Mobile_ID_vod__c,Number_vod__c,Order_vod__c,Question_Text_vod__c,Required_vod__c,Response_Hash_vod__c,Response_vod__c,Score_vod__c,Survey_Question_vod__c,Text_vod__c,Type_vod__c,Condition_vod__c,Inactive_Condition_vod__c,Source_ID_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,survey_target_vod__c,answer_choice_vod__c,date_vod__c,datetime_vod__c,external_id_vod__c,mobile_id_vod__c,number_vod__c,order_vod__c,question_text_vod__c,required_vod__c,response_hash_vod__c,response_vod__c,score_vod__c,survey_question_vod__c,text_vod__c,type_vod__c,condition_vod__c,inactive_condition_vod__c,source_id_vod__c +src02.crm_question_response_vod__c +org02.crm_question_response_vod__c + + diff --git a/s3/data/crm/settings/CRM_RecordType.txt b/s3/data/crm/settings/CRM_RecordType.txt new file mode 100644 index 00000000..78960693 --- /dev/null +++ b/s3/data/crm/settings/CRM_RecordType.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +14 +Id,Name,DeveloperName,NamespacePrefix,Description,BusinessProcessId,SobjectType,IsActive,IsPersonType,CreatedById,CreatedDate,LastModifiedById,LastModifiedDate,SystemModstamp +id,name,developer_name,namespace_prefix,description,business_process_id,sobject_type,is_active,is_person_type,created_by_id,created_date,last_modified_by_id,last_modified_date,system_modstamp +src02.crm_record_type +org02.crm_record_type + + diff --git a/s3/data/crm/settings/CRM_Remote_Meeting_vod__c.txt b/s3/data/crm/settings/CRM_Remote_Meeting_vod__c.txt new file mode 100644 index 00000000..1b4827c9 --- /dev/null +++ b/s3/data/crm/settings/CRM_Remote_Meeting_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +24 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Meeting_Id_vod__c,Meeting_Name_vod__c,Mobile_ID_vod__c,Scheduled_DateTime_vod__c,Scheduled_vod__c,Attendance_Report_Process_Status_vod__c,Latest_Meeting_Start_Datetime_vod__c,Meeting_Password_vod__c,Meeting_Outcome_Status_vod__c,Allow_for_Joining_via_Zoom_vod__c,Zoom_Join_Token_vod__c,VExternal_Id_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,meeting_id_vod__c,meeting_name_vod__c,mobile_id_vod__c,scheduled_date_time_vod__c,scheduled_vod__c,attendance_report_process_status_vod__c,latest_meeting_start_datetime_vod__c,meeting_password_vod__c,meeting_outcome_status_vod__c,allow_for_joining_via_zoom_vod__c,zoom_join_token_vod__c,vexternal_id_vod__c +src02.crm_remote_meeting_vod__c +org02.crm_remote_meeting_vod__c + + diff --git a/s3/data/crm/settings/CRM_Sent_Email_vod__c.txt b/s3/data/crm/settings/CRM_Sent_Email_vod__c.txt new file mode 100644 index 00000000..148ed757 --- /dev/null +++ b/s3/data/crm/settings/CRM_Sent_Email_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +65 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,Account_Email_vod__c,Account_vod__c,Approved_Email_Template_vod__c,Capture_Datetime_vod__c,Detail_Group_vod__c,Email_Config_Values_vod__c,Email_Content2_vod__c,Email_Content_vod__c,Email_Fragments_vod__c,Email_Sent_Date_vod__c,Failure_Msg_vod__c,Last_Activity_Date_vod__c,Last_Device_vod__c,MC_Capture_Datetime_vod__c,Mobile_ID_vod__c,Opened_vod__c,Product_Display_vod__c,Product_vod__c,Sender_Email_vod__c,Status_vod__c,Valid_Consent_Exists_vod__c,Approved_Document_Views_vod__c,Click_Count_vod__c,Last_Click_Date_vod__c,Last_Open_Date_vod__c,Open_Count_vod__c,Receipt_Entity_Type_vod__c,Receipt_Record_Id_vod__c,Territory_vod__c,Call2_vod__c,Medical_Inquiry_vod__c,Parent_Email_vod__c,Related_Transaction_ID_vod__c,Case_vod__c,Key_Message_vod__c,Suggestion_vod__c,EM_Attendee_vod__c,EM_Event_Speaker_vod__c,EM_Event_Team_Member_vod__c,Event_Attendee_vod__c,Event_vod__c,Medical_Event_vod__c,Scheduled_Send_Datetime_vod__c,User_vod__c,Content_Type_vod__c,Bcc_vod__c,Event_Attendee_Mobile_Id_vod__c,Event_Mobile_Id_vod__c,Activity_Tracking_Mode_vod__c,Email_Source_vod__c,Subject_vod__c,User_Input_Text_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,account_email_vod__c,account_vod__c,approved_email_template_vod__c,capture_datetime_vod__c,detail_group_vod__c,email_config_values_vod__c,email_content2_vod__c,email_content_vod__c,email_fragments_vod__c,email_sent_date_vod__c,failure_msg_vod__c,last_activity_date_vod__c,last_device_vod__c,mc_capture_datetime_vod__c,mobile_id_vod__c,opened_vod__c,product_display_vod__c,product_vod__c,sender_email_vod__c,status_vod__c,valid_consent_exists_vod__c,approved_document_views_vod__c,click_count_vod__c,last_click_date_vod__c,last_open_date_vod__c,open_count_vod__c,receipt_entity_type_vod__c,receipt_record_id_vod__c,territory_vod__c,call2_vod__c,medical_inquiry_vod__c,parent_email_vod__c,related_transaction_id_vod__c,case_vod__c,key_message_vod__c,suggestion_vod__c,em_attendee_vod__c,em_event_speaker_vod__c,em_event_team_member_vod__c,event_attendee_vod__c,event_vod__c,medical_event_vod__c,scheduled_send_datetime_vod__c,user_vod__c,content_type_vod__c,bcc_vod__c,event_attendee_mobile_id_vod__c,event_mobile_id_vod__c,activity_tracking_mode_vod__c,email_source_vod__c,subject_vod__c,user_input_text_vod__c +src02.crm_sent_email_vod__c +org02.crm_sent_email_vod__c + + diff --git a/s3/data/crm/settings/CRM_Sent_Fragment_vod__c.txt b/s3/data/crm/settings/CRM_Sent_Fragment_vod__c.txt new file mode 100644 index 00000000..c93f9f9e --- /dev/null +++ b/s3/data/crm/settings/CRM_Sent_Fragment_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +16 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,Sent_Email_vod__c,Account_vod__c,Email_Template_vod__c,Sent_Fragment_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,sent_email_vod__c,account_vod__c,email_template_vod__c,sent_fragment_vod__c +src02.crm_sent_fragment_vod__c +org02.crm_sent_fragment_vod__c + + diff --git a/s3/data/crm/settings/CRM_Survey_Question_vod__c.txt b/s3/data/crm/settings/CRM_Survey_Question_vod__c.txt new file mode 100644 index 00000000..3a636f0a --- /dev/null +++ b/s3/data/crm/settings/CRM_Survey_Question_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +23 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Survey_vod__c,Answer_Choice_vod__c,External_ID_vod__c,Max_Score_vod__c,Min_Score_vod__c,Order_vod__c,Question_vod__c,Required_vod__c,Text_vod__c,Condition_vod__c,Source_ID_vod__c,MSJ_External_ID__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,survey_vod__c,answer_choice_vod__c,external_id_vod__c,max_score_vod__c,min_score_vod__c,order_vod__c,question_vod__c,required_vod__c,text_vod__c,condition_vod__c,source_id_vod__c,msj_external_id__c +src02.crm_survey_question_vod__c +org02.crm_survey_question_vod__c + + diff --git a/s3/data/crm/settings/CRM_Survey_Target_vod__c.txt b/s3/data/crm/settings/CRM_Survey_Target_vod__c.txt new file mode 100644 index 00000000..f9f80626 --- /dev/null +++ b/s3/data/crm/settings/CRM_Survey_Target_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +39 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_Display_Name_vod__c,Account_vod__c,Channels_vod__c,End_Date_vod__c,Entity_Reference_Id_vod__c,External_ID_vod__c,Language_vod__c,Lock_vod__c,Mobile_ID_vod__c,No_Autoassign_vod__c,Not_Completed_vod__c,Region_vod__c,Segment_vod__c,Start_Date_vod__c,Status_vod__c,Survey_vod__c,Territory_vod__c,zvod_Address_vod__c,zvod_Specialty_vod__c,Score_vod__c,User_vod__c,Child_Account_vod__c,Location_Entity_Reference_Id_vod__c,Location_vod__c,Target_Type_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_display_name_vod__c,account_vod__c,channels_vod__c,end_date_vod__c,entity_reference_id_vod__c,external_id_vod__c,language_vod__c,lock_vod__c,mobile_id_vod__c,no_autoassign_vod__c,not_completed_vod__c,region_vod__c,segment_vod__c,start_date_vod__c,status_vod__c,survey_vod__c,territory_vod__c,zvod_address_vod__c,zvod_specialty_vod__c,score_vod__c,user_vod__c,child_account_vod__c,location_entity_reference_id_vod__c,location_vod__c,target_type_vod__c +src02.crm_survey_target_vod__c +org02.crm_survey_target_vod__c + + diff --git a/s3/data/crm/settings/CRM_Survey_vod__c.txt b/s3/data/crm/settings/CRM_Survey_vod__c.txt new file mode 100644 index 00000000..bfd04a2e --- /dev/null +++ b/s3/data/crm/settings/CRM_Survey_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +37 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Assignment_Type_vod__c,Channels_vod__c,End_Date_vod__c,Expired_vod__c,External_ID_vod__c,Language_vod__c,Lock_vod__c,Open_vod__c,Product_vod__c,Region_vod__c,Segment_vod__c,Start_Date_vod__c,Status_vod__c,Territory_vod__c,zvod_Questions_vod__c,zvod_Segments_vod__c,zvod_Targets_vod__c,Max_Score_vod__c,Min_Score_vod__c,Autotarget_vod__c,Territories_vod__c,Target_Type_vod__c,MSJ_External_ID__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,assignment_type_vod__c,channels_vod__c,end_date_vod__c,expired_vod__c,external_id_vod__c,language_vod__c,lock_vod__c,open_vod__c,product_vod__c,region_vod__c,segment_vod__c,start_date_vod__c,status_vod__c,territory_vod__c,zvod_questions_vod__c,zvod_segments_vod__c,zvod_targets_vod__c,max_score_vod__c,min_score_vod__c,autotarget_vod__c,territories_vod__c,target_type_vod__c,msj_external_id__c +src02.crm_survey_vod__c +org02.crm_survey_vod__c + + diff --git a/s3/data/crm/settings/CRM_Territory2.txt b/s3/data/crm/settings/CRM_Territory2.txt new file mode 100644 index 00000000..5486b657 --- /dev/null +++ b/s3/data/crm/settings/CRM_Territory2.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +15 +Id,Name,Territory2TypeId,Territory2ModelId,ParentTerritory2Id,Description,ForecastUserId,AccountAccessLevel,OpportunityAccessLevel,CaseAccessLevel,ContactAccessLevel,LastModifiedDate,LastModifiedById,SystemModstamp,DeveloperName,MSJ_Territory_Type__c,MSJ_Level__c +id,name,territory2_type_id,territory2_model_id,parent_territory2_id,description,forecast_user_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,last_modified_date,last_modified_by_id,system_modstamp,developer_name,msj_territory_type__c,msj_level__c +src02.crm_territory2 +org02.crm_territory2 +CRM_Territory2_ex.sql + diff --git a/s3/data/crm/settings/CRM_Territory2_ALL.txt b/s3/data/crm/settings/CRM_Territory2_ALL.txt new file mode 100644 index 00000000..525e1796 --- /dev/null +++ b/s3/data/crm/settings/CRM_Territory2_ALL.txt @@ -0,0 +1,14 @@ +CRM +, +utf-8 +" +CRLF +1 +15 +Id,Name,Territory2TypeId,Territory2ModelId,ParentTerritory2Id,Description,ForecastUserId,AccountAccessLevel,OpportunityAccessLevel,CaseAccessLevel,ContactAccessLevel,LastModifiedDate,LastModifiedById,SystemModstamp,DeveloperName +id,name,territory2_type_id,territory2_model_id,parent_territory2_id,description,forecast_user_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,last_modified_date,last_modified_by_id,system_modstamp,developer_name +src02.crm_territory2_all +org02.crm_territory2_all + + +truncate_src_table:src02.crm_territory2_all diff --git a/s3/data/crm/settings/CRM_Territory2_ex.sql b/s3/data/crm/settings/CRM_Territory2_ex.sql new file mode 100644 index 00000000..a30ccd91 --- /dev/null +++ b/s3/data/crm/settings/CRM_Territory2_ex.sql @@ -0,0 +1,2 @@ +CALL crm_data_sync('src02.crm_territory2', 'src02.crm_territory2_all', 'system_modstamp'); +CALL crm_history('src02.crm_territory2', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Time_Off_Territory_vod__c.txt b/s3/data/crm/settings/CRM_Time_Off_Territory_vod__c.txt new file mode 100644 index 00000000..e2b1b9b6 --- /dev/null +++ b/s3/data/crm/settings/CRM_Time_Off_Territory_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +25 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Reason_vod__c,Territory_vod__c,Date_vod__c,Status_vod__c,Time_vod__c,Hours_vod__c,Mobile_ID_vod__c,Hours_off_vod__c,Start_Time_vod__c,MSJ_Day__c,MSJ_Comment__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,reason_vod__c,territory_vod__c,date_vod__c,status_vod__c,time_vod__c,hours_vod__c,mobile_id_vod__c,hours_off_vod__c,start_time_vod__c,msj_day__c,msj_comment__c +src02.crm_time_off_territory_vod__c +org02.crm_time_off_territory_vod__c + + diff --git a/s3/data/crm/settings/CRM_User.txt b/s3/data/crm/settings/CRM_User.txt new file mode 100644 index 00000000..317a5e14 --- /dev/null +++ b/s3/data/crm/settings/CRM_User.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +197 +Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,LastPasswordChangeDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,NumberOfFailedLogins,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesNativeEmailClient,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c +id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,last_password_change_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,number_of_failed_logins,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_native_email_client,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c +src02.crm_user +org02.crm_user + + diff --git a/s3/data/crm/settings/CRM_UserRole.txt b/s3/data/crm/settings/CRM_UserRole.txt new file mode 100644 index 00000000..4b10b6c2 --- /dev/null +++ b/s3/data/crm/settings/CRM_UserRole.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +16 +Id,Name,ParentRoleId,RollupDescription,OpportunityAccessForAccountOwner,CaseAccessForAccountOwner,ContactAccessForAccountOwner,ForecastUserId,MayForecastManagerShare,LastModifiedDate,LastModifiedById,SystemModstamp,DeveloperName,PortalAccountId,PortalType,PortalAccountOwnerId +id,name,parent_role_id,rollup_description,opportunity_access_for_account_owner,case_access_for_account_owner,contact_access_for_account_owner,forecast_user_id,may_forecast_manager_share,last_modified_date,last_modified_by_id,system_modstamp,developer_name,portal_account_id,portal_type,portal_account_owner_id +src02.crm_user_role +org02.crm_user_role + + diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association.txt b/s3/data/crm/settings/CRM_UserTerritory2Association.txt new file mode 100644 index 00000000..039757b9 --- /dev/null +++ b/s3/data/crm/settings/CRM_UserTerritory2Association.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +8 +Id,UserId,Territory2Id,IsActive,RoleInTerritory2,LastModifiedDate,LastModifiedById,SystemModstamp +id,user_id,territory2_id,is_active,role_in_territory2,last_modified_date,last_modified_by_id,system_modstamp +src02.crm_user_territory2_association +org02.crm_user_territory2_association +CRM_UserTerritory2Association_ex.sql + diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt b/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt new file mode 100644 index 00000000..4617829d --- /dev/null +++ b/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt @@ -0,0 +1,14 @@ +CRM +, +utf-8 +" +CRLF +1 +8 +Id,UserId,Territory2Id,IsActive,RoleInTerritory2,LastModifiedDate,LastModifiedById,SystemModstamp +id,user_id,territory2_id,is_active,role_in_territory2,last_modified_date,last_modified_by_id,system_modstamp +src02.crm_user_territory2_association_all +org02.crm_user_territory2_association_all + + +truncate_src_table:src02.crm_user_territory2_association_all diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql new file mode 100644 index 00000000..6a9aaece --- /dev/null +++ b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql @@ -0,0 +1,2 @@ +CALL crm_data_sync('src02.crm_user_territory2_association', 'src02.crm_user_territory2_association_all', 'system_modstamp'); +CALL crm_history('src02.crm_user_territory2_association', 'system_modstamp'); diff --git a/s3/data/crm/settings/configmap.config b/s3/data/crm/settings/configmap.config new file mode 100644 index 00000000..89993b8b --- /dev/null +++ b/s3/data/crm/settings/configmap.config @@ -0,0 +1,58 @@ +/* 【CRMデータ 差分連携】 */ +CRM_Clm_Presentation_vod__c_[0-9]{14}\.(CSV|csv) CRM_Clm_Presentation_vod__c.txt +CRM_Clm_Presentation_Slide_vod__c_[0-9]{14}\.(CSV|csv) CRM_Clm_Presentation_Slide_vod__c.txt +CRM_Medical_Insight_vod__c_[0-9]{14}\.(CSV|csv) CRM_Medical_Insight_vod__c.txt +CRM_MSJ_MR_Weekly_Report__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_MR_Weekly_Report__c.txt +CRM_Account_Territory_Loader_vod__c_[0-9]{14}\.(CSV|csv) CRM_Account_Territory_Loader_vod__c.txt +CRM_Event_Attendee_vod__c_[0-9]{14}\.(CSV|csv) CRM_Event_Attendee_vod__c.txt +CRM_ObjectTerritory2Association_[0-9]{14}\.(CSV|csv) CRM_ObjectTerritory2Association.txt +CRM_Key_Message_vod__c_[0-9]{14}\.(CSV|csv) CRM_Key_Message_vod__c.txt +CRM_Group_[0-9]{14}\.(CSV|csv) CRM_Group.txt +CRM_Medical_Event_vod__c_[0-9]{14}\.(CSV|csv) CRM_Medical_Event_vod__c.txt +CRM_MSJ_Medical_Event_Evaluation__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Medical_Event_Evaluation__c.txt +CRM_Coaching_Report_vod__c_[0-9]{14}\.(CSV|csv) CRM_Coaching_Report_vod__c.txt +CRM_Call2_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call2_vod__c.txt +CRM_Call2_Detail_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call2_Detail_vod__c.txt +CRM_Call2_Key_Message_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call2_Key_Message_vod__c.txt +CRM_Call_Clickstream_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call_Clickstream_vod__c.txt +CRM_Call2_Discussion_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call2_Discussion_vod__c.txt +CRM_Time_Off_Territory_vod__c_[0-9]{14}\.(CSV|csv) CRM_Time_Off_Territory_vod__c.txt +CRM_Dynamic_Attribute_vod__c_[0-9]{14}\.(CSV|csv) CRM_Dynamic_Attribute_vod__c.txt +CRM_Dynamic_Attribute_Configuration_vod__c_[0-9]{14}\.(CSV|csv) CRM_Dynamic_Attribute_Configuration_vod__c.txt +CRM_Territory2_[0-9]{14}\.(CSV|csv) CRM_Territory2.txt +CRM_Profile_[0-9]{14}\.(CSV|csv) CRM_Profile.txt +CRM_My_Setup_Products_vod__c_[0-9]{14}\.(CSV|csv) CRM_My_Setup_Products_vod__c.txt +CRM_Multichannel_Activity_vod__c_[0-9]{14}\.(CSV|csv) CRM_Multichannel_Activity_vod__c.txt +CRM_Multichannel_Activity_Line_vod__c_[0-9]{14}\.(CSV|csv) CRM_Multichannel_Activity_Line_vod__c.txt +CRM_Multichannel_Consent_vod__c_[0-9]{14}\.(CSV|csv) CRM_Multichannel_Consent_vod__c.txt +CRM_Medical_Inquiry_vod__c_[0-9]{14}\.(CSV|csv) CRM_Medical_Inquiry_vod__c.txt +CRM_Email_Activity_vod__c_[0-9]{14}\.(CSV|csv) CRM_Email_Activity_vod__c.txt +CRM_User_[0-9]{14}\.(CSV|csv) CRM_User.txt +CRM_UserTerritory2Association_[0-9]{14}\.(CSV|csv) CRM_UserTerritory2Association.txt +CRM_Remote_Meeting_vod__c_[0-9]{14}\.(CSV|csv) CRM_Remote_Meeting_vod__c.txt +CRM_RecordType_[0-9]{14}\.(CSV|csv) CRM_RecordType.txt +CRM_UserRole_[0-9]{14}\.(CSV|csv) CRM_UserRole.txt +CRM_Account_[0-9]{14}\.(CSV|csv) CRM_Account.txt +CRM_AccountShare_[0-9]{14}\.(CSV|csv) CRM_AccountShare.txt +CRM_Contact_[0-9]{14}\.(CSV|csv) CRM_Contact.txt +CRM_Consent_Type_vod__c_[0-9]{14}\.(CSV|csv) CRM_Consent_Type_vod__c.txt +CRM_Consent_Header_vod__c_[0-9]{14}\.(CSV|csv) CRM_Consent_Header_vod__c.txt +CRM_Consent_Line_vod__c_[0-9]{14}\.(CSV|csv) CRM_Consent_Line_vod__c.txt +CRM_MSJ_Inquiry_Assignment__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Inquiry_Assignment__c.txt +CRM_Approved_Document_vod__c_[0-9]{14}\.(CSV|csv) CRM_Approved_Document_vod__c.txt +CRM_Child_Account_vod__c_[0-9]{14}\.(CSV|csv) CRM_Child_Account_vod__c.txt +CRM_MSJ_Hospital_Medical_Regimen__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Hospital_Medical_Regimen__c.txt +CRM_MSJ_Medical_Regimen__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Medical_Regimen__c.txt +CRM_MSJ_Patient__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Patient__c.txt +CRM_Product_vod__c_[0-9]{14}\.(CSV|csv) CRM_Product_vod__c.txt +CRM_Product_Group_vod__c_[0-9]{14}\.(CSV|csv) CRM_Product_Group_vod__c.txt +CRM_Product_Metrics_vod__c_[0-9]{14}\.(CSV|csv) CRM_Product_Metrics_vod__c.txt +CRM_Survey_vod__c_[0-9]{14}\.(CSV|csv) CRM_Survey_vod__c.txt +CRM_Survey_Target_vod__c_[0-9]{14}\.(CSV|csv) CRM_Survey_Target_vod__c.txt +CRM_Survey_Question_vod__c_[0-9]{14}\.(CSV|csv) CRM_Survey_Question_vod__c.txt +CRM_Question_Response_vod__c_[0-9]{14}\.(CSV|csv) CRM_Question_Response_vod__c.txt +CRM_Sent_Fragment_vod__c_[0-9]{14}\.(CSV|csv) CRM_Sent_Fragment_vod__c.txt +CRM_Sent_Email_vod__c_[0-9]{14}\.(CSV|csv) CRM_Sent_Email_vod__c.txt +/* 【CRMデータ 全件連携】 */ +CRM_Territory2_ALL_[0-9]{14}\.(CSV|csv) CRM_Territory2_ALL.txt +CRM_UserTerritory2Association_ALL_[0-9]{14}\.(CSV|csv) CRM_UserTerritory2Association_ALL.txt From afcca943ca451fc1eaa34c7762d6d2f239d88b76 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 28 Jul 2022 22:43:32 +0900 Subject: [PATCH 045/275] =?UTF-8?q?fix:=20=E3=82=BD=E3=83=BC=E3=82=B9?= =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=89=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E3=81=AE=E8=BF=BD=E5=8A=A0=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/aws/s3.py | 32 ++-- .../src/backup_crm_csv_data_process.py | 27 +--- .../src/backup_crm_data_process.py | 25 +-- .../src/check_object_info_process.py | 26 ++-- ecs/crm-datafetch/src/config/objects.py | 60 +++---- ecs/crm-datafetch/src/controller.py | 59 +++---- .../src/convert_crm_csv_data_process.py | 29 ++-- .../src/converter/convert_factory.py | 29 ++-- ecs/crm-datafetch/src/converter/converter.py | 3 +- .../src/copy_crm_csv_data_process.py | 25 +-- .../src/fetch_crm_data_process.py | 146 +++++++++--------- ecs/crm-datafetch/src/parser/json_parse.py | 2 +- .../src/prepare_data_fetch_process.py | 36 ++--- .../src/salesforce/salesforce_api.py | 12 +- .../src/set_datetime_period_process.py | 36 ++--- .../src/system_var/environments.py | 30 ++-- .../src/upload_last_fetch_datetime_process.py | 24 +-- .../src/upload_result_data_process.py | 22 +-- ecs/crm-datafetch/src/util/counter_object.py | 14 ++ ecs/crm-datafetch/src/util/dict_checker.py | 2 +- .../src/util/execute_datetime.py | 5 +- ecs/crm-datafetch/src/util/logger.py | 2 +- .../crm/last_fetch_datetime/Account.json | 4 + .../crm/last_fetch_datetime/AccountShare.json | 4 + .../Account_Territory_Loader_vod__c.json | 4 + .../Approved_Document_vod__c.json | 4 + .../Call2_Detail_vod__c.json | 4 + .../Call2_Discussion_vod__c.json | 4 + .../Call2_Key_Message_vod__c.json | 4 + .../crm/last_fetch_datetime/Call2_vod__c.json | 4 + .../Call_Clickstream_vod__c.json | 4 + .../Child_Account_vod__c.json | 4 + .../Clm_Presentation_Slide_vod__c.json | 4 + .../Clm_Presentation_vod__c.json | 4 + .../Coaching_Report_vod__c.json | 4 + .../Consent_Header_vod__c.json | 4 + .../Consent_Line_vod__c.json | 4 + .../Consent_Type_vod__c.json | 4 + .../crm/last_fetch_datetime/Contact.json | 4 + ...ynamic_Attribute_Configuration_vod__c.json | 4 + .../Dynamic_Attribute_vod__c.json | 4 + .../Email_Activity_vod__c.json | 4 + .../Event_Attendee_vod__c.json | 4 + s3/config/crm/last_fetch_datetime/Group.json | 4 + .../Key_Message_vod__c.json | 4 + .../MSJ_Hospital_Medical_Regimen__c.json | 4 + .../MSJ_Inquiry_Assignment__c.json | 4 + .../MSJ_MR_Weekly_Report__c.json | 4 + .../MSJ_Medical_Event_Evaluation__c.json | 4 + .../MSJ_Medical_Regimen__c.json | 4 + .../last_fetch_datetime/MSJ_Patient__c.json | 4 + .../Medical_Event_vod__c.json | 4 + .../Medical_Inquiry_vod__c.json | 4 + .../Medical_Insight_vod__c.json | 4 + .../Multichannel_Activity_Line_vod__c.json | 4 + .../Multichannel_Activity_vod__c.json | 4 + .../Multichannel_Consent_vod__c.json | 4 + .../My_Setup_Products_vod__c.json | 4 + .../ObjectTerritory2Association.json | 4 + .../Product_Group_vod__c.json | 4 + .../Product_Metrics_vod__c.json | 4 + .../last_fetch_datetime/Product_vod__c.json | 4 + .../crm/last_fetch_datetime/Profile.json | 4 + .../Question_Response_vod__c.json | 4 + .../crm/last_fetch_datetime/RecordType.json | 4 + .../Remote_Meeting_vod__c.json | 4 + .../Sent_Email_vod__c.json | 4 + .../Sent_Fragment_vod__c.json | 4 + .../Survey_Question_vod__c.json | 4 + .../Survey_Target_vod__c.json | 4 + .../last_fetch_datetime/Survey_vod__c.json | 4 + .../crm/last_fetch_datetime/Territory2.json | 4 + .../Time_Off_Territory_vod__c.json | 4 + s3/config/crm/last_fetch_datetime/User.json | 4 + .../crm/last_fetch_datetime/UserRole.json | 4 + .../UserTerritory2Association.json | 4 + .../crm/object_info/crm_object_list_all.json | 4 +- .../crm/object_info/crm_object_list_diff.json | 3 +- 78 files changed, 491 insertions(+), 378 deletions(-) create mode 100644 ecs/crm-datafetch/src/util/counter_object.py create mode 100644 s3/config/crm/last_fetch_datetime/Account.json create mode 100644 s3/config/crm/last_fetch_datetime/AccountShare.json create mode 100644 s3/config/crm/last_fetch_datetime/Account_Territory_Loader_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Approved_Document_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Call2_Detail_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Call2_Discussion_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Call2_Key_Message_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Call2_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Call_Clickstream_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Child_Account_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Clm_Presentation_Slide_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Clm_Presentation_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Coaching_Report_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Consent_Header_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Consent_Line_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Consent_Type_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Contact.json create mode 100644 s3/config/crm/last_fetch_datetime/Dynamic_Attribute_Configuration_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Dynamic_Attribute_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Email_Activity_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Event_Attendee_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Group.json create mode 100644 s3/config/crm/last_fetch_datetime/Key_Message_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/MSJ_Hospital_Medical_Regimen__c.json create mode 100644 s3/config/crm/last_fetch_datetime/MSJ_Inquiry_Assignment__c.json create mode 100644 s3/config/crm/last_fetch_datetime/MSJ_MR_Weekly_Report__c.json create mode 100644 s3/config/crm/last_fetch_datetime/MSJ_Medical_Event_Evaluation__c.json create mode 100644 s3/config/crm/last_fetch_datetime/MSJ_Medical_Regimen__c.json create mode 100644 s3/config/crm/last_fetch_datetime/MSJ_Patient__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Medical_Inquiry_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Medical_Insight_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Multichannel_Activity_Line_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Multichannel_Activity_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Multichannel_Consent_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/My_Setup_Products_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/ObjectTerritory2Association.json create mode 100644 s3/config/crm/last_fetch_datetime/Product_Group_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Product_Metrics_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Product_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Profile.json create mode 100644 s3/config/crm/last_fetch_datetime/Question_Response_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/RecordType.json create mode 100644 s3/config/crm/last_fetch_datetime/Remote_Meeting_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Sent_Email_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Sent_Fragment_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Survey_Question_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Survey_Target_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Survey_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Territory2.json create mode 100644 s3/config/crm/last_fetch_datetime/Time_Off_Territory_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/User.json create mode 100644 s3/config/crm/last_fetch_datetime/UserRole.json create mode 100644 s3/config/crm/last_fetch_datetime/UserTerritory2Association.json diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index e1d0dc70..d5bd03f0 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -1,13 +1,17 @@ import json import boto3 -from src.system_var.constants import AWS_RESOURCE_S3, S3_RESPONSE_BODY, S3_CHAR_CODE +from src.system_var.constants import (AWS_RESOURCE_S3, S3_CHAR_CODE, + S3_RESPONSE_BODY) from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, - CRM_IMPORT_DATA_BACKUP_FOLDER, - CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET, - LAST_FETCH_DATE_FOLDER, OBJECT_INFO_FILENAME, - OBJECT_INFO_FOLDER, PROCESS_RESULT_FOLDER, - RESPONSE_JSON_BACKUP_FOLDER) + CRM_IMPORT_DATA_BACKUP_FOLDER, + CRM_IMPORT_DATA_FOLDER, + IMPORT_DATA_BUCKET, + LAST_FETCH_DATE_FOLDER, + OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER, + PROCESS_RESULT_FOLDER, + RESPONSE_JSON_BACKUP_FOLDER) class S3Resource: @@ -31,16 +35,6 @@ class S3Resource: return -#class S3ResourceNonBucket: -# def __init__(self) -> None: -# self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) -# -# def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: -# copy_source = {'Bucket': src_bucket, 'Key': src_key} -# self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) -# return - - class ConfigBucket: __s3_resource: S3Resource = None @@ -77,11 +71,11 @@ class DataBucket: return def put_csv_from(self, src_bucket: str, src_key: str): - self.__s3_resource.copy(src_bucket, src_key, str(self), CRM_IMPORT_DATA_FOLDER) + dest_filename = src_key.split('/')[-1] + self.__s3_resource.copy(src_bucket, src_key, str(self), f'{CRM_IMPORT_DATA_FOLDER}/{dest_filename}') return - class BackupBucket: __s3_resource: S3Resource = None @@ -104,4 +98,4 @@ class BackupBucket: def put_result_json(self, file_path: str, data: dict) -> None: object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, json.dumps(data)) - return \ No newline at end of file + return diff --git a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py index f14490cf..d40c7a13 100644 --- a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py @@ -1,32 +1,21 @@ from src.aws.s3 import BackupBucket from src.config.objects import TargetObject -from src.system_var.constants import CSVBK_JP_NAME from src.error.exceptions import FileUploadException +from src.system_var.constants import CSVBK_JP_NAME from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger def backup_crm_csv_data_process(target_object: TargetObject, exetute_datetime: ExecuteDateTime, csv_string: str): - """ - CSVバックアップ処理 + """CSVバックアップ処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス - csv_string : str - csvデータ + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + exetute_datetime (ExecuteDateTime): 実行日次取得インスタンス + csv_string (str): csvデータ - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① CSVバックアップ処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/backup_crm_data_process.py b/ecs/crm-datafetch/src/backup_crm_data_process.py index a37eb583..0375d927 100644 --- a/ecs/crm-datafetch/src/backup_crm_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_data_process.py @@ -6,26 +6,15 @@ from src.util.logger import logger_instance as logger def backup_crm_data_process(object_name: str, sf_object_dict: dict, execute_datetime: ExecuteDateTime): - """ - CRM電文データバックアップ処理 + """CRM電文データバックアップ処理 - Parameters - ---------- - object_name : str - 取得対象オブジェクト情報インスタンス - sf_object_dict : dict - Salesforceオブジェクトデータ - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス + Args: + object_name (str): 取得対象オブジェクト名 + sf_object_dict (dict): Salesforceオブジェクトデータ + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① CRM電文データバックアップ処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/check_object_info_process.py b/ecs/crm-datafetch/src/check_object_info_process.py index ed2a34da..84f0db40 100644 --- a/ecs/crm-datafetch/src/check_object_info_process.py +++ b/ecs/crm-datafetch/src/check_object_info_process.py @@ -1,30 +1,22 @@ from src.config.objects import TargetObject from src.error.exceptions import InvalidConfigException -from src.util.execute_datetime import ExecuteDateTime from src.system_var.constants import CHK_JP_NAME +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger def check_object_info_process(object_info: dict, execute_datetime: ExecuteDateTime): - """ - オブジェクト情報形式チェック処理 + """オブジェクト情報形式チェック処理 - Parameters - ---------- - object_info : dict - 取得対象オブジェクト情報 - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス + Args: + object_info (dict): 取得対象オブジェクト情報 + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス - Returns - ------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス + Raises: + InvalidConfigException: オブジェクト情報定義が不正だった場合 - Raises - ------ - InvalidConfigException - オブジェクト情報定義が不正だった場合 + Returns: + target_object: 取得対象オブジェクト情報インスタンス """ # ① オブジェクト情報形式チェック処理開始ログを出力する diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 464f7b0e..3ac08851 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,25 +1,19 @@ -from src.system_var.constants import ( DATE_PATTERN_YYYYMMDDTHHMMSSTZ, - OBJECTS_KEY, - OBJECTS_TYPE, - OBJECT_NAME_KEY, - OBJECT_NAME_TYPE, - COLUMNS_KEY, - COLUMNS_TYPE, - IS_SKIP_KEY, - IS_SKIP_TYPE, - IS_UPDATE_LAST_FETCH_DATETIME_KEY, - IS_UPDATE_LAST_FETCH_DATETIME_TYPE, - LAST_FETCH_DATETIME_FILE_NAME_KEY, - LAST_FETCH_DATETIME_FILE_NAME_TYPE, - UPLOAD_FILE_NAME_KEY, - UPLOAD_FILE_NAME_TYPE, - DATETIME_COLUMN_KEY, - DATETIME_COLUMN_TYPE, - LAST_FETCH_DATETIME_FROM_KEY, - LAST_FETCH_DATETIME_TO_KEY, - DATETIME_COLUMN_DEFAULT_VALUE -) - +from src.system_var.constants import (COLUMNS_KEY, COLUMNS_TYPE, + DATE_PATTERN_YYYYMMDDTHHMMSSTZ, + DATETIME_COLUMN_DEFAULT_VALUE, + DATETIME_COLUMN_KEY, + DATETIME_COLUMN_TYPE, IS_SKIP_KEY, + IS_SKIP_TYPE, + IS_UPDATE_LAST_FETCH_DATETIME_KEY, + IS_UPDATE_LAST_FETCH_DATETIME_TYPE, + LAST_FETCH_DATETIME_FILE_NAME_KEY, + LAST_FETCH_DATETIME_FILE_NAME_TYPE, + LAST_FETCH_DATETIME_FROM_KEY, + LAST_FETCH_DATETIME_TO_KEY, + OBJECT_NAME_KEY, OBJECT_NAME_TYPE, + OBJECTS_KEY, OBJECTS_TYPE, + UPLOAD_FILE_NAME_KEY, + UPLOAD_FILE_NAME_TYPE) from src.util.dict_checker import DictChecker @@ -98,16 +92,24 @@ class TargetObject(): @property def is_update_last_fetch_datetime(self) -> bool: - return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY) else False + if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY): + return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] + else: + return False @property def last_fetch_datetime_file_name(self) -> str: - return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY] if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}.json' - + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY): + return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY] + else: + return f'{self.__object_info[OBJECT_NAME_KEY]}.json' @property def upload_file_name(self) -> str: - return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' + if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): + return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) + else: + return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' @property def datetime_column(self) -> str: @@ -126,7 +128,6 @@ class LastFetchDatetime(): self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) - return @property @@ -135,4 +136,7 @@ class LastFetchDatetime(): @property def last_fetch_datetime_to(self) -> str: - return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FROM_KEY) else self.__execute_datetime + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): + return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY] + else: + return self.__execute_datetime diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index acbd2383..3c92277b 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -7,13 +7,14 @@ from src.config.objects import FetchTargetObjects from src.convert_crm_csv_data_process import convert_crm_csv_data_process from src.copy_crm_csv_data_process import copy_crm_csv_data_process from src.error.exceptions import MeDaCaCRMDataFetchException -from src.util.execute_datetime import ExecuteDateTime from src.fetch_crm_data_process import fetch_crm_data_process from src.prepare_data_fetch_process import prepare_data_fetch_process from src.set_datetime_period_process import set_datetime_period_process from src.system_var.constants import OBJECT_NAME_KEY -from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime_process +from src.upload_last_fetch_datetime_process import \ + upload_last_fetch_datetime_process from src.upload_result_data_process import upload_result_data_process +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger @@ -63,29 +64,22 @@ def controller() -> None: def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: ExecuteDateTime, process_result: dict): + """取得対象オブジェクト情報をループし、１オブジェクトごとのデータを取得する + + Args: + fetch_target_objects (FetchTargetObjects): CRMオブジェクト情報インスタンス + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス + process_result (dict): 取得処理実行結果辞書オブジェクト + + Returns: + process_result: 取得処理実行結果辞書オブジェクト """ - 取得対象オブジェクト情報をループし、１オブジェクトごとのデータを取得する - Parameters - ---------- - fetch_target_objects : FetchTargetObjects - CRMオブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス - process_result : dict - 取得処理実行結果辞書オブジェクト - - Returns - ------- - process_result : dict - 取得処理実行結果辞書オブジェクト - - """ for object_info in fetch_target_objects: try: process_result[object_info.get(OBJECT_NAME_KEY)] = 'fail' - fetch_crm_data_per_object(object_info, execute_datetime, process_result) + fetch_crm_data_per_object(object_info, execute_datetime) process_result[object_info.get(OBJECT_NAME_KEY)] = 'success' @@ -102,28 +96,13 @@ def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: E return process_result + def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTime) -> None: - """ - オブジェクトごとにCRMのデータを取得し、取込フォルダにアップロードする - - Parameters - ---------- - object_info : dict - 取得対象オブジェクト情報 - execute_datetime : object - 実行日次取得インスタンス - - Returns - ------- - なし - - Raises - ------ - FileNotFoundException - S3上のファイルが存在しない場合 - InvalidConfigException - オブジェクト情報定義が不正だった場合 + """オブジェクトごとにCRMのデータを取得し、取込フォルダにアップロードする + Args: + object_info (dict): 取得対象オブジェクト情報 + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス """ # 1. オブジェクト処理結果の初期化 @@ -194,4 +173,4 @@ def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTi # 14. オブジェクトのアップロードが完了した旨をログに出力する logger.info(f'I-CTRL-16 [{target_object_name}] 処理正常終了') - return \ No newline at end of file + return diff --git a/ecs/crm-datafetch/src/convert_crm_csv_data_process.py b/ecs/crm-datafetch/src/convert_crm_csv_data_process.py index 1ed05c3a..fd6138b1 100644 --- a/ecs/crm-datafetch/src/convert_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/convert_crm_csv_data_process.py @@ -6,25 +6,17 @@ from src.util.logger import logger_instance as logger def convert_crm_csv_data_process(target_object: TargetObject, crm_data_response: dict): - """ - CSV変換処理 + """CSV変換処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - crm_data_response : dict - Salesforceオブジェクトデータ + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + crm_data_response (dict): Salesforceオブジェクトデータ - Returns - ------- - csv_string : str - csvデータ + Raises: + DataConvertException: データ変換が失敗した場合 - Raises - ------ - DataConvertException - データ変換が失敗した場合 + Returns: + csv_string: csvデータ """ # ① CSV変換処理の開始ログを出力する @@ -34,9 +26,8 @@ def convert_crm_csv_data_process(target_object: TargetObject, crm_data_response: try: # ② CSV変換 - CSVStringConverter(target_object, crm_data_response) - csv_string = CSVStringConverter.convert() - + csv_string_converter = CSVStringConverter(target_object, crm_data_response) + csv_string = csv_string_converter.convert() logger.debug(f'D-CONV-02 [{target_object_name}] のCSV変換処理 正常終了') diff --git a/ecs/crm-datafetch/src/converter/convert_factory.py b/ecs/crm-datafetch/src/converter/convert_factory.py index 48673f7e..bf949a6e 100644 --- a/ecs/crm-datafetch/src/converter/convert_factory.py +++ b/ecs/crm-datafetch/src/converter/convert_factory.py @@ -1,9 +1,10 @@ import re from datetime import datetime -from src.config.objects import TargetObject from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, - CSV_TRUE_VALUE, YYYYMMDDHHMMSS, DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC) + CSV_TRUE_VALUE, + DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, + YYYYMMDDHHMMSS) class ConvertStrategyFactory: @@ -12,30 +13,30 @@ class ConvertStrategyFactory: self.__float_convert_strategy = FloatConvertStrategy() self.__boolean_convert_strategy = BooleanConvertStrategy() self.__datetime_convert_strategy = DatatimeConvertStrategy() + self.__non_convert_strategy = NonConvertStrategy() def create(self, value): - converted_value = value - if value is None: - converted_value = self.__none_value_convert_strategy.convert_value() + convert_strategy = self.__none_value_convert_strategy - # 指数表記で取得できるパターン。指数表記を整数表記に変換する。 elif type(value) == float: - converted_value = self.__float_convert_strategy.convert_value(value) + convert_strategy = self.__float_convert_strategy - # SQLの真偽値に対応するために変換する elif type(value) == bool: - converted_value = self.__boolean_convert_strategy.convert_value(value) + convert_strategy = self.__boolean_convert_strategy elif type(value) == str and re.fullmatch(DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, value): - converted_value = self.__datetime_convert_strategy.convert_value(value) + convert_strategy = self.__datetime_convert_strategy - return converted_value + else: + convert_strategy = self.__non_convert_strategy + + return convert_strategy class NoneValueConvertStrategy: - def convert_value(self) -> str: + def convert_value(self, convert_value: None) -> str: return '' @@ -53,3 +54,7 @@ class FloatConvertStrategy: def convert_value(self, convert_value: str) -> int: return int(convert_value) + +class NonConvertStrategy: + def convert_value(self, convert_value: str): + return convert_value diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index bf7faaee..9c223182 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -52,7 +52,8 @@ class CSVStringConverter: for column in columns: v = json_object[column.upper()] - converted_value = self.__convert_strategy_factory.create(v) + convert_strategy = self.__convert_strategy_factory.create(v) + converted_value = convert_strategy.convert_value(v) csv_row.append(converted_value) diff --git a/ecs/crm-datafetch/src/copy_crm_csv_data_process.py b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py index 57a3659f..ba2831ea 100644 --- a/ecs/crm-datafetch/src/copy_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py @@ -2,31 +2,20 @@ from src.aws.s3 import BackupBucket, DataBucket from src.config.objects import TargetObject from src.error.exceptions import FileUploadException from src.system_var.constants import UPLD_JP_NAME -from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, - CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) +from src.system_var.environments import CRM_IMPORT_DATA_BACKUP_FOLDER from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger def copy_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime): - """ - CSVアップロード処理 + """CSVアップロード処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① CSVデータアップロード処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/fetch_crm_data_process.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py index 3a89da1e..d22efb7e 100644 --- a/ecs/crm-datafetch/src/fetch_crm_data_process.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -2,102 +2,91 @@ from requests.exceptions import ConnectTimeout, ReadTimeout from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential -from src.config.objects import TargetObject, LastFetchDatetime +from src.config.objects import LastFetchDatetime, TargetObject from src.error.exceptions import DataConvertException, SalesforceAPIException from src.salesforce.salesforce_api import SalesforceApiClient from src.salesforce.soql_builder import SOQLBuilder from src.system_var.constants import FETCH_JP_NAME -from src.system_var.environments import (CRM_AUTH_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, - CRM_GET_RECORD_COUNT_TIMEOUT) +from src.system_var.environments import ( + CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, CRM_FETCH_RECORD_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_TIMEOUT) +from src.util.counter_object import CounterObject from src.util.logger import logger_instance as logger def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): - """ - CRMデータ取得処理 + """CRMデータ取得処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - last_fetch_datetime : LastFetchDatetime - 取得対象オブジェクト情報インスタンス + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + last_fetch_datetime (LastFetchDatetime): データ取得期間設定インスタンス - Returns - ------- - sf_object_dict : dict - Salesforceオブジェクトデータ + Raises: + SalesforceAPIException: SalseforceのAPI実行失敗が発生した場合 + DataConvertException: データ変換が失敗した場合 - Raises - ------ - SalesforceAPIException - SalseforceのAPI実行失敗が発生した場合 - DataConvertException - データ変換が失敗した場合 + Returns: + crm_data_response: Salesforceオブジェクトデータ """ # ① CRMデータ取得処理開始ログを出力する logger.info( f'I-FETCH-01 [{target_object.object_name}] のCRMからのデータ取得処理を開始します') - object_name = target_object.object_name + target_object_name = target_object.object_name - global count_contime_counter, count_readtime_counter, count_counter, data_contime_counter, data_readtime_counter, data_counter - count_contime_counter = 1 - count_readtime_counter = 1 - count_counter = 1 - data_contime_counter = 1 - data_readtime_counter = 1 - data_counter = 1 + # リトライ回数判定用のカウンタオブジェクトを生成(@retryデコレータを利用したことによるリトライ対象の関数内でのカウント変数保持不可の対策のためオブジェクト化する) + count_contime_counter = CounterObject(1) + count_readtime_counter = CounterObject(1) + count_counter = CounterObject(1) + data_contime_counter = CounterObject(1) + data_readtime_counter = CounterObject(1) + data_counter = CounterObject(1) try: # ② 取得対象オブジェクトの取得期間内のレコード件数を取得する - logger.info(f'I-FETCH-02 [{object_name}] の件数取得を開始します') + logger.info(f'I-FETCH-02 [{target_object_name}] の件数取得を開始します') soql_builder = SOQLBuilder(target_object, last_fetch_datetime) count_soql = soql_builder.create_count_soql() - record_count = fetch_record_count_retry(count_soql, object_name) + record_count = fetch_record_count_retry(count_soql, target_object_name, count_contime_counter, count_readtime_counter, count_counter) - logger.info(f'I-FETCH-03 [{object_name}] の件数：[{record_count}]') + logger.info(f'I-FETCH-03 [{target_object_name}] の件数：[{record_count}]') except Exception as e: raise SalesforceAPIException( - 'E-FETCH-01', FETCH_JP_NAME, f'[{object_name}] の件数取得に失敗しました エラー内容：[{e}]') + 'E-FETCH-01', FETCH_JP_NAME, f'[{target_object_name}] の件数取得に失敗しました エラー内容：[{e}]') try: # ③ 取得対象オブジェクトのレコードを取得する - logger.info(f'I-FETCH-04 [{object_name}] のレコード取得を開始します') + logger.info(f'I-FETCH-04 [{target_object_name}] のレコード取得を開始します') fetch_soql = soql_builder.create_fetch_soql() - record_all = fetch_sf_data_retry(fetch_soql, object_name) + record_all = fetch_sf_data_retry(fetch_soql, target_object_name, data_contime_counter, data_readtime_counter, data_counter) except Exception as e: raise SalesforceAPIException( - 'E-FETCH-02', FETCH_JP_NAME, f'[{object_name}] のレコード取得に失敗しました エラー内容：[{e}]') + 'E-FETCH-02', FETCH_JP_NAME, f'[{target_object_name}] のレコード取得に失敗しました エラー内容：[{e}]') try: # ④ 取得対象オブジェクトをJSONに変換 - logger.info(f'I-FETCH-05 [{object_name}] のレコードをJSONに変換します') + logger.info(f'I-FETCH-05 [{target_object_name}] のレコードをJSONに変換します') crm_data_response = [record for record in record_all] except Exception as e: raise DataConvertException( - 'E-FETCH-03', FETCH_JP_NAME, f'[{object_name}] のレコードのJSON変換に失敗しました エラー内容：[{e}]') + 'E-FETCH-03', FETCH_JP_NAME, f'[{target_object_name}] のレコードのJSON変換に失敗しました エラー内容：[{e}]') # ⑤ CRMデータ取得処理終了ログを出力する - logger.info(f'I-FETCH-06 [{object_name}] のレコード取得が成功しました') + logger.info(f'I-FETCH-06 [{target_object_name}] のレコード取得が成功しました') # ⑥ 次の処理へ移行する return crm_data_response @@ -107,31 +96,34 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) -def fetch_record_count_retry(soql: str, object_name: str): +def fetch_record_count_retry(soql: str, target_object_name: str, + count_contime_counter: CounterObject, count_readtime_counter: CounterObject, count_counter: CounterObject): try: - global count_contime_counter, count_readtime_counter, count_counter - salesforce_api_client = SalesforceApiClient() return salesforce_api_client.fetch_sf_count(soql) except ConnectTimeout as e: - if count_contime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_contime_counter += 1 - logger.warn(f'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if count_contime_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_contime_counter.increment(1) + logger.warning(f'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') raise e except ReadTimeout as e: - if count_readtime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_readtime_counter += 1 - logger.warn( - f'W-FETCH-02 [{object_name}] の件数取得処理がタイムアウトしたため、リトライします：[{CRM_GET_RECORD_COUNT_TIMEOUT}] エラー内容：[{e}]') + + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if count_readtime_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_readtime_counter.increment(1) + logger.warning( + f'W-FETCH-02 [{target_object_name}] の件数取得処理がタイムアウトしたため、リトライします：[{CRM_GET_RECORD_COUNT_TIMEOUT}] エラー内容：[{e}]') raise e except Exception as e: - if count_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_counter += 1 - logger.warn( - f'W-FETCH-03 [{object_name}] の件数取得に失敗したため、リトライします エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if count_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_counter.increment(1) + logger.warning( + f'W-FETCH-03 [{target_object_name}] の件数取得に失敗したため、リトライします エラー内容：[{e}]') raise e @@ -139,29 +131,31 @@ def fetch_record_count_retry(soql: str, object_name: str): wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) -def fetch_sf_data_retry(soql: str, object_name: str): +def fetch_sf_data_retry(soql: str, target_object_name: str, + data_contime_counter: CounterObject, data_readtime_counter: CounterObject, data_counter: CounterObject): try: - global data_contime_counter, data_readtime_counter, data_counter - salesforce_api_client = SalesforceApiClient() return salesforce_api_client.fetch_sf_data(soql) except ConnectTimeout as e: - if data_contime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_contime_counter += 1 - logger.warn(f'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if data_contime_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_contime_counter.increment(1) + logger.warning(f'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') raise e except ReadTimeout as e: - if data_readtime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_readtime_counter += 1 - logger.warn( - f'W-FETCH-05 [{object_name}] のレコード取得処理がタイムアウトしたため、リトライします：[{CRM_FETCH_RECORD_TIMEOUT}] エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if data_readtime_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_readtime_counter.increment(1) + logger.warning( + f'W-FETCH-05 [{target_object_name}] のレコード取得処理がタイムアウトしたため、リトライします：[{CRM_FETCH_RECORD_TIMEOUT}] エラー内容：[{e}]') raise e except Exception as e: - if data_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_counter += 1 - logger.warn( - f'W-FETCH-06 [{object_name}] のレコード取得に失敗したため、リトライします エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if data_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_counter.increment(1) + logger.warning( + f'W-FETCH-06 [{target_object_name}] のレコード取得に失敗したため、リトライします エラー内容：[{e}]') raise e diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parse.py index 5e7249b9..ca1d8ed2 100644 --- a/ecs/crm-datafetch/src/parser/json_parse.py +++ b/ecs/crm-datafetch/src/parser/json_parse.py @@ -8,7 +8,7 @@ class JsonParser(): def __init__(self, json_str) -> None: self.__json_str = json_str - def json_parser(self) -> dict: + def parse(self) -> dict: for symbol in EXCLUDE_SYMBOL: # コメントアウトシンボルを含む部分を置き換える正規表現 replace_comment_regex = rf'\s(?!\"){symbol}[\s\S]*?.*' diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index 409651ad..3488c71f 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -3,36 +3,24 @@ from src.config.objects import FetchTargetObjects from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser from src.system_var.constants import PRE_JP_NAME -from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, - OBJECT_INFO_FOLDER) +from src.system_var.environments import (CRM_CONFIG_BUCKET, + OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER) from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger def prepare_data_fetch_process(): - """ - データ取得準備処理 + """データ取得準備処理 - Parameters - ---------- - なし - - Returns - ------- - fetch_target_objects : FetchTargetObjects - CRMオブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス - process_result : dict - 取得処理実行結果辞書オブジェクト - - Raises - ------ - FileNotFoundException - S3上のファイルが存在しない場合 - InvalidConfigException - オブジェクト情報定義が不正だった場合 + Raises: + FileNotFoundException: S3上のファイルが存在しない場合 + InvalidConfigException: オブジェクト情報定義が不正だった場合 + Returns: + fetch_target_objects : CRMオブジェクト情報インスタンス + execute_datetime : 実行日次取得インスタンス + process_result : 取得処理実行結果辞書オブジェクト """ # ① データ取得準備処理の開始ログを出力する @@ -64,7 +52,7 @@ def prepare_data_fetch_process(): logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします') json_parser = JsonParser(object_info_file_str) - object_info_file_dict = json_parser.json_parser() + object_info_file_dict = json_parser.parse() logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました') diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index 0c073b9a..4e4bb267 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,10 +1,9 @@ from simple_salesforce import Salesforce - from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_TIMEOUT, - CRM_FETCH_RECORD_TIMEOUT, - CRM_GET_RECORD_COUNT_TIMEOUT, CRM_USER_NAME, - CRM_USER_PASSWORD, CRM_USER_SECURITY_TOKEN) - + CRM_FETCH_RECORD_TIMEOUT, + CRM_GET_RECORD_COUNT_TIMEOUT, + CRM_USER_NAME, CRM_USER_PASSWORD, + CRM_USER_SECURITY_TOKEN) class SalesforceApiClient(): @@ -25,4 +24,5 @@ class SalesforceApiClient(): return count_res.get('records')[0].get('expr0') def fetch_sf_data(self, soql: str): - return self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + data_res = self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + return data_res.get('records') diff --git a/ecs/crm-datafetch/src/set_datetime_period_process.py b/ecs/crm-datafetch/src/set_datetime_period_process.py index 9795ab2e..bfebd5c2 100644 --- a/ecs/crm-datafetch/src/set_datetime_period_process.py +++ b/ecs/crm-datafetch/src/set_datetime_period_process.py @@ -1,37 +1,27 @@ from src.aws.s3 import ConfigBucket -from src.config.objects import TargetObject, LastFetchDatetime +from src.config.objects import LastFetchDatetime, TargetObject from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser from src.system_var.constants import DATE_JP_NAME -from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.system_var.environments import (CRM_CONFIG_BUCKET, + LAST_FETCH_DATE_FOLDER) from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger - - def set_datetime_period_process(target_object: TargetObject, execute_datetime: ExecuteDateTime): - """ - データ取得期間設定処理 + """データ取得期間設定処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス - Returns - ------- - last_fetch_datetime : LastFetchDatetime - 取得対象オブジェクト情報インスタンス + Raises: + FileNotFoundException: S3上のファイルが存在しない場合 + InvalidConfigException: オブジェクト情報定義が不正だった場合 - Raises - ------ - FileNotFoundException - S3上のファイルが存在しない場合 - InvalidConfigException - オブジェクト情報定義が不正だった場合 + Returns: + last_fetch_datetime: データ取得期間設定インスタンス """ # ① データ取得期間設定処理の開始ログを出力する @@ -59,7 +49,7 @@ def set_datetime_period_process(target_object: TargetObject, execute_datetime: E logger.debug(f'D-DATE-04 前回取得日時ファイルの形式チェックを開始します') json_parser = JsonParser(last_fetch_datetime_file_str) - last_fetch_datetime_file_dict = json_parser.json_parser() + last_fetch_datetime_file_dict = json_parser.parse() last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_file_dict, execute_datetime) diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index 5ff41929..7dc62c1b 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -6,35 +6,35 @@ import src.system_var.constants as constants # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_TIMEOUT = os.environ.get(constants.CRM_AUTH_TIMEOUT, 100) +CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)) # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3) +CRM_AUTH_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3)) # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5) +CRM_AUTH_RETRY_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5)) # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5) +CRM_AUTH_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5)) # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50) +CRM_AUTH_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50)) # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300) +CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)) # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3) +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3)) # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5) +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5)) # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, 5) +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, 5)) # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50) +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50)) # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300) +CRM_FETCH_RECORD_TIMEOUT = int(os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300)) # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3) +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3)) # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5) +CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5) +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50) +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50)) # environments(ECS Task Environment) # CRMのAPI実行のための認証エンドポイントのドメイン diff --git a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py index e83e58ab..e6b34c53 100644 --- a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py +++ b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py @@ -1,31 +1,21 @@ import json from src.aws.s3 import ConfigBucket -from src.config.objects import TargetObject, LastFetchDatetime +from src.config.objects import LastFetchDatetime, TargetObject from src.error.exceptions import FileUploadException from src.system_var.constants import UPD_JP_NAME from src.util.logger import logger_instance as logger def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): - """ - 前回取得日時ファイル更新 + """前回取得日時ファイル更新 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - last_fetch_datetime : LastFetchDatetime - 取得対象オブジェクト情報インスタンス + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + last_fetch_datetime (LastFetchDatetime): データ取得期間設定インスタンス - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① 前回取得日時ファイル更新処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/upload_result_data_process.py b/ecs/crm-datafetch/src/upload_result_data_process.py index 855ec900..8248fb78 100644 --- a/ecs/crm-datafetch/src/upload_result_data_process.py +++ b/ecs/crm-datafetch/src/upload_result_data_process.py @@ -7,24 +7,14 @@ from src.util.logger import logger_instance as logger def upload_result_data_process(process_result: dict, execute_datetime: ExecuteDateTime): - """ - 取得処理実施結果アップロード処理 + """取得処理実施結果アップロード処理 - Parameters - ---------- - process_result : dict - 取得処理実行結果辞書オブジェクト - last_fetch_datetime : LastFetchDatetime - 取得対象オブジェクト情報インスタンス + Args: + process_result (dict): 取得処理実行結果辞書オブジェクト + execute_datetime (ExecuteDateTime): データ取得期間設定インスタンス - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① 取得処理実施結果アップロード処理のログを出力する diff --git a/ecs/crm-datafetch/src/util/counter_object.py b/ecs/crm-datafetch/src/util/counter_object.py new file mode 100644 index 00000000..23da4f82 --- /dev/null +++ b/ecs/crm-datafetch/src/util/counter_object.py @@ -0,0 +1,14 @@ +class CounterObject: + def __init__(self, base_num=1) -> None: + self.__counter = base_num + + def describe(self) -> int: + return self.__counter + + def increment(self, num=1) -> int: + self.__counter += num + return self.__counter + + def decrement(self, num=1) -> int: + self.__counter -= num + return self.__counter diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py index 9d498b5e..1f97dded 100644 --- a/ecs/crm-datafetch/src/util/dict_checker.py +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -17,7 +17,7 @@ class DictChecker: """辞書型バリュー正規表現チェック""" return True if re.fullmatch(regex_str, self.__object_dict[check_key]) else False - def assert_key_exist(self, check_key: str, check_type: type) -> None: + def assert_key_exist(self, check_key: str) -> None: """辞書型キー存在検査""" if not self.check_key_exist(check_key): raise Exception(f'「{check_key}」キーは必須です') diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py index 2f07736a..8a804291 100644 --- a/ecs/crm-datafetch/src/util/execute_datetime.py +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -1,9 +1,6 @@ from datetime import datetime -from src.system_var.constants import( - YYYYMMDDTHHMMSSTZ, - MILLISEC_FORMAT -) +from src.system_var.constants import MILLISEC_FORMAT, YYYYMMDDTHHMMSSTZ class ExecuteDateTime: diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index ec1df277..47fd2e45 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -2,7 +2,7 @@ import logging from src.system_var.environments import LOG_LEVEL -"""boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する""" +# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する for name in ["boto3", "botocore", "s3transfer", "urllib3"]: logging.getLogger(name).setLevel(logging.WARNING) diff --git a/s3/config/crm/last_fetch_datetime/Account.json b/s3/config/crm/last_fetch_datetime/Account.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Account.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/AccountShare.json b/s3/config/crm/last_fetch_datetime/AccountShare.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/AccountShare.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Account_Territory_Loader_vod__c.json b/s3/config/crm/last_fetch_datetime/Account_Territory_Loader_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Account_Territory_Loader_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Approved_Document_vod__c.json b/s3/config/crm/last_fetch_datetime/Approved_Document_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Approved_Document_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call2_Detail_vod__c.json b/s3/config/crm/last_fetch_datetime/Call2_Detail_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call2_Detail_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call2_Discussion_vod__c.json b/s3/config/crm/last_fetch_datetime/Call2_Discussion_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call2_Discussion_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call2_Key_Message_vod__c.json b/s3/config/crm/last_fetch_datetime/Call2_Key_Message_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call2_Key_Message_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call2_vod__c.json b/s3/config/crm/last_fetch_datetime/Call2_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call2_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call_Clickstream_vod__c.json b/s3/config/crm/last_fetch_datetime/Call_Clickstream_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call_Clickstream_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Child_Account_vod__c.json b/s3/config/crm/last_fetch_datetime/Child_Account_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Child_Account_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Clm_Presentation_Slide_vod__c.json b/s3/config/crm/last_fetch_datetime/Clm_Presentation_Slide_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Clm_Presentation_Slide_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Clm_Presentation_vod__c.json b/s3/config/crm/last_fetch_datetime/Clm_Presentation_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Clm_Presentation_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Coaching_Report_vod__c.json b/s3/config/crm/last_fetch_datetime/Coaching_Report_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Coaching_Report_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Consent_Header_vod__c.json b/s3/config/crm/last_fetch_datetime/Consent_Header_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Consent_Header_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Consent_Line_vod__c.json b/s3/config/crm/last_fetch_datetime/Consent_Line_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Consent_Line_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Consent_Type_vod__c.json b/s3/config/crm/last_fetch_datetime/Consent_Type_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Consent_Type_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Contact.json b/s3/config/crm/last_fetch_datetime/Contact.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Contact.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_Configuration_vod__c.json b/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_Configuration_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_Configuration_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_vod__c.json b/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Email_Activity_vod__c.json b/s3/config/crm/last_fetch_datetime/Email_Activity_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Email_Activity_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Event_Attendee_vod__c.json b/s3/config/crm/last_fetch_datetime/Event_Attendee_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Event_Attendee_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Group.json b/s3/config/crm/last_fetch_datetime/Group.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Group.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Key_Message_vod__c.json b/s3/config/crm/last_fetch_datetime/Key_Message_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Key_Message_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Hospital_Medical_Regimen__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Hospital_Medical_Regimen__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Hospital_Medical_Regimen__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Inquiry_Assignment__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Inquiry_Assignment__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Inquiry_Assignment__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_MR_Weekly_Report__c.json b/s3/config/crm/last_fetch_datetime/MSJ_MR_Weekly_Report__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_MR_Weekly_Report__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Medical_Event_Evaluation__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Medical_Event_Evaluation__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Medical_Event_Evaluation__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Medical_Regimen__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Medical_Regimen__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Medical_Regimen__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Patient__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Patient__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Patient__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json b/s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Medical_Inquiry_vod__c.json b/s3/config/crm/last_fetch_datetime/Medical_Inquiry_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Medical_Inquiry_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Medical_Insight_vod__c.json b/s3/config/crm/last_fetch_datetime/Medical_Insight_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Medical_Insight_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Multichannel_Activity_Line_vod__c.json b/s3/config/crm/last_fetch_datetime/Multichannel_Activity_Line_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Multichannel_Activity_Line_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Multichannel_Activity_vod__c.json b/s3/config/crm/last_fetch_datetime/Multichannel_Activity_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Multichannel_Activity_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Multichannel_Consent_vod__c.json b/s3/config/crm/last_fetch_datetime/Multichannel_Consent_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Multichannel_Consent_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/My_Setup_Products_vod__c.json b/s3/config/crm/last_fetch_datetime/My_Setup_Products_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/My_Setup_Products_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/ObjectTerritory2Association.json b/s3/config/crm/last_fetch_datetime/ObjectTerritory2Association.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/ObjectTerritory2Association.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Product_Group_vod__c.json b/s3/config/crm/last_fetch_datetime/Product_Group_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Product_Group_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Product_Metrics_vod__c.json b/s3/config/crm/last_fetch_datetime/Product_Metrics_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Product_Metrics_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Product_vod__c.json b/s3/config/crm/last_fetch_datetime/Product_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Product_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Profile.json b/s3/config/crm/last_fetch_datetime/Profile.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Profile.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Question_Response_vod__c.json b/s3/config/crm/last_fetch_datetime/Question_Response_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Question_Response_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/RecordType.json b/s3/config/crm/last_fetch_datetime/RecordType.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/RecordType.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Remote_Meeting_vod__c.json b/s3/config/crm/last_fetch_datetime/Remote_Meeting_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Remote_Meeting_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Sent_Email_vod__c.json b/s3/config/crm/last_fetch_datetime/Sent_Email_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Sent_Email_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Sent_Fragment_vod__c.json b/s3/config/crm/last_fetch_datetime/Sent_Fragment_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Sent_Fragment_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Survey_Question_vod__c.json b/s3/config/crm/last_fetch_datetime/Survey_Question_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Survey_Question_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Survey_Target_vod__c.json b/s3/config/crm/last_fetch_datetime/Survey_Target_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Survey_Target_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Survey_vod__c.json b/s3/config/crm/last_fetch_datetime/Survey_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Survey_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Territory2.json b/s3/config/crm/last_fetch_datetime/Territory2.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Territory2.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Time_Off_Territory_vod__c.json b/s3/config/crm/last_fetch_datetime/Time_Off_Territory_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Time_Off_Territory_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/User.json b/s3/config/crm/last_fetch_datetime/User.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/User.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/UserRole.json b/s3/config/crm/last_fetch_datetime/UserRole.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/UserRole.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/UserTerritory2Association.json b/s3/config/crm/last_fetch_datetime/UserTerritory2Association.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/UserTerritory2Association.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/object_info/crm_object_list_all.json b/s3/config/crm/object_info/crm_object_list_all.json index a65cf1bb..0792da37 100644 --- a/s3/config/crm/object_info/crm_object_list_all.json +++ b/s3/config/crm/object_info/crm_object_list_all.json @@ -24,7 +24,7 @@ "is_skip": false, "is_update_last_fetch_datetime": false, "last_fetch_datetime_file_name": "Territory2_ALL.json", - "upload_file_name": "CRM_Territory2_ALL_{execute_datetime}.csv" + "upload_file_name": "CRM_Territory2_ALL_{execute_datetime}" }, { "object_name": "UserTerritory2Association", @@ -41,7 +41,7 @@ "is_skip": false, "is_update_last_fetch_datetime": false, "last_fetch_datetime_file_name": "UserTerritory2Association_ALL.json", - "upload_file_name": "CRM_UserTerritory2Association_ALL_{execute_datetime}.csv" + "upload_file_name": "CRM_UserTerritory2Association_ALL_{execute_datetime}" } ] } diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index 00f7fe89..aa20d187 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -2164,7 +2164,8 @@ "IsDeleted" ], "is_skip": false, - "is_update_last_fetch_datetime": true + "is_update_last_fetch_datetime": true, + "datetime_column": "LastModifiedDate" }, { "object_name": "Contact", From 939bdd6f0ac05439c05ee6a4cf2b3df4c2bad720 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 28 Jul 2022 22:52:12 +0900 Subject: [PATCH 046/275] =?UTF-8?q?feat:=20=E5=85=A8=E4=BB=B6=E5=8F=96?= =?UTF-8?q?=E5=BE=97=E3=82=AA=E3=83=96=E3=82=B8=E3=82=A7=E3=82=AF=E3=83=88?= =?UTF-8?q?=E7=94=A8=E3=81=AE=E6=9C=80=E7=B5=82=E5=8F=96=E5=BE=97=E6=97=A5?= =?UTF-8?q?=E6=99=82=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=92=E4=BD=9C?= =?UTF-8?q?=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/config/crm/last_fetch_datetime/Territory2_ALL.json | 4 ++++ .../last_fetch_datetime/UserTerritory2Association_ALL.json | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 s3/config/crm/last_fetch_datetime/Territory2_ALL.json create mode 100644 s3/config/crm/last_fetch_datetime/UserTerritory2Association_ALL.json diff --git a/s3/config/crm/last_fetch_datetime/Territory2_ALL.json b/s3/config/crm/last_fetch_datetime/Territory2_ALL.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Territory2_ALL.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/UserTerritory2Association_ALL.json b/s3/config/crm/last_fetch_datetime/UserTerritory2Association_ALL.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/UserTerritory2Association_ALL.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} From 70f03b4a6f7f02e6b34d7035debaca99c05161be Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 17:21:19 +0900 Subject: [PATCH 047/275] =?UTF-8?q?feat:=E3=80=8C"=E3=80=8D=E3=81=8C?= =?UTF-8?q?=E5=8F=96=E3=82=8A=E8=BE=BC=E3=82=81=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 05e5c353..d86ee84e 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -145,6 +145,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf if len(line[i]) > 0: # 0桁より大きい場合 replace_line = line[i].replace('\\', '\\\\') + replace_line = line[i].replace('"', '\\"') sql = f'{sql} "{replace_line}",' else: # 上記以外の場合 From fcac3d7b23a7da41e29ad832820823b1fe896ac8 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 1 Aug 2022 19:28:15 +0900 Subject: [PATCH 048/275] =?UTF-8?q?fix:=20=E6=8C=87=E6=91=98=E5=86=85?= =?UTF-8?q?=E5=AE=B9=E3=81=AE=E5=8F=8D=E6=98=A0=E3=80=81=E3=83=AD=E3=82=B0?= =?UTF-8?q?=E5=87=BA=E5=8A=9B=E3=81=AE=E5=BE=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/backup_crm_csv_data_process.py | 6 +-- ecs/crm-datafetch/src/config/objects.py | 12 ++--- ecs/crm-datafetch/src/controller.py | 10 ++-- ...convert_factory.py => convert_strategy.py} | 9 ++-- ecs/crm-datafetch/src/converter/converter.py | 2 +- ecs/crm-datafetch/src/error/exceptions.py | 2 +- .../src/fetch_crm_data_process.py | 54 +++++++------------ ecs/crm-datafetch/src/util/dict_checker.py | 6 ++- .../src/util/execute_datetime.py | 4 +- 9 files changed, 45 insertions(+), 60 deletions(-) rename ecs/crm-datafetch/src/converter/{convert_factory.py => convert_strategy.py} (83%) diff --git a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py index d40c7a13..ba7bb8ea 100644 --- a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py @@ -6,12 +6,12 @@ from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def backup_crm_csv_data_process(target_object: TargetObject, exetute_datetime: ExecuteDateTime, csv_string: str): +def backup_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime, csv_string: str): """CSVバックアップ処理 Args: target_object (TargetObject): 取得対象オブジェクト情報インスタンス - exetute_datetime (ExecuteDateTime): 実行日次取得インスタンス + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス csv_string (str): csvデータ Raises: @@ -29,7 +29,7 @@ def backup_crm_csv_data_process(target_object: TargetObject, exetute_datetime: E # ② CRMバックアップ保管用バケットに、変換後のCSVデータのバックアップを保管する backup_bucket = BackupBucket() backup_bucket.put_csv( - f'{exetute_datetime.to_path()}/{upload_file_name}.csv', csv_string) + f'{execute_datetime.to_path()}/{upload_file_name}.csv', csv_string) logger.debug( f'D-CSVBK-02 [{target_object_name}] のCSVデータバックアップ 正常終了') diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 3ac08851..3368a1db 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -94,22 +94,19 @@ class TargetObject(): def is_update_last_fetch_datetime(self) -> bool: if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY): return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] - else: - return False + return False @property def last_fetch_datetime_file_name(self) -> str: if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY): return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY] - else: - return f'{self.__object_info[OBJECT_NAME_KEY]}.json' + return f'{self.__object_info[OBJECT_NAME_KEY]}.json' @property def upload_file_name(self) -> str: if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) - else: - return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' + return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' @property def datetime_column(self) -> str: @@ -138,5 +135,4 @@ class LastFetchDatetime(): def last_fetch_datetime_to(self) -> str: if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY] - else: - return self.__execute_datetime + return self.__execute_datetime diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index 3c92277b..c531f113 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -39,17 +39,17 @@ def controller() -> None: logger.info(f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result}]') # ⑤ 取得処理実施結果アップロード処理を呼び出す - logger.info('I-CTRL-19 CRM_取得処理実施結果ファイルアップロード処理開始') + logger.info('I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始') upload_result_data_process(process_result, execute_datetime) # ⑥ 最終結果をチェックし、チェック結果をログに出力 if not all([v == 'success' for v in process_result.values()]): logger.error('E-CTRL-01 一部のデータ取得に失敗しています。詳細はログをご確認ください。') else: - logger.info('I-CTRL-20 すべてのデータの取得に成功しました。') + logger.info('I-CTRL-19 すべてのデータの取得に成功しました。') # ⑦ CRMデータ取得処理終了ログを出力する - logger.info('I-CTRL-21 CRMデータ取得処理を終了します') + logger.info('I-CTRL-20 CRMデータ取得処理を終了します') return exit(0) @@ -106,7 +106,7 @@ def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTi """ # 1. オブジェクト処理結果の初期化 - logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：{object_info}') + logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：[{object_info}]') # 2. オブジェクト情報形式チェック処理を呼び出す logger.info('I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') @@ -138,7 +138,7 @@ def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTi # 7. 出力ファイル名をログ出力する logger.info( - f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{target_object.upload_file_name}]となります') + f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{target_object.upload_file_name}] となります') # 8. CRM電文データバックアップ処理を呼び出す logger.info( diff --git a/ecs/crm-datafetch/src/converter/convert_factory.py b/ecs/crm-datafetch/src/converter/convert_strategy.py similarity index 83% rename from ecs/crm-datafetch/src/converter/convert_factory.py rename to ecs/crm-datafetch/src/converter/convert_strategy.py index bf949a6e..92500588 100644 --- a/ecs/crm-datafetch/src/converter/convert_factory.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -1,5 +1,5 @@ import re -from datetime import datetime +from datetime import datetime, timedelta, timezone from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, CSV_TRUE_VALUE, @@ -12,7 +12,7 @@ class ConvertStrategyFactory: self.__none_value_convert_strategy = NoneValueConvertStrategy() self.__float_convert_strategy = FloatConvertStrategy() self.__boolean_convert_strategy = BooleanConvertStrategy() - self.__datetime_convert_strategy = DatatimeConvertStrategy() + self.__datetime_convert_strategy = DatetimeConvertStrategy() self.__non_convert_strategy = NonConvertStrategy() def create(self, value): @@ -45,9 +45,10 @@ class BooleanConvertStrategy: return CSV_TRUE_VALUE if convert_value is True else CSV_FALSE_VALUE -class DatatimeConvertStrategy: +class DatetimeConvertStrategy: def convert_value(self, convert_value: str) -> str: - return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).strftime(YYYYMMDDHHMMSS) + # データ登録処理がJSTとして登録するため、変換処理内で事前にJSTの日次文字列に変換する + return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(timezone(timedelta(hours=+9))).strftime(YYYYMMDDHHMMSS) class FloatConvertStrategy: diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 9c223182..761f550e 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -2,7 +2,7 @@ import csv import io from src.config.objects import TargetObject -from src.converter.convert_factory import ConvertStrategyFactory +from src.converter.convert_strategy import ConvertStrategyFactory class CSVStringConverter: diff --git a/ecs/crm-datafetch/src/error/exceptions.py b/ecs/crm-datafetch/src/error/exceptions.py index f231804b..5085dbca 100644 --- a/ecs/crm-datafetch/src/error/exceptions.py +++ b/ecs/crm-datafetch/src/error/exceptions.py @@ -31,5 +31,5 @@ class DataConvertException(MeDaCaCRMDataFetchException): class SalesforceAPIException(MeDaCaCRMDataFetchException): - """SalseforceのAPI実行失敗が発生した場合の例外""" + """SalesforceのAPI実行失敗が発生した場合の例外""" pass diff --git a/ecs/crm-datafetch/src/fetch_crm_data_process.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py index d22efb7e..1c37f860 100644 --- a/ecs/crm-datafetch/src/fetch_crm_data_process.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -3,7 +3,7 @@ from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential from src.config.objects import LastFetchDatetime, TargetObject -from src.error.exceptions import DataConvertException, SalesforceAPIException +from src.error.exceptions import SalesforceAPIException from src.salesforce.salesforce_api import SalesforceApiClient from src.salesforce.soql_builder import SOQLBuilder from src.system_var.constants import FETCH_JP_NAME @@ -27,8 +27,7 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las last_fetch_datetime (LastFetchDatetime): データ取得期間設定インスタンス Raises: - SalesforceAPIException: SalseforceのAPI実行失敗が発生した場合 - DataConvertException: データ変換が失敗した場合 + SalesforceAPIException: SalesforceのAPI実行失敗が発生した場合 Returns: crm_data_response: Salesforceオブジェクトデータ @@ -40,12 +39,9 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las target_object_name = target_object.object_name - # リトライ回数判定用のカウンタオブジェクトを生成(@retryデコレータを利用したことによるリトライ対象の関数内でのカウント変数保持不可の対策のためオブジェクト化する) - count_contime_counter = CounterObject(1) - count_readtime_counter = CounterObject(1) + # リトライ回数判定用のカウンタオブジェクトを生成 + # @retryデコレータを利用した関数のリトライ処理で、基本データ型だとリトライ回数をカウントすることができないため、オブジェクト化する count_counter = CounterObject(1) - data_contime_counter = CounterObject(1) - data_readtime_counter = CounterObject(1) data_counter = CounterObject(1) try: @@ -55,7 +51,7 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las soql_builder = SOQLBuilder(target_object, last_fetch_datetime) count_soql = soql_builder.create_count_soql() - record_count = fetch_record_count_retry(count_soql, target_object_name, count_contime_counter, count_readtime_counter, count_counter) + record_count = fetch_record_count_retry(count_soql, target_object_name, count_counter) logger.info(f'I-FETCH-03 [{target_object_name}] の件数：[{record_count}]') @@ -69,26 +65,16 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las fetch_soql = soql_builder.create_fetch_soql() - record_all = fetch_sf_data_retry(fetch_soql, target_object_name, data_contime_counter, data_readtime_counter, data_counter) + crm_data_response = fetch_sf_data_retry(fetch_soql, target_object_name, data_counter) except Exception as e: raise SalesforceAPIException( 'E-FETCH-02', FETCH_JP_NAME, f'[{target_object_name}] のレコード取得に失敗しました エラー内容：[{e}]') - try: - # ④ 取得対象オブジェクトをJSONに変換 - logger.info(f'I-FETCH-05 [{target_object_name}] のレコードをJSONに変換します') + # ④ CRMデータ取得処理終了ログを出力する + logger.info(f'I-FETCH-05 [{target_object_name}] のレコード取得が成功しました') - crm_data_response = [record for record in record_all] - - except Exception as e: - raise DataConvertException( - 'E-FETCH-03', FETCH_JP_NAME, f'[{target_object_name}] のレコードのJSON変換に失敗しました エラー内容：[{e}]') - - # ⑤ CRMデータ取得処理終了ログを出力する - logger.info(f'I-FETCH-06 [{target_object_name}] のレコード取得が成功しました') - - # ⑥ 次の処理へ移行する + # ⑤ 次の処理へ移行する return crm_data_response @@ -96,24 +82,23 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) -def fetch_record_count_retry(soql: str, target_object_name: str, - count_contime_counter: CounterObject, count_readtime_counter: CounterObject, count_counter: CounterObject): +def fetch_record_count_retry(soql: str, target_object_name: str, count_counter: CounterObject): try: salesforce_api_client = SalesforceApiClient() return salesforce_api_client.fetch_sf_count(soql) except ConnectTimeout as e: # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う - if count_contime_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_contime_counter.increment(1) + if count_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_counter.increment(1) logger.warning(f'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') raise e except ReadTimeout as e: # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う - if count_readtime_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_readtime_counter.increment(1) + if count_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_counter.increment(1) logger.warning( f'W-FETCH-02 [{target_object_name}] の件数取得処理がタイムアウトしたため、リトライします：[{CRM_GET_RECORD_COUNT_TIMEOUT}] エラー内容：[{e}]') raise e @@ -131,23 +116,22 @@ def fetch_record_count_retry(soql: str, target_object_name: str, wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) -def fetch_sf_data_retry(soql: str, target_object_name: str, - data_contime_counter: CounterObject, data_readtime_counter: CounterObject, data_counter: CounterObject): +def fetch_sf_data_retry(soql: str, target_object_name: str, data_counter: CounterObject): try: salesforce_api_client = SalesforceApiClient() return salesforce_api_client.fetch_sf_data(soql) except ConnectTimeout as e: # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う - if data_contime_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_contime_counter.increment(1) + if data_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_counter.increment(1) logger.warning(f'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') raise e except ReadTimeout as e: # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う - if data_readtime_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_readtime_counter.increment(1) + if data_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_counter.increment(1) logger.warning( f'W-FETCH-05 [{target_object_name}] のレコード取得処理がタイムアウトしたため、リトライします：[{CRM_FETCH_RECORD_TIMEOUT}] エラー内容：[{e}]') raise e diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py index 1f97dded..39ebf9f0 100644 --- a/ecs/crm-datafetch/src/util/dict_checker.py +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -5,9 +5,13 @@ class DictChecker: def __init__(self, object_dict: dict) -> None: self.__object_dict = object_dict + def is_empty(self, check_key): + """辞書型バリュー空文字チェック""" + return self.__object_dict[check_key] != '' and self.__object_dict[check_key] is not None + def check_key_exist(self, check_key: str) -> bool: """辞書型キー存在チェック""" - return check_key in self.__object_dict and self.__object_dict[check_key] != '' + return check_key in self.__object_dict and self.is_empty(check_key) def check_data_type(self, check_key: str, check_type: type) -> bool: """辞書型バリュー型チェック""" diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py index 8a804291..bdf7f053 100644 --- a/ecs/crm-datafetch/src/util/execute_datetime.py +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -1,11 +1,11 @@ -from datetime import datetime +from datetime import datetime, timezone from src.system_var.constants import MILLISEC_FORMAT, YYYYMMDDTHHMMSSTZ class ExecuteDateTime: def __init__(self): - self.__execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) + self.__execute_datetime = datetime.now(timezone.utc).strftime(YYYYMMDDTHHMMSSTZ) def __str__(self) -> str: return self.__execute_datetime From 96487ccb2513a0e4c90e968cd9b59ace3d7472ee Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 1 Aug 2022 19:30:23 +0900 Subject: [PATCH 049/275] =?UTF-8?q?fix:=20=E6=8C=87=E6=91=98=E5=86=85?= =?UTF-8?q?=E5=AE=B9=E3=81=AE=E8=BF=BD=E5=8A=A0=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/system_var/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index b84d8f1d..56f49681 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -49,7 +49,7 @@ S3_CHAR_CODE = 'utf-8' # 正規表現チェック EXCLUDE_SYMBOL = ['#', '/'] DATE_PATTERN_YYYYMMDDTHHMMSSTZ = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' -DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.000\+0000' +DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.000\+0000' # logger LOG_FORMAT = '[%(levelname)s]\t%(asctime)s\t%(message)s\n' From fd8f792961c4b5e7beb225a40c4c29ca05132f6c Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 050/275] =?UTF-8?q?feat:crm=5Fhistory=E3=81=AE=E6=96=B0?= =?UTF-8?q?=E8=A6=8F=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 rds_mysql/stored_procedure/crm_history.sql diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql new file mode 100644 index 00000000..cf2ca1d8 --- /dev/null +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -0,0 +1,24 @@ +-- A5M2で実行時にSQL区切り文字を「;」以外にすること + +CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) +BEGIN + SET @new_history_save = 'UPDATE @@target_table SET start_datetime = @@target_column, end_datetime = "9999-12-31 00:00:00" WHERE start_datetime IS NULL AND end_datetime IS NULL'; + SET @new_history_save = REPLACE(@new_history_save, "@@target_table", target_table); + SET @new_history_save = REPLACE(@new_history_save, "@@target_column", target_column); + PREPARE new_history_save_stmt from @new_history_save; + EXECUTE new_history_save_stmt; + + SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id'; + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); + PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; + EXECUTE make_history_tmp_create_stmt; + + SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NULL'; + SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); + PREPARE update_end_datetime_stmt from @update_end_datetime; + EXECUTE update_end_datetime_stmt; + + DROP TEMPORARY TABLE make_history_tmp; +END \ No newline at end of file From 3e6b1ffd0ae0a933d71b61d603a7597bf660f67e Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 051/275] =?UTF-8?q?refactor:=E3=82=B3=E3=83=A1=E3=83=B3?= =?UTF-8?q?=E3=83=88=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index cf2ca1d8..3247da51 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -2,23 +2,27 @@ CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN + -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = 'UPDATE @@target_table SET start_datetime = @@target_column, end_datetime = "9999-12-31 00:00:00" WHERE start_datetime IS NULL AND end_datetime IS NULL'; SET @new_history_save = REPLACE(@new_history_save, "@@target_table", target_table); SET @new_history_save = REPLACE(@new_history_save, "@@target_column", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; + -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id'; SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; EXECUTE make_history_tmp_create_stmt; + -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NULL'; SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; + -- ②-3 「②-1」で作成した一時テーブルを削除する DROP TEMPORARY TABLE make_history_tmp; END \ No newline at end of file From 7b5821e38462c3d6f3704793a92dee088d5f30be Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 052/275] =?UTF-8?q?fix:=E4=B8=80=E6=99=82=E3=83=86?= =?UTF-8?q?=E3=83=BC=E3=83=96=E3=83=AB=E4=BD=9C=E6=88=90=E6=99=82=E3=81=AE?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E3=83=9F=E3=82=B9=E3=82=92=E4=BF=AE=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 3247da51..caeb09c0 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -10,7 +10,7 @@ BEGIN EXECUTE new_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 - SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id'; + SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id HAVING count(id) = 2'; SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; From 73cd6a96e243c9d3d7e41790b2a888f4c123a036 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 053/275] =?UTF-8?q?fix:=20=E2=91=A1-2=E3=81=AE=E6=9D=A1?= =?UTF-8?q?=E4=BB=B6=E3=83=9F=E3=82=B9=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index caeb09c0..7c183714 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -17,7 +17,7 @@ BEGIN EXECUTE make_history_tmp_create_stmt; -- ②-2 「②-1」で取得した全件に更新処理を行う - SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NULL'; + SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NOT NULL'; SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; From da08d37422bd6dd821ffd3b5cdfa240e2fe8077e Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 054/275] =?UTF-8?q?fix:=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E4=BF=AE=E6=AD=A3=E3=80=81=E4=BE=8B?= =?UTF-8?q?=E5=A4=96=E5=87=A6=E7=90=86=E3=81=AE=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 54 +++++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 7c183714..f9057a5e 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,28 +1,70 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること - CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN + -- 例外処理 + -- エラーが発生した場合に一時テーブルの削除を実施 + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + DROP TEMPORARY TABLE IF EXISTS make_history_tmp; + SIGNAL SQLSTATE '45000' + SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; + END; + -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する - SET @new_history_save = 'UPDATE @@target_table SET start_datetime = @@target_column, end_datetime = "9999-12-31 00:00:00" WHERE start_datetime IS NULL AND end_datetime IS NULL'; + SET @new_history_save = ' + UPDATE @@target_table + SET + start_datetime = @@target_column + , end_datetime = "9999-12-31 00:00:00" + WHERE + start_datetime IS NULL + AND end_datetime IS NULL + '; SET @new_history_save = REPLACE(@new_history_save, "@@target_table", target_table); SET @new_history_save = REPLACE(@new_history_save, "@@target_column", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 - SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id HAVING count(id) = 2'; + SET @make_history_tmp_create = ' + CREATE TEMPORARY TABLE make_history_tmp + SELECT + id + , MIN(@@target_column) AS min_start_datetime + , MAX(start_datetime) AS max_start_datetime + FROM + @@target_table + WHERE + end_datetime = "9999-12-31 00:00:00" + GROUP BY + id + HAVING + count(id) = 2 + '; SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; EXECUTE make_history_tmp_create_stmt; -- ②-2 「②-1」で取得した全件に更新処理を行う - SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NOT NULL'; + SET @update_end_datetime = ' + UPDATE @@target_table tt + LEFT JOIN make_history_tmp mht + ON tt.id = mht.id + AND tt.start_datetime = mht.min_start_datetime + SET + start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND + WHERE + mht.id IS NOT NULL + '; SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); + SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; -- ②-3 「②-1」で作成した一時テーブルを削除する DROP TEMPORARY TABLE make_history_tmp; + END \ No newline at end of file From d102117999dee7be561efd272d701f7112ecd8e6 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 055/275] =?UTF-8?q?refactor:=E5=8D=8A=E8=A7=92=E3=82=B9?= =?UTF-8?q?=E3=83=9A=E3=83=BC=E3=82=B9=E3=82=92=E3=82=BF=E3=83=96=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index f9057a5e..794abe38 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -9,15 +9,15 @@ BEGIN @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; DROP TEMPORARY TABLE IF EXISTS make_history_tmp; SIGNAL SQLSTATE '45000' - SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; + SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; END; -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = ' UPDATE @@target_table SET - start_datetime = @@target_column - , end_datetime = "9999-12-31 00:00:00" + start_datetime = @@target_column + , end_datetime = "9999-12-31 00:00:00" WHERE start_datetime IS NULL AND end_datetime IS NULL From 51ed74f1104e466421945fb01a74b95957718385 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 056/275] =?UTF-8?q?refactor:@@=E3=81=8B=E3=82=89=E5=A7=8B?= =?UTF-8?q?=E3=81=BE=E3=82=8B=E6=96=87=E5=AD=97=E3=81=AE=E8=AA=AC=E6=98=8E?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 794abe38..0777912f 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,4 +1,5 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること +-- @@から始まる文字は後からREPLACEする文字を示す独自ルール CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN -- 例外処理 From 9fb7e706ecaf1051b35c885111ffd4e5516ecb0e Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 057/275] =?UTF-8?q?fix:=E4=B8=8D=E8=B6=B3=E9=A0=85?= =?UTF-8?q?=E7=9B=AE=E3=81=A8=E8=A8=AD=E5=AE=9A=E9=A0=85=E7=9B=AE=E3=83=9F?= =?UTF-8?q?=E3=82=B9=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 0777912f..00279493 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -18,7 +18,9 @@ BEGIN UPDATE @@target_table SET start_datetime = @@target_column - , end_datetime = "9999-12-31 00:00:00" + , end_datetime = "9999-12-31 00:00:00" + , upd_user = CURRENT_USER() + , upd_date = CURRENT_TIMESTAMP() WHERE start_datetime IS NULL AND end_datetime IS NULL @@ -56,7 +58,9 @@ BEGIN ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET - start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND + end_datetime = mht.max_start_datetime - INTERVAL 1 SECOND + , upd_user = CURRENT_USER() + , upd_date = CURRENT_TIMESTAMP() WHERE mht.id IS NOT NULL '; From a7b2bb0c851a59aba50694e347536ff12e84ef35 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 058/275] =?UTF-8?q?refactor:@@=E2=86=92$$=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4=E3=80=81=E4=B8=80=E6=99=82=E3=83=86=E3=83=BC?= =?UTF-8?q?=E3=83=96=E3=83=AB=E3=81=AE=E3=83=86=E3=83=BC=E3=83=96=E3=83=AB?= =?UTF-8?q?=E5=90=8D=E4=BF=AE=E6=AD=A3=E3=80=81DEALLOCATE=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 45 ++++++++++++++-------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 00279493..a83cc581 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,5 +1,5 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること --- @@から始まる文字は後からREPLACEする文字を示す独自ルール +-- $$から始まる文字は後からREPLACEする文字を示す独自ルール CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN -- 例外処理 @@ -8,16 +8,25 @@ BEGIN BEGIN GET DIAGNOSTICS CONDITION 1 @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; - DROP TEMPORARY TABLE IF EXISTS make_history_tmp; + EXECUTE drop_tmp_table; + DEALLOCATE PREPARE drop_tmp_table; SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; END; + -- ②-3で利用する一時テーブル削除のSQLを準備 + -- 例外処理でも利用するため先に記述 + SET @drop_tmp_table = ' + DROP TEMPORARY TABLE IF EXISTS make_history_tmp_$$target_table + '; + SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table", target_table); + PREPARE new_history_save_stmt from @drop_tmp_table; + -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = ' - UPDATE @@target_table + UPDATE $$target_table SET - start_datetime = @@target_column + start_datetime = $$target_column , end_datetime = "9999-12-31 00:00:00" , upd_user = CURRENT_USER() , upd_date = CURRENT_TIMESTAMP() @@ -25,20 +34,21 @@ BEGIN start_datetime IS NULL AND end_datetime IS NULL '; - SET @new_history_save = REPLACE(@new_history_save, "@@target_table", target_table); - SET @new_history_save = REPLACE(@new_history_save, "@@target_column", target_column); + SET @new_history_save = REPLACE(@new_history_save, "$$target_table", target_table); + SET @new_history_save = REPLACE(@new_history_save, "$$target_column", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; + DEALLOCATE PREPARE ew_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 SET @make_history_tmp_create = ' - CREATE TEMPORARY TABLE make_history_tmp + CREATE TEMPORARY TABLE make_history_tmp_$$target_table SELECT id - , MIN(@@target_column) AS min_start_datetime + , MIN($$target_column) AS min_start_datetime , MAX(start_datetime) AS max_start_datetime FROM - @@target_table + $$target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY @@ -46,15 +56,16 @@ BEGIN HAVING count(id) = 2 '; - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_table", target_table); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_column", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; EXECUTE make_history_tmp_create_stmt; + DEALLOCATE PREPARE make_history_tmp_create_stmt; -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' - UPDATE @@target_table tt - LEFT JOIN make_history_tmp mht + UPDATE $$target_table tt + LEFT JOIN make_history_tmp_$$target_table mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET @@ -64,12 +75,14 @@ BEGIN WHERE mht.id IS NOT NULL '; - SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; + DEALLOCATE PREPARE update_end_datetime_stmt; -- ②-3 「②-1」で作成した一時テーブルを削除する - DROP TEMPORARY TABLE make_history_tmp; + EXECUTE drop_tmp_table; + DEALLOCATE PREPARE drop_tmp_table; END \ No newline at end of file From 76cf08ebdd1d21b8d4d9cdfdb90c8311fe78ef37 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 059/275] =?UTF-8?q?refactor:=E4=B8=80=E6=99=82=E3=83=86?= =?UTF-8?q?=E3=83=BC=E3=83=96=E3=83=AB=E5=90=8D=E3=81=AE=E5=A4=89=E6=9B=B4?= =?UTF-8?q?=E3=80=81=E7=B4=B0=E3=81=8B=E3=81=84=E3=83=9F=E3=82=B9=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index a83cc581..f361a850 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -8,19 +8,19 @@ BEGIN BEGIN GET DIAGNOSTICS CONDITION 1 @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; - EXECUTE drop_tmp_table; - DEALLOCATE PREPARE drop_tmp_table; + EXECUTE drop_tmp_table_stmt; + DEALLOCATE PREPARE drop_tmp_table_stmt; SIGNAL SQLSTATE '45000' - SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; END; -- ②-3で利用する一時テーブル削除のSQLを準備 -- 例外処理でも利用するため先に記述 SET @drop_tmp_table = ' - DROP TEMPORARY TABLE IF EXISTS make_history_tmp_$$target_table + DROP TEMPORARY TABLE IF EXISTS $$target_table_make_history_tmp '; SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table", target_table); - PREPARE new_history_save_stmt from @drop_tmp_table; + PREPARE drop_tmp_table_stmt from @drop_tmp_table; -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = ' @@ -38,11 +38,11 @@ BEGIN SET @new_history_save = REPLACE(@new_history_save, "$$target_column", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; - DEALLOCATE PREPARE ew_history_save_stmt; + DEALLOCATE PREPARE new_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 SET @make_history_tmp_create = ' - CREATE TEMPORARY TABLE make_history_tmp_$$target_table + CREATE TEMPORARY TABLE $$target_table_make_history_tmp SELECT id , MIN($$target_column) AS min_start_datetime @@ -65,7 +65,7 @@ BEGIN -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' UPDATE $$target_table tt - LEFT JOIN make_history_tmp_$$target_table mht + LEFT JOIN $$target_table_make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET @@ -82,7 +82,7 @@ BEGIN DEALLOCATE PREPARE update_end_datetime_stmt; -- ②-3 「②-1」で作成した一時テーブルを削除する - EXECUTE drop_tmp_table; - DEALLOCATE PREPARE drop_tmp_table; + EXECUTE drop_tmp_table_stmt; + DEALLOCATE PREPARE drop_tmp_table_stmt; END \ No newline at end of file From ba56ec8b5a7d6c3b4c57b36051665e7f77311127 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 060/275] =?UTF-8?q?fix:=E3=83=88=E3=83=A9=E3=83=B3?= =?UTF-8?q?=E3=82=B6=E3=82=AF=E3=82=B7=E3=83=A7=E3=83=B3=E5=87=A6=E7=90=86?= =?UTF-8?q?=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index f361a850..7573bf9b 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -10,10 +10,13 @@ BEGIN @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; EXECUTE drop_tmp_table_stmt; DEALLOCATE PREPARE drop_tmp_table_stmt; + ROLLBACK; SIGNAL SQLSTATE '45000' SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; END; + START TRANSACTION; + -- ②-3で利用する一時テーブル削除のSQLを準備 -- 例外処理でも利用するため先に記述 SET @drop_tmp_table = ' @@ -85,4 +88,6 @@ BEGIN EXECUTE drop_tmp_table_stmt; DEALLOCATE PREPARE drop_tmp_table_stmt; + COMMIT; + END \ No newline at end of file From a7257e1101ad55f316dd9f4211debe5b7d1dcde3 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 061/275] =?UTF-8?q?refactor:=E4=B8=A6=E5=88=97=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=81=A7=E3=81=8D=E3=81=AA=E3=81=84=E3=82=B3=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E8=BF=BD=E5=8A=A0=E3=80=81=E4=BE=8B=E5=A4=96?= =?UTF-8?q?=E3=81=A7=E4=BD=BF=E7=94=A8=E3=81=99=E3=82=8B=E3=83=A6=E3=83=BC?= =?UTF-8?q?=E3=82=B6=E5=A4=89=E6=95=B0=E3=81=AE=E5=88=9D=E6=9C=9F=E5=8C=96?= =?UTF-8?q?=E3=80=81$$=E7=B5=82=E7=AB=AF=E6=96=87=E5=AD=97=E3=81=AE?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 34 ++++++++++++---------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 7573bf9b..b60e2681 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,5 +1,6 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること --- $$から始まる文字は後からREPLACEする文字を示す独自ルール +-- $$から始まり$$で終わる文字は後からREPLACEする文字を示す独自ルール +-- crm_historyストアドプロシージャは、同一セッション内での並列処理を実行することができない CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN -- 例外処理 @@ -15,21 +16,22 @@ BEGIN SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; END; + SET @error_state = NULL, @error_msg = NULL; START TRANSACTION; -- ②-3で利用する一時テーブル削除のSQLを準備 -- 例外処理でも利用するため先に記述 SET @drop_tmp_table = ' - DROP TEMPORARY TABLE IF EXISTS $$target_table_make_history_tmp + DROP TEMPORARY TABLE IF EXISTS $$target_table$$_make_history_tmp '; - SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table", target_table); + SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table$$", target_table); PREPARE drop_tmp_table_stmt from @drop_tmp_table; -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = ' - UPDATE $$target_table + UPDATE $$target_table$$ SET - start_datetime = $$target_column + start_datetime = $$target_column$$ , end_datetime = "9999-12-31 00:00:00" , upd_user = CURRENT_USER() , upd_date = CURRENT_TIMESTAMP() @@ -37,21 +39,21 @@ BEGIN start_datetime IS NULL AND end_datetime IS NULL '; - SET @new_history_save = REPLACE(@new_history_save, "$$target_table", target_table); - SET @new_history_save = REPLACE(@new_history_save, "$$target_column", target_column); + SET @new_history_save = REPLACE(@new_history_save, "$$target_table$$", target_table); + SET @new_history_save = REPLACE(@new_history_save, "$$target_column$$", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; DEALLOCATE PREPARE new_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 SET @make_history_tmp_create = ' - CREATE TEMPORARY TABLE $$target_table_make_history_tmp + CREATE TEMPORARY TABLE $$target_table$$_make_history_tmp SELECT id - , MIN($$target_column) AS min_start_datetime + , MIN($$target_column$$) AS min_start_datetime , MAX(start_datetime) AS max_start_datetime FROM - $$target_table + $$target_table$$ WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY @@ -59,16 +61,16 @@ BEGIN HAVING count(id) = 2 '; - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_table", target_table); - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_column", target_column); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_table$$", target_table); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_column$$", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; EXECUTE make_history_tmp_create_stmt; DEALLOCATE PREPARE make_history_tmp_create_stmt; -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' - UPDATE $$target_table tt - LEFT JOIN $$target_table_make_history_tmp mht + UPDATE $$target_table$$ tt + LEFT JOIN $$target_table$$_make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET @@ -78,8 +80,8 @@ BEGIN WHERE mht.id IS NOT NULL '; - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column", target_column); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; DEALLOCATE PREPARE update_end_datetime_stmt; From c66d208c11d05a2ac412b388344e2cf5b833a277 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 062/275] =?UTF-8?q?feat:crm=5Fdata=5Fsync=E3=81=AE?= =?UTF-8?q?=E6=96=B0=E8=A6=8F=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 rds_mysql/stored_procedure/crm_data_sync.sql diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql new file mode 100644 index 00000000..c039a18a --- /dev/null +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -0,0 +1,41 @@ +-- A5M2で実行時にSQL区切り文字を「;」以外にすること +-- $$から始まる文字は後からREPLACEする文字を示す独自ルール +-- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない +CREATE PROCEDURE crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) +BEGIN + -- 例外処理 + -- エラーが発生した場合に一時テーブルの削除を実施 + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + ROLLBACK; + SIGNAL SQLSTATE '45000' + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; + END; + + SET @error_state = NULL, @error_msg = NULL; + START TRANSACTION; + + -- ①-1 Salesforce側で物理削除されたデータを検出し更新する + SET @update_end_datetime = ' + UPDATE $$target_table$$ tt + LEFT JOIN $$target_table_all$$ tta + ON tt.id = tta.id + AND tt.$$target_column$$ = tta.$$target_column$$ + SET + tt.end_datetime = CURRENT_TIMESTAMP() + , tt.upd_user = CURRENT_USER() + , tt.upd_date = CURRENT_TIMESTAMP() + WHERE + tta.id IS NULL + AND tt.end_datetime = "9999-12-31 00:00:00" + '; + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); + PREPARE update_end_datetime_stmt from @update_end_datetime; + EXECUTE update_end_datetime_stmt; + + COMMIT; +END \ No newline at end of file From 5d7962af70e28e4a572a0b48bf8aa1a84ca6a649 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 063/275] =?UTF-8?q?Revert=20"feat:crm=5Fdata=5Fsync?= =?UTF-8?q?=E3=81=AE=E6=96=B0=E8=A6=8F=E4=BD=9C=E6=88=90"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 9b8076451f14b1ec07d5751d47cb79bbc1b709db. --- rds_mysql/stored_procedure/crm_data_sync.sql | 41 -------------------- 1 file changed, 41 deletions(-) delete mode 100644 rds_mysql/stored_procedure/crm_data_sync.sql diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql deleted file mode 100644 index c039a18a..00000000 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ /dev/null @@ -1,41 +0,0 @@ --- A5M2で実行時にSQL区切り文字を「;」以外にすること --- $$から始まる文字は後からREPLACEする文字を示す独自ルール --- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない -CREATE PROCEDURE crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) -BEGIN - -- 例外処理 - -- エラーが発生した場合に一時テーブルの削除を実施 - DECLARE EXIT HANDLER FOR SQLEXCEPTION - BEGIN - GET DIAGNOSTICS CONDITION 1 - @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; - ROLLBACK; - SIGNAL SQLSTATE '45000' - SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; - END; - - SET @error_state = NULL, @error_msg = NULL; - START TRANSACTION; - - -- ①-1 Salesforce側で物理削除されたデータを検出し更新する - SET @update_end_datetime = ' - UPDATE $$target_table$$ tt - LEFT JOIN $$target_table_all$$ tta - ON tt.id = tta.id - AND tt.$$target_column$$ = tta.$$target_column$$ - SET - tt.end_datetime = CURRENT_TIMESTAMP() - , tt.upd_user = CURRENT_USER() - , tt.upd_date = CURRENT_TIMESTAMP() - WHERE - tta.id IS NULL - AND tt.end_datetime = "9999-12-31 00:00:00" - '; - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); - PREPARE update_end_datetime_stmt from @update_end_datetime; - EXECUTE update_end_datetime_stmt; - - COMMIT; -END \ No newline at end of file From 99a97658b4a2492e0f42d61b6221fa008de0971b Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 064/275] =?UTF-8?q?refactor:=E2=91=A1-2=E5=86=85=E9=83=A8?= =?UTF-8?q?=E7=B5=90=E5=90=88=E3=81=AB=E5=A4=89=E6=9B=B4=E3=81=97Where?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index b60e2681..366cde70 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -70,15 +70,13 @@ BEGIN -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' UPDATE $$target_table$$ tt - LEFT JOIN $$target_table$$_make_history_tmp mht + JOIN $$target_table$$_make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET end_datetime = mht.max_start_datetime - INTERVAL 1 SECOND , upd_user = CURRENT_USER() , upd_date = CURRENT_TIMESTAMP() - WHERE - mht.id IS NOT NULL '; SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); From 0d7d66ebe4d9e04dd13b356677ba5cb9e7d6ecde Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 065/275] =?UTF-8?q?refactor:INNER=E3=82=92=E6=98=8E?= =?UTF-8?q?=E7=A4=BA=E7=9A=84=E3=81=AB=E8=A8=98=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 366cde70..10bb0095 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -70,7 +70,7 @@ BEGIN -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' UPDATE $$target_table$$ tt - JOIN $$target_table$$_make_history_tmp mht + INNER JOIN $$target_table$$_make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET From 885fa7d274923dcb5dcab2f559f34be02e971121 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 066/275] =?UTF-8?q?feat:crm=5Fdata=5Fsync=E3=81=AE?= =?UTF-8?q?=E6=96=B0=E8=A6=8F=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 rds_mysql/stored_procedure/crm_data_sync.sql diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql new file mode 100644 index 00000000..c039a18a --- /dev/null +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -0,0 +1,41 @@ +-- A5M2で実行時にSQL区切り文字を「;」以外にすること +-- $$から始まる文字は後からREPLACEする文字を示す独自ルール +-- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない +CREATE PROCEDURE crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) +BEGIN + -- 例外処理 + -- エラーが発生した場合に一時テーブルの削除を実施 + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + ROLLBACK; + SIGNAL SQLSTATE '45000' + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; + END; + + SET @error_state = NULL, @error_msg = NULL; + START TRANSACTION; + + -- ①-1 Salesforce側で物理削除されたデータを検出し更新する + SET @update_end_datetime = ' + UPDATE $$target_table$$ tt + LEFT JOIN $$target_table_all$$ tta + ON tt.id = tta.id + AND tt.$$target_column$$ = tta.$$target_column$$ + SET + tt.end_datetime = CURRENT_TIMESTAMP() + , tt.upd_user = CURRENT_USER() + , tt.upd_date = CURRENT_TIMESTAMP() + WHERE + tta.id IS NULL + AND tt.end_datetime = "9999-12-31 00:00:00" + '; + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); + PREPARE update_end_datetime_stmt from @update_end_datetime; + EXECUTE update_end_datetime_stmt; + + COMMIT; +END \ No newline at end of file From 5a0b19f4d8e438723c1471aeec94660586f823d3 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 067/275] =?UTF-8?q?refactor:EXISTS=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 21 ++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql index c039a18a..01d27b9a 100644 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -20,16 +20,21 @@ BEGIN -- ①-1 Salesforce側で物理削除されたデータを検出し更新する SET @update_end_datetime = ' UPDATE $$target_table$$ tt - LEFT JOIN $$target_table_all$$ tta - ON tt.id = tta.id - AND tt.$$target_column$$ = tta.$$target_column$$ SET - tt.end_datetime = CURRENT_TIMESTAMP() - , tt.upd_user = CURRENT_USER() - , tt.upd_date = CURRENT_TIMESTAMP() + tt.end_datetime = CURRENT_TIMESTAMP () + , tt.upd_user = CURRENT_USER () + , tt.upd_date = CURRENT_TIMESTAMP () WHERE - tta.id IS NULL - AND tt.end_datetime = "9999-12-31 00:00:00" + tt.end_datetime = "9999-12-31 00:00:00" + AND NOT EXISTS ( + SELECT + * + FROM + $$target_table_all$$ tta + WHERE + tt.id = tta.id + AND tt.$$target_column$$ = tta.$$target_column$$ + ) '; SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); From 2d9d1308b56a2b406723684aa23434958ad3e092 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 068/275] =?UTF-8?q?refactor:EXISTS=E3=81=AE=E4=B8=AD?= =?UTF-8?q?=E8=BA=AB=E3=81=AESELECT=E5=8F=A5=E3=82=92ID=E3=81=AE=E3=81=BF?= =?UTF-8?q?=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql index 01d27b9a..155bfb99 100644 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -28,7 +28,7 @@ BEGIN tt.end_datetime = "9999-12-31 00:00:00" AND NOT EXISTS ( SELECT - * + tt.id FROM $$target_table_all$$ tta WHERE @@ -37,8 +37,8 @@ BEGIN ) '; SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; From f093a5df226568879dfc6d8a9ba383b16150e160 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 069/275] =?UTF-8?q?refactor:=E5=8F=82=E7=85=A7ID=E3=82=92t?= =?UTF-8?q?ta=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql index 155bfb99..e9986c0f 100644 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -28,7 +28,7 @@ BEGIN tt.end_datetime = "9999-12-31 00:00:00" AND NOT EXISTS ( SELECT - tt.id + tta.id FROM $$target_table_all$$ tta WHERE From 93717ae52843a267d0eff5067ad18535e8c76641 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Mon, 1 Aug 2022 20:02:29 +0900 Subject: [PATCH 070/275] =?UTF-8?q?feat:CRM=E3=81=AE=E6=8B=A1=E5=BC=B5SQL?= =?UTF-8?q?=E3=80=81=E8=A8=AD=E5=AE=9A=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/crm/settings/CRM_Account.txt | 13 +++++ s3/data/crm/settings/CRM_AccountShare.txt | 13 +++++ s3/data/crm/settings/CRM_AccountShare_ex.sql | 1 + .../CRM_Account_Territory_Loader_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Account_ex.sql | 1 + .../settings/CRM_Approved_Document_vod__c.txt | 13 +++++ .../crm/settings/CRM_Call2_Detail_vod__c.txt | 13 +++++ .../settings/CRM_Call2_Discussion_vod__c.txt | 13 +++++ .../settings/CRM_Call2_Key_Message_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Call2_vod__c.txt | 13 +++++ .../settings/CRM_Call_Clickstream_vod__c.txt | 13 +++++ .../crm/settings/CRM_Child_Account_vod__c.txt | 13 +++++ .../settings/CRM_Child_Account_vod__c_ex.sql | 1 + .../CRM_Clm_Presentation_Slide_vod__c.txt | 13 +++++ .../settings/CRM_Clm_Presentation_vod__c.txt | 13 +++++ .../settings/CRM_Coaching_Report_vod__c.txt | 13 +++++ .../settings/CRM_Consent_Header_vod__c.txt | 13 +++++ .../crm/settings/CRM_Consent_Line_vod__c.txt | 13 +++++ .../crm/settings/CRM_Consent_Type_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Contact.txt | 13 +++++ s3/data/crm/settings/CRM_Contact_ex.sql | 1 + ...Dynamic_Attribute_Configuration_vod__c.txt | 13 +++++ .../settings/CRM_Dynamic_Attribute_vod__c.txt | 13 +++++ .../settings/CRM_Email_Activity_vod__c.txt | 13 +++++ .../settings/CRM_Event_Attendee_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Group.txt | 13 +++++ s3/data/crm/settings/CRM_Group_ex.sql | 1 + .../crm/settings/CRM_Key_Message_vod__c.txt | 13 +++++ .../CRM_MSJ_Hospital_Medical_Regimen__c.txt | 13 +++++ .../CRM_MSJ_Inquiry_Assignment__c.txt | 13 +++++ .../settings/CRM_MSJ_MR_Weekly_Report__c.txt | 13 +++++ .../CRM_MSJ_Medical_Event_Evaluation__c.txt | 13 +++++ .../settings/CRM_MSJ_Medical_Regimen__c.txt | 13 +++++ s3/data/crm/settings/CRM_MSJ_Patient__c.txt | 13 +++++ .../crm/settings/CRM_Medical_Event_vod__c.txt | 13 +++++ .../settings/CRM_Medical_Inquiry_vod__c.txt | 13 +++++ .../settings/CRM_Medical_Insight_vod__c.txt | 13 +++++ .../CRM_Multichannel_Activity_Line_vod__c.txt | 13 +++++ .../CRM_Multichannel_Activity_vod__c.txt | 13 +++++ .../CRM_Multichannel_Consent_vod__c.txt | 13 +++++ .../settings/CRM_My_Setup_Products_vod__c.txt | 13 +++++ .../CRM_ObjectTerritory2Association.txt | 13 +++++ .../CRM_ObjectTerritory2Association_ex.sql | 1 + .../crm/settings/CRM_Product_Group_vod__c.txt | 13 +++++ .../settings/CRM_Product_Metrics_vod__c.txt | 13 +++++ .../CRM_Product_Metrics_vod__c_ex.sql | 1 + s3/data/crm/settings/CRM_Product_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Profile.txt | 13 +++++ s3/data/crm/settings/CRM_Profile_ex.sql | 1 + .../settings/CRM_Question_Response_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_RecordType.txt | 13 +++++ .../settings/CRM_Remote_Meeting_vod__c.txt | 13 +++++ .../crm/settings/CRM_Sent_Email_vod__c.txt | 13 +++++ .../crm/settings/CRM_Sent_Fragment_vod__c.txt | 13 +++++ .../settings/CRM_Survey_Question_vod__c.txt | 13 +++++ .../crm/settings/CRM_Survey_Target_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Survey_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Territory2.txt | 13 +++++ s3/data/crm/settings/CRM_Territory2_ALL.txt | 14 +++++ s3/data/crm/settings/CRM_Territory2_ex.sql | 2 + .../CRM_Time_Off_Territory_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_User.txt | 13 +++++ s3/data/crm/settings/CRM_UserRole.txt | 13 +++++ .../CRM_UserTerritory2Association.txt | 13 +++++ .../CRM_UserTerritory2Association_ALL.txt | 14 +++++ .../CRM_UserTerritory2Association_ex.sql | 2 + s3/data/crm/settings/configmap.config | 58 +++++++++++++++++++ 67 files changed, 800 insertions(+) create mode 100644 s3/data/crm/settings/CRM_Account.txt create mode 100644 s3/data/crm/settings/CRM_AccountShare.txt create mode 100644 s3/data/crm/settings/CRM_AccountShare_ex.sql create mode 100644 s3/data/crm/settings/CRM_Account_Territory_Loader_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Account_ex.sql create mode 100644 s3/data/crm/settings/CRM_Approved_Document_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call2_Detail_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call2_Discussion_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call2_Key_Message_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call2_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call_Clickstream_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Child_Account_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql create mode 100644 s3/data/crm/settings/CRM_Clm_Presentation_Slide_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Clm_Presentation_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Coaching_Report_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Consent_Header_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Consent_Line_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Consent_Type_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Contact.txt create mode 100644 s3/data/crm/settings/CRM_Contact_ex.sql create mode 100644 s3/data/crm/settings/CRM_Dynamic_Attribute_Configuration_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Dynamic_Attribute_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Email_Activity_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Group.txt create mode 100644 s3/data/crm/settings/CRM_Group_ex.sql create mode 100644 s3/data/crm/settings/CRM_Key_Message_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Hospital_Medical_Regimen__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Inquiry_Assignment__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_MR_Weekly_Report__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Medical_Event_Evaluation__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Medical_Regimen__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Patient__c.txt create mode 100644 s3/data/crm/settings/CRM_Medical_Event_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Medical_Inquiry_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Medical_Insight_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Multichannel_Activity_Line_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Multichannel_Activity_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Multichannel_Consent_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_My_Setup_Products_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_ObjectTerritory2Association.txt create mode 100644 s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql create mode 100644 s3/data/crm/settings/CRM_Product_Group_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Product_Metrics_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql create mode 100644 s3/data/crm/settings/CRM_Product_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Profile.txt create mode 100644 s3/data/crm/settings/CRM_Profile_ex.sql create mode 100644 s3/data/crm/settings/CRM_Question_Response_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_RecordType.txt create mode 100644 s3/data/crm/settings/CRM_Remote_Meeting_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Sent_Email_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Sent_Fragment_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Survey_Question_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Survey_Target_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Survey_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Territory2.txt create mode 100644 s3/data/crm/settings/CRM_Territory2_ALL.txt create mode 100644 s3/data/crm/settings/CRM_Territory2_ex.sql create mode 100644 s3/data/crm/settings/CRM_Time_Off_Territory_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_User.txt create mode 100644 s3/data/crm/settings/CRM_UserRole.txt create mode 100644 s3/data/crm/settings/CRM_UserTerritory2Association.txt create mode 100644 s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt create mode 100644 s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql create mode 100644 s3/data/crm/settings/configmap.config diff --git a/s3/data/crm/settings/CRM_Account.txt b/s3/data/crm/settings/CRM_Account.txt new file mode 100644 index 00000000..bee3992c --- /dev/null +++ b/s3/data/crm/settings/CRM_Account.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +248 +Id,IsDeleted,MasterRecordId,Name,LastName,FirstName,Salutation,RecordTypeId,Phone,Fax,Website,PhotoUrl,NumberOfEmployees,Ownership,OwnerId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,IsExcludedFromRealign,PersonContactId,IsPersonAccount,PersonMailingStreet,PersonMailingCity,PersonMailingState,PersonMailingPostalCode,PersonMailingCountry,PersonMailingLatitude,PersonMailingLongitude,PersonMailingGeocodeAccuracy,PersonMailingAddress,PersonOtherStreet,PersonOtherCity,PersonOtherState,PersonOtherPostalCode,PersonOtherCountry,PersonOtherLatitude,PersonOtherLongitude,PersonOtherGeocodeAccuracy,PersonOtherAddress,PersonMobilePhone,PersonHomePhone,PersonOtherPhone,PersonAssistantPhone,PersonEmail,PersonTitle,PersonDepartment,PersonAssistantName,PersonBirthdate,PersonHasOptedOutOfEmail,PersonHasOptedOutOfFax,PersonDoNotCall,PersonLastCURequestDate,PersonLastCUUpdateDate,PersonEmailBouncedReason,PersonEmailBouncedDate,PersonIndividualId,Jigsaw,JigsawCompanyId,AccountSource,SicDesc,External_ID_vod__c,Credentials_vod__c,Territory_vod__c,Exclude_from_Zip_to_Terr_Processing_vod__c,Group_Specialty_1_vod__c,Group_Specialty_2_vod__c,Specialty_1_vod__c,Specialty_2_vod__c,Formatted_Name_vod__c,Territory_Test_vod__c,Mobile_ID_vod__c,Gender_vod__c,ID_vod__c,Do_Not_Sync_Sales_Data_vod__c,ID2_vod__c,Preferred_Name_vod__c,Sample_Default_vod__c,Segmentations_vod__c,Restricted_Products_vod__c,Payer_Id_vod__c,Alternate_Name_vod__c,Do_Not_Call_vod__c,MSJ_Beds__c,Spend_Amount__c,PDRP_Opt_Out_vod__c,Spend_Status_Value_vod__c,PDRP_Opt_Out_Date_vod__c,Spend_Status_vod__c,Enable_Restricted_Products_vod__c,Call_Reminder_vod__c,Account_Group_vod__c,Primary_Parent_vod__c,Color_vod__c,Middle_vod__c,Suffix_vod__c,MSJ_Type__c,No_Orders_vod__c,MSJ_BU_ONC__c,MSJ_BU_FE__c,Account_Search_FirstLast_vod__c,Account_Search_LastFirst_vod__c,MSJ_Operation__c,Practice_at_Hospital_vod__c,Practice_Near_Hospital_vod__c,Do_Not_Create_Child_Account_vod__c,Total_MDs_DOs__c,AHA__c,Order_Type_vod__c,NPI_vod__c,ME__c,Speaker__c,Investigator_vod__c,Default_Order_Type_vod__c,Tax_Status__c,Model__c,Offerings__c,Departments__c,Account_Type__c,MSJ_ONC_Tier__c,Account_Search_Business_vod__c,Business_Professional_Person_vod__c,Hospital_Type_vod__c,Account_Class_vod__c,Furigana_vod__c,MSJ_JISART__c,Total_Revenue_000__c,Net_Income_Loss_000__c,PMPM_Income_Loss_000__c,Commercial_Premiums_PMPM__c,Medical_Loss_Ratio__c,Medical_Expenses_PMPM__c,Commercial_Patient_Days_1000__c,HMO_Market_Shr__c,HMO__c,HMO_POS__c,PPO__c,PPO_POS__c,Medicare__c,Medicaid__c,MSJ_HP_Name_E__c,MSJ_Department__c,MSJ_Date_Of_Birth__c,MSJ_FE_GF_Potential__c,MSJ_FE_SZ_Potential__c,MSJ_EB_CRC_Ladder__c,MSJ_EB_CRC_Segment__c,MSJ_EB_HN_Segment__c,Business_Description__c,Regional_Strategy__c,Contracts_Process__c,MSJ_GF_segment__c,MSJ_DCF_DR_Code__c,MSJ_SZ_Segment__c,MSJ_Remark__c,MSJ_Title__c,MSJ_Role__c,MSJ_Kana__c,MSJ_Specialism__c,MSJ_Graduated_from__c,MSJ_Year_Graduation__c,Target__c,KOL_vod__c,MSJ_EPPV_Code__c,MSJ_DCF_HP_Code__c,Total_Lives__c,Total_Physicians_Enrolled__c,MSJ_Delete__c,MSJ_KOL_LOL__c,MSJ_ONC_Status__c,Account_Identifier_vod__c,Approved_Email_Opt_Type_vod__c,Language_vod__c,MSJ_KRAS_Routine_Date__c,MSJ_KRAS_Routine__c,MSJ_DRP_Target__c,MSJ_Fertility_Evaluation_Score__c,MSJ_Fertility_Tracking_Last_Modify_Date__c,Total_Pharmacists__c,MSJ_Number_of_Gonadotropin__c,MSJ_Number_of_IUI_cycle__c,MSJ_Number_of_OI_monthly_cycle__c,MSJ_OI_Protocol_learning_level__c,MSJ_H_N_Tier__c,MSJ_XLK_Segment__c,MSJ_XLK_Tier__c,Career_Status_vod__c,Photo_vod__c,MSJ_EB_H_N_LA_Segment__c,MSJ_EB_H_N_RM_Segment__c,MSJ_FE_CE_Potential__c,MSJ_FE_1C_potential__c,MSJ_FE_OV_potential__c,MSJ_FE_Tech_potential__c,MSJ_CE_segment__c,MSJ_1C_segment__c,MSJ_OV_segment__c,MSJ_Tech_segment__c,MSJ_Target_Call_Num__c,MSJ_DR_Change_Log__c,MSJ_Global_scientific_exposure__c,MSJ_H_index__c,MSJ_Num_of_Article_3Y__c,MSJ_Num_of_Article__c,MSJ_Num_of_Article_as_1st_Author_3Y__c,MSJ_Num_of_article_growth_rate_3Y__c,MSJ_Num_of_cited_3Y__c,MSJ_Num_of_impact_factor_3Y__c,MSJ_impact_factor_as_1st_Author_3Y__c,EMDS_Has_Pipeline_Opportunity__c,EMDS_Pipeline_Count__c,MSJ_BVC_Segment__c,MSJ_BVC_Tier__c,MSJ_BVC_AcctOpen__c,MSJ_BVC_MCC_Patients__c,MSJ_ONC_HP_Segment__c,MSJ_AE_Department__c,MSJ_AE_Facility__c,MSJ_AE_Name__c,MSJ_AE_Title__c,MSJ_Email__c,MSJ_FE_GF2_Potential__c,MSJ_FE_Location_potential__c,MSJ_GF2_segment__c,MSJ_OPTIN_target__c,MSJ_Merck_Specialty1__c,MSJ_Merck_Specialty2__c,MSJ_Marketing_Cloud_Integration__c,MSJ_Marketing_Cloud1__c,MSJ_Marketing_Cloud2__c,MSJ_Marketing_Cloud3__c,MSJ_Marketing_Cloud4__c,MSJ_Medical_Department__c,MSJ_Marketing_Cloud0__c,Mobile_ID_vod__pc,H1Insights__H1_NPI_Value_for_Testing__pc,H1Insights__H1_Person_ID__pc,H1Insights__H1_Request_Status__pc,H1Insights__H1_URL__pc,H1Insights__NPI_Number__pc,H1Insights__NPI_Number_for_H1_Insights__pc,MSJ_Marketing_Cloud_Integration__pc +id,is_deleted,master_record_id,name,last_name,first_name,salutation,record_type_id,phone,fax,website,photo_url,number_of_employees,ownership,owner_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,is_excluded_from_realign,person_contact_id,is_person_account,person_mailing_street,person_mailing_city,person_mailing_state,person_mailing_postal_code,person_mailing_country,person_mailing_latitude,person_mailing_longitude,person_mailing_geocode_accuracy,person_mailing_address,person_other_street,person_other_city,person_other_state,person_other_postal_code,person_other_country,person_other_latitude,person_other_longitude,person_other_geocode_accuracy,person_other_address,person_mobile_phone,person_home_phone,person_other_phone,person_assistant_phone,person_email,person_title,person_department,person_assistant_name,person_birthdate,person_has_opted_out_of_email,person_has_opted_out_of_fax,person_do_not_call,person_last_curequest_date,person_last_cuupdate_date,person_email_bounced_reason,person_email_bounced_date,person_individual_id,jigsaw,jigsaw_company_id,account_source,sic_desc,external_id_vod__c,credentials_vod__c,territory_vod__c,exclude_from_zip_to_terr_processing_vod__c,group_specialty_1_vod__c,group_specialty_2_vod__c,specialty_1_vod__c,specialty_2_vod__c,formatted_name_vod__c,territory_test_vod__c,mobile_id_vod__c,gender_vod__c,id_vod__c,do_not_sync_sales_data_vod__c,id2_vod__c,preferred_name_vod__c,sample_default_vod__c,segmentations_vod__c,restricted_products_vod__c,payer_id_vod__c,alternate_name_vod__c,do_not_call_vod__c,msj_beds__c,spend_amount__c,pdrp_opt_out_vod__c,spend_status_value_vod__c,pdrp_opt_out_date_vod__c,spend_status_vod__c,enable_restricted_products_vod__c,call_reminder_vod__c,account_group_vod__c,primary_parent_vod__c,color_vod__c,middle_vod__c,suffix_vod__c,msj_type__c,no_orders_vod__c,msj_bu_onc__c,msj_bu_fe__c,account_search_first_last_vod__c,account_search_last_first_vod__c,msj_operation__c,practice_at_hospital_vod__c,practice_near_hospital_vod__c,do_not_create_child_account_vod__c,total_mds_dos__c,aha__c,order_type_vod__c,npi_vod__c,me__c,speaker__c,investigator_vod__c,default_order_type_vod__c,tax_status__c,model__c,offerings__c,departments__c,account_type__c,msj_onc_tier__c,account_search_business_vod__c,business_professional_person_vod__c,hospital_type_vod__c,account_class_vod__c,furigana_vod__c,msj_jisart__c,total_revenue_000__c,net_income_loss_000__c,pmpm_income_loss_000__c,commercial_premiums_pmpm__c,medical_loss_ratio__c,medical_expenses_pmpm__c,commercial_patient_days_1000__c,hmo_market_shr__c,hmo__c,hmo_pos__c,ppo__c,ppo_pos__c,medicare__c,medicaid__c,msj_hp_name_e__c,msj_department__c,msj_date_of_birth__c,msj_fe_gf_potential__c,msj_fe_sz_potential__c,msj_eb_crc_ladder__c,msj_eb_crc_segment__c,msj_eb_hn_segment__c,business_description__c,regional_strategy__c,contracts_process__c,msj_gf_segment__c,msj_dcf_dr_code__c,msj_sz_segment__c,msj_remark__c,msj_title__c,msj_role__c,msj_kana__c,msj_specialism__c,msj_graduated_from__c,msj_year_graduation__c,target__c,kol_vod__c,msj_eppv_code__c,msj_dcf_hp_code__c,total_lives__c,total_physicians_enrolled__c,msj_delete__c,msj_kol_lol__c,msj_onc_status__c,account_identifier_vod__c,approved_email_opt_type_vod__c,language_vod__c,msj_kras_routine_date__c,msj_kras_routine__c,msj_drp_target__c,msj_fertility_evaluation_score__c,msj_fertility_tracking_last_modify_date__c,total_pharmacists__c,msj_number_of_gonadotropin__c,msj_number_of_iui_cycle__c,msj_number_of_oi_monthly_cycle__c,msj_oi_protocol_learning_level__c,msj_h_n_tier__c,msj_xlk_segment__c,msj_xlk_tier__c,career_status_vod__c,photo_vod__c,msj_eb_h_n_la_segment__c,msj_eb_h_n_rm_segment__c,msj_fe_ce_potential__c,msj_fe_1_c_potential__c,msj_fe_ov_potential__c,msj_fe_tech_potential__c,msj_ce_segment__c,msj_1_c_segment__c,msj_ov_segment__c,msj_tech_segment__c,msj_target_call_num__c,msj_dr_change_log__c,msj_global_scientific_exposure__c,msj_h_index__c,msj_num_of_article_3_y__c,msj_num_of_article__c,msj_num_of_article_as_1st_author_3_y__c,msj_num_of_article_growth_rate_3_y__c,msj_num_of_cited_3_y__c,msj_num_of_impact_factor_3_y__c,msj_impact_factor_as_1st_author_3_y__c,emds_has_pipeline_opportunity__c,emds_pipeline_count__c,msj_bvc_segment__c,msj_bvc_tier__c,msj_bvc_acct_open__c,msj_bvc_mcc_patients__c,msj_onc_hp_segment__c,msj_ae_department__c,msj_ae_facility__c,msj_ae_name__c,msj_ae_title__c,msj_email__c,msj_fe_gf2_potential__c,msj_fe_location_potential__c,msj_gf2_segment__c,msj_optin_target__c,msj_merck_specialty1__c,msj_merck_specialty2__c,msj_marketing_cloud_integration__c,msj_marketing_cloud1__c,msj_marketing_cloud2__c,msj_marketing_cloud3__c,msj_marketing_cloud4__c,msj_medical_department__c,msj_marketing_cloud0__c,mobile_id_vod__pc,h1_insights__h1_npi_value_for_testing__pc,h1_insights__h1_person_id__pc,h1_insights__h1_request_status__pc,h1_insights__h1_url__pc,h1_insights__npi_number__pc,h1_insights__npi_number_for_h1_insights__pc,msj_marketing_cloud_integration__pc +src02.crm_account +org02.crm_account +CRM_Account_ex.sql + diff --git a/s3/data/crm/settings/CRM_AccountShare.txt b/s3/data/crm/settings/CRM_AccountShare.txt new file mode 100644 index 00000000..8176af1e --- /dev/null +++ b/s3/data/crm/settings/CRM_AccountShare.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +11 +Id,AccountId,UserOrGroupId,AccountAccessLevel,OpportunityAccessLevel,CaseAccessLevel,ContactAccessLevel,RowCause,LastModifiedDate,LastModifiedById,IsDeleted +id,account_id,user_or_group_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,row_cause,last_modified_date,last_modified_by_id,is_deleted +src02.crm_account_share +org02.crm_account_share +CRM_AccountShare_ex.sql + diff --git a/s3/data/crm/settings/CRM_AccountShare_ex.sql b/s3/data/crm/settings/CRM_AccountShare_ex.sql new file mode 100644 index 00000000..d18e68db --- /dev/null +++ b/s3/data/crm/settings/CRM_AccountShare_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_account_share', 'last_modified_date'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Account_Territory_Loader_vod__c.txt b/s3/data/crm/settings/CRM_Account_Territory_Loader_vod__c.txt new file mode 100644 index 00000000..676e8b39 --- /dev/null +++ b/s3/data/crm/settings/CRM_Account_Territory_Loader_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +19 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,External_ID_vod__c,Territory_vod__c,Mobile_ID_vod__c,Territory_To_Add_vod__c,Territory_to_Drop_vod__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,external_id_vod__c,territory_vod__c,mobile_id_vod__c,territory_to_add_vod__c,territory_to_drop_vod__c +src02.crm_account_territory_loader_vod__c +org02.crm_account_territory_loader_vod__c + + diff --git a/s3/data/crm/settings/CRM_Account_ex.sql b/s3/data/crm/settings/CRM_Account_ex.sql new file mode 100644 index 00000000..792d7a28 --- /dev/null +++ b/s3/data/crm/settings/CRM_Account_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_account', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Approved_Document_vod__c.txt b/s3/data/crm/settings/CRM_Approved_Document_vod__c.txt new file mode 100644 index 00000000..23683a8c --- /dev/null +++ b/s3/data/crm/settings/CRM_Approved_Document_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +53 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Detail_Group_vod__c,Document_Description_vod__c,Document_Host_URL_vod__c,Document_ID_vod__c,Document_Last_Mod_DateTime_vod__c,Email_Allows_Documents_vod__c,Email_Domain_vod__c,Email_Fragment_HTML_vod__c,Email_From_Address_vod__c,Email_From_Name_vod__c,Email_HTML_1_vod__c,Email_HTML_2_vod__c,Email_ReplyTo_Address_vod__c,Email_ReplyTo_Name_vod__c,Email_Subject_vod__c,Email_Template_Fragment_Document_ID_vod__c,Email_Template_Fragment_HTML_vod__c,ISI_Document_ID_vod__c,Language_vod__c,Other_Document_ID_List_vod__c,PI_Document_ID_vod__c,Piece_Document_ID_vod__c,Product_vod__c,Status_vod__c,Territory_vod__c,Vault_Instance_ID_vod__c,Allow_Any_Product_Fragment_vod__c,Allowed_Document_IDs_vod__c,Engage_Document_Id_vod__c,Vault_Document_ID_vod__c,Key_Message_vod__c,Events_Management_Subtype_vod__c,Survey_vod__c,Content_Type_vod__c,Bcc_vod__c,Audience_vod__c,WeChat_Template_ID_vod__c,Check_Consent_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,detail_group_vod__c,document_description_vod__c,document_host_url_vod__c,document_id_vod__c,document_last_mod_date_time_vod__c,email_allows_documents_vod__c,email_domain_vod__c,email_fragment_html_vod__c,email_from_address_vod__c,email_from_name_vod__c,email_html_1_vod__c,email_html_2_vod__c,email_reply_to_address_vod__c,email_reply_to_name_vod__c,email_subject_vod__c,email_template_fragment_document_id_vod__c,email_template_fragment_html_vod__c,isi_document_id_vod__c,language_vod__c,other_document_id_list_vod__c,pi_document_id_vod__c,piece_document_id_vod__c,product_vod__c,status_vod__c,territory_vod__c,vault_instance_id_vod__c,allow_any_product_fragment_vod__c,allowed_document_ids_vod__c,engage_document_id_vod__c,vault_document_id_vod__c,key_message_vod__c,events_management_subtype_vod__c,survey_vod__c,content_type_vod__c,bcc_vod__c,audience_vod__c,we_chat_template_id_vod__c,check_consent_vod__c +src02.crm_approved_document_vod__c +org02.crm_approved_document_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call2_Detail_vod__c.txt b/s3/data/crm/settings/CRM_Call2_Detail_vod__c.txt new file mode 100644 index 00000000..67a53029 --- /dev/null +++ b/s3/data/crm/settings/CRM_Call2_Detail_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +17 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Is_Parent_Call_vod__c,Call2_vod__c,Product_vod__c,Detail_Priority_vod__c,Mobile_ID_vod__c,Override_Lock_vod__c,Type_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,is_parent_call_vod__c,call2_vod__c,product_vod__c,detail_priority_vod__c,mobile_id_vod__c,override_lock_vod__c,type_vod__c +src02.crm_call2_detail_vod__c +org02.crm_call2_detail_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call2_Discussion_vod__c.txt b/s3/data/crm/settings/CRM_Call2_Discussion_vod__c.txt new file mode 100644 index 00000000..c86c840c --- /dev/null +++ b/s3/data/crm/settings/CRM_Call2_Discussion_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +44 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Account_vod__c,Call2_vod__c,Activity__c,Comments__c,Contact_vod__c,Call_Date_vod__c,Product_Strategy_vod__c,Product_Tactic_vod__c,Restricted_Comments__c,Product_vod__c,Presentation__c,Discussion_Topics__c,Slides__c,User_vod__c,Indication__c,Mobile_ID_vod__c,Medical_Event_vod__c,Is_Parent_Call_vod__c,Override_Lock_vod__c,zvod_Product_Map_vod__c,Attendee_Type_vod__c,Entity_Reference_Id_vod__c,Account_Tactic_vod__c,MSJ_Material_Type__c,MSJ_Discussion_Contents__c,MSJ_IST_Minutes__c,MSJ_Off_Label_Minutes__c,MSJ_Discussion_Objectives__c,MSJ_Insight__c,EMDS_Materials__c,EMDS_Topic__c,MSJ_Visit_Purpose__c,MSJ_Insight_Count__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,account_vod__c,call2_vod__c,activity__c,comments__c,contact_vod__c,call_date_vod__c,product_strategy_vod__c,product_tactic_vod__c,restricted_comments__c,product_vod__c,presentation__c,discussion_topics__c,slides__c,user_vod__c,indication__c,mobile_id_vod__c,medical_event_vod__c,is_parent_call_vod__c,override_lock_vod__c,zvod_product_map_vod__c,attendee_type_vod__c,entity_reference_id_vod__c,account_tactic_vod__c,msj_material_type__c,msj_discussion_contents__c,msj_ist_minutes__c,msj_off_label_minutes__c,msj_discussion_objectives__c,msj_insight__c,emds_materials__c,emds_topic__c,msj_visit_purpose__c,msj_insight_count__c +src02.crm_call2_discussion_vod__c +org02.crm_call2_discussion_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call2_Key_Message_vod__c.txt b/s3/data/crm/settings/CRM_Call2_Key_Message_vod__c.txt new file mode 100644 index 00000000..40267bd0 --- /dev/null +++ b/s3/data/crm/settings/CRM_Call2_Key_Message_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +35 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Account_vod__c,Call2_vod__c,Reaction_vod__c,Product_vod__c,Key_Message_vod__c,Mobile_ID_vod__c,Contact_vod__c,Call_Date_vod__c,User_vod__c,Category_vod__c,Vehicle_vod__c,Is_Parent_Call_vod__c,Override_Lock_vod__c,CLM_ID_vod__c,Slide_Version_vod__c,Duration_vod__c,Presentation_ID_vod__c,Start_Time_vod__c,Attendee_Type_vod__c,Entity_Reference_Id_vod__c,Segment_vod__c,Display_Order_vod__c,Clm_Presentation_Name_vod__c,Clm_Presentation_Version_vod__c,Clm_Presentation_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,account_vod__c,call2_vod__c,reaction_vod__c,product_vod__c,key_message_vod__c,mobile_id_vod__c,contact_vod__c,call_date_vod__c,user_vod__c,category_vod__c,vehicle_vod__c,is_parent_call_vod__c,override_lock_vod__c,clm_id_vod__c,slide_version_vod__c,duration_vod__c,presentation_id_vod__c,start_time_vod__c,attendee_type_vod__c,entity_reference_id_vod__c,segment_vod__c,display_order_vod__c,clm_presentation_name_vod__c,clm_presentation_version_vod__c,clm_presentation_vod__c +src02.crm_call2_key_message_vod__c +org02.crm_call2_key_message_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call2_vod__c.txt b/s3/data/crm/settings/CRM_Call2_vod__c.txt new file mode 100644 index 00000000..6b2c330e --- /dev/null +++ b/s3/data/crm/settings/CRM_Call2_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +205 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Call_Comments_vod__c,Sample_Card_vod__c,Add_Detail_vod__c,Property_vod__c,Account_vod__c,zvod_Product_Discussion_vod__c,Status_vod__c,Parent_Address_vod__c,Account_Plan_vod__c,zvod_SaveNew_vod__c,Next_Call_Notes_vod__c,Pre_Call_Notes_vod__c,Mobile_ID_vod__c,zvod_Account_Credentials_vod_c_vod__c,zvod_Account_Preferred_Name_vod_c_vod__c,zvod_Account_Sample_Status_vod_c_vod__c,zvod_Attendees_vod__c,zvod_Key_Messages_vod__c,zvod_Detailing_vod__c,zvod_Expenses_vod__c,zvod_Followup_vod__c,zvod_Samples_vod__c,zvod_Save_vod__c,zvod_Submit_vod__c,zvod_Delete_vod__c,Activity_Type__c,Significant_Event__c,Location_vod__c,Subject_vod__c,Unlock_vod__c,Call_Datetime_vod__c,Disbursed_To_vod__c,Disclaimer_vod__c,Request_Receipt_vod__c,Signature_Date_vod__c,Signature_vod__c,Territory_vod__c,Submitted_By_Mobile_vod__c,Call_Type_vod__c,Add_Key_Message_vod__c,Address_vod__c,Attendees_vod__c,Attendee_Type_vod__c,Call_Date_vod__c,Detailed_Products_vod__c,No_Disbursement_vod__c,Parent_Call_vod__c,User_vod__c,Contact_vod__c,zvod_Entity_vod__c,Medical_Event_vod__c,Mobile_Created_Datetime_vod__c,Mobile_Last_Modified_Datetime_vod__c,License_vod__c,Is_Parent_Call_vod__c,Entity_Display_Name_vod__c,Override_Lock_vod__c,Last_Device_vod__c,Ship_Address_Line_1_vod__c,Ship_Address_Line_2_vod__c,Ship_City_vod__c,Ship_Country_vod__c,Ship_License_Expiration_Date_vod__c,Ship_License_Status_vod__c,Ship_License_vod__c,Ship_State_vod__c,Ship_To_Address_vod__c,Ship_Zip_vod__c,Ship_To_Address_Text_vod__c,CLM_vod__c,zvod_CLMDetails_vod__c,Is_Sampled_Call_vod__c,zvod_Surveys_vod__c,Presentations_vod__c,Entity_Reference_Id_vod__c,Error_Reference_Call_vod__c,Duration_vod__c,Color_vod__c,Allowed_Products_vod__c,zvod_Attachments_vod__c,Sample_Card_Reason_vod__c,ASSMCA_vod__c,Address_Line_1_vod__c,Address_Line_2_vod__c,City_vod__c,DEA_Address_Line_1_vod__c,DEA_Address_Line_2_vod__c,DEA_Address_vod__c,DEA_City_vod__c,DEA_Expiration_Date_vod__c,DEA_State_vod__c,DEA_Zip_4_vod__c,DEA_Zip_vod__c,DEA_vod__c,Ship_Zip_4_vod__c,State_vod__c,Zip_4_vod__c,Zip_vod__c,Sample_Send_Card_vod__c,zvod_Address_vod_c_DEA_Status_vod_c_vod__c,Signature_Page_Image_vod__c,Credentials_vod__c,Salutation_vod__c,zvod_Account_Call_Reminder_vod_c_vod__c,MSJ_Meeting_Duration__c,MSJ_Double_Visit_AM__c,zvod_Business_Account_vod__c,Product_Priority_1_vod__c,Product_Priority_2_vod__c,Product_Priority_3_vod__c,Product_Priority_4_vod__c,Product_Priority_5_vod__c,zvod_More_Actions_vod__c,zvod_Call_Conflict_Status_vod__c,Signature_Timestamp_vod__c,Expense_Amount_vod__c,Total_Expense_Attendees_Count_vod__c,Attendee_list_vod__c,Expense_Post_Status_vod__c,Attendee_Post_Status_vod__c,Expense_System_External_ID_vod__c,Incurred_Expense_vod__c,Assigner_vod__c,Assignment_Datetime_vod__c,zvod_Call_Objective_vod__c,Signature_Location_Longitude_vod__c,Signature_Location_Latitude_vod__c,Location_Services_Status_vod__c,MSJ_Double_Visit_Other__c,MSJ_Comment__c,MSJ_For_Reporting__c,MSJ_Number_of_Attendees__c,MSJ_Main_Dept__c,Planned_Type_vjh__c,Cobrowse_URL_Participant_vod__c,MSJ_Activity_Method_Text__c,MSJ_Activity_Method__c,MSJ_Classification__c,MSJ_Double_Visit_MSL__c,MSJ_MSL_Comment_for_MR__c,MSJ_APD__c,Medical_Inquiry_vod__c,MSJ_Call_Type_MSJ__c,MSJ_Prescription_Request__c,MSJ_Patient_Follow__c,Child_Account_Id_vod__c,Child_Account_vod__c,Location_Id_vod__c,Location_Name_vod__c,MSJ_Comments_about_technology__c,Remote_Meeting_vod__c,Veeva_Remote_Meeting_Id_vod__c,MSJ_Activity_Type_Report__c,MSJ_Activity_Type__c,MSJ_Activity__c,MSJ_Comments__c,MSJ_Therapy__c,MSJ_Time_Hrs__c,EMDS_CO_Reference__c,EMDS_Call_Sub_Type__c,EMDS_Call_Type__c,EMDS_Call_Unsuccessful__c,EMDS_Congress_Type__c,EMDS_Date_of_Service__c,EMDS_Fertility_DisInterest__c,EMDS_Fertility_Interest__c,EMDS_Installed_Equipment__c,EMDS_Pipeline_Stage_Value__c,EMDS_Pipeline_Stage__c,EMDS_Pipeline__c,EMDS_Reason_for_Call__c,EMDS_Training_Completed__c,MSJ_BrainStorming__c,MSJ_SIPAGL_1A__c,MSJ_SIPAGL_1B__c,MSJ_SIPAGL_2__c,MSJ_SIPAGL_3__c,MSJ_SIPAGL_4A__c,MSJ_SIPAGL_5A__c,MSJ_SIPAGL_comment__c,MSJ_SIPAGL_4B__c,MSJ_SIPAGL_5B__c,Location_Text_vod__c,Call_Channel_vod__c,MSJ_Scientific_Interaction__c,MSJ_Activity_Email_Reply__c,MSJ_Interaction_Duration__c,MSJ_SIPAGL_1A_date__c,MSJ_CoPromotion__c,Call_Channel_Formula_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,call_comments_vod__c,sample_card_vod__c,add_detail_vod__c,property_vod__c,account_vod__c,zvod_product_discussion_vod__c,status_vod__c,parent_address_vod__c,account_plan_vod__c,zvod_save_new_vod__c,next_call_notes_vod__c,pre_call_notes_vod__c,mobile_id_vod__c,zvod_account_credentials_vod_c_vod__c,zvod_account_preferred_name_vod_c_vod__c,zvod_account_sample_status_vod_c_vod__c,zvod_attendees_vod__c,zvod_key_messages_vod__c,zvod_detailing_vod__c,zvod_expenses_vod__c,zvod_followup_vod__c,zvod_samples_vod__c,zvod_save_vod__c,zvod_submit_vod__c,zvod_delete_vod__c,activity_type__c,significant_event__c,location_vod__c,subject_vod__c,unlock_vod__c,call_datetime_vod__c,disbursed_to_vod__c,disclaimer_vod__c,request_receipt_vod__c,signature_date_vod__c,signature_vod__c,territory_vod__c,submitted_by_mobile_vod__c,call_type_vod__c,add_key_message_vod__c,address_vod__c,attendees_vod__c,attendee_type_vod__c,call_date_vod__c,detailed_products_vod__c,no_disbursement_vod__c,parent_call_vod__c,user_vod__c,contact_vod__c,zvod_entity_vod__c,medical_event_vod__c,mobile_created_datetime_vod__c,mobile_last_modified_datetime_vod__c,license_vod__c,is_parent_call_vod__c,entity_display_name_vod__c,override_lock_vod__c,last_device_vod__c,ship_address_line_1_vod__c,ship_address_line_2_vod__c,ship_city_vod__c,ship_country_vod__c,ship_license_expiration_date_vod__c,ship_license_status_vod__c,ship_license_vod__c,ship_state_vod__c,ship_to_address_vod__c,ship_zip_vod__c,ship_to_address_text_vod__c,clm_vod__c,zvod_clmdetails_vod__c,is_sampled_call_vod__c,zvod_surveys_vod__c,presentations_vod__c,entity_reference_id_vod__c,error_reference_call_vod__c,duration_vod__c,color_vod__c,allowed_products_vod__c,zvod_attachments_vod__c,sample_card_reason_vod__c,assmca_vod__c,address_line_1_vod__c,address_line_2_vod__c,city_vod__c,dea_address_line_1_vod__c,dea_address_line_2_vod__c,dea_address_vod__c,dea_city_vod__c,dea_expiration_date_vod__c,dea_state_vod__c,dea_zip_4_vod__c,dea_zip_vod__c,dea_vod__c,ship_zip_4_vod__c,state_vod__c,zip_4_vod__c,zip_vod__c,sample_send_card_vod__c,zvod_address_vod_c_dea_status_vod_c_vod__c,signature_page_image_vod__c,credentials_vod__c,salutation_vod__c,zvod_account_call_reminder_vod_c_vod__c,msj_meeting_duration__c,msj_double_visit_am__c,zvod_business_account_vod__c,product_priority_1_vod__c,product_priority_2_vod__c,product_priority_3_vod__c,product_priority_4_vod__c,product_priority_5_vod__c,zvod_more_actions_vod__c,zvod_call_conflict_status_vod__c,signature_timestamp_vod__c,expense_amount_vod__c,total_expense_attendees_count_vod__c,attendee_list_vod__c,expense_post_status_vod__c,attendee_post_status_vod__c,expense_system_external_id_vod__c,incurred_expense_vod__c,assigner_vod__c,assignment_datetime_vod__c,zvod_call_objective_vod__c,signature_location_longitude_vod__c,signature_location_latitude_vod__c,location_services_status_vod__c,msj_double_visit_other__c,msj_comment__c,msj_for_reporting__c,msj_number_of_attendees__c,msj_main_dept__c,planned_type_vjh__c,cobrowse_url_participant_vod__c,msj_activity_method_text__c,msj_activity_method__c,msj_classification__c,msj_double_visit_msl__c,msj_msl_comment_for_mr__c,msj_apd__c,medical_inquiry_vod__c,msj_call_type_msj__c,msj_prescription_request__c,msj_patient_follow__c,child_account_id_vod__c,child_account_vod__c,location_id_vod__c,location_name_vod__c,msj_comments_about_technology__c,remote_meeting_vod__c,veeva_remote_meeting_id_vod__c,msj_activity_type_report__c,msj_activity_type__c,msj_activity__c,msj_comments__c,msj_therapy__c,msj_time_hrs__c,emds_co_reference__c,emds_call_sub_type__c,emds_call_type__c,emds_call_unsuccessful__c,emds_congress_type__c,emds_date_of_service__c,emds_fertility_dis_interest__c,emds_fertility_interest__c,emds_installed_equipment__c,emds_pipeline_stage_value__c,emds_pipeline_stage__c,emds_pipeline__c,emds_reason_for_call__c,emds_training_completed__c,msj_brain_storming__c,msj_sipagl_1_a__c,msj_sipagl_1_b__c,msj_sipagl_2__c,msj_sipagl_3__c,msj_sipagl_4_a__c,msj_sipagl_5_a__c,msj_sipagl_comment__c,msj_sipagl_4_b__c,msj_sipagl_5_b__c,location_text_vod__c,call_channel_vod__c,msj_scientific_interaction__c,msj_activity_email_reply__c,msj_interaction_duration__c,msj_sipagl_1_a_date__c,msj_co_promotion__c,call_channel_formula_vod__c +src02.crm_call2_vod__c +org02.crm_call2_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call_Clickstream_vod__c.txt b/s3/data/crm/settings/CRM_Call_Clickstream_vod__c.txt new file mode 100644 index 00000000..e2b56535 --- /dev/null +++ b/s3/data/crm/settings/CRM_Call_Clickstream_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +34 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Answer_vod__c,Call_vod__c,Key_Message_vod__c,Mobile_ID_vod__c,Popup_Opened_vod__c,Possible_Answers_vod__c,Presentation_ID_vod__c,Product_vod__c,Range_Value_vod__c,Rollover_Entered_vod__c,Selected_Items_vod__c,CLM_ID_vod__c,Question_vod__c,Survey_Type_vod__c,Text_Entered_vod__c,Toggle_Button_On_vod__c,Track_Element_Description_vod__c,Track_Element_Id_vod__c,Track_Element_Type_vod__c,Usage_Duration_vod__c,Usage_Start_Time_vod__c,AuxillaryId_vod__c,ParentId_vod__c,Revision_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,answer_vod__c,call_vod__c,key_message_vod__c,mobile_id_vod__c,popup_opened_vod__c,possible_answers_vod__c,presentation_id_vod__c,product_vod__c,range_value_vod__c,rollover_entered_vod__c,selected_items_vod__c,clm_id_vod__c,question_vod__c,survey_type_vod__c,text_entered_vod__c,toggle_button_on_vod__c,track_element_description_vod__c,track_element_id_vod__c,track_element_type_vod__c,usage_duration_vod__c,usage_start_time_vod__c,auxillary_id_vod__c,parent_id_vod__c,revision_vod__c +src02.crm_call_clickstream_vod__c +org02.crm_call_clickstream_vod__c + + diff --git a/s3/data/crm/settings/CRM_Child_Account_vod__c.txt b/s3/data/crm/settings/CRM_Child_Account_vod__c.txt new file mode 100644 index 00000000..876bef05 --- /dev/null +++ b/s3/data/crm/settings/CRM_Child_Account_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +57 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Parent_Account_vod__c,Child_Account_vod__c,External_ID_vod__c,Mobile_ID_vod__c,Primary_vod__c,Copy_Address_vod__c,Child_Name_vod__c,Parent_Name_vod__c,Parent_Child_Name_vod__c,Account_Code__c,Child_Department__c,Child_Role__c,Child_Title__c,Child_Remark__c,MSJ_1C_segment__c,MSJ_BU_FE__c,MSJ_BU_ONC__c,MSJ_BVC_Segment__c,MSJ_CE_segment__c,MSJ_Child_Account_Link__c,MSJ_DCF_DR_Code__c,MSJ_DCF_HP_Code__c,MSJ_DR_Change_Log__c,MSJ_Delete__c,MSJ_Department__c,MSJ_EB_CRC_Segment__c,MSJ_EB_HN_Segment__c,MSJ_EB_H_N_LA_Segment__c,MSJ_EB_H_N_RM_Segment__c,MSJ_External_ID__c,MSJ_Fax__c,MSJ_GF2_segment__c,MSJ_GF_segment__c,MSJ_KOL_LOL__c,MSJ_KOL__c,MSJ_ONC_HP_Segment__c,MSJ_OPTIN_target__c,MSJ_OV_segment__c,MSJ_Parent_Child_Name__c,MSJ_Phone__c,MSJ_Remark__c,MSJ_Target_Call_Num__c,MSJ_Tech_segment__c,MSJ_Title__c,MSJ_XLK_Segment__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,parent_account_vod__c,child_account_vod__c,external_id_vod__c,mobile_id_vod__c,primary_vod__c,copy_address_vod__c,child_name_vod__c,parent_name_vod__c,parent_child_name_vod__c,account_code__c,child_department__c,child_role__c,child_title__c,child_remark__c,msj_1_c_segment__c,msj_bu_fe__c,msj_bu_onc__c,msj_bvc_segment__c,msj_ce_segment__c,msj_child_account_link__c,msj_dcf_dr_code__c,msj_dcf_hp_code__c,msj_dr_change_log__c,msj_delete__c,msj_department__c,msj_eb_crc_segment__c,msj_eb_hn_segment__c,msj_eb_h_n_la_segment__c,msj_eb_h_n_rm_segment__c,msj_external_id__c,msj_fax__c,msj_gf2_segment__c,msj_gf_segment__c,msj_kol_lol__c,msj_kol__c,msj_onc_hp_segment__c,msj_optin_target__c,msj_ov_segment__c,msj_parent_child_name__c,msj_phone__c,msj_remark__c,msj_target_call_num__c,msj_tech_segment__c,msj_title__c,msj_xlk_segment__c +src02.crm_child_account_vod__c +org02.crm_child_account_vod__c +CRM_Child_Account_vod__c_ex.sql + diff --git a/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql b/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql new file mode 100644 index 00000000..cac33f47 --- /dev/null +++ b/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_child_account_vod__c', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Clm_Presentation_Slide_vod__c.txt b/s3/data/crm/settings/CRM_Clm_Presentation_Slide_vod__c.txt new file mode 100644 index 00000000..422a6926 --- /dev/null +++ b/s3/data/crm/settings/CRM_Clm_Presentation_Slide_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +17 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Clm_Presentation_vod__c,Key_Message_vod__c,Display_Order_vod__c,Sub_Presentation_vod__c,Mobile_ID_vod__c,External_ID_vod__c,VExternal_Id_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,clm_presentation_vod__c,key_message_vod__c,display_order_vod__c,sub_presentation_vod__c,mobile_id_vod__c,external_id_vod__c,vexternal_id_vod__c +src02.crm_clm_presentation_slide_vod__c +org02.crm_clm_presentation_slide_vod__c + + diff --git a/s3/data/crm/settings/CRM_Clm_Presentation_vod__c.txt b/s3/data/crm/settings/CRM_Clm_Presentation_vod__c.txt new file mode 100644 index 00000000..364a308b --- /dev/null +++ b/s3/data/crm/settings/CRM_Clm_Presentation_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +46 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Mobile_ID_vod__c,Presentation_Id_vod__c,Product_vod__c,Default_Presentation_vod__c,Training_vod__c,ParentId_vod__c,Hidden_vod__c,Type_vod__c,Approved_vod__c,Copied_From_vod__c,Copy_Date_vod__c,Survey_vod__c,Original_Record_ID_vod__c,Directory_vod__c,End_Date_vod__c,Start_Date_vod__c,Status_vod__c,VExternal_Id_vod__c,Vault_DNS_vod__c,Vault_Doc_Id_vod__c,Vault_External_Id_vod__c,Vault_GUID_vod__c,Vault_Last_Modified_Date_Time_vod__c,Version_vod__c,Enable_Survey_Overlay_vod__c,Description_vod__c,Keywords_vod__c,Content_Channel_vod__c,original_material_approved_in_veritas__c,keywords__c,trade_team__c,ewizard_link__c,business_function__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,mobile_id_vod__c,presentation_id_vod__c,product_vod__c,default_presentation_vod__c,training_vod__c,parent_id_vod__c,hidden_vod__c,type_vod__c,approved_vod__c,copied_from_vod__c,copy_date_vod__c,survey_vod__c,original_record_id_vod__c,directory_vod__c,end_date_vod__c,start_date_vod__c,status_vod__c,vexternal_id_vod__c,vault_dns_vod__c,vault_doc_id_vod__c,vault_external_id_vod__c,vault_guid_vod__c,vault_last_modified_date_time_vod__c,version_vod__c,enable_survey_overlay_vod__c,description_vod__c,keywords_vod__c,content_channel_vod__c,original_material_approved_in_veritas__c,keywords__c,trade_team__c,ewizard_link__c,business_function__c +src02.crm_clm_presentation_vod__c +org02.crm_clm_presentation_vod__c + + diff --git a/s3/data/crm/settings/CRM_Coaching_Report_vod__c.txt b/s3/data/crm/settings/CRM_Coaching_Report_vod__c.txt new file mode 100644 index 00000000..fd64a473 --- /dev/null +++ b/s3/data/crm/settings/CRM_Coaching_Report_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +144 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Mobile_ID_vod__c,Manager_vod__c,Employee_vod__c,Review_Date__c,Review_Period__c,Status__c,Comments__c,Strategic_Planning__c,Customer_Focus__c,Knowledge_Expertise__c,Business_Account_Planning__c,Call_Productivity__c,Overall_Rating__c,MSJ_A01__c,MSJ_A02__c,MSJ_A03__c,MSJ_AM_Memo__c,MSJ_Aid_Total__c,MSJ_C0_GC__c,MSJ_C1_GC__c,MSJ_C2_GC__c,MSJ_Countermeasure__c,MSJ_Deadline__c,MSJ_Double_Visit_Time__c,MSJ_Hospital__c,MSJ_K01_FE__c,MSJ_K01_ONC__c,MSJ_K02_FE__c,MSJ_K02_ONC__c,MSJ_K03_FE__c,MSJ_K03_ONC__c,MSJ_K04_FE__c,MSJ_K04_ONC__c,MSJ_K05_FE__c,MSJ_K05_ONC__c,MSJ_K06_FE__c,MSJ_K06_ONC__c,MSJ_K0_GC__c,MSJ_K1_GC__c,MSJ_K2_GC__c,MSJ_Knowledge_Total__c,MSJ_L0_GC__c,MSJ_L1_GC__c,MSJ_L2_GC__c,MSJ_MR_GC__c,MSJ_MR_Problems__c,MSJ_N0_GC__c,MSJ_N1_GC__c,MSJ_N2_GC__c,MSJ_Num_of_DTL__c,MSJ_P01__c,MSJ_P02__c,MSJ_P03__c,MSJ_P04__c,MSJ_P05__c,MSJ_P0_GC__c,MSJ_P1_GC__c,MSJ_P2_GC__c,MSJ_PlanningTotal__c,MSJ_R0_GC__c,MSJ_R1_GC__c,MSJ_R2_GC__c,MSJ_S01__c,MSJ_S02__c,MSJ_S03__c,MSJ_S04__c,MSJ_S05__c,MSJ_S06__c,MSJ_S07__c,MSJ_S08__c,MSJ_S09__c,MSJ_S10__c,MSJ_S11__c,MSJ_S12__c,MSJ_Skill_Total__c,MSJ_After_Call_01__c,MSJ_After_Call_02__c,MSJ_After_Call_03__c,MSJ_After_Call_04__c,MSJ_Closing__c,MSJ_Comment_by_MR__c,MSJ_Confirmed_by_MR__c,MSJ_Createdby__c,MSJ_FT_AM_Name__c,MSJ_Interview_Preparation__c,MSJ_Interview_Reflection__c,MSJ_Notify_To_MR__c,MSJ_Opening__c,MSJ_Others_01_Result__c,MSJ_Others_01__c,MSJ_Others_02_Result__c,MSJ_Others_02__c,MSJ_Patient_Thinking__c,MSJ_Probing__c,MSJ_Supporting__c,MSJ_Patient_Thinking_for_FE__c,MSJ_After_Call_05__c,MSJ_After_Call_06__c,MSJ_After_Call_07__c,MSJ_After_Call_08__c,MSJ_Createdby_FE__c,MSJ_Createdby_ONC__c,MSJ_Development_Level__c,MSJ_Interview_Prep_01__c,MSJ_Interview_Prep_02__c,MSJ_Leadership_Style__c,MSJ_Overcome_01__c,MSJ_Overcome_02__c,MSJ_Overcome_03__c,MSJ_Overcome_04__c,MSJ_Review_01__c,MSJ_Review_02__c,MSJ_SK_01__c,MSJ_SK_02__c,MSJ_SK_03__c,MSJ_SK_04__c,MSJ_SK_05__c,MSJ_SK_06__c,MSJ_SK_07__c,MSJ_SK_08__c,MSJ_SK_09__c,MSJ_SK_10__c,MSJ_Specific_Action__c,MSJ_Training_Point__c,MSJ_Efforts_of_Year__c,MSJ_Efforts_of_Month__c,MSJ_Skill_Task__c,MSJ_Action_of_This_Month__c,MSJ_Achievement_of_This_Month__c,MSJ_Comment_from_AM__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,mobile_id_vod__c,manager_vod__c,employee_vod__c,review_date__c,review_period__c,status__c,comments__c,strategic_planning__c,customer_focus__c,knowledge_expertise__c,business_account_planning__c,call_productivity__c,overall_rating__c,msj_a01__c,msj_a02__c,msj_a03__c,msj_am_memo__c,msj_aid_total__c,msj_c0_gc__c,msj_c1_gc__c,msj_c2_gc__c,msj_countermeasure__c,msj_deadline__c,msj_double_visit_time__c,msj_hospital__c,msj_k01_fe__c,msj_k01_onc__c,msj_k02_fe__c,msj_k02_onc__c,msj_k03_fe__c,msj_k03_onc__c,msj_k04_fe__c,msj_k04_onc__c,msj_k05_fe__c,msj_k05_onc__c,msj_k06_fe__c,msj_k06_onc__c,msj_k0_gc__c,msj_k1_gc__c,msj_k2_gc__c,msj_knowledge_total__c,msj_l0_gc__c,msj_l1_gc__c,msj_l2_gc__c,msj_mr_gc__c,msj_mr_problems__c,msj_n0_gc__c,msj_n1_gc__c,msj_n2_gc__c,msj_num_of_dtl__c,msj_p01__c,msj_p02__c,msj_p03__c,msj_p04__c,msj_p05__c,msj_p0_gc__c,msj_p1_gc__c,msj_p2_gc__c,msj_planning_total__c,msj_r0_gc__c,msj_r1_gc__c,msj_r2_gc__c,msj_s01__c,msj_s02__c,msj_s03__c,msj_s04__c,msj_s05__c,msj_s06__c,msj_s07__c,msj_s08__c,msj_s09__c,msj_s10__c,msj_s11__c,msj_s12__c,msj_skill_total__c,msj_after_call_01__c,msj_after_call_02__c,msj_after_call_03__c,msj_after_call_04__c,msj_closing__c,msj_comment_by_mr__c,msj_confirmed_by_mr__c,msj_createdby__c,msj_ft_am_name__c,msj_interview_preparation__c,msj_interview_reflection__c,msj_notify_to_mr__c,msj_opening__c,msj_others_01_result__c,msj_others_01__c,msj_others_02_result__c,msj_others_02__c,msj_patient_thinking__c,msj_probing__c,msj_supporting__c,msj_patient_thinking_for_fe__c,msj_after_call_05__c,msj_after_call_06__c,msj_after_call_07__c,msj_after_call_08__c,msj_createdby_fe__c,msj_createdby_onc__c,msj_development_level__c,msj_interview_prep_01__c,msj_interview_prep_02__c,msj_leadership_style__c,msj_overcome_01__c,msj_overcome_02__c,msj_overcome_03__c,msj_overcome_04__c,msj_review_01__c,msj_review_02__c,msj_sk_01__c,msj_sk_02__c,msj_sk_03__c,msj_sk_04__c,msj_sk_05__c,msj_sk_06__c,msj_sk_07__c,msj_sk_08__c,msj_sk_09__c,msj_sk_10__c,msj_specific_action__c,msj_training_point__c,msj_efforts_of_year__c,msj_efforts_of_month__c,msj_skill_task__c,msj_action_of_this_month__c,msj_achievement_of_this_month__c,msj_comment_from_am__c +src02.crm_coaching_report_vod__c +org02.crm_coaching_report_vod__c + + diff --git a/s3/data/crm/settings/CRM_Consent_Header_vod__c.txt b/s3/data/crm/settings/CRM_Consent_Header_vod__c.txt new file mode 100644 index 00000000..729c3452 --- /dev/null +++ b/s3/data/crm/settings/CRM_Consent_Header_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +22 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consent_Header_Help_Text_vod__c,Country_vod__c,Inactive_Datetime_vod__c,Language_vod__c,Status_vod__c,Signature_Required_On_Opt_Out_vod__c,Request_Receipt_vod__c,Subscription_Option_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consent_header_help_text_vod__c,country_vod__c,inactive_datetime_vod__c,language_vod__c,status_vod__c,signature_required_on_opt_out_vod__c,request_receipt_vod__c,subscription_option_vod__c +src02.crm_consent_header_vod__c +org02.crm_consent_header_vod__c + + diff --git a/s3/data/crm/settings/CRM_Consent_Line_vod__c.txt b/s3/data/crm/settings/CRM_Consent_Line_vod__c.txt new file mode 100644 index 00000000..3cc95e31 --- /dev/null +++ b/s3/data/crm/settings/CRM_Consent_Line_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +25 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consent_Type_vod__c,Detail_Group_Display_Name_vod__c,Detail_Group_vod__c,Display_Order_vod__c,End_Date_vod__c,Group_By_vod__c,Product_Display_Name_vod__c,Product_vod__c,Start_Date_vod__c,Sub_Channel_Description_vod__c,Sub_Channel_Display_Name_vod__c,Sub_Channel_Key_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consent_type_vod__c,detail_group_display_name_vod__c,detail_group_vod__c,display_order_vod__c,end_date_vod__c,group_by_vod__c,product_display_name_vod__c,product_vod__c,start_date_vod__c,sub_channel_description_vod__c,sub_channel_display_name_vod__c,sub_channel_key_vod__c +src02.crm_consent_line_vod__c +org02.crm_consent_line_vod__c + + diff --git a/s3/data/crm/settings/CRM_Consent_Type_vod__c.txt b/s3/data/crm/settings/CRM_Consent_Type_vod__c.txt new file mode 100644 index 00000000..2a3451a0 --- /dev/null +++ b/s3/data/crm/settings/CRM_Consent_Type_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +26 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consent_Header_vod__c,Channel_Label_vod__c,Channel_Source_vod__c,Consent_Expires_In_vod__c,Default_Consent_Type_vod__c,Disclaimer_Text_vod__c,Display_Order_vod__c,Product_Preference_vod__c,zvod_Consent_Default_Consent_Text_vod__c,zvod_Consent_Line_vod__c,zvod_Signature_Capture_vod__c,Double_Opt_In_vod__c,zvod_Consent_Activity_Tracking_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consent_header_vod__c,channel_label_vod__c,channel_source_vod__c,consent_expires_in_vod__c,default_consent_type_vod__c,disclaimer_text_vod__c,display_order_vod__c,product_preference_vod__c,zvod_consent_default_consent_text_vod__c,zvod_consent_line_vod__c,zvod_signature_capture_vod__c,double_opt_in_vod__c,zvod_consent_activity_tracking_vod__c +src02.crm_consent_type_vod__c +org02.crm_consent_type_vod__c + + diff --git a/s3/data/crm/settings/CRM_Contact.txt b/s3/data/crm/settings/CRM_Contact.txt new file mode 100644 index 00000000..21a9c5dd --- /dev/null +++ b/s3/data/crm/settings/CRM_Contact.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +71 +Id,IsDeleted,MasterRecordId,AccountId,IsPersonAccount,LastName,FirstName,Salutation,Name,OtherStreet,OtherCity,OtherState,OtherPostalCode,OtherCountry,OtherLatitude,OtherLongitude,OtherGeocodeAccuracy,OtherAddress,MailingStreet,MailingCity,MailingState,MailingPostalCode,MailingCountry,MailingLatitude,MailingLongitude,MailingGeocodeAccuracy,MailingAddress,Phone,Fax,MobilePhone,HomePhone,OtherPhone,AssistantPhone,ReportsToId,Email,Title,Department,AssistantName,Birthdate,Description,OwnerId,HasOptedOutOfEmail,HasOptedOutOfFax,DoNotCall,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,LastCURequestDate,LastCUUpdateDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,EmailBouncedReason,EmailBouncedDate,IsEmailBounced,PhotoUrl,Jigsaw,JigsawContactId,IndividualId,Mobile_ID_vod__c,H1Insights__H1_NPI_Value_for_Testing__c,H1Insights__H1_Person_ID__c,H1Insights__H1_Request_Status__c,H1Insights__H1_URL__c,H1Insights__NPI_Number__c,H1Insights__NPI_Number_for_H1_Insights__c,MSJ_Marketing_Cloud_Integration__c +id,is_deleted,master_record_id,account_id,is_person_account,last_name,first_name,salutation,name,other_street,other_city,other_state,other_postal_code,other_country,other_latitude,other_longitude,other_geocode_accuracy,other_address,mailing_street,mailing_city,mailing_state,mailing_postal_code,mailing_country,mailing_latitude,mailing_longitude,mailing_geocode_accuracy,mailing_address,phone,fax,mobile_phone,home_phone,other_phone,assistant_phone,reports_to_id,email,title,department,assistant_name,birthdate,description,owner_id,has_opted_out_of_email,has_opted_out_of_fax,do_not_call,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,last_curequest_date,last_cuupdate_date,may_edit,is_locked,last_viewed_date,last_referenced_date,email_bounced_reason,email_bounced_date,is_email_bounced,photo_url,jigsaw,jigsaw_contact_id,individual_id,mobile_id_vod__c,h1_insights__h1_npi_value_for_testing__c,h1_insights__h1_person_id__c,h1_insights__h1_request_status__c,h1_insights__h1_url__c,h1_insights__npi_number__c,h1_insights__npi_number_for_h1_insights__c,msj_marketing_cloud_integration__c +src02.crm_contact +org02.crm_contact +CRM_Contact_ex.sql + diff --git a/s3/data/crm/settings/CRM_Contact_ex.sql b/s3/data/crm/settings/CRM_Contact_ex.sql new file mode 100644 index 00000000..4b916e23 --- /dev/null +++ b/s3/data/crm/settings/CRM_Contact_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_contact', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Dynamic_Attribute_Configuration_vod__c.txt b/s3/data/crm/settings/CRM_Dynamic_Attribute_Configuration_vod__c.txt new file mode 100644 index 00000000..d48828a7 --- /dev/null +++ b/s3/data/crm/settings/CRM_Dynamic_Attribute_Configuration_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +29 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Applies_To_vod__c,Attribute_Label_vod__c,Attribute_Name_vod__c,Available_Values_vod__c,Description_vod__c,Detail_Group_vod__c,Display_Order_vod__c,External_ID_vod__c,Help_Text_vod__c,Product_vod__c,Read_Only_vod__c,Section_Name_vod__c,Sharing_Group_vod__c,Status_vod__c,Track_Changes_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,applies_to_vod__c,attribute_label_vod__c,attribute_name_vod__c,available_values_vod__c,description_vod__c,detail_group_vod__c,display_order_vod__c,external_id_vod__c,help_text_vod__c,product_vod__c,read_only_vod__c,section_name_vod__c,sharing_group_vod__c,status_vod__c,track_changes_vod__c +src02.crm_dynamic_attribute_configuration_vod__c +org02.crm_dynamic_attribute_configuration_vod__c + + diff --git a/s3/data/crm/settings/CRM_Dynamic_Attribute_vod__c.txt b/s3/data/crm/settings/CRM_Dynamic_Attribute_vod__c.txt new file mode 100644 index 00000000..adea9267 --- /dev/null +++ b/s3/data/crm/settings/CRM_Dynamic_Attribute_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +26 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Account_vod__c,Active_vod__c,Dynamic_Attribute_Configuration_vod__c,Dynamic_Attribute_Description_vod__c,Dynamic_Attribute_Help_Text_vod__c,Dynamic_Attribute_Label_vod__c,Dynamic_Attribute_Name_vod__c,Dynamic_Attribute_Record_Type_vod__c,Dynamic_Attribute_Value_Checkbox_vod__c,Dynamic_Attribute_Value_Date_Time_vod__c,Dynamic_Attribute_Value_Date_vod__c,Dynamic_Attribute_Value_Number_vod__c,Dynamic_Attribute_Value_Text_Area_vod__c,Dynamic_Attribute_Value_Text_vod__c,Mobile_ID_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,account_vod__c,active_vod__c,dynamic_attribute_configuration_vod__c,dynamic_attribute_description_vod__c,dynamic_attribute_help_text_vod__c,dynamic_attribute_label_vod__c,dynamic_attribute_name_vod__c,dynamic_attribute_record_type_vod__c,dynamic_attribute_value_checkbox_vod__c,dynamic_attribute_value_date_time_vod__c,dynamic_attribute_value_date_vod__c,dynamic_attribute_value_number_vod__c,dynamic_attribute_value_text_area_vod__c,dynamic_attribute_value_text_vod__c,mobile_id_vod__c +src02.crm_dynamic_attribute_vod__c +org02.crm_dynamic_attribute_vod__c + + diff --git a/s3/data/crm/settings/CRM_Email_Activity_vod__c.txt b/s3/data/crm/settings/CRM_Email_Activity_vod__c.txt new file mode 100644 index 00000000..2c801e06 --- /dev/null +++ b/s3/data/crm/settings/CRM_Email_Activity_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +36 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,Sent_Email_vod__c,Activity_DateTime_vod__c,City_vod__c,Click_URL_vod__c,Client_Name_vod__c,Client_OS_vod__c,Client_Type_vod__c,Country_vod__c,Device_Type_vod__c,Event_Msg_vod__c,Event_type_vod__c,IP_Address_vod__c,Region_vod__c,User_Agent_vod__c,Vault_Doc_ID_vod__c,Vault_Doc_Name_vod__c,Vault_Document_Major_Version_vod__c,Vault_Document_Minor_Version_vod__c,Vault_Document_Number_vod__c,Vault_Document_Title_vod__c,Vault_Instance_ID_vod__c,Preference_Modification_vod__c,Approved_Document_vod__c,Link_Name_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,sent_email_vod__c,activity_date_time_vod__c,city_vod__c,click_url_vod__c,client_name_vod__c,client_os_vod__c,client_type_vod__c,country_vod__c,device_type_vod__c,event_msg_vod__c,event_type_vod__c,ip_address_vod__c,region_vod__c,user_agent_vod__c,vault_doc_id_vod__c,vault_doc_name_vod__c,vault_document_major_version_vod__c,vault_document_minor_version_vod__c,vault_document_number_vod__c,vault_document_title_vod__c,vault_instance_id_vod__c,preference_modification_vod__c,approved_document_vod__c,link_name_vod__c +src02.crm_email_activity_vod__c +org02.crm_email_activity_vod__c + + diff --git a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt new file mode 100644 index 00000000..d5d6c0dc --- /dev/null +++ b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +31 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c +src02.crm_event_attendee_vod__c +org02.crm_event_attendee_vod__c + + diff --git a/s3/data/crm/settings/CRM_Group.txt b/s3/data/crm/settings/CRM_Group.txt new file mode 100644 index 00000000..94f4d0ca --- /dev/null +++ b/s3/data/crm/settings/CRM_Group.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +14 +Id,Name,DeveloperName,RelatedId,Type,Email,OwnerId,DoesSendEmailToMembers,DoesIncludeBosses,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp +id,name,developer_name,related_id,type,email,owner_id,does_send_email_to_members,does_include_bosses,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp +src02.crm_group +org02.crm_group +CRM_Group_ex.sql + diff --git a/s3/data/crm/settings/CRM_Group_ex.sql b/s3/data/crm/settings/CRM_Group_ex.sql new file mode 100644 index 00000000..5e50dab6 --- /dev/null +++ b/s3/data/crm/settings/CRM_Group_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_group', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Key_Message_vod__c.txt b/s3/data/crm/settings/CRM_Key_Message_vod__c.txt new file mode 100644 index 00000000..1c1f1df0 --- /dev/null +++ b/s3/data/crm/settings/CRM_Key_Message_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +48 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Description_vod__c,Product_vod__c,Product_Strategy_vod__c,Display_Order_vod__c,Active_vod__c,Category_vod__c,Vehicle_vod__c,CLM_ID_vod__c,Custom_Reaction_vod__c,Slide_Version_vod__c,Language_vod__c,Media_File_CRC_vod__c,Media_File_Name_vod__c,Media_File_Size_vod__c,Segment_vod__c,Disable_Actions_vod__c,VExternal_Id_vod__c,CDN_Path_vod__c,Status_vod__c,Vault_DNS_vod__c,Vault_Doc_Id_vod__c,Vault_External_Id_vod__c,Vault_GUID_vod__c,Vault_Last_Modified_Date_Time_vod__c,Is_Shared_Resource_vod__c,iOS_Viewer_vod__c,iOS_Resolution_vod__c,approved_for_distribution_date__c,approved_for_use_date__c,ewizard_link__c,expiration_date__c,keywords__c,trade_team__c,business_function__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,description_vod__c,product_vod__c,product_strategy_vod__c,display_order_vod__c,active_vod__c,category_vod__c,vehicle_vod__c,clm_id_vod__c,custom_reaction_vod__c,slide_version_vod__c,language_vod__c,media_file_crc_vod__c,media_file_name_vod__c,media_file_size_vod__c,segment_vod__c,disable_actions_vod__c,vexternal_id_vod__c,cdn_path_vod__c,status_vod__c,vault_dns_vod__c,vault_doc_id_vod__c,vault_external_id_vod__c,vault_guid_vod__c,vault_last_modified_date_time_vod__c,is_shared_resource_vod__c,i_os_viewer_vod__c,i_os_resolution_vod__c,approved_for_distribution_date__c,approved_for_use_date__c,ewizard_link__c,expiration_date__c,keywords__c,trade_team__c,business_function__c +src02.crm_key_message_vod__c +org02.crm_key_message_vod__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Hospital_Medical_Regimen__c.txt b/s3/data/crm/settings/CRM_MSJ_Hospital_Medical_Regimen__c.txt new file mode 100644 index 00000000..c295fc5e --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Hospital_Medical_Regimen__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +18 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,MSJ_Account_Name__c,MSJ_Delete_Date__c,MSJ_Delete_Flag__c,MSJ_Indication__c,MSJ_Line__c,MSJ_Medical_Regimen__c,Mobile_ID_vod__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,msj_account_name__c,msj_delete_date__c,msj_delete_flag__c,msj_indication__c,msj_line__c,msj_medical_regimen__c,mobile_id_vod__c +src02.crm_msj_hospital_medical_regimen__c +org02.crm_msj_hospital_medical_regimen__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Inquiry_Assignment__c.txt b/s3/data/crm/settings/CRM_MSJ_Inquiry_Assignment__c.txt new file mode 100644 index 00000000..a50bf1f3 --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Inquiry_Assignment__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +47 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,MSJ_Medical_Inquiry__c,MSJ_Close__c,MSJ_Doctor_Name__c,MSJ_Hospital_Name__c,MSJ_Indication__c,MSJ_Inquiry_Text__c,MSJ_MEC_User__c,MSJ_MSL_Manager__c,MSJ_MSL_User__c,MSJ_Notice_to_MR__c,MSJ_Product_for_MEC__c,MSJ_Product_for_MR__c,MSJ_Reply_Date__c,MSJ_Reply__c,MSJ_AE_Infomation__c,MSJ_Cancel__c,MSJ_FAQ_number_c__c,MSJ_Return_Call__c,MSJ_Inquiry_Origin__c,First_Response__c,Inquiry_Created_Date__c,Inquiry_Type_1__c,Inquiry_Type_2__c,MSJ_First_User__c,MSJ_MEC_Comment__c,MSJ_Send_Email__c,MSJ_Temp_Aggregated_Info__c,MSJ_AE_Report__c,MSJ_Background__c,MSJ_Inquiry_Date__c,MSJ_MSL_Support__c,MSJ_Handover_Comment__c,MSJ_Handover_Email__c,MSJ_Material_Requirement__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,msj_medical_inquiry__c,msj_close__c,msj_doctor_name__c,msj_hospital_name__c,msj_indication__c,msj_inquiry_text__c,msj_mec_user__c,msj_msl_manager__c,msj_msl_user__c,msj_notice_to_mr__c,msj_product_for_mec__c,msj_product_for_mr__c,msj_reply_date__c,msj_reply__c,msj_ae_infomation__c,msj_cancel__c,msj_faq_number_c__c,msj_return_call__c,msj_inquiry_origin__c,first_response__c,inquiry_created_date__c,inquiry_type_1__c,inquiry_type_2__c,msj_first_user__c,msj_mec_comment__c,msj_send_email__c,msj_temp_aggregated_info__c,msj_ae_report__c,msj_background__c,msj_inquiry_date__c,msj_msl_support__c,msj_handover_comment__c,msj_handover_email__c,msj_material_requirement__c +src02.crm_msj_inquiry_assignment__c +org02.crm_msj_inquiry_assignment__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_MR_Weekly_Report__c.txt b/s3/data/crm/settings/CRM_MSJ_MR_Weekly_Report__c.txt new file mode 100644 index 00000000..89df0b85 --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_MR_Weekly_Report__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +35 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,MSJ_Account_Name__c,MSJ_Activity_Results_Summary__c,MSJ_MUID__c,MSJ_Next_Week_Action_What__c,MSJ_Next_Week_Action_When__c,MSJ_Next_Week_Action_Where__c,MSJ_Next_Week_Action_Who__c,MSJ_Report_Week__c,MSJ_Target_Patient_Count__c,Mobile_ID_vod__c,MSJ_Activity_Results_Summary_HN__c,MSJ_Next_Week_Action_Where_HN__c,MSJ_Next_Week_Action_Who_HN__c,MSJ_Next_Week_Action_What_HN__c,MSJ_Next_Week_Action_When_HN__c,MSJ_Target_Patient_Count_HN__c,MSJ_Activity_Results_Summary_MCC__c,MSJ_Next_Week_Action_Where_MCC__c,MSJ_Next_Week_Action_Who_MCC__c,MSJ_Next_Week_Action_What_MCC__c,MSJ_Next_Week_Action_When_MCC__c,MSJ_Target_Patient_Count_MCC__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,msj_account_name__c,msj_activity_results_summary__c,msj_muid__c,msj_next_week_action_what__c,msj_next_week_action_when__c,msj_next_week_action_where__c,msj_next_week_action_who__c,msj_report_week__c,msj_target_patient_count__c,mobile_id_vod__c,msj_activity_results_summary_hn__c,msj_next_week_action_where_hn__c,msj_next_week_action_who_hn__c,msj_next_week_action_what_hn__c,msj_next_week_action_when_hn__c,msj_target_patient_count_hn__c,msj_activity_results_summary_mcc__c,msj_next_week_action_where_mcc__c,msj_next_week_action_who_mcc__c,msj_next_week_action_what_mcc__c,msj_next_week_action_when_mcc__c,msj_target_patient_count_mcc__c +src02.crm_msj_mr_weekly_report__c +org02.crm_msj_mr_weekly_report__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Medical_Event_Evaluation__c.txt b/s3/data/crm/settings/CRM_MSJ_Medical_Event_Evaluation__c.txt new file mode 100644 index 00000000..5f1546f2 --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Medical_Event_Evaluation__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +14 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,MSJ_Medical_Event__c,MSJ_Evaluation_Comment__c,MSJ_Evaluation__c,Mobile_ID_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,msj_medical_event__c,msj_evaluation_comment__c,msj_evaluation__c,mobile_id_vod__c +src02.crm_msj_medical_event_evaluation__c +org02.crm_msj_medical_event_evaluation__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Medical_Regimen__c.txt b/s3/data/crm/settings/CRM_MSJ_Medical_Regimen__c.txt new file mode 100644 index 00000000..321b8a5e --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Medical_Regimen__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +17 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,MSJ_Delete_Date__c,MSJ_Delete_Flag__c,MSJ_Indication__c,MSJ_Remark__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,msj_delete_date__c,msj_delete_flag__c,msj_indication__c,msj_remark__c +src02.crm_msj_medical_regimen__c +org02.crm_msj_medical_regimen__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Patient__c.txt b/s3/data/crm/settings/CRM_MSJ_Patient__c.txt new file mode 100644 index 00000000..8b1bac96 --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Patient__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +37 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,MSJ_Account_Name__c,MSJ_CRC_Group__c,MSJ_Casus_or_Transfer_Point__c,MSJ_Entry_Date__c,MSJ_IST_Name__c,MSJ_Indication__c,MSJ_Line__c,MSJ_MR_Comments__c,MSJ_MUID__c,MSJ_Medical_Regimen__c,MSJ_Month__c,MSJ_Report_Comments__c,MSJ_Start_Date_Of_Administration__c,MSJ_Year__c,Mobile_ID_vod__c,MSJ_CRC_RAS_KRAS__c,MSJ_End_Date_Of_Administration__c,MSJ_End_Date_of_Stop_Administration__c,MSJ_HN_Hospitalized_Type__c,MSJ_Start_Date_of_Stop_Administration__c,MSJ_Patient_Status__c,MSJ_Patient_TA__c,MSJ_Child_Account_Name__c,MSJ_Child_Account__c,MSJ_Parent_Account_Name__c,MSJ_Parent_Child_Name__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,msj_account_name__c,msj_crc_group__c,msj_casus_or_transfer_point__c,msj_entry_date__c,msj_ist_name__c,msj_indication__c,msj_line__c,msj_mr_comments__c,msj_muid__c,msj_medical_regimen__c,msj_month__c,msj_report_comments__c,msj_start_date_of_administration__c,msj_year__c,mobile_id_vod__c,msj_crc_ras_kras__c,msj_end_date_of_administration__c,msj_end_date_of_stop_administration__c,msj_hn_hospitalized_type__c,msj_start_date_of_stop_administration__c,msj_patient_status__c,msj_patient_ta__c,msj_child_account_name__c,msj_child_account__c,msj_parent_account_name__c,msj_parent_child_name__c +src02.crm_msj_patient__c +org02.crm_msj_patient__c + + diff --git a/s3/data/crm/settings/CRM_Medical_Event_vod__c.txt b/s3/data/crm/settings/CRM_Medical_Event_vod__c.txt new file mode 100644 index 00000000..159a475b --- /dev/null +++ b/s3/data/crm/settings/CRM_Medical_Event_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +52 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Primary_Product__c,Description_vod__c,Start_Date_vod__c,Location__c,End_Date_vod__c,Secondary_Product__c,Website__c,Active_vod__c,Event_Type__c,MSJ_Area__c,MSJ_Business_Unit__c,MSJ_Comment__c,MSJ_Company__c,MSJ_Expense_App_No__c,MSJ_Form__c,MSJ_HQ_Area__c,MSJ_Location__c,MSJ_MR__c,MSJ_Number_of_Attendee__c,MSJ_Product__c,MSJ_Site__c,MSJ_Type__c,MSJ_Number_of_Attendee_Auto_Calc__c,MSJ_Number_of_Attendee_Invited__c,Account_vod__c,MSJ_MUID__c,Country_Name_vod__c,MSJ_CE_SIPAGL_Updater__c,MSJ_CE_SIPAGL_1A__c,MSJ_CE_SIPAGL_1B__c,MSJ_CE_SIPAGL_2__c,MSJ_CE_SIPAGL_3__c,MSJ_CE_SIPAGL_4__c,MSJ_CE_SIPAGL_5__c,MSJ_CE_SIPAGL_Comment__c,MSJ_CE_SIPAGL_1A_date__c,MSJ_CE_SIPAGL_6__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,primary_product__c,description_vod__c,start_date_vod__c,location__c,end_date_vod__c,secondary_product__c,website__c,active_vod__c,event_type__c,msj_area__c,msj_business_unit__c,msj_comment__c,msj_company__c,msj_expense_app_no__c,msj_form__c,msj_hq_area__c,msj_location__c,msj_mr__c,msj_number_of_attendee__c,msj_product__c,msj_site__c,msj_type__c,msj_number_of_attendee_auto_calc__c,msj_number_of_attendee_invited__c,account_vod__c,msj_muid__c,country_name_vod__c,msj_ce_sipagl_updater__c,msj_ce_sipagl_1_a__c,msj_ce_sipagl_1_b__c,msj_ce_sipagl_2__c,msj_ce_sipagl_3__c,msj_ce_sipagl_4__c,msj_ce_sipagl_5__c,msj_ce_sipagl_comment__c,msj_ce_sipagl_1_a_date__c,msj_ce_sipagl_6__c +src02.crm_medical_event_vod__c +org02.crm_medical_event_vod__c + + diff --git a/s3/data/crm/settings/CRM_Medical_Inquiry_vod__c.txt b/s3/data/crm/settings/CRM_Medical_Inquiry_vod__c.txt new file mode 100644 index 00000000..cafca6d9 --- /dev/null +++ b/s3/data/crm/settings/CRM_Medical_Inquiry_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +67 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,Address_Line_1_vod__c,Address_Line_2_vod__c,City_vod__c,Delivery_Method_vod__c,Email_vod__c,Fax_Number_vod__c,Inquiry_Text__c,Lock_vod__c,Mobile_ID_vod__c,Phone_Number_vod__c,Product__c,Rush_Delivery__c,Signature_Date_vod__c,Signature_vod__c,State_vod__c,Status_vod__c,Zip_vod__c,zvod_Delivery_Method_vod__c,zvod_Disclaimer_vod__c,Submitted_By_Mobile_vod__c,Disclaimer_vod__c,Entity_Reference_Id_vod__c,Call2_vod__c,Country_vod__c,Override_Lock_vod__c,MSJ_Department__c,MSJ_Doctor_Name__c,MSJ_Hospital_Name__c,MSJ_Indication__c,MSJ_Inquiry_Assignment__c,MSJ_Inquiry_Date__c,MSJ_Inquiry_Input_Manager__c,MSJ_Inquiry_Input_User__c,MSJ_MSL_Manager__c,MSJ_Notice_to_MR__c,MSJ_Person_in_charge_1__c,MSJ_Person_in_charge_2__c,MSJ_Product_for_MEC__c,MSJ_Product_for_MR__c,MSJ_Reply_Date__c,MSJ_Reply_User__c,MSJ_Reply__c,MSJ_Title__c,MSJ_AE_Infomation__c,MSJ_FAQ_Number_Report__c,MSJ_Return_Call_Report__c,MSJ_Inquiry_Origin_Report__c,MSJ_AE_Report__c,MSJ_Background__c,MSJ_MSL_Support__c,MSJ_Material_Requirement__c,MSJ_Hospital_Name_Disp__c,MSJ_Hospital__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,address_line_1_vod__c,address_line_2_vod__c,city_vod__c,delivery_method_vod__c,email_vod__c,fax_number_vod__c,inquiry_text__c,lock_vod__c,mobile_id_vod__c,phone_number_vod__c,product__c,rush_delivery__c,signature_date_vod__c,signature_vod__c,state_vod__c,status_vod__c,zip_vod__c,zvod_delivery_method_vod__c,zvod_disclaimer_vod__c,submitted_by_mobile_vod__c,disclaimer_vod__c,entity_reference_id_vod__c,call2_vod__c,country_vod__c,override_lock_vod__c,msj_department__c,msj_doctor_name__c,msj_hospital_name__c,msj_indication__c,msj_inquiry_assignment__c,msj_inquiry_date__c,msj_inquiry_input_manager__c,msj_inquiry_input_user__c,msj_msl_manager__c,msj_notice_to_mr__c,msj_person_in_charge_1__c,msj_person_in_charge_2__c,msj_product_for_mec__c,msj_product_for_mr__c,msj_reply_date__c,msj_reply_user__c,msj_reply__c,msj_title__c,msj_ae_infomation__c,msj_faq_number_report__c,msj_return_call_report__c,msj_inquiry_origin_report__c,msj_ae_report__c,msj_background__c,msj_msl_support__c,msj_material_requirement__c,msj_hospital_name_disp__c,msj_hospital__c +src02.crm_medical_inquiry_vod__c +org02.crm_medical_inquiry_vod__c + + diff --git a/s3/data/crm/settings/CRM_Medical_Insight_vod__c.txt b/s3/data/crm/settings/CRM_Medical_Insight_vod__c.txt new file mode 100644 index 00000000..0c477d5d --- /dev/null +++ b/s3/data/crm/settings/CRM_Medical_Insight_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +54 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,Clinical_Trial_vod__c,Date_vod__c,Description_vod__c,Entity_Reference_Id_vod__c,Interaction_vod__c,Medical_Event_vod__c,Mobile_ID_vod__c,Other_Source_vod__c,Override_Lock_vod__c,Publication_vod__c,Status_vod__c,Summary_vod__c,Unlock_vod__c,Commercial_Medical__c,MSJ_Level_1A__c,MSJ_Level_1B__c,MSJ_Level_2A__c,MSJ_Level_2B__c,MSJ_Level_3A__c,MSJ_Level_3B__c,MSJ_Level_4A__c,MSJ_Level_4B__c,MSJ_SubStatus__c,MSJ_Type_A__c,MSJ_Type_B__c,MSJ_Description_Backup__c,MSJ_Country__c,MSJ_Received_at_Boomi__c,MSJ_Level_1A_Value__c,MSJ_Level_1B_Value__c,MSJ_Level_2A_Value__c,MSJ_Level_2B_Value__c,MSJ_Level_3A_Value__c,MSJ_Level_3B_Value__c,MSJ_Level_4A_Value__c,MSJ_Level_4B_Value__c,MSJ_Hospital_ID__c,MSJ_Hospital_Name__c,MSJ_Hospital__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,clinical_trial_vod__c,date_vod__c,description_vod__c,entity_reference_id_vod__c,interaction_vod__c,medical_event_vod__c,mobile_id_vod__c,other_source_vod__c,override_lock_vod__c,publication_vod__c,status_vod__c,summary_vod__c,unlock_vod__c,commercial_medical__c,msj_level_1_a__c,msj_level_1_b__c,msj_level_2_a__c,msj_level_2_b__c,msj_level_3_a__c,msj_level_3_b__c,msj_level_4_a__c,msj_level_4_b__c,msj_sub_status__c,msj_type_a__c,msj_type_b__c,msj_description_backup__c,msj_country__c,msj_received_at_boomi__c,msj_level_1_a_value__c,msj_level_1_b_value__c,msj_level_2_a_value__c,msj_level_2_b_value__c,msj_level_3_a_value__c,msj_level_3_b_value__c,msj_level_4_a_value__c,msj_level_4_b_value__c,msj_hospital_id__c,msj_hospital_name__c,msj_hospital__c +src02.crm_medical_insight_vod__c +org02.crm_medical_insight_vod__c + + diff --git a/s3/data/crm/settings/CRM_Multichannel_Activity_Line_vod__c.txt b/s3/data/crm/settings/CRM_Multichannel_Activity_Line_vod__c.txt new file mode 100644 index 00000000..1b4ed66a --- /dev/null +++ b/s3/data/crm/settings/CRM_Multichannel_Activity_Line_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +35 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Multichannel_Activity_vod__c,Call_vod__c,Custom_vod__c,DateTime_vod__c,Debug_vod__c,Detail_Group_VExternal_Id_vod__c,Detail_Group_vod__c,Duration_vod__c,Event_Subtype_vod__c,Event_Type_vod__c,Key_Message_VExternal_Id_vod__c,Key_Message_vod__c,Multichannel_Content_Asset_Id_vod__c,Multichannel_Content_Asset_Version_vod__c,Multichannel_Content_Asset_vod__c,Multichannel_Content_vod__c,Product_VExternal_Id_vod__c,Product_vod__c,Sent_Email_vod__c,VExternal_Id_vod__c,Video_Last_Viewed_Time_vod__c,Video_Length_vod__c,Video_Total_Time_Spent_vod__c,View_Order_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,multichannel_activity_vod__c,call_vod__c,custom_vod__c,date_time_vod__c,debug_vod__c,detail_group_vexternal_id_vod__c,detail_group_vod__c,duration_vod__c,event_subtype_vod__c,event_type_vod__c,key_message_vexternal_id_vod__c,key_message_vod__c,multichannel_content_asset_id_vod__c,multichannel_content_asset_version_vod__c,multichannel_content_asset_vod__c,multichannel_content_vod__c,product_vexternal_id_vod__c,product_vod__c,sent_email_vod__c,vexternal_id_vod__c,video_last_viewed_time_vod__c,video_length_vod__c,video_total_time_spent_vod__c,view_order_vod__c +src02.crm_multichannel_activity_line_vod__c +org02.crm_multichannel_activity_line_vod__c + + diff --git a/s3/data/crm/settings/CRM_Multichannel_Activity_vod__c.txt b/s3/data/crm/settings/CRM_Multichannel_Activity_vod__c.txt new file mode 100644 index 00000000..9072d9e4 --- /dev/null +++ b/s3/data/crm/settings/CRM_Multichannel_Activity_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +47 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_External_ID_Map_vod__c,Account_vod__c,Call_vod__c,City_vod__c,Client_Name_vod__c,Client_OS_vod__c,Client_Type_vod__c,Country_vod__c,Debug_vod__c,Device_vod__c,IP_Address_vod__c,Multichannel_Activity_vod__c,Referring_Site_vod__c,Region_vod__c,Sent_Email_vod__c,Session_Id_vod__c,Site_vod__c,Start_DateTime_vod__c,Total_Duration_vod__c,URL_vod__c,User_Agent_vod__c,VExternal_Id_vod__c,Viewport_Height_vod__c,Viewport_Width_vod__c,Color_vod__c,Icon_vod__c,MCD_Primary_Key_vod__c,Record_Type_Name_vod__c,MSJ_Date_Opened__c,MSJ_Sent_Date__c,MSJ_Email_Subject__c,MSJ_Opens__c,MSJ_Email_Status__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_external_id_map_vod__c,account_vod__c,call_vod__c,city_vod__c,client_name_vod__c,client_os_vod__c,client_type_vod__c,country_vod__c,debug_vod__c,device_vod__c,ip_address_vod__c,multichannel_activity_vod__c,referring_site_vod__c,region_vod__c,sent_email_vod__c,session_id_vod__c,site_vod__c,start_date_time_vod__c,total_duration_vod__c,url_vod__c,user_agent_vod__c,vexternal_id_vod__c,viewport_height_vod__c,viewport_width_vod__c,color_vod__c,icon_vod__c,mcd_primary_key_vod__c,record_type_name_vod__c,msj_date_opened__c,msj_sent_date__c,msj_email_subject__c,msj_opens__c,msj_email_status__c +src02.crm_multichannel_activity_vod__c +org02.crm_multichannel_activity_vod__c + + diff --git a/s3/data/crm/settings/CRM_Multichannel_Consent_vod__c.txt b/s3/data/crm/settings/CRM_Multichannel_Consent_vod__c.txt new file mode 100644 index 00000000..be8d8b57 --- /dev/null +++ b/s3/data/crm/settings/CRM_Multichannel_Consent_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +48 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,Capture_Datetime_vod__c,Channel_Value_vod__c,Detail_Group_vod__c,External_ID_vod__c,Last_Device_vod__c,Mobile_ID_vod__c,Opt_Expiration_Date_vod__c,Opt_Type_vod__c,Optout_Event_Type_vod__c,Product_vod__c,Signature_Datetime_vod__c,Signature_ID_vod__c,Signature_vod__c,Sample_Consent_Template_Data_vod__c,Sample_Consent_Template_vod__c,Consent_Line_vod__c,Consent_Type_vod__c,Default_Consent_Text_vod__c,Disclaimer_Text_vod__c,Sub_Channel_Key_vod__c,Consent_Confirm_Datetime_vod__c,Related_Transaction_Id_vod__c,Sent_Email_vod__c,Content_Type_vod__c,Receipt_Email_vod__c,Receipt_Sent_Email_Transaction_Id_vod__c,Receipt_Sent_Email_vod__c,Captured_By_vod__c,Opt_Out_Disclaimer_Text_vod__c,Channel_Source_vod__c,Union_Id_vod__c,User_Last_Notified_vod__c,Sub_Channel_Display_Name__c,MSJ_Consent_Source__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,capture_datetime_vod__c,channel_value_vod__c,detail_group_vod__c,external_id_vod__c,last_device_vod__c,mobile_id_vod__c,opt_expiration_date_vod__c,opt_type_vod__c,optout_event_type_vod__c,product_vod__c,signature_datetime_vod__c,signature_id_vod__c,signature_vod__c,sample_consent_template_data_vod__c,sample_consent_template_vod__c,consent_line_vod__c,consent_type_vod__c,default_consent_text_vod__c,disclaimer_text_vod__c,sub_channel_key_vod__c,consent_confirm_datetime_vod__c,related_transaction_id_vod__c,sent_email_vod__c,content_type_vod__c,receipt_email_vod__c,receipt_sent_email_transaction_id_vod__c,receipt_sent_email_vod__c,captured_by_vod__c,opt_out_disclaimer_text_vod__c,channel_source_vod__c,union_id_vod__c,user_last_notified_vod__c,sub_channel_display_name__c,msj_consent_source__c +src02.crm_multichannel_consent_vod__c +org02.crm_multichannel_consent_vod__c + + diff --git a/s3/data/crm/settings/CRM_My_Setup_Products_vod__c.txt b/s3/data/crm/settings/CRM_My_Setup_Products_vod__c.txt new file mode 100644 index 00000000..47c71f53 --- /dev/null +++ b/s3/data/crm/settings/CRM_My_Setup_Products_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +12 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Product_vod__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,product_vod__c +src02.crm_my_setup_products_vod__c +org02.crm_my_setup_products_vod__c + + diff --git a/s3/data/crm/settings/CRM_ObjectTerritory2Association.txt b/s3/data/crm/settings/CRM_ObjectTerritory2Association.txt new file mode 100644 index 00000000..fff7b033 --- /dev/null +++ b/s3/data/crm/settings/CRM_ObjectTerritory2Association.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +9 +Id,ObjectId,Territory2Id,AssociationCause,SobjectType,IsDeleted,LastModifiedDate,LastModifiedById,SystemModstamp +id,object_id,territory2_id,association_cause,sobject_type,is_deleted,last_modified_date,last_modified_by_id,system_modstamp +src02.crm_object_territory2_association +org02.crm_object_territory2_association +CRM_ObjectTerritory2Association_ex.sql + diff --git a/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql new file mode 100644 index 00000000..d142c071 --- /dev/null +++ b/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_object_territory2_association', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Product_Group_vod__c.txt b/s3/data/crm/settings/CRM_Product_Group_vod__c.txt new file mode 100644 index 00000000..7b6c6cff --- /dev/null +++ b/s3/data/crm/settings/CRM_Product_Group_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +17 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Description_vod__c,Product_vod__c,Product_Catalog_vod__c,Start_Date_vod__c,End_Date_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,description_vod__c,product_vod__c,product_catalog_vod__c,start_date_vod__c,end_date_vod__c +src02.crm_product_group_vod__c +org02.crm_product_group_vod__c + + diff --git a/s3/data/crm/settings/CRM_Product_Metrics_vod__c.txt b/s3/data/crm/settings/CRM_Product_Metrics_vod__c.txt new file mode 100644 index 00000000..4629f0eb --- /dev/null +++ b/s3/data/crm/settings/CRM_Product_Metrics_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +52 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,Awareness__c,Selling_Stage__c,Formulary_Status__c,Movement__c,Products_vod__c,Segment__c,X12_mo_trx_chg__c,Speaker_Skills__c,Investigator_Readiness__c,Engagements__c,Mobile_ID_vod__c,External_ID_vod__c,MSJ_Patient__c,Detail_Group_vod__c,MSJ_EB_1st_Line_Liver_Meta__c,MSJ_EB_1st_Line_Multi_Meta__c,MSJ_EB_2nd_Line_Mono__c,MSJ_EB_2nd_Line_Combination__c,MSJ_EB_3rd_Line_Mono__c,MSJ_EB_3rd_Line_Combination__c,EMDS_Ability__c,EMDS_Brand_Loyalty__c,EMDS_Decision_Maker__c,EMDS_Early_Tech_Adopter__c,EMDS_Influence__c,EMDS_Main_Driver__c,EMDS_Priority__c,EMDS_Willingness__c,MSJ_KTL_Type__c,MSJ_KTL_Tier__c,MSJ_Publications__c,MSJ_Clinical_Trials__c,MSJ_Speaker_for_Medical_Events__c,MSJ_Advisor_to_Medical_Affairs__c,MSJ_Guidelines_Treatment_Standards__c,MSJ_Therapeutic_Area_Expertise__c,MSJ_MAP_GAP__c,MSJ_Associations__c,MSJ_Tier_Score__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,awareness__c,selling_stage__c,formulary_status__c,movement__c,products_vod__c,segment__c,x12_mo_trx_chg__c,speaker_skills__c,investigator_readiness__c,engagements__c,mobile_id_vod__c,external_id_vod__c,msj_patient__c,detail_group_vod__c,msj_eb_1st_line_liver_meta__c,msj_eb_1st_line_multi_meta__c,msj_eb_2nd_line_mono__c,msj_eb_2nd_line_combination__c,msj_eb_3rd_line_mono__c,msj_eb_3rd_line_combination__c,emds_ability__c,emds_brand_loyalty__c,emds_decision_maker__c,emds_early_tech_adopter__c,emds_influence__c,emds_main_driver__c,emds_priority__c,emds_willingness__c,msj_ktl_type__c,msj_ktl_tier__c,msj_publications__c,msj_clinical_trials__c,msj_speaker_for_medical_events__c,msj_advisor_to_medical_affairs__c,msj_guidelines_treatment_standards__c,msj_therapeutic_area_expertise__c,msj_map_gap__c,msj_associations__c,msj_tier_score__c +src02.crm_product_metrics_vod__c +org02.crm_product_metrics_vod__c +CRM_Product_Metrics_vod__c_ex.sql + diff --git a/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql b/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql new file mode 100644 index 00000000..af2e678e --- /dev/null +++ b/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_product_metrics_vod__c', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Product_vod__c.txt b/s3/data/crm/settings/CRM_Product_vod__c.txt new file mode 100644 index 00000000..843cec84 --- /dev/null +++ b/s3/data/crm/settings/CRM_Product_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +46 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consumer_site__c,Product_info__c,Therapeutic_Class_vod__c,Parent_Product_vod__c,Therapeutic_Area_vod__c,Product_Type_vod__c,Require_Key_Message_vod__c,Cost_vod__c,External_ID_vod__c,Manufacturer_vod__c,Company_Product_vod__c,Controlled_Substance_vod__c,Description_vod__c,Sample_Quantity_Picklist_vod__c,Display_Order_vod__c,No_Metrics_vod__c,Distributor_vod__c,Sample_Quantity_Bound_vod__c,Sample_U_M_vod__c,No_Details_vod__c,Quantity_Per_Case_vod__c,Schedule_vod__c,Restricted_vod__c,Pricing_Rule_Quantity_Bound_vod__c,No_Promo_Items_vod__c,User_Aligned_vod__c,Restricted_States_vod__c,Sort_Code_vod__c,No_Cycle_Plans_vod__c,Inventory_Order_UOM_vod__c,Inventory_Quantity_Per_Case_vod__c,VExternal_Id_vod__c,Country__c,MSJ_Product_Classification__c,MSJ_Indication__c,MSJ_Therapeutic_Area__c,MSJ_Global_Brand__c,MSJ_Global_Business_Unit__c,MSJ_Molecules__c,MSJ_SBU__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consumer_site__c,product_info__c,therapeutic_class_vod__c,parent_product_vod__c,therapeutic_area_vod__c,product_type_vod__c,require_key_message_vod__c,cost_vod__c,external_id_vod__c,manufacturer_vod__c,company_product_vod__c,controlled_substance_vod__c,description_vod__c,sample_quantity_picklist_vod__c,display_order_vod__c,no_metrics_vod__c,distributor_vod__c,sample_quantity_bound_vod__c,sample_u_m_vod__c,no_details_vod__c,quantity_per_case_vod__c,schedule_vod__c,restricted_vod__c,pricing_rule_quantity_bound_vod__c,no_promo_items_vod__c,user_aligned_vod__c,restricted_states_vod__c,sort_code_vod__c,no_cycle_plans_vod__c,inventory_order_uom_vod__c,inventory_quantity_per_case_vod__c,vexternal_id_vod__c,country__c,msj_product_classification__c,msj_indication__c,msj_therapeutic_area__c,msj_global_brand__c,msj_global_business_unit__c,msj_molecules__c,msj_sbu__c +src02.crm_product_vod__c +org02.crm_product_vod__c + + diff --git a/s3/data/crm/settings/CRM_Profile.txt b/s3/data/crm/settings/CRM_Profile.txt new file mode 100644 index 00000000..558ce629 --- /dev/null +++ b/s3/data/crm/settings/CRM_Profile.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +237 +Id,Name,PermissionsEmailSingle,PermissionsEmailMass,PermissionsEditTask,PermissionsEditEvent,PermissionsExportReport,PermissionsImportPersonal,PermissionsDataExport,PermissionsManageUsers,PermissionsEditPublicFilters,PermissionsEditPublicTemplates,PermissionsModifyAllData,PermissionsManageCases,PermissionsManageSolutions,PermissionsCustomizeApplication,PermissionsEditReadonlyFields,PermissionsRunReports,PermissionsViewSetup,PermissionsTransferAnyEntity,PermissionsNewReportBuilder,PermissionsManageSelfService,PermissionsManageCssUsers,PermissionsActivateContract,PermissionsApproveContract,PermissionsImportLeads,PermissionsManageLeads,PermissionsTransferAnyLead,PermissionsViewAllData,PermissionsEditPublicDocuments,PermissionsViewEncryptedData,PermissionsEditBrandTemplates,PermissionsEditHtmlTemplates,PermissionsManageTranslation,PermissionsDeleteActivatedContract,PermissionsSendSitRequests,PermissionsApiUserOnly,PermissionsManageRemoteAccess,PermissionsCanUseNewDashboardBuilder,PermissionsManageCategories,PermissionsConvertLeads,PermissionsTestInstanceCreate,PermissionsPasswordNeverExpires,PermissionsUseTeamReassignWizards,PermissionsInstallMultiforce,PermissionsPublishMultiforce,PermissionsEditOppLineItemUnitPrice,PermissionsManageTerritories,PermissionsCreateMultiforce,PermissionsBulkApiHardDelete,PermissionsInboundMigrationToolsUser,PermissionsSolutionImport,PermissionsManageCallCenters,PermissionsManageSynonyms,PermissionsOutboundMigrationToolsUser,PermissionsViewContent,PermissionsManageEmailClientConfig,PermissionsEnableNotifications,PermissionsManageDataIntegrations,PermissionsDistributeFromPersWksp,PermissionsViewDataCategories,PermissionsManageDataCategories,PermissionsAuthorApex,PermissionsManageMobile,PermissionsApiEnabled,PermissionsManageCustomReportTypes,PermissionsEditCaseComments,PermissionsTransferAnyCase,PermissionsContentAdministrator,PermissionsCreateWorkspaces,PermissionsManageContentPermissions,PermissionsManageContentProperties,PermissionsManageContentTypes,PermissionsScheduleJob,PermissionsManageExchangeConfig,PermissionsManageAnalyticSnapshots,PermissionsScheduleReports,PermissionsManageBusinessHourHolidays,PermissionsManageDynamicDashboards,PermissionsManageInteraction,PermissionsViewMyTeamsDashboards,PermissionsResetPasswords,PermissionsFlowUFLRequired,PermissionsActivitiesAccess,PermissionsEmailTemplateManagement,PermissionsEmailAdministration,PermissionsChatterFileLink,PermissionsForceTwoFactor,PermissionsViewEventLogFiles,PermissionsManageNetworks,PermissionsManageAuthProviders,PermissionsRunFlow,PermissionsCreateCustomizeDashboards,PermissionsCreateDashboardFolders,PermissionsViewPublicDashboards,PermissionsManageDashbdsInPubFolders,PermissionsCreateCustomizeReports,PermissionsCreateReportFolders,PermissionsViewPublicReports,PermissionsManageReportsInPubFolders,PermissionsEditMyDashboards,PermissionsEditMyReports,PermissionsViewAllUsers,PermissionsConnectOrgToEnvironmentHub,PermissionsCreateCustomizeFilters,PermissionsContentHubUser,PermissionsGovernNetworks,PermissionsSalesConsole,PermissionsTwoFactorApi,PermissionsDeleteTopics,PermissionsEditTopics,PermissionsCreateTopics,PermissionsAssignTopics,PermissionsIdentityEnabled,PermissionsIdentityConnect,PermissionsContentWorkspaces,PermissionsCustomMobileAppsAccess,PermissionsViewHelpLink,PermissionsManageProfilesPermissionsets,PermissionsAssignPermissionSets,PermissionsManageRoles,PermissionsManageIpAddresses,PermissionsManageSharing,PermissionsManageInternalUsers,PermissionsManagePasswordPolicies,PermissionsManageLoginAccessPolicies,PermissionsManageCustomPermissions,PermissionsStdAutomaticActivityCapture,PermissionsManageTwoFactor,PermissionsDebugApex,PermissionsLightningExperienceUser,PermissionsConfigCustomRecs,PermissionsSubmitMacrosAllowed,PermissionsBulkMacrosAllowed,PermissionsManageSessionPermissionSets,PermissionsCreateAuditFields,PermissionsUpdateWithInactiveOwner,PermissionsManageSandboxes,PermissionsAutomaticActivityCapture,PermissionsImportCustomObjects,PermissionsDelegatedTwoFactor,PermissionsSelectFilesFromSalesforce,PermissionsModerateNetworkUsers,PermissionsMergeTopics,PermissionsSubscribeToLightningReports,PermissionsManagePvtRptsAndDashbds,PermissionsAllowLightningLogin,PermissionsCampaignInfluence2,PermissionsViewDataAssessment,PermissionsCanApproveFeedPost,PermissionsAllowViewEditConvertedLeads,PermissionsShowCompanyNameAsUserBadge,PermissionsAccessCMC,PermissionsViewHealthCheck,PermissionsManageHealthCheck,PermissionsPackaging2,PermissionsManageCertificates,PermissionsCreateReportInLightning,PermissionsPreventClassicExperience,PermissionsListEmailSend,PermissionsChangeDashboardColors,PermissionsManageRecommendationStrategies,PermissionsManagePropositions,PermissionsSubscribeReportRolesGrps,PermissionsSubscribeDashboardRolesGrps,PermissionsUseWebLink,PermissionsHasUnlimitedNBAExecutions,PermissionsViewOnlyEmbeddedAppUser,PermissionsViewAllActivities,PermissionsSubscribeReportToOtherUsers,PermissionsLightningConsoleAllowedForUser,PermissionsSubscribeReportsRunAsUser,PermissionsSubscribeToLightningDashboards,PermissionsSubscribeDashboardToOtherUsers,PermissionsCreateLtngTempInPub,PermissionsTransactionalEmailSend,PermissionsViewPrivateStaticResources,PermissionsCreateLtngTempFolder,PermissionsApexRestServices,PermissionsEnableCommunityAppLauncher,PermissionsGiveRecognitionBadge,PermissionsUseMySearch,PermissionsLtngPromoReserved01UserPerm,PermissionsManageSubscriptions,PermissionsManageSurveys,PermissionsUseAssistantDialog,PermissionsUseQuerySuggestions,PermissionsViewRoles,PermissionsLMOutboundMessagingUserPerm,PermissionsModifyDataClassification,PermissionsPrivacyDataAccess,PermissionsQueryAllFiles,PermissionsModifyMetadata,PermissionsManageCMS,PermissionsSandboxTestingInCommunityApp,PermissionsCanEditPrompts,PermissionsViewUserPII,PermissionsManageHubConnections,PermissionsB2BMarketingAnalyticsUser,PermissionsTraceXdsQueries,PermissionsViewAllCustomSettings,PermissionsViewAllForeignKeyNames,PermissionsHeadlessCMSAccess,PermissionsLMEndMessagingSessionUserPerm,PermissionsConsentApiUpdate,PermissionsAccessContentBuilder,PermissionsAccountSwitcherUser,PermissionsManageC360AConnections,PermissionsManageReleaseUpdates,PermissionsViewAllProfiles,PermissionsSkipIdentityConfirmation,PermissionsSendCustomNotifications,PermissionsPackaging2Delete,PermissionsFSCComprehensiveUserAccess,PermissionsManageTrustMeasures,PermissionsViewTrustMeasures,PermissionsIsotopeCToCUser,PermissionsIsotopeAccess,PermissionsIsotopeLEX,PermissionsQuipMetricsAccess,PermissionsQuipUserEngagementMetrics,PermissionsManageExternalConnections,PermissionsAIViewInsightObjects,PermissionsAICreateInsightObjects,PermissionsNativeWebviewScrolling,PermissionsViewDeveloperName,Type,UserLicenseId,UserType,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,Description,LastViewedDate,LastReferencedDate +id,name,permissions_email_single,permissions_email_mass,permissions_edit_task,permissions_edit_event,permissions_export_report,permissions_import_personal,permissions_data_export,permissions_manage_users,permissions_edit_public_filters,permissions_edit_public_templates,permissions_modify_all_data,permissions_manage_cases,permissions_manage_solutions,permissions_customize_application,permissions_edit_readonly_fields,permissions_run_reports,permissions_view_setup,permissions_transfer_any_entity,permissions_new_report_builder,permissions_manage_self_service,permissions_manage_css_users,permissions_activate_contract,permissions_approve_contract,permissions_import_leads,permissions_manage_leads,permissions_transfer_any_lead,permissions_view_all_data,permissions_edit_public_documents,permissions_view_encrypted_data,permissions_edit_brand_templates,permissions_edit_html_templates,permissions_manage_translation,permissions_delete_activated_contract,permissions_send_sit_requests,permissions_api_user_only,permissions_manage_remote_access,permissions_can_use_new_dashboard_builder,permissions_manage_categories,permissions_convert_leads,permissions_test_instance_create,permissions_password_never_expires,permissions_use_team_reassign_wizards,permissions_install_multiforce,permissions_publish_multiforce,permissions_edit_opp_line_item_unit_price,permissions_manage_territories,permissions_create_multiforce,permissions_bulk_api_hard_delete,permissions_inbound_migration_tools_user,permissions_solution_import,permissions_manage_call_centers,permissions_manage_synonyms,permissions_outbound_migration_tools_user,permissions_view_content,permissions_manage_email_client_config,permissions_enable_notifications,permissions_manage_data_integrations,permissions_distribute_from_pers_wksp,permissions_view_data_categories,permissions_manage_data_categories,permissions_author_apex,permissions_manage_mobile,permissions_api_enabled,permissions_manage_custom_report_types,permissions_edit_case_comments,permissions_transfer_any_case,permissions_content_administrator,permissions_create_workspaces,permissions_manage_content_permissions,permissions_manage_content_properties,permissions_manage_content_types,permissions_schedule_job,permissions_manage_exchange_config,permissions_manage_analytic_snapshots,permissions_schedule_reports,permissions_manage_business_hour_holidays,permissions_manage_dynamic_dashboards,permissions_manage_interaction,permissions_view_my_teams_dashboards,permissions_reset_passwords,permissions_flow_uflrequired,permissions_activities_access,permissions_email_template_management,permissions_email_administration,permissions_chatter_file_link,permissions_force_two_factor,permissions_view_event_log_files,permissions_manage_networks,permissions_manage_auth_providers,permissions_run_flow,permissions_create_customize_dashboards,permissions_create_dashboard_folders,permissions_view_public_dashboards,permissions_manage_dashbds_in_pub_folders,permissions_create_customize_reports,permissions_create_report_folders,permissions_view_public_reports,permissions_manage_reports_in_pub_folders,permissions_edit_my_dashboards,permissions_edit_my_reports,permissions_view_all_users,permissions_connect_org_to_environment_hub,permissions_create_customize_filters,permissions_content_hub_user,permissions_govern_networks,permissions_sales_console,permissions_two_factor_api,permissions_delete_topics,permissions_edit_topics,permissions_create_topics,permissions_assign_topics,permissions_identity_enabled,permissions_identity_connect,permissions_content_workspaces,permissions_custom_mobile_apps_access,permissions_view_help_link,permissions_manage_profiles_permissionsets,permissions_assign_permission_sets,permissions_manage_roles,permissions_manage_ip_addresses,permissions_manage_sharing,permissions_manage_internal_users,permissions_manage_password_policies,permissions_manage_login_access_policies,permissions_manage_custom_permissions,permissions_std_automatic_activity_capture,permissions_manage_two_factor,permissions_debug_apex,permissions_lightning_experience_user,permissions_config_custom_recs,permissions_submit_macros_allowed,permissions_bulk_macros_allowed,permissions_manage_session_permission_sets,permissions_create_audit_fields,permissions_update_with_inactive_owner,permissions_manage_sandboxes,permissions_automatic_activity_capture,permissions_import_custom_objects,permissions_delegated_two_factor,permissions_select_files_from_salesforce,permissions_moderate_network_users,permissions_merge_topics,permissions_subscribe_to_lightning_reports,permissions_manage_pvt_rpts_and_dashbds,permissions_allow_lightning_login,permissions_campaign_influence2,permissions_view_data_assessment,permissions_can_approve_feed_post,permissions_allow_view_edit_converted_leads,permissions_show_company_name_as_user_badge,permissions_access_cmc,permissions_view_health_check,permissions_manage_health_check,permissions_packaging2,permissions_manage_certificates,permissions_create_report_in_lightning,permissions_prevent_classic_experience,permissions_list_email_send,permissions_change_dashboard_colors,permissions_manage_recommendation_strategies,permissions_manage_propositions,permissions_subscribe_report_roles_grps,permissions_subscribe_dashboard_roles_grps,permissions_use_web_link,permissions_has_unlimited_nbaexecutions,permissions_view_only_embedded_app_user,permissions_view_all_activities,permissions_subscribe_report_to_other_users,permissions_lightning_console_allowed_for_user,permissions_subscribe_reports_run_as_user,permissions_subscribe_to_lightning_dashboards,permissions_subscribe_dashboard_to_other_users,permissions_create_ltng_temp_in_pub,permissions_transactional_email_send,permissions_view_private_static_resources,permissions_create_ltng_temp_folder,permissions_apex_rest_services,permissions_enable_community_app_launcher,permissions_give_recognition_badge,permissions_use_my_search,permissions_ltng_promo_reserved01_user_perm,permissions_manage_subscriptions,permissions_manage_surveys,permissions_use_assistant_dialog,permissions_use_query_suggestions,permissions_view_roles,permissions_lmoutbound_messaging_user_perm,permissions_modify_data_classification,permissions_privacy_data_access,permissions_query_all_files,permissions_modify_metadata,permissions_manage_cms,permissions_sandbox_testing_in_community_app,permissions_can_edit_prompts,permissions_view_user_pii,permissions_manage_hub_connections,permissions_b2_bmarketing_analytics_user,permissions_trace_xds_queries,permissions_view_all_custom_settings,permissions_view_all_foreign_key_names,permissions_headless_cmsaccess,permissions_lmend_messaging_session_user_perm,permissions_consent_api_update,permissions_access_content_builder,permissions_account_switcher_user,permissions_manage_c360_aconnections,permissions_manage_release_updates,permissions_view_all_profiles,permissions_skip_identity_confirmation,permissions_send_custom_notifications,permissions_packaging2_delete,permissions_fsccomprehensive_user_access,permissions_manage_trust_measures,permissions_view_trust_measures,permissions_isotope_cto_cuser,permissions_isotope_access,permissions_isotope_lex,permissions_quip_metrics_access,permissions_quip_user_engagement_metrics,permissions_manage_external_connections,permissions_aiview_insight_objects,permissions_aicreate_insight_objects,permissions_native_webview_scrolling,permissions_view_developer_name,type,user_license_id,user_type,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,description,last_viewed_date,last_referenced_date +src02.crm_profile +org02.crm_profile +CRM_Profile_ex.sql + diff --git a/s3/data/crm/settings/CRM_Profile_ex.sql b/s3/data/crm/settings/CRM_Profile_ex.sql new file mode 100644 index 00000000..b858ab02 --- /dev/null +++ b/s3/data/crm/settings/CRM_Profile_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_profile', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Question_Response_vod__c.txt b/s3/data/crm/settings/CRM_Question_Response_vod__c.txt new file mode 100644 index 00000000..a994ab4a --- /dev/null +++ b/s3/data/crm/settings/CRM_Question_Response_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +30 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Survey_Target_vod__c,Answer_Choice_vod__c,Date_vod__c,Datetime_vod__c,External_ID_vod__c,Mobile_ID_vod__c,Number_vod__c,Order_vod__c,Question_Text_vod__c,Required_vod__c,Response_Hash_vod__c,Response_vod__c,Score_vod__c,Survey_Question_vod__c,Text_vod__c,Type_vod__c,Condition_vod__c,Inactive_Condition_vod__c,Source_ID_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,survey_target_vod__c,answer_choice_vod__c,date_vod__c,datetime_vod__c,external_id_vod__c,mobile_id_vod__c,number_vod__c,order_vod__c,question_text_vod__c,required_vod__c,response_hash_vod__c,response_vod__c,score_vod__c,survey_question_vod__c,text_vod__c,type_vod__c,condition_vod__c,inactive_condition_vod__c,source_id_vod__c +src02.crm_question_response_vod__c +org02.crm_question_response_vod__c + + diff --git a/s3/data/crm/settings/CRM_RecordType.txt b/s3/data/crm/settings/CRM_RecordType.txt new file mode 100644 index 00000000..78960693 --- /dev/null +++ b/s3/data/crm/settings/CRM_RecordType.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +14 +Id,Name,DeveloperName,NamespacePrefix,Description,BusinessProcessId,SobjectType,IsActive,IsPersonType,CreatedById,CreatedDate,LastModifiedById,LastModifiedDate,SystemModstamp +id,name,developer_name,namespace_prefix,description,business_process_id,sobject_type,is_active,is_person_type,created_by_id,created_date,last_modified_by_id,last_modified_date,system_modstamp +src02.crm_record_type +org02.crm_record_type + + diff --git a/s3/data/crm/settings/CRM_Remote_Meeting_vod__c.txt b/s3/data/crm/settings/CRM_Remote_Meeting_vod__c.txt new file mode 100644 index 00000000..1b4827c9 --- /dev/null +++ b/s3/data/crm/settings/CRM_Remote_Meeting_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +24 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Meeting_Id_vod__c,Meeting_Name_vod__c,Mobile_ID_vod__c,Scheduled_DateTime_vod__c,Scheduled_vod__c,Attendance_Report_Process_Status_vod__c,Latest_Meeting_Start_Datetime_vod__c,Meeting_Password_vod__c,Meeting_Outcome_Status_vod__c,Allow_for_Joining_via_Zoom_vod__c,Zoom_Join_Token_vod__c,VExternal_Id_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,meeting_id_vod__c,meeting_name_vod__c,mobile_id_vod__c,scheduled_date_time_vod__c,scheduled_vod__c,attendance_report_process_status_vod__c,latest_meeting_start_datetime_vod__c,meeting_password_vod__c,meeting_outcome_status_vod__c,allow_for_joining_via_zoom_vod__c,zoom_join_token_vod__c,vexternal_id_vod__c +src02.crm_remote_meeting_vod__c +org02.crm_remote_meeting_vod__c + + diff --git a/s3/data/crm/settings/CRM_Sent_Email_vod__c.txt b/s3/data/crm/settings/CRM_Sent_Email_vod__c.txt new file mode 100644 index 00000000..148ed757 --- /dev/null +++ b/s3/data/crm/settings/CRM_Sent_Email_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +65 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,Account_Email_vod__c,Account_vod__c,Approved_Email_Template_vod__c,Capture_Datetime_vod__c,Detail_Group_vod__c,Email_Config_Values_vod__c,Email_Content2_vod__c,Email_Content_vod__c,Email_Fragments_vod__c,Email_Sent_Date_vod__c,Failure_Msg_vod__c,Last_Activity_Date_vod__c,Last_Device_vod__c,MC_Capture_Datetime_vod__c,Mobile_ID_vod__c,Opened_vod__c,Product_Display_vod__c,Product_vod__c,Sender_Email_vod__c,Status_vod__c,Valid_Consent_Exists_vod__c,Approved_Document_Views_vod__c,Click_Count_vod__c,Last_Click_Date_vod__c,Last_Open_Date_vod__c,Open_Count_vod__c,Receipt_Entity_Type_vod__c,Receipt_Record_Id_vod__c,Territory_vod__c,Call2_vod__c,Medical_Inquiry_vod__c,Parent_Email_vod__c,Related_Transaction_ID_vod__c,Case_vod__c,Key_Message_vod__c,Suggestion_vod__c,EM_Attendee_vod__c,EM_Event_Speaker_vod__c,EM_Event_Team_Member_vod__c,Event_Attendee_vod__c,Event_vod__c,Medical_Event_vod__c,Scheduled_Send_Datetime_vod__c,User_vod__c,Content_Type_vod__c,Bcc_vod__c,Event_Attendee_Mobile_Id_vod__c,Event_Mobile_Id_vod__c,Activity_Tracking_Mode_vod__c,Email_Source_vod__c,Subject_vod__c,User_Input_Text_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,account_email_vod__c,account_vod__c,approved_email_template_vod__c,capture_datetime_vod__c,detail_group_vod__c,email_config_values_vod__c,email_content2_vod__c,email_content_vod__c,email_fragments_vod__c,email_sent_date_vod__c,failure_msg_vod__c,last_activity_date_vod__c,last_device_vod__c,mc_capture_datetime_vod__c,mobile_id_vod__c,opened_vod__c,product_display_vod__c,product_vod__c,sender_email_vod__c,status_vod__c,valid_consent_exists_vod__c,approved_document_views_vod__c,click_count_vod__c,last_click_date_vod__c,last_open_date_vod__c,open_count_vod__c,receipt_entity_type_vod__c,receipt_record_id_vod__c,territory_vod__c,call2_vod__c,medical_inquiry_vod__c,parent_email_vod__c,related_transaction_id_vod__c,case_vod__c,key_message_vod__c,suggestion_vod__c,em_attendee_vod__c,em_event_speaker_vod__c,em_event_team_member_vod__c,event_attendee_vod__c,event_vod__c,medical_event_vod__c,scheduled_send_datetime_vod__c,user_vod__c,content_type_vod__c,bcc_vod__c,event_attendee_mobile_id_vod__c,event_mobile_id_vod__c,activity_tracking_mode_vod__c,email_source_vod__c,subject_vod__c,user_input_text_vod__c +src02.crm_sent_email_vod__c +org02.crm_sent_email_vod__c + + diff --git a/s3/data/crm/settings/CRM_Sent_Fragment_vod__c.txt b/s3/data/crm/settings/CRM_Sent_Fragment_vod__c.txt new file mode 100644 index 00000000..c93f9f9e --- /dev/null +++ b/s3/data/crm/settings/CRM_Sent_Fragment_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +16 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,Sent_Email_vod__c,Account_vod__c,Email_Template_vod__c,Sent_Fragment_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,sent_email_vod__c,account_vod__c,email_template_vod__c,sent_fragment_vod__c +src02.crm_sent_fragment_vod__c +org02.crm_sent_fragment_vod__c + + diff --git a/s3/data/crm/settings/CRM_Survey_Question_vod__c.txt b/s3/data/crm/settings/CRM_Survey_Question_vod__c.txt new file mode 100644 index 00000000..3a636f0a --- /dev/null +++ b/s3/data/crm/settings/CRM_Survey_Question_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +23 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Survey_vod__c,Answer_Choice_vod__c,External_ID_vod__c,Max_Score_vod__c,Min_Score_vod__c,Order_vod__c,Question_vod__c,Required_vod__c,Text_vod__c,Condition_vod__c,Source_ID_vod__c,MSJ_External_ID__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,survey_vod__c,answer_choice_vod__c,external_id_vod__c,max_score_vod__c,min_score_vod__c,order_vod__c,question_vod__c,required_vod__c,text_vod__c,condition_vod__c,source_id_vod__c,msj_external_id__c +src02.crm_survey_question_vod__c +org02.crm_survey_question_vod__c + + diff --git a/s3/data/crm/settings/CRM_Survey_Target_vod__c.txt b/s3/data/crm/settings/CRM_Survey_Target_vod__c.txt new file mode 100644 index 00000000..f9f80626 --- /dev/null +++ b/s3/data/crm/settings/CRM_Survey_Target_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +39 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_Display_Name_vod__c,Account_vod__c,Channels_vod__c,End_Date_vod__c,Entity_Reference_Id_vod__c,External_ID_vod__c,Language_vod__c,Lock_vod__c,Mobile_ID_vod__c,No_Autoassign_vod__c,Not_Completed_vod__c,Region_vod__c,Segment_vod__c,Start_Date_vod__c,Status_vod__c,Survey_vod__c,Territory_vod__c,zvod_Address_vod__c,zvod_Specialty_vod__c,Score_vod__c,User_vod__c,Child_Account_vod__c,Location_Entity_Reference_Id_vod__c,Location_vod__c,Target_Type_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_display_name_vod__c,account_vod__c,channels_vod__c,end_date_vod__c,entity_reference_id_vod__c,external_id_vod__c,language_vod__c,lock_vod__c,mobile_id_vod__c,no_autoassign_vod__c,not_completed_vod__c,region_vod__c,segment_vod__c,start_date_vod__c,status_vod__c,survey_vod__c,territory_vod__c,zvod_address_vod__c,zvod_specialty_vod__c,score_vod__c,user_vod__c,child_account_vod__c,location_entity_reference_id_vod__c,location_vod__c,target_type_vod__c +src02.crm_survey_target_vod__c +org02.crm_survey_target_vod__c + + diff --git a/s3/data/crm/settings/CRM_Survey_vod__c.txt b/s3/data/crm/settings/CRM_Survey_vod__c.txt new file mode 100644 index 00000000..bfd04a2e --- /dev/null +++ b/s3/data/crm/settings/CRM_Survey_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +37 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Assignment_Type_vod__c,Channels_vod__c,End_Date_vod__c,Expired_vod__c,External_ID_vod__c,Language_vod__c,Lock_vod__c,Open_vod__c,Product_vod__c,Region_vod__c,Segment_vod__c,Start_Date_vod__c,Status_vod__c,Territory_vod__c,zvod_Questions_vod__c,zvod_Segments_vod__c,zvod_Targets_vod__c,Max_Score_vod__c,Min_Score_vod__c,Autotarget_vod__c,Territories_vod__c,Target_Type_vod__c,MSJ_External_ID__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,assignment_type_vod__c,channels_vod__c,end_date_vod__c,expired_vod__c,external_id_vod__c,language_vod__c,lock_vod__c,open_vod__c,product_vod__c,region_vod__c,segment_vod__c,start_date_vod__c,status_vod__c,territory_vod__c,zvod_questions_vod__c,zvod_segments_vod__c,zvod_targets_vod__c,max_score_vod__c,min_score_vod__c,autotarget_vod__c,territories_vod__c,target_type_vod__c,msj_external_id__c +src02.crm_survey_vod__c +org02.crm_survey_vod__c + + diff --git a/s3/data/crm/settings/CRM_Territory2.txt b/s3/data/crm/settings/CRM_Territory2.txt new file mode 100644 index 00000000..5486b657 --- /dev/null +++ b/s3/data/crm/settings/CRM_Territory2.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +15 +Id,Name,Territory2TypeId,Territory2ModelId,ParentTerritory2Id,Description,ForecastUserId,AccountAccessLevel,OpportunityAccessLevel,CaseAccessLevel,ContactAccessLevel,LastModifiedDate,LastModifiedById,SystemModstamp,DeveloperName,MSJ_Territory_Type__c,MSJ_Level__c +id,name,territory2_type_id,territory2_model_id,parent_territory2_id,description,forecast_user_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,last_modified_date,last_modified_by_id,system_modstamp,developer_name,msj_territory_type__c,msj_level__c +src02.crm_territory2 +org02.crm_territory2 +CRM_Territory2_ex.sql + diff --git a/s3/data/crm/settings/CRM_Territory2_ALL.txt b/s3/data/crm/settings/CRM_Territory2_ALL.txt new file mode 100644 index 00000000..525e1796 --- /dev/null +++ b/s3/data/crm/settings/CRM_Territory2_ALL.txt @@ -0,0 +1,14 @@ +CRM +, +utf-8 +" +CRLF +1 +15 +Id,Name,Territory2TypeId,Territory2ModelId,ParentTerritory2Id,Description,ForecastUserId,AccountAccessLevel,OpportunityAccessLevel,CaseAccessLevel,ContactAccessLevel,LastModifiedDate,LastModifiedById,SystemModstamp,DeveloperName +id,name,territory2_type_id,territory2_model_id,parent_territory2_id,description,forecast_user_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,last_modified_date,last_modified_by_id,system_modstamp,developer_name +src02.crm_territory2_all +org02.crm_territory2_all + + +truncate_src_table:src02.crm_territory2_all diff --git a/s3/data/crm/settings/CRM_Territory2_ex.sql b/s3/data/crm/settings/CRM_Territory2_ex.sql new file mode 100644 index 00000000..a30ccd91 --- /dev/null +++ b/s3/data/crm/settings/CRM_Territory2_ex.sql @@ -0,0 +1,2 @@ +CALL crm_data_sync('src02.crm_territory2', 'src02.crm_territory2_all', 'system_modstamp'); +CALL crm_history('src02.crm_territory2', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Time_Off_Territory_vod__c.txt b/s3/data/crm/settings/CRM_Time_Off_Territory_vod__c.txt new file mode 100644 index 00000000..e2b1b9b6 --- /dev/null +++ b/s3/data/crm/settings/CRM_Time_Off_Territory_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +25 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Reason_vod__c,Territory_vod__c,Date_vod__c,Status_vod__c,Time_vod__c,Hours_vod__c,Mobile_ID_vod__c,Hours_off_vod__c,Start_Time_vod__c,MSJ_Day__c,MSJ_Comment__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,reason_vod__c,territory_vod__c,date_vod__c,status_vod__c,time_vod__c,hours_vod__c,mobile_id_vod__c,hours_off_vod__c,start_time_vod__c,msj_day__c,msj_comment__c +src02.crm_time_off_territory_vod__c +org02.crm_time_off_territory_vod__c + + diff --git a/s3/data/crm/settings/CRM_User.txt b/s3/data/crm/settings/CRM_User.txt new file mode 100644 index 00000000..317a5e14 --- /dev/null +++ b/s3/data/crm/settings/CRM_User.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +197 +Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,LastPasswordChangeDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,NumberOfFailedLogins,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesNativeEmailClient,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c +id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,last_password_change_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,number_of_failed_logins,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_native_email_client,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c +src02.crm_user +org02.crm_user + + diff --git a/s3/data/crm/settings/CRM_UserRole.txt b/s3/data/crm/settings/CRM_UserRole.txt new file mode 100644 index 00000000..4b10b6c2 --- /dev/null +++ b/s3/data/crm/settings/CRM_UserRole.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +16 +Id,Name,ParentRoleId,RollupDescription,OpportunityAccessForAccountOwner,CaseAccessForAccountOwner,ContactAccessForAccountOwner,ForecastUserId,MayForecastManagerShare,LastModifiedDate,LastModifiedById,SystemModstamp,DeveloperName,PortalAccountId,PortalType,PortalAccountOwnerId +id,name,parent_role_id,rollup_description,opportunity_access_for_account_owner,case_access_for_account_owner,contact_access_for_account_owner,forecast_user_id,may_forecast_manager_share,last_modified_date,last_modified_by_id,system_modstamp,developer_name,portal_account_id,portal_type,portal_account_owner_id +src02.crm_user_role +org02.crm_user_role + + diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association.txt b/s3/data/crm/settings/CRM_UserTerritory2Association.txt new file mode 100644 index 00000000..039757b9 --- /dev/null +++ b/s3/data/crm/settings/CRM_UserTerritory2Association.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +8 +Id,UserId,Territory2Id,IsActive,RoleInTerritory2,LastModifiedDate,LastModifiedById,SystemModstamp +id,user_id,territory2_id,is_active,role_in_territory2,last_modified_date,last_modified_by_id,system_modstamp +src02.crm_user_territory2_association +org02.crm_user_territory2_association +CRM_UserTerritory2Association_ex.sql + diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt b/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt new file mode 100644 index 00000000..4617829d --- /dev/null +++ b/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt @@ -0,0 +1,14 @@ +CRM +, +utf-8 +" +CRLF +1 +8 +Id,UserId,Territory2Id,IsActive,RoleInTerritory2,LastModifiedDate,LastModifiedById,SystemModstamp +id,user_id,territory2_id,is_active,role_in_territory2,last_modified_date,last_modified_by_id,system_modstamp +src02.crm_user_territory2_association_all +org02.crm_user_territory2_association_all + + +truncate_src_table:src02.crm_user_territory2_association_all diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql new file mode 100644 index 00000000..6a9aaece --- /dev/null +++ b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql @@ -0,0 +1,2 @@ +CALL crm_data_sync('src02.crm_user_territory2_association', 'src02.crm_user_territory2_association_all', 'system_modstamp'); +CALL crm_history('src02.crm_user_territory2_association', 'system_modstamp'); diff --git a/s3/data/crm/settings/configmap.config b/s3/data/crm/settings/configmap.config new file mode 100644 index 00000000..89993b8b --- /dev/null +++ b/s3/data/crm/settings/configmap.config @@ -0,0 +1,58 @@ +/* 【CRMデータ 差分連携】 */ +CRM_Clm_Presentation_vod__c_[0-9]{14}\.(CSV|csv) CRM_Clm_Presentation_vod__c.txt +CRM_Clm_Presentation_Slide_vod__c_[0-9]{14}\.(CSV|csv) CRM_Clm_Presentation_Slide_vod__c.txt +CRM_Medical_Insight_vod__c_[0-9]{14}\.(CSV|csv) CRM_Medical_Insight_vod__c.txt +CRM_MSJ_MR_Weekly_Report__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_MR_Weekly_Report__c.txt +CRM_Account_Territory_Loader_vod__c_[0-9]{14}\.(CSV|csv) CRM_Account_Territory_Loader_vod__c.txt +CRM_Event_Attendee_vod__c_[0-9]{14}\.(CSV|csv) CRM_Event_Attendee_vod__c.txt +CRM_ObjectTerritory2Association_[0-9]{14}\.(CSV|csv) CRM_ObjectTerritory2Association.txt +CRM_Key_Message_vod__c_[0-9]{14}\.(CSV|csv) CRM_Key_Message_vod__c.txt +CRM_Group_[0-9]{14}\.(CSV|csv) CRM_Group.txt +CRM_Medical_Event_vod__c_[0-9]{14}\.(CSV|csv) CRM_Medical_Event_vod__c.txt +CRM_MSJ_Medical_Event_Evaluation__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Medical_Event_Evaluation__c.txt +CRM_Coaching_Report_vod__c_[0-9]{14}\.(CSV|csv) CRM_Coaching_Report_vod__c.txt +CRM_Call2_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call2_vod__c.txt +CRM_Call2_Detail_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call2_Detail_vod__c.txt +CRM_Call2_Key_Message_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call2_Key_Message_vod__c.txt +CRM_Call_Clickstream_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call_Clickstream_vod__c.txt +CRM_Call2_Discussion_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call2_Discussion_vod__c.txt +CRM_Time_Off_Territory_vod__c_[0-9]{14}\.(CSV|csv) CRM_Time_Off_Territory_vod__c.txt +CRM_Dynamic_Attribute_vod__c_[0-9]{14}\.(CSV|csv) CRM_Dynamic_Attribute_vod__c.txt +CRM_Dynamic_Attribute_Configuration_vod__c_[0-9]{14}\.(CSV|csv) CRM_Dynamic_Attribute_Configuration_vod__c.txt +CRM_Territory2_[0-9]{14}\.(CSV|csv) CRM_Territory2.txt +CRM_Profile_[0-9]{14}\.(CSV|csv) CRM_Profile.txt +CRM_My_Setup_Products_vod__c_[0-9]{14}\.(CSV|csv) CRM_My_Setup_Products_vod__c.txt +CRM_Multichannel_Activity_vod__c_[0-9]{14}\.(CSV|csv) CRM_Multichannel_Activity_vod__c.txt +CRM_Multichannel_Activity_Line_vod__c_[0-9]{14}\.(CSV|csv) CRM_Multichannel_Activity_Line_vod__c.txt +CRM_Multichannel_Consent_vod__c_[0-9]{14}\.(CSV|csv) CRM_Multichannel_Consent_vod__c.txt +CRM_Medical_Inquiry_vod__c_[0-9]{14}\.(CSV|csv) CRM_Medical_Inquiry_vod__c.txt +CRM_Email_Activity_vod__c_[0-9]{14}\.(CSV|csv) CRM_Email_Activity_vod__c.txt +CRM_User_[0-9]{14}\.(CSV|csv) CRM_User.txt +CRM_UserTerritory2Association_[0-9]{14}\.(CSV|csv) CRM_UserTerritory2Association.txt +CRM_Remote_Meeting_vod__c_[0-9]{14}\.(CSV|csv) CRM_Remote_Meeting_vod__c.txt +CRM_RecordType_[0-9]{14}\.(CSV|csv) CRM_RecordType.txt +CRM_UserRole_[0-9]{14}\.(CSV|csv) CRM_UserRole.txt +CRM_Account_[0-9]{14}\.(CSV|csv) CRM_Account.txt +CRM_AccountShare_[0-9]{14}\.(CSV|csv) CRM_AccountShare.txt +CRM_Contact_[0-9]{14}\.(CSV|csv) CRM_Contact.txt +CRM_Consent_Type_vod__c_[0-9]{14}\.(CSV|csv) CRM_Consent_Type_vod__c.txt +CRM_Consent_Header_vod__c_[0-9]{14}\.(CSV|csv) CRM_Consent_Header_vod__c.txt +CRM_Consent_Line_vod__c_[0-9]{14}\.(CSV|csv) CRM_Consent_Line_vod__c.txt +CRM_MSJ_Inquiry_Assignment__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Inquiry_Assignment__c.txt +CRM_Approved_Document_vod__c_[0-9]{14}\.(CSV|csv) CRM_Approved_Document_vod__c.txt +CRM_Child_Account_vod__c_[0-9]{14}\.(CSV|csv) CRM_Child_Account_vod__c.txt +CRM_MSJ_Hospital_Medical_Regimen__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Hospital_Medical_Regimen__c.txt +CRM_MSJ_Medical_Regimen__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Medical_Regimen__c.txt +CRM_MSJ_Patient__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Patient__c.txt +CRM_Product_vod__c_[0-9]{14}\.(CSV|csv) CRM_Product_vod__c.txt +CRM_Product_Group_vod__c_[0-9]{14}\.(CSV|csv) CRM_Product_Group_vod__c.txt +CRM_Product_Metrics_vod__c_[0-9]{14}\.(CSV|csv) CRM_Product_Metrics_vod__c.txt +CRM_Survey_vod__c_[0-9]{14}\.(CSV|csv) CRM_Survey_vod__c.txt +CRM_Survey_Target_vod__c_[0-9]{14}\.(CSV|csv) CRM_Survey_Target_vod__c.txt +CRM_Survey_Question_vod__c_[0-9]{14}\.(CSV|csv) CRM_Survey_Question_vod__c.txt +CRM_Question_Response_vod__c_[0-9]{14}\.(CSV|csv) CRM_Question_Response_vod__c.txt +CRM_Sent_Fragment_vod__c_[0-9]{14}\.(CSV|csv) CRM_Sent_Fragment_vod__c.txt +CRM_Sent_Email_vod__c_[0-9]{14}\.(CSV|csv) CRM_Sent_Email_vod__c.txt +/* 【CRMデータ 全件連携】 */ +CRM_Territory2_ALL_[0-9]{14}\.(CSV|csv) CRM_Territory2_ALL.txt +CRM_UserTerritory2Association_ALL_[0-9]{14}\.(CSV|csv) CRM_UserTerritory2Association_ALL.txt From 78ac0ba11411d607f3566d778d962cf0cdca93a9 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 1 Aug 2022 20:02:30 +0900 Subject: [PATCH 071/275] =?UTF-8?q?feat:=20S3=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E3=80=81=E9=9B=9B=E5=BD=A2?= =?UTF-8?q?=E3=82=92=E4=BD=9C=E3=81=A3=E3=81=A6=E3=81=BF=E3=81=9F=E3=80=82?= =?UTF-8?q?=E5=8B=95=E4=BD=9C=E3=81=97=E3=81=A6=E3=81=84=E3=82=8B=E3=81=93?= =?UTF-8?q?=E3=81=A8=E3=81=AF=E7=A2=BA=E8=AA=8D=E3=81=A7=E3=81=8D=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + ecs/crm-datafetch/Pipfile | 7 + ecs/crm-datafetch/Pipfile.lock | 471 ++++++++++++++++++++++-- ecs/crm-datafetch/tests/__init__.py | 0 ecs/crm-datafetch/tests/aws/__init__.py | 0 ecs/crm-datafetch/tests/aws/test_s3.py | 20 + ecs/crm-datafetch/tests/conftest.py | 21 ++ 7 files changed, 488 insertions(+), 32 deletions(-) create mode 100644 ecs/crm-datafetch/tests/__init__.py create mode 100644 ecs/crm-datafetch/tests/aws/__init__.py create mode 100644 ecs/crm-datafetch/tests/aws/test_s3.py create mode 100644 ecs/crm-datafetch/tests/conftest.py diff --git a/.gitignore b/.gitignore index 0e397fc3..88a052ff 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* __pycache__/ .env **/.vscode/settings.json +.coverage diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 33501d98..3d7988e7 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -3,6 +3,10 @@ url = "https://pypi.org/simple" verify_ssl = true name = "pypi" +[scripts] +test = "pytest tests/" +"test:cov" = "pytest --cov=src tests/" + [packages] boto3 = "*" simple-salesforce = "*" @@ -11,6 +15,9 @@ tenacity = "*" [dev-packages] autopep8 = "*" flake8 = "*" +pytest = "*" +pytest-cov = "*" +moto = "*" [requires] python_version = "3.8" diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index c8c60540..7c13f0c9 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ec1d83143aff859500979be73f67196dcfe2298ad3553a7d81ad0605a277d672" + "sha256": "f1433a55f486f24bb14d6447713a0cdeeb704542a695103debd9514face495cc" }, "pipfile-spec": 6, "requires": { @@ -18,11 +18,11 @@ "default": { "attrs": { "hashes": [ - "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", - "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6", + "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==21.4.0" + "markers": "python_version >= '3.5'", + "version": "==22.1.0" }, "authlib": { "hashes": [ @@ -33,19 +33,19 @@ }, "boto3": { "hashes": [ - "sha256:5c775dcb12ca5d6be3f5aa3c49d77783faa64eb30fd3f4af93ff116bb42f9ffb", - "sha256:5d9bcc355cf6edd7f3849fedac4252e12a0aa2b436cdbc0d4371b16a0f852a30" + "sha256:aec404d06690a0ca806592efc6bbe30a9ac88fa2ad73b6d907f7794a8b3b1459", + "sha256:df3d6ef02304bd7c3711090936c092d5db735dda109decac1e236c3ef7fdb7af" ], "index": "pypi", - "version": "==1.24.34" + "version": "==1.24.42" }, "botocore": { "hashes": [ - "sha256:0d824a5315f5f5c3bea53c14107a69695ef43190edf647f1281bac8f172ca77c", - "sha256:9c695d47f1f1212f3e306e51f7bacdf67e58055194ddcf7d8296660b124cf135" + "sha256:38a180a6666c5a9b069a75ec3cf374ff2a64c3e90c9f24a916858bcdeb04456d", + "sha256:f8e6c2f69a9d577fb9c69e4e74f49f4315a48decee0e7dc88b6e470772110884" ], "markers": "python_version >= '3.7'", - "version": "==1.27.34" + "version": "==1.27.42" }, "cached-property": { "hashes": [ @@ -59,7 +59,7 @@ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2022.6.15" }, "cffi": { @@ -136,7 +136,7 @@ "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.1.0" }, "cryptography": { @@ -164,7 +164,7 @@ "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==37.0.4" }, "idna": { @@ -352,22 +352,30 @@ }, "urllib3": { "hashes": [ - "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", - "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" + "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc", + "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", - "version": "==1.26.10" + "version": "==1.26.11" }, "zeep": { "hashes": [ "sha256:5867f2eadd6b028d9751f4155af590d3aaf9280e3a0ed5e15a53343921c956e5", "sha256:81c491092b71f5b276de8c63dfd452be3f322622c48a54f3a497cf913bdfb2f4" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==4.1.0" } }, "develop": { + "attrs": { + "hashes": [ + "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6", + "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c" + ], + "markers": "python_version >= '3.5'", + "version": "==22.1.0" + }, "autopep8": { "hashes": [ "sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979", @@ -376,36 +384,403 @@ "index": "pypi", "version": "==1.6.0" }, - "flake8": { + "boto3": { "hashes": [ - "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", - "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" + "sha256:aec404d06690a0ca806592efc6bbe30a9ac88fa2ad73b6d907f7794a8b3b1459", + "sha256:df3d6ef02304bd7c3711090936c092d5db735dda109decac1e236c3ef7fdb7af" ], "index": "pypi", - "version": "==4.0.1" + "version": "==1.24.42" + }, + "botocore": { + "hashes": [ + "sha256:38a180a6666c5a9b069a75ec3cf374ff2a64c3e90c9f24a916858bcdeb04456d", + "sha256:f8e6c2f69a9d577fb9c69e4e74f49f4315a48decee0e7dc88b6e470772110884" + ], + "markers": "python_version >= '3.7'", + "version": "==1.27.42" + }, + "certifi": { + "hashes": [ + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==2022.6.15" + }, + "cffi": { + "hashes": [ + "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", + "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", + "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", + "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", + "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", + "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", + "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", + "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", + "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", + "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", + "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", + "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", + "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", + "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", + "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", + "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", + "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", + "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", + "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", + "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", + "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", + "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", + "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", + "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", + "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", + "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", + "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", + "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", + "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", + "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", + "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", + "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", + "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", + "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", + "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", + "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", + "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", + "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", + "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", + "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", + "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", + "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", + "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", + "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", + "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", + "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", + "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", + "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", + "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", + "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", + "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", + "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", + "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", + "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", + "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", + "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", + "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", + "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", + "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", + "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", + "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", + "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", + "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", + "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" + ], + "version": "==1.15.1" + }, + "charset-normalizer": { + "hashes": [ + "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", + "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==2.1.0" + }, + "coverage": { + "extras": [ + "toml" + ], + "hashes": [ + "sha256:0895ea6e6f7f9939166cc835df8fa4599e2d9b759b02d1521b574e13b859ac32", + "sha256:0f211df2cba951ffcae210ee00e54921ab42e2b64e0bf2c0befc977377fb09b7", + "sha256:147605e1702d996279bb3cc3b164f408698850011210d133a2cb96a73a2f7996", + "sha256:24b04d305ea172ccb21bee5bacd559383cba2c6fcdef85b7701cf2de4188aa55", + "sha256:25b7ec944f114f70803d6529394b64f8749e93cbfac0fe6c5ea1b7e6c14e8a46", + "sha256:2b20286c2b726f94e766e86a3fddb7b7e37af5d0c635bdfa7e4399bc523563de", + "sha256:2dff52b3e7f76ada36f82124703f4953186d9029d00d6287f17c68a75e2e6039", + "sha256:2f8553878a24b00d5ab04b7a92a2af50409247ca5c4b7a2bf4eabe94ed20d3ee", + "sha256:3def6791adf580d66f025223078dc84c64696a26f174131059ce8e91452584e1", + "sha256:422fa44070b42fef9fb8dabd5af03861708cdd6deb69463adc2130b7bf81332f", + "sha256:4f89d8e03c8a3757aae65570d14033e8edf192ee9298303db15955cadcff0c63", + "sha256:5336e0352c0b12c7e72727d50ff02557005f79a0b8dcad9219c7c4940a930083", + "sha256:54d8d0e073a7f238f0666d3c7c0d37469b2aa43311e4024c925ee14f5d5a1cbe", + "sha256:5ef42e1db047ca42827a85e34abe973971c635f83aed49611b7f3ab49d0130f0", + "sha256:5f65e5d3ff2d895dab76b1faca4586b970a99b5d4b24e9aafffc0ce94a6022d6", + "sha256:6c3ccfe89c36f3e5b9837b9ee507472310164f352c9fe332120b764c9d60adbe", + "sha256:6d0b48aff8e9720bdec315d67723f0babd936a7211dc5df453ddf76f89c59933", + "sha256:6fe75dcfcb889b6800f072f2af5a331342d63d0c1b3d2bf0f7b4f6c353e8c9c0", + "sha256:79419370d6a637cb18553ecb25228893966bd7935a9120fa454e7076f13b627c", + "sha256:7bb00521ab4f99fdce2d5c05a91bddc0280f0afaee0e0a00425e28e209d4af07", + "sha256:80db4a47a199c4563d4a25919ff29c97c87569130375beca3483b41ad5f698e8", + "sha256:866ebf42b4c5dbafd64455b0a1cd5aa7b4837a894809413b930026c91e18090b", + "sha256:8af6c26ba8df6338e57bedbf916d76bdae6308e57fc8f14397f03b5da8622b4e", + "sha256:a13772c19619118903d65a91f1d5fea84be494d12fd406d06c849b00d31bf120", + "sha256:a697977157adc052284a7160569b36a8bbec09db3c3220642e6323b47cec090f", + "sha256:a9032f9b7d38bdf882ac9f66ebde3afb8145f0d4c24b2e600bc4c6304aafb87e", + "sha256:b5e28db9199dd3833cc8a07fa6cf429a01227b5d429facb56eccd765050c26cd", + "sha256:c77943ef768276b61c96a3eb854eba55633c7a3fddf0a79f82805f232326d33f", + "sha256:d230d333b0be8042ac34808ad722eabba30036232e7a6fb3e317c49f61c93386", + "sha256:d4548be38a1c810d79e097a38107b6bf2ff42151900e47d49635be69943763d8", + "sha256:d4e7ced84a11c10160c0697a6cc0b214a5d7ab21dfec1cd46e89fbf77cc66fae", + "sha256:d56f105592188ce7a797b2bd94b4a8cb2e36d5d9b0d8a1d2060ff2a71e6b9bbc", + "sha256:d714af0bdba67739598849c9f18efdcc5a0412f4993914a0ec5ce0f1e864d783", + "sha256:d774d9e97007b018a651eadc1b3970ed20237395527e22cbeb743d8e73e0563d", + "sha256:e0524adb49c716ca763dbc1d27bedce36b14f33e6b8af6dba56886476b42957c", + "sha256:e2618cb2cf5a7cc8d698306e42ebcacd02fb7ef8cfc18485c59394152c70be97", + "sha256:e36750fbbc422c1c46c9d13b937ab437138b998fe74a635ec88989afb57a3978", + "sha256:edfdabe7aa4f97ed2b9dd5dde52d2bb29cb466993bb9d612ddd10d0085a683cf", + "sha256:f22325010d8824594820d6ce84fa830838f581a7fd86a9235f0d2ed6deb61e29", + "sha256:f23876b018dfa5d3e98e96f5644b109090f16a4acb22064e0f06933663005d39", + "sha256:f7bd0ffbcd03dc39490a1f40b2669cc414fae0c4e16b77bb26806a4d0b7d1452" + ], + "markers": "python_version >= '3.7'", + "version": "==6.4.2" + }, + "cryptography": { + "hashes": [ + "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", + "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596", + "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", + "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", + "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab", + "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", + "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", + "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b", + "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441", + "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa", + "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d", + "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", + "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a", + "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", + "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", + "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", + "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282", + "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67", + "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", + "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", + "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", + "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==37.0.4" + }, + "flake8": { + "hashes": [ + "sha256:44e3ecd719bba1cb2ae65d1b54212cc9df4f5db15ac271f8856e5e6c2eebefed", + "sha256:9c51d3d1426379fd444d3b79eabbeb887849368bd053039066439523d8486961" + ], + "index": "pypi", + "version": "==5.0.1" + }, + "idna": { + "hashes": [ + "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + ], + "markers": "python_version >= '3.5'", + "version": "==3.3" + }, + "iniconfig": { + "hashes": [ + "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", + "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" + ], + "version": "==1.1.1" + }, + "jinja2": { + "hashes": [ + "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", + "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" + ], + "markers": "python_version >= '3.7'", + "version": "==3.1.2" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "markupsafe": { + "hashes": [ + "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003", + "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88", + "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5", + "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7", + "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a", + "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603", + "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1", + "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135", + "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247", + "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6", + "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601", + "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77", + "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02", + "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e", + "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63", + "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f", + "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980", + "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b", + "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812", + "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff", + "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96", + "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1", + "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925", + "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a", + "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6", + "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e", + "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f", + "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4", + "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f", + "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3", + "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c", + "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a", + "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417", + "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a", + "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a", + "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37", + "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452", + "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933", + "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a", + "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.1" }, "mccabe": { "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" ], - "version": "==0.6.1" + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "moto": { + "hashes": [ + "sha256:8bb8e267d9b948509d4739d81d995615a193d2c459f5c0a979aaeb0d3bd4b381", + "sha256:cbe8ad8a949f519771e5d25b670738604757fb67cd474d75d14c20677582e81f" + ], + "index": "pypi", + "version": "==3.1.16" + }, + "packaging": { + "hashes": [ + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + ], + "markers": "python_version >= '3.6'", + "version": "==21.3" + }, + "pluggy": { + "hashes": [ + "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", + "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" + ], + "markers": "python_version >= '3.6'", + "version": "==1.0.0" + }, + "py": { + "hashes": [ + "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719", + "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.11.0" }, "pycodestyle": { "hashes": [ - "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", - "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" + "sha256:289cdc0969d589d90752582bef6dff57c5fbc6949ee8b013ad6d6449a8ae9437", + "sha256:beaba44501f89d785be791c9462553f06958a221d166c64e1f107320f839acc2" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==2.8.0" + "markers": "python_version >= '3.6'", + "version": "==2.9.0" + }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" }, "pyflakes": { "hashes": [ - "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", - "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" + "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2", + "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3" + ], + "markers": "python_version >= '3.6'", + "version": "==2.5.0" + }, + "pyparsing": { + "hashes": [ + "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", + "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" + ], + "markers": "python_full_version >= '3.6.8'", + "version": "==3.0.9" + }, + "pytest": { + "hashes": [ + "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c", + "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45" + ], + "index": "pypi", + "version": "==7.1.2" + }, + "pytest-cov": { + "hashes": [ + "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6", + "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470" + ], + "index": "pypi", + "version": "==3.0.0" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.4.0" + "version": "==2.8.2" + }, + "pytz": { + "hashes": [ + "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", + "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" + ], + "version": "==2022.1" + }, + "requests": { + "hashes": [ + "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" + ], + "markers": "python_version >= '3.7' and python_version < '4'", + "version": "==2.28.1" + }, + "responses": { + "hashes": [ + "sha256:2dcc863ba63963c0c3d9ee3fa9507cbe36b7d7b0fccb4f0bdfd9e96c539b1487", + "sha256:b82502eb5f09a0289d8e209e7bad71ef3978334f56d09b444253d5ad67bf5253" + ], + "markers": "python_version >= '3.7'", + "version": "==0.21.0" + }, + "s3transfer": { + "hashes": [ + "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", + "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" + ], + "markers": "python_version >= '3.7'", + "version": "==0.6.0" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" }, "toml": { "hashes": [ @@ -414,6 +789,38 @@ ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version >= '3.7'", + "version": "==2.0.1" + }, + "urllib3": { + "hashes": [ + "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc", + "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.11" + }, + "werkzeug": { + "hashes": [ + "sha256:4d7013ef96fd197d1cdeb03e066c6c5a491ccb44758a5b2b91137319383e5a5a", + "sha256:7e1db6a5ba6b9a8be061e47e900456355b8714c0f238b0313f53afce1a55a79a" + ], + "markers": "python_version >= '3.7'", + "version": "==2.2.1" + }, + "xmltodict": { + "hashes": [ + "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56", + "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852" + ], + "markers": "python_version >= '3.4'", + "version": "==0.13.0" } } } diff --git a/ecs/crm-datafetch/tests/__init__.py b/ecs/crm-datafetch/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/tests/aws/__init__.py b/ecs/crm-datafetch/tests/aws/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py new file mode 100644 index 00000000..61c3d352 --- /dev/null +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -0,0 +1,20 @@ +import pytest +from src.aws.s3 import S3Resource + + +@pytest.fixture +def bucket_name(): + return 'test-bucket' + + +@pytest.fixture +def s3_test(s3_client, bucket_name): + s3_client.create_bucket(Bucket=bucket_name) + yield + + +class TestS3Resource: + + def test_put_object(self, s3_test, s3_client, bucket_name): + s3_resource = S3Resource(bucket_name) + s3_resource.put_object('hogehoge', 'aaaaaaaaaaaaaaa') diff --git a/ecs/crm-datafetch/tests/conftest.py b/ecs/crm-datafetch/tests/conftest.py new file mode 100644 index 00000000..1120ced6 --- /dev/null +++ b/ecs/crm-datafetch/tests/conftest.py @@ -0,0 +1,21 @@ +import os + +import boto3 +import pytest +from moto import mock_s3 + + +@pytest.fixture +def aws_credentials(): + """Mocked AWS Credentials for moto.""" + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + + +@pytest.fixture +def s3_client(aws_credentials): + with mock_s3(): + conn = boto3.client("s3", region_name="us-east-1") + yield conn From deb44cd1e2e4cf968294eb9c421274a59c0941ba Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 1 Aug 2022 20:02:30 +0900 Subject: [PATCH 072/275] =?UTF-8?q?feat:=20S3=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/aws/test_s3.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index 61c3d352..f341cd2c 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -15,6 +15,28 @@ def s3_test(s3_client, bucket_name): class TestS3Resource: + def test_get_object(self, s3_test, s3_client, bucket_name): + s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') + s3_resource = S3Resource(bucket_name) + actual = s3_resource.get_object('hogehoge/test.txt') + assert actual == 'aaaaaaaaaaaaaaa' + def test_put_object(self, s3_test, s3_client, bucket_name): s3_resource = S3Resource(bucket_name) - s3_resource.put_object('hogehoge', 'aaaaaaaaaaaaaaa') + s3_resource.put_object('hogehoge/test.txt', 'aaaaaaaaaaaaaaa') + actual = s3_client.get_object(Bucket=bucket_name, Key='hogehoge/test.txt') + assert actual['Body'].read() == b'aaaaaaaaaaaaaaa' + + def test_copy(self, s3_test, s3_client, bucket_name): + for_copy_bucket = 'for-copy-bucket' + s3_client.create_bucket(Bucket=for_copy_bucket) + s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') + s3_resource = S3Resource(bucket_name) + s3_resource.copy(bucket_name, 'hogehoge/test.txt', for_copy_bucket, 'test.txt') + actual = s3_client.get_object(Bucket=for_copy_bucket, Key='test.txt') + assert actual['Body'].read() == b'aaaaaaaaaaaaaaa' + + def test_init_raise_no_provide_bucket_name(self): + with pytest.raises(Exception) as e: + S3Resource() + assert e.value.args[0] == "__init__() missing 1 required positional argument: 'bucket_name'" From 6c31ff800722957bd0db7de48d4a2d26bdd1bfc3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 1 Aug 2022 20:02:30 +0900 Subject: [PATCH 073/275] =?UTF-8?q?feat:=20S3=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E8=BF=BD=E5=8A=A0=20Config=E3=83=90=E3=82=B1=E3=83=83?= =?UTF-8?q?=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/aws/test_s3.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index f341cd2c..db4f1c09 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -1,5 +1,7 @@ import pytest -from src.aws.s3 import S3Resource +from src.aws.s3 import ConfigBucket, S3Resource +from src.system_var import environments +from src.system_var.constants import OBJECT_INFO_FILENAME, OBJECT_INFO_FOLDER @pytest.fixture @@ -40,3 +42,18 @@ class TestS3Resource: with pytest.raises(Exception) as e: S3Resource() assert e.value.args[0] == "__init__() missing 1 required positional argument: 'bucket_name'" + + +class TestConfigBucket: + + def test_get_object_info_file(self, s3_test, s3_client, bucket_name, monkeypatch): + # = bucket_name + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', 'crm') + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'objects.json') + s3_client.put_object(Bucket=bucket_name, Key=f'crm/objects.json', Body=b'aaaaaaaaaaaaaaa') + config_bucket = ConfigBucket() + print('*' * 50, str(config_bucket), '*' * 50) + actual = config_bucket.get_object_info_file() + print(actual) + assert actual == 'aaaaaaaaaaaaaaa' From e3421996ec6be7500ba42dfc46bab3a4c4a80122 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 1 Aug 2022 20:02:30 +0900 Subject: [PATCH 074/275] =?UTF-8?q?feat:=20=E3=82=B3=E3=83=B3=E3=83=95?= =?UTF-8?q?=E3=82=A3=E3=82=B0=E3=83=90=E3=82=B1=E3=83=83=E3=83=88=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/aws/test_s3.py | 29 ++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index db4f1c09..db97825a 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -1,7 +1,5 @@ import pytest from src.aws.s3 import ConfigBucket, S3Resource -from src.system_var import environments -from src.system_var.constants import OBJECT_INFO_FILENAME, OBJECT_INFO_FOLDER @pytest.fixture @@ -27,7 +25,7 @@ class TestS3Resource: s3_resource = S3Resource(bucket_name) s3_resource.put_object('hogehoge/test.txt', 'aaaaaaaaaaaaaaa') actual = s3_client.get_object(Bucket=bucket_name, Key='hogehoge/test.txt') - assert actual['Body'].read() == b'aaaaaaaaaaaaaaa' + assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' def test_copy(self, s3_test, s3_client, bucket_name): for_copy_bucket = 'for-copy-bucket' @@ -47,7 +45,6 @@ class TestS3Resource: class TestConfigBucket: def test_get_object_info_file(self, s3_test, s3_client, bucket_name, monkeypatch): - # = bucket_name monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', 'crm') monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'objects.json') @@ -57,3 +54,27 @@ class TestConfigBucket: actual = config_bucket.get_object_info_file() print(actual) assert actual == 'aaaaaaaaaaaaaaa' + + def test_get_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') + s3_client.put_object(Bucket=bucket_name, Key=f'crm/Object.json', Body=b'aaaaaaaaaaaaaaa') + config_bucket = ConfigBucket() + print('*' * 50, str(config_bucket), '*' * 50) + actual = config_bucket.get_last_fetch_datetime_file('Object.json') + print(actual) + assert actual == 'aaaaaaaaaaaaaaa' + + def test_put_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') + config_bucket = ConfigBucket() + print('*' * 50, str(config_bucket), '*' * 50) + config_bucket.put_last_fetch_datetime_file('Object.json', 'aaaaaaaaaaaaaaa') + actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/Object.json') + assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' + + def test_config_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + config_bucket = ConfigBucket() + assert str(config_bucket) == bucket_name From 75187e54e9b13b1f54934ed9a8ab8049c4f9218d Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 2 Aug 2022 09:55:30 +0900 Subject: [PATCH 075/275] =?UTF-8?q?fix:=20=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E5=90=8D=E3=81=AE=E6=97=A5=E4=BB=98=E3=83=95=E3=82=A9?= =?UTF-8?q?=E3=83=BC=E3=83=9E=E3=83=83=E3=83=88=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/config/objects.py | 7 ++++--- ecs/crm-datafetch/src/util/execute_datetime.py | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 3368a1db..65a7bc9a 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -15,6 +15,7 @@ from src.system_var.constants import (COLUMNS_KEY, COLUMNS_TYPE, UPLOAD_FILE_NAME_KEY, UPLOAD_FILE_NAME_TYPE) from src.util.dict_checker import DictChecker +from src.util.execute_datetime import ExecuteDateTime class FetchTargetObjects(): @@ -40,7 +41,7 @@ class FetchTargetObjects(): class TargetObject(): - def __init__(self, object_info, execute_datetime) -> None: + def __init__(self, object_info, execute_datetime: ExecuteDateTime) -> None: self.__dict_checker = DictChecker(object_info) self.__object_info = object_info self.__execute_datetime = execute_datetime @@ -105,8 +106,8 @@ class TargetObject(): @property def upload_file_name(self) -> str: if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): - return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) - return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' + return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime.to_YYYYMMDDHHMMSS) + return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.to_YYYYMMDDHHMMSS}' @property def datetime_column(self) -> str: diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py index bdf7f053..e87ef861 100644 --- a/ecs/crm-datafetch/src/util/execute_datetime.py +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -12,3 +12,6 @@ class ExecuteDateTime: def to_path(self) -> str: return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) + + def to_YYYYMMDDHHMMSS(self) -> str: + return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': None, 'T': None, ':': None, '.': None})) \ No newline at end of file From 223e97241f0738d975d8c586bd5f3d0140d4a66c Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 11:55:53 +0900 Subject: [PATCH 076/275] =?UTF-8?q?feat:=20=E3=83=89=E3=82=AD=E3=83=A5?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=81=8B=E3=82=89=E3=83=AC=E3=83=9D=E3=83=BC=E3=83=88=E3=82=92?= =?UTF-8?q?=E7=94=9F=E6=88=90=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 + .../.vscode/python.code-snippets | 17 ++++ ecs/crm-datafetch/Pipfile | 2 + ecs/crm-datafetch/Pipfile.lock | 32 +++++-- ecs/crm-datafetch/tests/aws/test_s3.py | 89 +++++++++++++++++-- ecs/crm-datafetch/tests/conftest.py | 41 +++++++++ ecs/crm-datafetch/tests/docstring_parser.py | 32 +++++++ 7 files changed, 203 insertions(+), 13 deletions(-) create mode 100644 ecs/crm-datafetch/.vscode/python.code-snippets create mode 100644 ecs/crm-datafetch/tests/docstring_parser.py diff --git a/.gitignore b/.gitignore index 88a052ff..16a48619 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* __pycache__/ .env **/.vscode/settings.json + +# python test .coverage +.report/ \ No newline at end of file diff --git a/ecs/crm-datafetch/.vscode/python.code-snippets b/ecs/crm-datafetch/.vscode/python.code-snippets new file mode 100644 index 00000000..9f01c623 --- /dev/null +++ b/ecs/crm-datafetch/.vscode/python.code-snippets @@ -0,0 +1,17 @@ +{ + "Generate Test docstring": { + "scope": "python", + "prefix": "\"\"\"\"\"\"", + "body": [ + "\"\"\"", + "Tests:", + " $1", + "Arranges:", + " $2", + "Expects:", + " $3", + "\"\"\"" + ], + "description": "Test docstring (User Snipets)" + } +} diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 3d7988e7..990beb4b 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -6,6 +6,7 @@ name = "pypi" [scripts] test = "pytest tests/" "test:cov" = "pytest --cov=src tests/" +"test:report" = "pytest --cov=src --html=.report/test_result.html tests/" [packages] boto3 = "*" @@ -17,6 +18,7 @@ autopep8 = "*" flake8 = "*" pytest = "*" pytest-cov = "*" +pytest-html = "*" moto = "*" [requires] diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index 7c13f0c9..cfd16ed2 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f1433a55f486f24bb14d6447713a0cdeeb704542a695103debd9514face495cc" + "sha256": "7006de596d6123ecd56760b584ab75430fa6bcfc0ecd3fdf49f08368ff53477d" }, "pipfile-spec": 6, "requires": { @@ -59,7 +59,7 @@ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2022.6.15" }, "cffi": { @@ -136,7 +136,7 @@ "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2.1.0" }, "cryptography": { @@ -164,7 +164,7 @@ "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==37.0.4" }, "idna": { @@ -363,7 +363,7 @@ "sha256:5867f2eadd6b028d9751f4155af590d3aaf9280e3a0ed5e15a53343921c956e5", "sha256:81c491092b71f5b276de8c63dfd452be3f322622c48a54f3a497cf913bdfb2f4" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==4.1.0" } }, @@ -405,7 +405,7 @@ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2022.6.15" }, "cffi": { @@ -482,7 +482,7 @@ "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2.1.0" }, "coverage": { @@ -560,7 +560,7 @@ "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==37.0.4" }, "flake8": { @@ -735,6 +735,22 @@ "index": "pypi", "version": "==3.0.0" }, + "pytest-html": { + "hashes": [ + "sha256:3ee1cf319c913d19fe53aeb0bc400e7b0bc2dbeb477553733db1dad12eb75ee3", + "sha256:b7f82f123936a3f4d2950bc993c2c1ca09ce262c9ae12f9ac763a2401380b455" + ], + "index": "pypi", + "version": "==3.1.1" + }, + "pytest-metadata": { + "hashes": [ + "sha256:39261ee0086f17649b180baf2a8633e1922a4c4b6fcc28a2de7d8127a82541bf", + "sha256:fcd2f416f15be295943527b3c8ba16a44ae5a7141939c90c3dc5ce9d167cf2a5" + ], + "markers": "python_version >= '3.7' and python_version < '4'", + "version": "==2.0.2" + }, "python-dateutil": { "hashes": [ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index db97825a..b6016b35 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -16,27 +16,71 @@ def s3_test(s3_client, bucket_name): class TestS3Resource: def test_get_object(self, s3_test, s3_client, bucket_name): + """ + Cases: + S3からオブジェクトが取得できるか + Arranges: + - S3をモック化する + - 期待値となるファイルを配置する + Expects: + オブジェクトが取得でき、期待値と正しいこと + """ + # Arrange s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') + + # ActAssert s3_resource = S3Resource(bucket_name) actual = s3_resource.get_object('hogehoge/test.txt') + + # Assert assert actual == 'aaaaaaaaaaaaaaa' def test_put_object(self, s3_test, s3_client, bucket_name): + """ + Cases: + S3にオブジェクトをPUTできるか + Arranges: + - S3をモック化する + Expects: + オブジェクトがPUTできること + """ s3_resource = S3Resource(bucket_name) + s3_resource.put_object('hogehoge/test.txt', 'aaaaaaaaaaaaaaa') actual = s3_client.get_object(Bucket=bucket_name, Key='hogehoge/test.txt') + assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' def test_copy(self, s3_test, s3_client, bucket_name): + """ + Cases: + S3内のオブジェクトを別バケットにコピーできるか + Arranges: + - S3をモック化する + - 期待値となるファイルをコピー元バケットに配置する + Expects: + - コピーされたファイルが存在する + - コピーされたファイルの内容が期待値と一致する + """ for_copy_bucket = 'for-copy-bucket' s3_client.create_bucket(Bucket=for_copy_bucket) s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') + s3_resource = S3Resource(bucket_name) s3_resource.copy(bucket_name, 'hogehoge/test.txt', for_copy_bucket, 'test.txt') + actual = s3_client.get_object(Bucket=for_copy_bucket, Key='test.txt') assert actual['Body'].read() == b'aaaaaaaaaaaaaaa' def test_init_raise_no_provide_bucket_name(self): + """ + Cases: + バケット名を指定しない場合、例外となること + Arranges: + + Expects: + 例外が発生すること + """ with pytest.raises(Exception) as e: S3Resource() assert e.value.args[0] == "__init__() missing 1 required positional argument: 'bucket_name'" @@ -45,36 +89,71 @@ class TestS3Resource: class TestConfigBucket: def test_get_object_info_file(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + オブジェクト情報ファイルが取得できること + Arranges: + オブジェクト情報ファイルを配置する + Expects: + オブジェクト情報ファイルが文字列として取得でき、期待値と一致する + """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', 'crm') monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'objects.json') s3_client.put_object(Bucket=bucket_name, Key=f'crm/objects.json', Body=b'aaaaaaaaaaaaaaa') + config_bucket = ConfigBucket() - print('*' * 50, str(config_bucket), '*' * 50) actual = config_bucket.get_object_info_file() - print(actual) + assert actual == 'aaaaaaaaaaaaaaa' def test_get_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + オブジェクト最終更新日時ファイルが取得できること + Arranges: + オブジェクト最終更新日時ファイルを配置する + Expects: + オブジェクト最終更新日時ファイルが文字列として取得でき、期待値と一致する + """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') s3_client.put_object(Bucket=bucket_name, Key=f'crm/Object.json', Body=b'aaaaaaaaaaaaaaa') + config_bucket = ConfigBucket() - print('*' * 50, str(config_bucket), '*' * 50) actual = config_bucket.get_last_fetch_datetime_file('Object.json') - print(actual) + assert actual == 'aaaaaaaaaaaaaaa' def test_put_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + オブジェクト最終更新日時ファイルをPUTできること + Arranges: + + Expects: + オブジェクト最終更新日時ファイルが存在する + """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') + config_bucket = ConfigBucket() - print('*' * 50, str(config_bucket), '*' * 50) config_bucket.put_last_fetch_datetime_file('Object.json', 'aaaaaaaaaaaaaaa') + actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/Object.json') assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' def test_config_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + 設定ファイル配置バケットを文字列化したとき、バケット名が取得できること + Arranges: + + Expects: + 環境変数の設定ファイル配置バケット名と一致する + """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + config_bucket = ConfigBucket() + assert str(config_bucket) == bucket_name diff --git a/ecs/crm-datafetch/tests/conftest.py b/ecs/crm-datafetch/tests/conftest.py index 1120ced6..e46746bb 100644 --- a/ecs/crm-datafetch/tests/conftest.py +++ b/ecs/crm-datafetch/tests/conftest.py @@ -1,8 +1,12 @@ import os +from datetime import datetime import boto3 import pytest from moto import mock_s3 +from py.xml import html # type: ignore + +from . import docstring_parser @pytest.fixture @@ -19,3 +23,40 @@ def s3_client(aws_credentials): with mock_s3(): conn = boto3.client("s3", region_name="us-east-1") yield conn + + +# 以下、レポート出力用の設定 + +def pytest_html_report_title(report): + # レポートタイトル + report.title = "CRMデータ連携 CRMデータ取得機能 単体機能テスト結果報告書" + + +# # def pytest_configure(config): +# # config._metadata["結果確認者"] = "" # Version情報を追加 + + +def pytest_html_results_table_header(cells): + del cells[2:] + cells.insert(3, html.th("Cases")) + cells.insert(4, html.th("Arranges")) + cells.insert(5, html.th("Expects")) + cells.append(html.th("Time", class_="sortable time", col="time")) + + +def pytest_html_results_table_row(report, cells): + del cells[2:] + cells.insert(3, html.td(html.pre(report.cases))) # 「テスト内容」をレポートに出力 + cells.insert(4, html.td(html.pre(report.arranges))) # 「期待結果」をレポートに出力 + cells.insert(5, html.td(html.pre(report.expects))) # 「期待結果」をレポートに出力 + cells.append(html.td(datetime.now(), class_="col-time")) # ついでに「時間」もレポートに出力 + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + docstring = docstring_parser.parse(str(item.function.__doc__)) + report.cases = docstring.get("Cases", '') # 「テスト内容」を`report`に追加 + report.arranges = docstring.get("Arranges", '') # 「準備作業」を`report`に追加 + report.expects = docstring.get("Expects", '') # 「期待結果」を`report`に追加 diff --git a/ecs/crm-datafetch/tests/docstring_parser.py b/ecs/crm-datafetch/tests/docstring_parser.py new file mode 100644 index 00000000..32ac1579 --- /dev/null +++ b/ecs/crm-datafetch/tests/docstring_parser.py @@ -0,0 +1,32 @@ +import re +from itertools import takewhile + +_section_rgx = re.compile(r"^\s*[a-zA-Z]+:\s*$") +_lspace_rgx = re.compile(r"^\s*") + + +def _parse_section(lines: list) -> list: + matches = map(lambda x: _section_rgx.match(x), lines) + indexes = [i for i, x in enumerate(matches) if x is not None] + return list(map(lambda x: (x, lines[x].strip()[: -1]), indexes)) + + +def _count_lspace(s: str) -> int: + rgx = _lspace_rgx.match(s) + if rgx is not None: + return rgx.end() + return 0 + + +def _parse_content(index: int, lines: list) -> str: + lspace = _count_lspace(lines[index]) + i = index + 1 + contents = takewhile(lambda x: _count_lspace(x) > lspace, lines[i:]) + return "\n".join(map(lambda x: x.strip(), contents)) + + +def parse(docstring: str) -> dict: + """🚧sloppy docstring parser🚧""" + lines = docstring.splitlines() + sections = _parse_section(lines) + return dict(map(lambda x: (x[1], _parse_content(x[0], lines)), sections)) From 070763ec8e4393f0821528623c2efeb5511804ad Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 2 Aug 2022 15:10:14 +0900 Subject: [PATCH 077/275] =?UTF-8?q?fix:=20=E3=83=AD=E3=82=B0=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=80=81=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E5=90=8D=E3=81=AE=E6=97=A5=E4=BB=98=E5=8F=96=E5=BE=97=E3=81=AE?= =?UTF-8?q?=E4=B8=8D=E5=85=B7=E5=90=88=E4=BF=AE=E6=AD=A3=E3=80=81=E3=82=BF?= =?UTF-8?q?=E3=82=A4=E3=83=A0=E3=82=BE=E3=83=BC=E3=83=B3=E5=88=A9=E7=94=A8?= =?UTF-8?q?=E5=87=A6=E7=90=86=E3=81=AE=E5=8F=96=E3=82=8A=E8=BE=BC=E3=81=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/config/objects.py | 4 ++-- ecs/crm-datafetch/src/controller.py | 4 ++-- ecs/crm-datafetch/src/converter/convert_strategy.py | 9 +++++---- ecs/crm-datafetch/src/fetch_crm_data_process.py | 4 +++- ecs/crm-datafetch/src/system_var/constants.py | 1 + ecs/crm-datafetch/src/system_var/environments.py | 2 ++ ecs/crm-datafetch/src/util/execute_datetime.py | 2 +- s3/config/crm/task_settings/task_settings.env | 3 ++- 8 files changed, 18 insertions(+), 11 deletions(-) diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 65a7bc9a..20673d70 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -106,8 +106,8 @@ class TargetObject(): @property def upload_file_name(self) -> str: if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): - return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime.to_YYYYMMDDHHMMSS) - return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.to_YYYYMMDDHHMMSS}' + return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime.to_YYYYMMDDHHMMSS()) + return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.to_YYYYMMDDHHMMSS()}' @property def datetime_column(self) -> str: diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index c531f113..3d0537b4 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -44,9 +44,9 @@ def controller() -> None: # ⑥ 最終結果をチェックし、チェック結果をログに出力 if not all([v == 'success' for v in process_result.values()]): - logger.error('E-CTRL-01 一部のデータ取得に失敗しています。詳細はログをご確認ください。') + logger.error('E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') else: - logger.info('I-CTRL-19 すべてのデータの取得に成功しました。') + logger.info('I-CTRL-19 すべてのデータの取得に成功しました') # ⑦ CRMデータ取得処理終了ログを出力する logger.info('I-CTRL-20 CRMデータ取得処理を終了します') diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 92500588..307fbb9a 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -1,8 +1,9 @@ import re -from datetime import datetime, timedelta, timezone +from datetime import datetime -from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, - CSV_TRUE_VALUE, +from dateutil.tz import gettz +from src.system_var.constants import (CONVERT_TZ, CRM_DATETIME_FORMAT, + CSV_FALSE_VALUE, CSV_TRUE_VALUE, DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, YYYYMMDDHHMMSS) @@ -48,7 +49,7 @@ class BooleanConvertStrategy: class DatetimeConvertStrategy: def convert_value(self, convert_value: str) -> str: # データ登録処理がJSTとして登録するため、変換処理内で事前にJSTの日次文字列に変換する - return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(timezone(timedelta(hours=+9))).strftime(YYYYMMDDHHMMSS) + return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS) class FloatConvertStrategy: diff --git a/ecs/crm-datafetch/src/fetch_crm_data_process.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py index 1c37f860..8c66bf59 100644 --- a/ecs/crm-datafetch/src/fetch_crm_data_process.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -67,12 +67,14 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las crm_data_response = fetch_sf_data_retry(fetch_soql, target_object_name, data_counter) + logger.info(f'I-FETCH-05 [{target_object_name}] のレコード取得が成功しました') + except Exception as e: raise SalesforceAPIException( 'E-FETCH-02', FETCH_JP_NAME, f'[{target_object_name}] のレコード取得に失敗しました エラー内容：[{e}]') # ④ CRMデータ取得処理終了ログを出力する - logger.info(f'I-FETCH-05 [{target_object_name}] のレコード取得が成功しました') + logger.info(f'I-FETCH-06 [{target_object_name}] のCRMからのデータ取得処理を終了します') # ⑤ 次の処理へ移行する return crm_data_response diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index 56f49681..bf65010c 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -15,6 +15,7 @@ CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = 'CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT' CRM_FETCH_RECORD_RETRY_INTERVAL = 'CRM_FETCH_RECORD_RETRY_INTERVAL' # CRMのレコード取得処理のリトライ時の初回待ち秒数 CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MIN_INTERVAL' # CRMのレコード取得処理のリトライ時の最小待ち秒数 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MAX_INTERVAL' # CRMのレコード取得処理のリトライ時の最大待ち秒数 +CONVERT_TZ = 'CONVERT_TZ' # CRMデータの日時を変換するときのタイムゾーン # environments(ECS Task Environment) CRM_AUTH_DOMAIN = 'CRM_AUTH_DOMAIN' # CRMのAPI実行のための認証エンドポイントのドメイン diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index 7dc62c1b..b290cc7b 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -35,6 +35,8 @@ CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_ CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最大待ち秒数 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50)) +# CRMデータの日時を変換するときのタイムゾーン +CONVERT_TZ = os.environ.get(constants.CONVERT_TZ, 'Asia/Tokyo') # environments(ECS Task Environment) # CRMのAPI実行のための認証エンドポイントのドメイン diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py index e87ef861..82a451e2 100644 --- a/ecs/crm-datafetch/src/util/execute_datetime.py +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -14,4 +14,4 @@ class ExecuteDateTime: return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) def to_YYYYMMDDHHMMSS(self) -> str: - return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': None, 'T': None, ':': None, '.': None})) \ No newline at end of file + return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': None, 'T': None, ':': None, '.': None})) diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env index d48ff51a..1998012c 100644 --- a/s3/config/crm/task_settings/task_settings.env +++ b/s3/config/crm/task_settings/task_settings.env @@ -13,4 +13,5 @@ CRM_FETCH_RECORD_TIMEOUT=300 CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3 CRM_FETCH_RECORD_RETRY_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 \ No newline at end of file +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 +CONVERT_TZ='Asia/Tokyo' \ No newline at end of file From 96326b1a463163fbbbd79b07a6c3d17a3b09b86c Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 15:32:23 +0900 Subject: [PATCH 078/275] =?UTF-8?q?docs:=20README=E3=81=AB=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=81=AB=E3=81=A4=E3=81=84=E3=81=A6=E8=BF=BD?= =?UTF-8?q?=E8=A8=98=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.env.example | 17 +++++ .../.vscode/python.code-snippets | 2 +- ecs/crm-datafetch/README.md | 72 +++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 ecs/crm-datafetch/.env.example diff --git a/ecs/crm-datafetch/.env.example b/ecs/crm-datafetch/.env.example new file mode 100644 index 00000000..aa6a3d18 --- /dev/null +++ b/ecs/crm-datafetch/.env.example @@ -0,0 +1,17 @@ +CRM_AUTH_DOMAIN=test +CRM_USER_NAME=test +CRM_USER_PASSWORD=test +CRM_USER_SECURITY_TOKEN=test +CRM_CONFIG_BUCKET=test +CRM_BACKUP_BUCKET=test +IMPORT_DATA_BUCKET=test +OBJECT_INFO_FOLDER=test +OBJECT_INFO_FILENAME=test +PROCESS_RESULT_FOLDER=test +PROCESS_RESULT_FILENAME=test +LAST_FETCH_DATE_FOLDER=test +CRM_IMPORT_DATA_FOLDER=test +LAST_FETCH_DATE_BACKUP_FOLDER=test +RESPONSE_JSON_BACKUP_FOLDER=test +CRM_IMPORT_DATA_BACKUP_FOLDER=test +TZ=Asia/Tokyo diff --git a/ecs/crm-datafetch/.vscode/python.code-snippets b/ecs/crm-datafetch/.vscode/python.code-snippets index 9f01c623..aa7b3501 100644 --- a/ecs/crm-datafetch/.vscode/python.code-snippets +++ b/ecs/crm-datafetch/.vscode/python.code-snippets @@ -12,6 +12,6 @@ " $3", "\"\"\"" ], - "description": "Test docstring (User Snipets)" + "description": "Test docstring (User Snippets)" } } diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md index 5a23e966..f0188f5f 100644 --- a/ecs/crm-datafetch/README.md +++ b/ecs/crm-datafetch/README.md @@ -67,3 +67,75 @@ - 環境変数が必要な場合、直接設定するか、上記JSONの`"envFile"`に設定されたパスに`.env`ファイルを作成し、環境変数を入力する - キーボードの「F5」キーを押して起動する - デバッグモードで実行されるため、適当なところにブレークポイントを置いてデバッグすることができる + +## 単体テストについて + +### 前提 + +- Pytestを使用する + - +- カバレッジも取得したいため、pytest-covも使う + - +- レポートを出力するため、pytest-htmlを使う + - +- S3をモック化したいため、motoをつかう + - +- CRMはテスト用の環境を使いたいため、newdwh_opeのアドレスでDeveloper組織を登録する + +### テスト環境構築 + +- Pipenvの仮想環境下で、以下のコマンドを実行する + +```sh +pipenv install --dev +``` + +- `.env.example`をコピーし、同じ階層に`.env`を作成する +- `.env`の以下に示す環境変数の値をDeveloper組織のものに書き換える + - CRM_AUTH_DOMAIN + - CRM_USER_NAME + - CRM_USER_PASSWORD + - CRM_USER_SECURITY_TOKEN +- 以下のコマンドを実行して単体テストを起動する + +```sh +pipenv run test:cov +``` + +#### 各コマンドの説明 + +- `pipenv run test` + - pytestを使用してテストを実行する + - `tests`フォルダに配置されているテストモジュールを対象に、単体テストを実行する +- `pipenv run test:cov` + - pytestのテスト終了時にカバレッジを収集する + - 標準出力とカバレッジファイル(`.coverage`)に出力される +- `pipenv run test:report` + - pytestのテスト終了時にテスト結果をHTMLで出力する + - `.report/test_result.html`が出力される + +## 単体テストの追加方法 + +- `tests`フォルダ内に、`src`フォルダの構成と同じようにフォルダを作り、`test_<テスト対象のモジュール名.py>`というファイルを作成する + - 例：`src/aws/s3.py`をテストする場合は`tests/aws/test_s3.py`というファイル名にする +- テスト関数はクラスにまとめて、テストスイートとする +- テスト関数にはドキュメントコメントを付け、テスト観点・準備作業・期待値を記載すること + - 上記が出力されるスニペットを用意してある。 + - `""""""`と入力し、「Test docstring (User Snippets)」を選択し、ドキュメントコメントを挿入できる + +```python + +from src.aws.s3 import S3Resource +class TestS3Resource: + def test_get_object(self, s3_test, s3_): + """ + Cases: + S3からオブジェクトが取得できるか + Arranges: + - S3をモック化する + - 期待値となるファイルを配置する + Expects: + オブジェクトが取得でき、期待値と正しいこと + """ + # more code... +``` From f1f7cb6bbfb277980e4d75e38698fd2abf2dd1f1 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 2 Aug 2022 15:55:39 +0900 Subject: [PATCH 079/275] =?UTF-8?q?fix:=20=E9=96=A2=E6=95=B0=E5=90=8D?= =?UTF-8?q?=E3=81=AE=E4=BF=AE=E6=AD=A3=E3=80=81=E3=82=BF=E3=82=A4=E3=83=A0?= =?UTF-8?q?=E3=82=BE=E3=83=BC=E3=83=B3=E3=82=92=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E3=81=8B=E3=82=89=E8=AA=AD=E3=82=80=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/config/objects.py | 4 ++-- ecs/crm-datafetch/src/converter/convert_strategy.py | 5 +++-- ecs/crm-datafetch/src/util/execute_datetime.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 20673d70..41ca753b 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -106,8 +106,8 @@ class TargetObject(): @property def upload_file_name(self) -> str: if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): - return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime.to_YYYYMMDDHHMMSS()) - return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.to_YYYYMMDDHHMMSS()}' + return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime.format_date()) + return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.format_date()}' @property def datetime_column(self) -> str: diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 307fbb9a..89ad2a2e 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -2,10 +2,11 @@ import re from datetime import datetime from dateutil.tz import gettz -from src.system_var.constants import (CONVERT_TZ, CRM_DATETIME_FORMAT, - CSV_FALSE_VALUE, CSV_TRUE_VALUE, +from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, + CSV_TRUE_VALUE, DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, YYYYMMDDHHMMSS) +from src.system_var.environments import CONVERT_TZ class ConvertStrategyFactory: diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py index 82a451e2..5a608013 100644 --- a/ecs/crm-datafetch/src/util/execute_datetime.py +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -13,5 +13,5 @@ class ExecuteDateTime: def to_path(self) -> str: return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) - def to_YYYYMMDDHHMMSS(self) -> str: + def format_date(self) -> str: return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': None, 'T': None, ':': None, '.': None})) From d5abe856cf7944834019603d8bc04da25833c0d1 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:31 +0900 Subject: [PATCH 080/275] =?UTF-8?q?feat:crm=5Fhistory=E3=81=AE=E6=96=B0?= =?UTF-8?q?=E8=A6=8F=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 24 ++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 rds_mysql/stored_procedure/crm_history.sql diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql new file mode 100644 index 00000000..cf2ca1d8 --- /dev/null +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -0,0 +1,24 @@ +-- A5M2で実行時にSQL区切り文字を「;」以外にすること + +CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) +BEGIN + SET @new_history_save = 'UPDATE @@target_table SET start_datetime = @@target_column, end_datetime = "9999-12-31 00:00:00" WHERE start_datetime IS NULL AND end_datetime IS NULL'; + SET @new_history_save = REPLACE(@new_history_save, "@@target_table", target_table); + SET @new_history_save = REPLACE(@new_history_save, "@@target_column", target_column); + PREPARE new_history_save_stmt from @new_history_save; + EXECUTE new_history_save_stmt; + + SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id'; + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); + PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; + EXECUTE make_history_tmp_create_stmt; + + SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NULL'; + SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); + PREPARE update_end_datetime_stmt from @update_end_datetime; + EXECUTE update_end_datetime_stmt; + + DROP TEMPORARY TABLE make_history_tmp; +END \ No newline at end of file From 16d0b9803b1d95197799591e4c66b14ff696bcdd Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:31 +0900 Subject: [PATCH 081/275] =?UTF-8?q?refactor:=E3=82=B3=E3=83=A1=E3=83=B3?= =?UTF-8?q?=E3=83=88=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index cf2ca1d8..3247da51 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -2,23 +2,27 @@ CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN + -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = 'UPDATE @@target_table SET start_datetime = @@target_column, end_datetime = "9999-12-31 00:00:00" WHERE start_datetime IS NULL AND end_datetime IS NULL'; SET @new_history_save = REPLACE(@new_history_save, "@@target_table", target_table); SET @new_history_save = REPLACE(@new_history_save, "@@target_column", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; + -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id'; SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; EXECUTE make_history_tmp_create_stmt; + -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NULL'; SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; + -- ②-3 「②-1」で作成した一時テーブルを削除する DROP TEMPORARY TABLE make_history_tmp; END \ No newline at end of file From 0238c401dcca853acab9cea0c338ba63723f01b0 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:31 +0900 Subject: [PATCH 082/275] =?UTF-8?q?fix:=E4=B8=80=E6=99=82=E3=83=86?= =?UTF-8?q?=E3=83=BC=E3=83=96=E3=83=AB=E4=BD=9C=E6=88=90=E6=99=82=E3=81=AE?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E3=83=9F=E3=82=B9=E3=82=92=E4=BF=AE=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 3247da51..caeb09c0 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -10,7 +10,7 @@ BEGIN EXECUTE new_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 - SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id'; + SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id HAVING count(id) = 2'; SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; From 119731b20353700161b3423f98b39946990455db Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:31 +0900 Subject: [PATCH 083/275] =?UTF-8?q?fix:=20=E2=91=A1-2=E3=81=AE=E6=9D=A1?= =?UTF-8?q?=E4=BB=B6=E3=83=9F=E3=82=B9=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index caeb09c0..7c183714 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -17,7 +17,7 @@ BEGIN EXECUTE make_history_tmp_create_stmt; -- ②-2 「②-1」で取得した全件に更新処理を行う - SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NULL'; + SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NOT NULL'; SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; From 61dd2a4e0a199545d5126c7ce370d8ec47d7dc47 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:31 +0900 Subject: [PATCH 084/275] =?UTF-8?q?fix:=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E4=BF=AE=E6=AD=A3=E3=80=81=E4=BE=8B?= =?UTF-8?q?=E5=A4=96=E5=87=A6=E7=90=86=E3=81=AE=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 54 +++++++++++++++++++--- 1 file changed, 48 insertions(+), 6 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 7c183714..f9057a5e 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,28 +1,70 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること - CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN + -- 例外処理 + -- エラーが発生した場合に一時テーブルの削除を実施 + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + DROP TEMPORARY TABLE IF EXISTS make_history_tmp; + SIGNAL SQLSTATE '45000' + SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; + END; + -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する - SET @new_history_save = 'UPDATE @@target_table SET start_datetime = @@target_column, end_datetime = "9999-12-31 00:00:00" WHERE start_datetime IS NULL AND end_datetime IS NULL'; + SET @new_history_save = ' + UPDATE @@target_table + SET + start_datetime = @@target_column + , end_datetime = "9999-12-31 00:00:00" + WHERE + start_datetime IS NULL + AND end_datetime IS NULL + '; SET @new_history_save = REPLACE(@new_history_save, "@@target_table", target_table); SET @new_history_save = REPLACE(@new_history_save, "@@target_column", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 - SET @make_history_tmp_create = 'CREATE TEMPORARY TABLE make_history_tmp SELECT id, MIN(@@target_column) AS min_start_datetime, MAX(start_datetime) AS max_start_datetime FROM @@target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY id HAVING count(id) = 2'; + SET @make_history_tmp_create = ' + CREATE TEMPORARY TABLE make_history_tmp + SELECT + id + , MIN(@@target_column) AS min_start_datetime + , MAX(start_datetime) AS max_start_datetime + FROM + @@target_table + WHERE + end_datetime = "9999-12-31 00:00:00" + GROUP BY + id + HAVING + count(id) = 2 + '; SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; EXECUTE make_history_tmp_create_stmt; -- ②-2 「②-1」で取得した全件に更新処理を行う - SET @update_end_datetime = 'UPDATE @@target_table tt LEFT JOIN make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND WHERE mht.id IS NOT NULL'; + SET @update_end_datetime = ' + UPDATE @@target_table tt + LEFT JOIN make_history_tmp mht + ON tt.id = mht.id + AND tt.start_datetime = mht.min_start_datetime + SET + start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND + WHERE + mht.id IS NOT NULL + '; SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); + SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; -- ②-3 「②-1」で作成した一時テーブルを削除する DROP TEMPORARY TABLE make_history_tmp; + END \ No newline at end of file From a4cd7f946842f529ef96081b048d2710c318235d Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:31 +0900 Subject: [PATCH 085/275] =?UTF-8?q?refactor:=E5=8D=8A=E8=A7=92=E3=82=B9?= =?UTF-8?q?=E3=83=9A=E3=83=BC=E3=82=B9=E3=82=92=E3=82=BF=E3=83=96=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index f9057a5e..794abe38 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -9,15 +9,15 @@ BEGIN @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; DROP TEMPORARY TABLE IF EXISTS make_history_tmp; SIGNAL SQLSTATE '45000' - SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; + SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; END; -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = ' UPDATE @@target_table SET - start_datetime = @@target_column - , end_datetime = "9999-12-31 00:00:00" + start_datetime = @@target_column + , end_datetime = "9999-12-31 00:00:00" WHERE start_datetime IS NULL AND end_datetime IS NULL From a11f30c057f41e32d786c0ba514a808ceb0f7a82 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:31 +0900 Subject: [PATCH 086/275] =?UTF-8?q?refactor:@@=E3=81=8B=E3=82=89=E5=A7=8B?= =?UTF-8?q?=E3=81=BE=E3=82=8B=E6=96=87=E5=AD=97=E3=81=AE=E8=AA=AC=E6=98=8E?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 794abe38..0777912f 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,4 +1,5 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること +-- @@から始まる文字は後からREPLACEする文字を示す独自ルール CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN -- 例外処理 From 7dfa1cd727d618fa5b79b6307d76fb0ecb6e3483 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:31 +0900 Subject: [PATCH 087/275] =?UTF-8?q?fix:=E4=B8=8D=E8=B6=B3=E9=A0=85?= =?UTF-8?q?=E7=9B=AE=E3=81=A8=E8=A8=AD=E5=AE=9A=E9=A0=85=E7=9B=AE=E3=83=9F?= =?UTF-8?q?=E3=82=B9=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 0777912f..00279493 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -18,7 +18,9 @@ BEGIN UPDATE @@target_table SET start_datetime = @@target_column - , end_datetime = "9999-12-31 00:00:00" + , end_datetime = "9999-12-31 00:00:00" + , upd_user = CURRENT_USER() + , upd_date = CURRENT_TIMESTAMP() WHERE start_datetime IS NULL AND end_datetime IS NULL @@ -56,7 +58,9 @@ BEGIN ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET - start_datetime = mht.max_start_datetime - INTERVAL 1 SECOND + end_datetime = mht.max_start_datetime - INTERVAL 1 SECOND + , upd_user = CURRENT_USER() + , upd_date = CURRENT_TIMESTAMP() WHERE mht.id IS NOT NULL '; From 3f0bbc5b578d01f1944181b985d6e70122462d02 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:31 +0900 Subject: [PATCH 088/275] =?UTF-8?q?refactor:@@=E2=86=92$$=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=9B=B4=E3=80=81=E4=B8=80=E6=99=82=E3=83=86=E3=83=BC?= =?UTF-8?q?=E3=83=96=E3=83=AB=E3=81=AE=E3=83=86=E3=83=BC=E3=83=96=E3=83=AB?= =?UTF-8?q?=E5=90=8D=E4=BF=AE=E6=AD=A3=E3=80=81DEALLOCATE=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 45 ++++++++++++++-------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 00279493..a83cc581 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,5 +1,5 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること --- @@から始まる文字は後からREPLACEする文字を示す独自ルール +-- $$から始まる文字は後からREPLACEする文字を示す独自ルール CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN -- 例外処理 @@ -8,16 +8,25 @@ BEGIN BEGIN GET DIAGNOSTICS CONDITION 1 @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; - DROP TEMPORARY TABLE IF EXISTS make_history_tmp; + EXECUTE drop_tmp_table; + DEALLOCATE PREPARE drop_tmp_table; SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; END; + -- ②-3で利用する一時テーブル削除のSQLを準備 + -- 例外処理でも利用するため先に記述 + SET @drop_tmp_table = ' + DROP TEMPORARY TABLE IF EXISTS make_history_tmp_$$target_table + '; + SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table", target_table); + PREPARE new_history_save_stmt from @drop_tmp_table; + -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = ' - UPDATE @@target_table + UPDATE $$target_table SET - start_datetime = @@target_column + start_datetime = $$target_column , end_datetime = "9999-12-31 00:00:00" , upd_user = CURRENT_USER() , upd_date = CURRENT_TIMESTAMP() @@ -25,20 +34,21 @@ BEGIN start_datetime IS NULL AND end_datetime IS NULL '; - SET @new_history_save = REPLACE(@new_history_save, "@@target_table", target_table); - SET @new_history_save = REPLACE(@new_history_save, "@@target_column", target_column); + SET @new_history_save = REPLACE(@new_history_save, "$$target_table", target_table); + SET @new_history_save = REPLACE(@new_history_save, "$$target_column", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; + DEALLOCATE PREPARE ew_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 SET @make_history_tmp_create = ' - CREATE TEMPORARY TABLE make_history_tmp + CREATE TEMPORARY TABLE make_history_tmp_$$target_table SELECT id - , MIN(@@target_column) AS min_start_datetime + , MIN($$target_column) AS min_start_datetime , MAX(start_datetime) AS max_start_datetime FROM - @@target_table + $$target_table WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY @@ -46,15 +56,16 @@ BEGIN HAVING count(id) = 2 '; - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_table", target_table); - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "@@target_column", target_column); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_table", target_table); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_column", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; EXECUTE make_history_tmp_create_stmt; + DEALLOCATE PREPARE make_history_tmp_create_stmt; -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' - UPDATE @@target_table tt - LEFT JOIN make_history_tmp mht + UPDATE $$target_table tt + LEFT JOIN make_history_tmp_$$target_table mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET @@ -64,12 +75,14 @@ BEGIN WHERE mht.id IS NOT NULL '; - SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_table", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "@@target_column", target_column); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; + DEALLOCATE PREPARE update_end_datetime_stmt; -- ②-3 「②-1」で作成した一時テーブルを削除する - DROP TEMPORARY TABLE make_history_tmp; + EXECUTE drop_tmp_table; + DEALLOCATE PREPARE drop_tmp_table; END \ No newline at end of file From 6052fd267ea81dd074f4c104ac0bc5f56c034726 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:31 +0900 Subject: [PATCH 089/275] =?UTF-8?q?refactor:=E4=B8=80=E6=99=82=E3=83=86?= =?UTF-8?q?=E3=83=BC=E3=83=96=E3=83=AB=E5=90=8D=E3=81=AE=E5=A4=89=E6=9B=B4?= =?UTF-8?q?=E3=80=81=E7=B4=B0=E3=81=8B=E3=81=84=E3=83=9F=E3=82=B9=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index a83cc581..f361a850 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -8,19 +8,19 @@ BEGIN BEGIN GET DIAGNOSTICS CONDITION 1 @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; - EXECUTE drop_tmp_table; - DEALLOCATE PREPARE drop_tmp_table; + EXECUTE drop_tmp_table_stmt; + DEALLOCATE PREPARE drop_tmp_table_stmt; SIGNAL SQLSTATE '45000' - SET MESSAGE_TEXT = @error_msg, MYSQL_ERRNO = @error_state; + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; END; -- ②-3で利用する一時テーブル削除のSQLを準備 -- 例外処理でも利用するため先に記述 SET @drop_tmp_table = ' - DROP TEMPORARY TABLE IF EXISTS make_history_tmp_$$target_table + DROP TEMPORARY TABLE IF EXISTS $$target_table_make_history_tmp '; SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table", target_table); - PREPARE new_history_save_stmt from @drop_tmp_table; + PREPARE drop_tmp_table_stmt from @drop_tmp_table; -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = ' @@ -38,11 +38,11 @@ BEGIN SET @new_history_save = REPLACE(@new_history_save, "$$target_column", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; - DEALLOCATE PREPARE ew_history_save_stmt; + DEALLOCATE PREPARE new_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 SET @make_history_tmp_create = ' - CREATE TEMPORARY TABLE make_history_tmp_$$target_table + CREATE TEMPORARY TABLE $$target_table_make_history_tmp SELECT id , MIN($$target_column) AS min_start_datetime @@ -65,7 +65,7 @@ BEGIN -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' UPDATE $$target_table tt - LEFT JOIN make_history_tmp_$$target_table mht + LEFT JOIN $$target_table_make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET @@ -82,7 +82,7 @@ BEGIN DEALLOCATE PREPARE update_end_datetime_stmt; -- ②-3 「②-1」で作成した一時テーブルを削除する - EXECUTE drop_tmp_table; - DEALLOCATE PREPARE drop_tmp_table; + EXECUTE drop_tmp_table_stmt; + DEALLOCATE PREPARE drop_tmp_table_stmt; END \ No newline at end of file From 1df639363aa772c3028ad321679741faf07ac16d Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:31 +0900 Subject: [PATCH 090/275] =?UTF-8?q?fix:=E3=83=88=E3=83=A9=E3=83=B3?= =?UTF-8?q?=E3=82=B6=E3=82=AF=E3=82=B7=E3=83=A7=E3=83=B3=E5=87=A6=E7=90=86?= =?UTF-8?q?=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index f361a850..7573bf9b 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -10,10 +10,13 @@ BEGIN @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; EXECUTE drop_tmp_table_stmt; DEALLOCATE PREPARE drop_tmp_table_stmt; + ROLLBACK; SIGNAL SQLSTATE '45000' SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; END; + START TRANSACTION; + -- ②-3で利用する一時テーブル削除のSQLを準備 -- 例外処理でも利用するため先に記述 SET @drop_tmp_table = ' @@ -85,4 +88,6 @@ BEGIN EXECUTE drop_tmp_table_stmt; DEALLOCATE PREPARE drop_tmp_table_stmt; + COMMIT; + END \ No newline at end of file From 374093bcc707ecc06b84058ef0c7aa6f096f7048 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:31 +0900 Subject: [PATCH 091/275] =?UTF-8?q?refactor:=E4=B8=A6=E5=88=97=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=81=A7=E3=81=8D=E3=81=AA=E3=81=84=E3=82=B3=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E8=BF=BD=E5=8A=A0=E3=80=81=E4=BE=8B=E5=A4=96?= =?UTF-8?q?=E3=81=A7=E4=BD=BF=E7=94=A8=E3=81=99=E3=82=8B=E3=83=A6=E3=83=BC?= =?UTF-8?q?=E3=82=B6=E5=A4=89=E6=95=B0=E3=81=AE=E5=88=9D=E6=9C=9F=E5=8C=96?= =?UTF-8?q?=E3=80=81$$=E7=B5=82=E7=AB=AF=E6=96=87=E5=AD=97=E3=81=AE?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 34 ++++++++++++---------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 7573bf9b..b60e2681 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,5 +1,6 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること --- $$から始まる文字は後からREPLACEする文字を示す独自ルール +-- $$から始まり$$で終わる文字は後からREPLACEする文字を示す独自ルール +-- crm_historyストアドプロシージャは、同一セッション内での並列処理を実行することができない CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN -- 例外処理 @@ -15,21 +16,22 @@ BEGIN SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; END; + SET @error_state = NULL, @error_msg = NULL; START TRANSACTION; -- ②-3で利用する一時テーブル削除のSQLを準備 -- 例外処理でも利用するため先に記述 SET @drop_tmp_table = ' - DROP TEMPORARY TABLE IF EXISTS $$target_table_make_history_tmp + DROP TEMPORARY TABLE IF EXISTS $$target_table$$_make_history_tmp '; - SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table", target_table); + SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table$$", target_table); PREPARE drop_tmp_table_stmt from @drop_tmp_table; -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する SET @new_history_save = ' - UPDATE $$target_table + UPDATE $$target_table$$ SET - start_datetime = $$target_column + start_datetime = $$target_column$$ , end_datetime = "9999-12-31 00:00:00" , upd_user = CURRENT_USER() , upd_date = CURRENT_TIMESTAMP() @@ -37,21 +39,21 @@ BEGIN start_datetime IS NULL AND end_datetime IS NULL '; - SET @new_history_save = REPLACE(@new_history_save, "$$target_table", target_table); - SET @new_history_save = REPLACE(@new_history_save, "$$target_column", target_column); + SET @new_history_save = REPLACE(@new_history_save, "$$target_table$$", target_table); + SET @new_history_save = REPLACE(@new_history_save, "$$target_column$$", target_column); PREPARE new_history_save_stmt from @new_history_save; EXECUTE new_history_save_stmt; DEALLOCATE PREPARE new_history_save_stmt; -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 SET @make_history_tmp_create = ' - CREATE TEMPORARY TABLE $$target_table_make_history_tmp + CREATE TEMPORARY TABLE $$target_table$$_make_history_tmp SELECT id - , MIN($$target_column) AS min_start_datetime + , MIN($$target_column$$) AS min_start_datetime , MAX(start_datetime) AS max_start_datetime FROM - $$target_table + $$target_table$$ WHERE end_datetime = "9999-12-31 00:00:00" GROUP BY @@ -59,16 +61,16 @@ BEGIN HAVING count(id) = 2 '; - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_table", target_table); - SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_column", target_column); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_table$$", target_table); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_column$$", target_column); PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; EXECUTE make_history_tmp_create_stmt; DEALLOCATE PREPARE make_history_tmp_create_stmt; -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' - UPDATE $$target_table tt - LEFT JOIN $$target_table_make_history_tmp mht + UPDATE $$target_table$$ tt + LEFT JOIN $$target_table$$_make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET @@ -78,8 +80,8 @@ BEGIN WHERE mht.id IS NOT NULL '; - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column", target_column); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; DEALLOCATE PREPARE update_end_datetime_stmt; From 1b6420f970bd03d40d1d1db50bab3270c260d85f Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 092/275] =?UTF-8?q?feat:crm=5Fdata=5Fsync=E3=81=AE?= =?UTF-8?q?=E6=96=B0=E8=A6=8F=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 rds_mysql/stored_procedure/crm_data_sync.sql diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql new file mode 100644 index 00000000..c039a18a --- /dev/null +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -0,0 +1,41 @@ +-- A5M2で実行時にSQL区切り文字を「;」以外にすること +-- $$から始まる文字は後からREPLACEする文字を示す独自ルール +-- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない +CREATE PROCEDURE crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) +BEGIN + -- 例外処理 + -- エラーが発生した場合に一時テーブルの削除を実施 + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + ROLLBACK; + SIGNAL SQLSTATE '45000' + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; + END; + + SET @error_state = NULL, @error_msg = NULL; + START TRANSACTION; + + -- ①-1 Salesforce側で物理削除されたデータを検出し更新する + SET @update_end_datetime = ' + UPDATE $$target_table$$ tt + LEFT JOIN $$target_table_all$$ tta + ON tt.id = tta.id + AND tt.$$target_column$$ = tta.$$target_column$$ + SET + tt.end_datetime = CURRENT_TIMESTAMP() + , tt.upd_user = CURRENT_USER() + , tt.upd_date = CURRENT_TIMESTAMP() + WHERE + tta.id IS NULL + AND tt.end_datetime = "9999-12-31 00:00:00" + '; + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); + PREPARE update_end_datetime_stmt from @update_end_datetime; + EXECUTE update_end_datetime_stmt; + + COMMIT; +END \ No newline at end of file From fc3ce92e21ed01eab3331eea796e1f0da907280b Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 093/275] =?UTF-8?q?Revert=20"feat:crm=5Fdata=5Fsync?= =?UTF-8?q?=E3=81=AE=E6=96=B0=E8=A6=8F=E4=BD=9C=E6=88=90"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 9b8076451f14b1ec07d5751d47cb79bbc1b709db. --- rds_mysql/stored_procedure/crm_data_sync.sql | 41 -------------------- 1 file changed, 41 deletions(-) delete mode 100644 rds_mysql/stored_procedure/crm_data_sync.sql diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql deleted file mode 100644 index c039a18a..00000000 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ /dev/null @@ -1,41 +0,0 @@ --- A5M2で実行時にSQL区切り文字を「;」以外にすること --- $$から始まる文字は後からREPLACEする文字を示す独自ルール --- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない -CREATE PROCEDURE crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) -BEGIN - -- 例外処理 - -- エラーが発生した場合に一時テーブルの削除を実施 - DECLARE EXIT HANDLER FOR SQLEXCEPTION - BEGIN - GET DIAGNOSTICS CONDITION 1 - @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; - ROLLBACK; - SIGNAL SQLSTATE '45000' - SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; - END; - - SET @error_state = NULL, @error_msg = NULL; - START TRANSACTION; - - -- ①-1 Salesforce側で物理削除されたデータを検出し更新する - SET @update_end_datetime = ' - UPDATE $$target_table$$ tt - LEFT JOIN $$target_table_all$$ tta - ON tt.id = tta.id - AND tt.$$target_column$$ = tta.$$target_column$$ - SET - tt.end_datetime = CURRENT_TIMESTAMP() - , tt.upd_user = CURRENT_USER() - , tt.upd_date = CURRENT_TIMESTAMP() - WHERE - tta.id IS NULL - AND tt.end_datetime = "9999-12-31 00:00:00" - '; - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); - PREPARE update_end_datetime_stmt from @update_end_datetime; - EXECUTE update_end_datetime_stmt; - - COMMIT; -END \ No newline at end of file From c182d377f41edb29402bfd0c9482a8b3c334574d Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 094/275] =?UTF-8?q?refactor:=E2=91=A1-2=E5=86=85=E9=83=A8?= =?UTF-8?q?=E7=B5=90=E5=90=88=E3=81=AB=E5=A4=89=E6=9B=B4=E3=81=97Where?= =?UTF-8?q?=E6=9D=A1=E4=BB=B6=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index b60e2681..366cde70 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -70,15 +70,13 @@ BEGIN -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' UPDATE $$target_table$$ tt - LEFT JOIN $$target_table$$_make_history_tmp mht + JOIN $$target_table$$_make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET end_datetime = mht.max_start_datetime - INTERVAL 1 SECOND , upd_user = CURRENT_USER() , upd_date = CURRENT_TIMESTAMP() - WHERE - mht.id IS NOT NULL '; SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); From 580b94ee86b7c018779f6c28f5f98d5fb2b804e3 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 095/275] =?UTF-8?q?refactor:INNER=E3=82=92=E6=98=8E?= =?UTF-8?q?=E7=A4=BA=E7=9A=84=E3=81=AB=E8=A8=98=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_history.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 366cde70..10bb0095 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -70,7 +70,7 @@ BEGIN -- ②-2 「②-1」で取得した全件に更新処理を行う SET @update_end_datetime = ' UPDATE $$target_table$$ tt - JOIN $$target_table$$_make_history_tmp mht + INNER JOIN $$target_table$$_make_history_tmp mht ON tt.id = mht.id AND tt.start_datetime = mht.min_start_datetime SET From d92c5418cf2c349dfaf879e0c83378830e017cf9 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 096/275] =?UTF-8?q?feat:crm=5Fdata=5Fsync=E3=81=AE?= =?UTF-8?q?=E6=96=B0=E8=A6=8F=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 41 ++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 rds_mysql/stored_procedure/crm_data_sync.sql diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql new file mode 100644 index 00000000..c039a18a --- /dev/null +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -0,0 +1,41 @@ +-- A5M2で実行時にSQL区切り文字を「;」以外にすること +-- $$から始まる文字は後からREPLACEする文字を示す独自ルール +-- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない +CREATE PROCEDURE crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) +BEGIN + -- 例外処理 + -- エラーが発生した場合に一時テーブルの削除を実施 + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + ROLLBACK; + SIGNAL SQLSTATE '45000' + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; + END; + + SET @error_state = NULL, @error_msg = NULL; + START TRANSACTION; + + -- ①-1 Salesforce側で物理削除されたデータを検出し更新する + SET @update_end_datetime = ' + UPDATE $$target_table$$ tt + LEFT JOIN $$target_table_all$$ tta + ON tt.id = tta.id + AND tt.$$target_column$$ = tta.$$target_column$$ + SET + tt.end_datetime = CURRENT_TIMESTAMP() + , tt.upd_user = CURRENT_USER() + , tt.upd_date = CURRENT_TIMESTAMP() + WHERE + tta.id IS NULL + AND tt.end_datetime = "9999-12-31 00:00:00" + '; + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); + PREPARE update_end_datetime_stmt from @update_end_datetime; + EXECUTE update_end_datetime_stmt; + + COMMIT; +END \ No newline at end of file From 7447d54323f7fb39cd45203c6647ce565b5b31f0 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 097/275] =?UTF-8?q?refactor:EXISTS=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 21 ++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql index c039a18a..01d27b9a 100644 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -20,16 +20,21 @@ BEGIN -- ①-1 Salesforce側で物理削除されたデータを検出し更新する SET @update_end_datetime = ' UPDATE $$target_table$$ tt - LEFT JOIN $$target_table_all$$ tta - ON tt.id = tta.id - AND tt.$$target_column$$ = tta.$$target_column$$ SET - tt.end_datetime = CURRENT_TIMESTAMP() - , tt.upd_user = CURRENT_USER() - , tt.upd_date = CURRENT_TIMESTAMP() + tt.end_datetime = CURRENT_TIMESTAMP () + , tt.upd_user = CURRENT_USER () + , tt.upd_date = CURRENT_TIMESTAMP () WHERE - tta.id IS NULL - AND tt.end_datetime = "9999-12-31 00:00:00" + tt.end_datetime = "9999-12-31 00:00:00" + AND NOT EXISTS ( + SELECT + * + FROM + $$target_table_all$$ tta + WHERE + tt.id = tta.id + AND tt.$$target_column$$ = tta.$$target_column$$ + ) '; SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); From fa1c2b7f83a2604e6e4d01830157131481753dad Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 098/275] =?UTF-8?q?refactor:EXISTS=E3=81=AE=E4=B8=AD?= =?UTF-8?q?=E8=BA=AB=E3=81=AESELECT=E5=8F=A5=E3=82=92ID=E3=81=AE=E3=81=BF?= =?UTF-8?q?=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql index 01d27b9a..155bfb99 100644 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -28,7 +28,7 @@ BEGIN tt.end_datetime = "9999-12-31 00:00:00" AND NOT EXISTS ( SELECT - * + tt.id FROM $$target_table_all$$ tta WHERE @@ -37,8 +37,8 @@ BEGIN ) '; SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); - SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); PREPARE update_end_datetime_stmt from @update_end_datetime; EXECUTE update_end_datetime_stmt; From 150a83670d2a1dbec2c7bbf14da6c7310a2dcf24 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 099/275] =?UTF-8?q?refactor:=E5=8F=82=E7=85=A7ID=E3=82=92t?= =?UTF-8?q?ta=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql index 155bfb99..e9986c0f 100644 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -28,7 +28,7 @@ BEGIN tt.end_datetime = "9999-12-31 00:00:00" AND NOT EXISTS ( SELECT - tt.id + tta.id FROM $$target_table_all$$ tta WHERE From 93fedc6a30225ab0c5ddc4966edbc590371acdd5 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 100/275] =?UTF-8?q?feat:CRM=E3=81=AE=E6=8B=A1=E5=BC=B5SQL?= =?UTF-8?q?=E3=80=81=E8=A8=AD=E5=AE=9A=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/crm/settings/CRM_Account.txt | 13 +++++ s3/data/crm/settings/CRM_AccountShare.txt | 13 +++++ s3/data/crm/settings/CRM_AccountShare_ex.sql | 1 + .../CRM_Account_Territory_Loader_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Account_ex.sql | 1 + .../settings/CRM_Approved_Document_vod__c.txt | 13 +++++ .../crm/settings/CRM_Call2_Detail_vod__c.txt | 13 +++++ .../settings/CRM_Call2_Discussion_vod__c.txt | 13 +++++ .../settings/CRM_Call2_Key_Message_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Call2_vod__c.txt | 13 +++++ .../settings/CRM_Call_Clickstream_vod__c.txt | 13 +++++ .../crm/settings/CRM_Child_Account_vod__c.txt | 13 +++++ .../settings/CRM_Child_Account_vod__c_ex.sql | 1 + .../CRM_Clm_Presentation_Slide_vod__c.txt | 13 +++++ .../settings/CRM_Clm_Presentation_vod__c.txt | 13 +++++ .../settings/CRM_Coaching_Report_vod__c.txt | 13 +++++ .../settings/CRM_Consent_Header_vod__c.txt | 13 +++++ .../crm/settings/CRM_Consent_Line_vod__c.txt | 13 +++++ .../crm/settings/CRM_Consent_Type_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Contact.txt | 13 +++++ s3/data/crm/settings/CRM_Contact_ex.sql | 1 + ...Dynamic_Attribute_Configuration_vod__c.txt | 13 +++++ .../settings/CRM_Dynamic_Attribute_vod__c.txt | 13 +++++ .../settings/CRM_Email_Activity_vod__c.txt | 13 +++++ .../settings/CRM_Event_Attendee_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Group.txt | 13 +++++ s3/data/crm/settings/CRM_Group_ex.sql | 1 + .../crm/settings/CRM_Key_Message_vod__c.txt | 13 +++++ .../CRM_MSJ_Hospital_Medical_Regimen__c.txt | 13 +++++ .../CRM_MSJ_Inquiry_Assignment__c.txt | 13 +++++ .../settings/CRM_MSJ_MR_Weekly_Report__c.txt | 13 +++++ .../CRM_MSJ_Medical_Event_Evaluation__c.txt | 13 +++++ .../settings/CRM_MSJ_Medical_Regimen__c.txt | 13 +++++ s3/data/crm/settings/CRM_MSJ_Patient__c.txt | 13 +++++ .../crm/settings/CRM_Medical_Event_vod__c.txt | 13 +++++ .../settings/CRM_Medical_Inquiry_vod__c.txt | 13 +++++ .../settings/CRM_Medical_Insight_vod__c.txt | 13 +++++ .../CRM_Multichannel_Activity_Line_vod__c.txt | 13 +++++ .../CRM_Multichannel_Activity_vod__c.txt | 13 +++++ .../CRM_Multichannel_Consent_vod__c.txt | 13 +++++ .../settings/CRM_My_Setup_Products_vod__c.txt | 13 +++++ .../CRM_ObjectTerritory2Association.txt | 13 +++++ .../CRM_ObjectTerritory2Association_ex.sql | 1 + .../crm/settings/CRM_Product_Group_vod__c.txt | 13 +++++ .../settings/CRM_Product_Metrics_vod__c.txt | 13 +++++ .../CRM_Product_Metrics_vod__c_ex.sql | 1 + s3/data/crm/settings/CRM_Product_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Profile.txt | 13 +++++ s3/data/crm/settings/CRM_Profile_ex.sql | 1 + .../settings/CRM_Question_Response_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_RecordType.txt | 13 +++++ .../settings/CRM_Remote_Meeting_vod__c.txt | 13 +++++ .../crm/settings/CRM_Sent_Email_vod__c.txt | 13 +++++ .../crm/settings/CRM_Sent_Fragment_vod__c.txt | 13 +++++ .../settings/CRM_Survey_Question_vod__c.txt | 13 +++++ .../crm/settings/CRM_Survey_Target_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Survey_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_Territory2.txt | 13 +++++ s3/data/crm/settings/CRM_Territory2_ALL.txt | 14 +++++ s3/data/crm/settings/CRM_Territory2_ex.sql | 2 + .../CRM_Time_Off_Territory_vod__c.txt | 13 +++++ s3/data/crm/settings/CRM_User.txt | 13 +++++ s3/data/crm/settings/CRM_UserRole.txt | 13 +++++ .../CRM_UserTerritory2Association.txt | 13 +++++ .../CRM_UserTerritory2Association_ALL.txt | 14 +++++ .../CRM_UserTerritory2Association_ex.sql | 2 + s3/data/crm/settings/configmap.config | 58 +++++++++++++++++++ 67 files changed, 800 insertions(+) create mode 100644 s3/data/crm/settings/CRM_Account.txt create mode 100644 s3/data/crm/settings/CRM_AccountShare.txt create mode 100644 s3/data/crm/settings/CRM_AccountShare_ex.sql create mode 100644 s3/data/crm/settings/CRM_Account_Territory_Loader_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Account_ex.sql create mode 100644 s3/data/crm/settings/CRM_Approved_Document_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call2_Detail_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call2_Discussion_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call2_Key_Message_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call2_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Call_Clickstream_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Child_Account_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql create mode 100644 s3/data/crm/settings/CRM_Clm_Presentation_Slide_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Clm_Presentation_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Coaching_Report_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Consent_Header_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Consent_Line_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Consent_Type_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Contact.txt create mode 100644 s3/data/crm/settings/CRM_Contact_ex.sql create mode 100644 s3/data/crm/settings/CRM_Dynamic_Attribute_Configuration_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Dynamic_Attribute_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Email_Activity_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Group.txt create mode 100644 s3/data/crm/settings/CRM_Group_ex.sql create mode 100644 s3/data/crm/settings/CRM_Key_Message_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Hospital_Medical_Regimen__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Inquiry_Assignment__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_MR_Weekly_Report__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Medical_Event_Evaluation__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Medical_Regimen__c.txt create mode 100644 s3/data/crm/settings/CRM_MSJ_Patient__c.txt create mode 100644 s3/data/crm/settings/CRM_Medical_Event_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Medical_Inquiry_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Medical_Insight_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Multichannel_Activity_Line_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Multichannel_Activity_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Multichannel_Consent_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_My_Setup_Products_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_ObjectTerritory2Association.txt create mode 100644 s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql create mode 100644 s3/data/crm/settings/CRM_Product_Group_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Product_Metrics_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql create mode 100644 s3/data/crm/settings/CRM_Product_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Profile.txt create mode 100644 s3/data/crm/settings/CRM_Profile_ex.sql create mode 100644 s3/data/crm/settings/CRM_Question_Response_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_RecordType.txt create mode 100644 s3/data/crm/settings/CRM_Remote_Meeting_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Sent_Email_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Sent_Fragment_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Survey_Question_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Survey_Target_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Survey_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_Territory2.txt create mode 100644 s3/data/crm/settings/CRM_Territory2_ALL.txt create mode 100644 s3/data/crm/settings/CRM_Territory2_ex.sql create mode 100644 s3/data/crm/settings/CRM_Time_Off_Territory_vod__c.txt create mode 100644 s3/data/crm/settings/CRM_User.txt create mode 100644 s3/data/crm/settings/CRM_UserRole.txt create mode 100644 s3/data/crm/settings/CRM_UserTerritory2Association.txt create mode 100644 s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt create mode 100644 s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql create mode 100644 s3/data/crm/settings/configmap.config diff --git a/s3/data/crm/settings/CRM_Account.txt b/s3/data/crm/settings/CRM_Account.txt new file mode 100644 index 00000000..bee3992c --- /dev/null +++ b/s3/data/crm/settings/CRM_Account.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +248 +Id,IsDeleted,MasterRecordId,Name,LastName,FirstName,Salutation,RecordTypeId,Phone,Fax,Website,PhotoUrl,NumberOfEmployees,Ownership,OwnerId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,IsExcludedFromRealign,PersonContactId,IsPersonAccount,PersonMailingStreet,PersonMailingCity,PersonMailingState,PersonMailingPostalCode,PersonMailingCountry,PersonMailingLatitude,PersonMailingLongitude,PersonMailingGeocodeAccuracy,PersonMailingAddress,PersonOtherStreet,PersonOtherCity,PersonOtherState,PersonOtherPostalCode,PersonOtherCountry,PersonOtherLatitude,PersonOtherLongitude,PersonOtherGeocodeAccuracy,PersonOtherAddress,PersonMobilePhone,PersonHomePhone,PersonOtherPhone,PersonAssistantPhone,PersonEmail,PersonTitle,PersonDepartment,PersonAssistantName,PersonBirthdate,PersonHasOptedOutOfEmail,PersonHasOptedOutOfFax,PersonDoNotCall,PersonLastCURequestDate,PersonLastCUUpdateDate,PersonEmailBouncedReason,PersonEmailBouncedDate,PersonIndividualId,Jigsaw,JigsawCompanyId,AccountSource,SicDesc,External_ID_vod__c,Credentials_vod__c,Territory_vod__c,Exclude_from_Zip_to_Terr_Processing_vod__c,Group_Specialty_1_vod__c,Group_Specialty_2_vod__c,Specialty_1_vod__c,Specialty_2_vod__c,Formatted_Name_vod__c,Territory_Test_vod__c,Mobile_ID_vod__c,Gender_vod__c,ID_vod__c,Do_Not_Sync_Sales_Data_vod__c,ID2_vod__c,Preferred_Name_vod__c,Sample_Default_vod__c,Segmentations_vod__c,Restricted_Products_vod__c,Payer_Id_vod__c,Alternate_Name_vod__c,Do_Not_Call_vod__c,MSJ_Beds__c,Spend_Amount__c,PDRP_Opt_Out_vod__c,Spend_Status_Value_vod__c,PDRP_Opt_Out_Date_vod__c,Spend_Status_vod__c,Enable_Restricted_Products_vod__c,Call_Reminder_vod__c,Account_Group_vod__c,Primary_Parent_vod__c,Color_vod__c,Middle_vod__c,Suffix_vod__c,MSJ_Type__c,No_Orders_vod__c,MSJ_BU_ONC__c,MSJ_BU_FE__c,Account_Search_FirstLast_vod__c,Account_Search_LastFirst_vod__c,MSJ_Operation__c,Practice_at_Hospital_vod__c,Practice_Near_Hospital_vod__c,Do_Not_Create_Child_Account_vod__c,Total_MDs_DOs__c,AHA__c,Order_Type_vod__c,NPI_vod__c,ME__c,Speaker__c,Investigator_vod__c,Default_Order_Type_vod__c,Tax_Status__c,Model__c,Offerings__c,Departments__c,Account_Type__c,MSJ_ONC_Tier__c,Account_Search_Business_vod__c,Business_Professional_Person_vod__c,Hospital_Type_vod__c,Account_Class_vod__c,Furigana_vod__c,MSJ_JISART__c,Total_Revenue_000__c,Net_Income_Loss_000__c,PMPM_Income_Loss_000__c,Commercial_Premiums_PMPM__c,Medical_Loss_Ratio__c,Medical_Expenses_PMPM__c,Commercial_Patient_Days_1000__c,HMO_Market_Shr__c,HMO__c,HMO_POS__c,PPO__c,PPO_POS__c,Medicare__c,Medicaid__c,MSJ_HP_Name_E__c,MSJ_Department__c,MSJ_Date_Of_Birth__c,MSJ_FE_GF_Potential__c,MSJ_FE_SZ_Potential__c,MSJ_EB_CRC_Ladder__c,MSJ_EB_CRC_Segment__c,MSJ_EB_HN_Segment__c,Business_Description__c,Regional_Strategy__c,Contracts_Process__c,MSJ_GF_segment__c,MSJ_DCF_DR_Code__c,MSJ_SZ_Segment__c,MSJ_Remark__c,MSJ_Title__c,MSJ_Role__c,MSJ_Kana__c,MSJ_Specialism__c,MSJ_Graduated_from__c,MSJ_Year_Graduation__c,Target__c,KOL_vod__c,MSJ_EPPV_Code__c,MSJ_DCF_HP_Code__c,Total_Lives__c,Total_Physicians_Enrolled__c,MSJ_Delete__c,MSJ_KOL_LOL__c,MSJ_ONC_Status__c,Account_Identifier_vod__c,Approved_Email_Opt_Type_vod__c,Language_vod__c,MSJ_KRAS_Routine_Date__c,MSJ_KRAS_Routine__c,MSJ_DRP_Target__c,MSJ_Fertility_Evaluation_Score__c,MSJ_Fertility_Tracking_Last_Modify_Date__c,Total_Pharmacists__c,MSJ_Number_of_Gonadotropin__c,MSJ_Number_of_IUI_cycle__c,MSJ_Number_of_OI_monthly_cycle__c,MSJ_OI_Protocol_learning_level__c,MSJ_H_N_Tier__c,MSJ_XLK_Segment__c,MSJ_XLK_Tier__c,Career_Status_vod__c,Photo_vod__c,MSJ_EB_H_N_LA_Segment__c,MSJ_EB_H_N_RM_Segment__c,MSJ_FE_CE_Potential__c,MSJ_FE_1C_potential__c,MSJ_FE_OV_potential__c,MSJ_FE_Tech_potential__c,MSJ_CE_segment__c,MSJ_1C_segment__c,MSJ_OV_segment__c,MSJ_Tech_segment__c,MSJ_Target_Call_Num__c,MSJ_DR_Change_Log__c,MSJ_Global_scientific_exposure__c,MSJ_H_index__c,MSJ_Num_of_Article_3Y__c,MSJ_Num_of_Article__c,MSJ_Num_of_Article_as_1st_Author_3Y__c,MSJ_Num_of_article_growth_rate_3Y__c,MSJ_Num_of_cited_3Y__c,MSJ_Num_of_impact_factor_3Y__c,MSJ_impact_factor_as_1st_Author_3Y__c,EMDS_Has_Pipeline_Opportunity__c,EMDS_Pipeline_Count__c,MSJ_BVC_Segment__c,MSJ_BVC_Tier__c,MSJ_BVC_AcctOpen__c,MSJ_BVC_MCC_Patients__c,MSJ_ONC_HP_Segment__c,MSJ_AE_Department__c,MSJ_AE_Facility__c,MSJ_AE_Name__c,MSJ_AE_Title__c,MSJ_Email__c,MSJ_FE_GF2_Potential__c,MSJ_FE_Location_potential__c,MSJ_GF2_segment__c,MSJ_OPTIN_target__c,MSJ_Merck_Specialty1__c,MSJ_Merck_Specialty2__c,MSJ_Marketing_Cloud_Integration__c,MSJ_Marketing_Cloud1__c,MSJ_Marketing_Cloud2__c,MSJ_Marketing_Cloud3__c,MSJ_Marketing_Cloud4__c,MSJ_Medical_Department__c,MSJ_Marketing_Cloud0__c,Mobile_ID_vod__pc,H1Insights__H1_NPI_Value_for_Testing__pc,H1Insights__H1_Person_ID__pc,H1Insights__H1_Request_Status__pc,H1Insights__H1_URL__pc,H1Insights__NPI_Number__pc,H1Insights__NPI_Number_for_H1_Insights__pc,MSJ_Marketing_Cloud_Integration__pc +id,is_deleted,master_record_id,name,last_name,first_name,salutation,record_type_id,phone,fax,website,photo_url,number_of_employees,ownership,owner_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,is_excluded_from_realign,person_contact_id,is_person_account,person_mailing_street,person_mailing_city,person_mailing_state,person_mailing_postal_code,person_mailing_country,person_mailing_latitude,person_mailing_longitude,person_mailing_geocode_accuracy,person_mailing_address,person_other_street,person_other_city,person_other_state,person_other_postal_code,person_other_country,person_other_latitude,person_other_longitude,person_other_geocode_accuracy,person_other_address,person_mobile_phone,person_home_phone,person_other_phone,person_assistant_phone,person_email,person_title,person_department,person_assistant_name,person_birthdate,person_has_opted_out_of_email,person_has_opted_out_of_fax,person_do_not_call,person_last_curequest_date,person_last_cuupdate_date,person_email_bounced_reason,person_email_bounced_date,person_individual_id,jigsaw,jigsaw_company_id,account_source,sic_desc,external_id_vod__c,credentials_vod__c,territory_vod__c,exclude_from_zip_to_terr_processing_vod__c,group_specialty_1_vod__c,group_specialty_2_vod__c,specialty_1_vod__c,specialty_2_vod__c,formatted_name_vod__c,territory_test_vod__c,mobile_id_vod__c,gender_vod__c,id_vod__c,do_not_sync_sales_data_vod__c,id2_vod__c,preferred_name_vod__c,sample_default_vod__c,segmentations_vod__c,restricted_products_vod__c,payer_id_vod__c,alternate_name_vod__c,do_not_call_vod__c,msj_beds__c,spend_amount__c,pdrp_opt_out_vod__c,spend_status_value_vod__c,pdrp_opt_out_date_vod__c,spend_status_vod__c,enable_restricted_products_vod__c,call_reminder_vod__c,account_group_vod__c,primary_parent_vod__c,color_vod__c,middle_vod__c,suffix_vod__c,msj_type__c,no_orders_vod__c,msj_bu_onc__c,msj_bu_fe__c,account_search_first_last_vod__c,account_search_last_first_vod__c,msj_operation__c,practice_at_hospital_vod__c,practice_near_hospital_vod__c,do_not_create_child_account_vod__c,total_mds_dos__c,aha__c,order_type_vod__c,npi_vod__c,me__c,speaker__c,investigator_vod__c,default_order_type_vod__c,tax_status__c,model__c,offerings__c,departments__c,account_type__c,msj_onc_tier__c,account_search_business_vod__c,business_professional_person_vod__c,hospital_type_vod__c,account_class_vod__c,furigana_vod__c,msj_jisart__c,total_revenue_000__c,net_income_loss_000__c,pmpm_income_loss_000__c,commercial_premiums_pmpm__c,medical_loss_ratio__c,medical_expenses_pmpm__c,commercial_patient_days_1000__c,hmo_market_shr__c,hmo__c,hmo_pos__c,ppo__c,ppo_pos__c,medicare__c,medicaid__c,msj_hp_name_e__c,msj_department__c,msj_date_of_birth__c,msj_fe_gf_potential__c,msj_fe_sz_potential__c,msj_eb_crc_ladder__c,msj_eb_crc_segment__c,msj_eb_hn_segment__c,business_description__c,regional_strategy__c,contracts_process__c,msj_gf_segment__c,msj_dcf_dr_code__c,msj_sz_segment__c,msj_remark__c,msj_title__c,msj_role__c,msj_kana__c,msj_specialism__c,msj_graduated_from__c,msj_year_graduation__c,target__c,kol_vod__c,msj_eppv_code__c,msj_dcf_hp_code__c,total_lives__c,total_physicians_enrolled__c,msj_delete__c,msj_kol_lol__c,msj_onc_status__c,account_identifier_vod__c,approved_email_opt_type_vod__c,language_vod__c,msj_kras_routine_date__c,msj_kras_routine__c,msj_drp_target__c,msj_fertility_evaluation_score__c,msj_fertility_tracking_last_modify_date__c,total_pharmacists__c,msj_number_of_gonadotropin__c,msj_number_of_iui_cycle__c,msj_number_of_oi_monthly_cycle__c,msj_oi_protocol_learning_level__c,msj_h_n_tier__c,msj_xlk_segment__c,msj_xlk_tier__c,career_status_vod__c,photo_vod__c,msj_eb_h_n_la_segment__c,msj_eb_h_n_rm_segment__c,msj_fe_ce_potential__c,msj_fe_1_c_potential__c,msj_fe_ov_potential__c,msj_fe_tech_potential__c,msj_ce_segment__c,msj_1_c_segment__c,msj_ov_segment__c,msj_tech_segment__c,msj_target_call_num__c,msj_dr_change_log__c,msj_global_scientific_exposure__c,msj_h_index__c,msj_num_of_article_3_y__c,msj_num_of_article__c,msj_num_of_article_as_1st_author_3_y__c,msj_num_of_article_growth_rate_3_y__c,msj_num_of_cited_3_y__c,msj_num_of_impact_factor_3_y__c,msj_impact_factor_as_1st_author_3_y__c,emds_has_pipeline_opportunity__c,emds_pipeline_count__c,msj_bvc_segment__c,msj_bvc_tier__c,msj_bvc_acct_open__c,msj_bvc_mcc_patients__c,msj_onc_hp_segment__c,msj_ae_department__c,msj_ae_facility__c,msj_ae_name__c,msj_ae_title__c,msj_email__c,msj_fe_gf2_potential__c,msj_fe_location_potential__c,msj_gf2_segment__c,msj_optin_target__c,msj_merck_specialty1__c,msj_merck_specialty2__c,msj_marketing_cloud_integration__c,msj_marketing_cloud1__c,msj_marketing_cloud2__c,msj_marketing_cloud3__c,msj_marketing_cloud4__c,msj_medical_department__c,msj_marketing_cloud0__c,mobile_id_vod__pc,h1_insights__h1_npi_value_for_testing__pc,h1_insights__h1_person_id__pc,h1_insights__h1_request_status__pc,h1_insights__h1_url__pc,h1_insights__npi_number__pc,h1_insights__npi_number_for_h1_insights__pc,msj_marketing_cloud_integration__pc +src02.crm_account +org02.crm_account +CRM_Account_ex.sql + diff --git a/s3/data/crm/settings/CRM_AccountShare.txt b/s3/data/crm/settings/CRM_AccountShare.txt new file mode 100644 index 00000000..8176af1e --- /dev/null +++ b/s3/data/crm/settings/CRM_AccountShare.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +11 +Id,AccountId,UserOrGroupId,AccountAccessLevel,OpportunityAccessLevel,CaseAccessLevel,ContactAccessLevel,RowCause,LastModifiedDate,LastModifiedById,IsDeleted +id,account_id,user_or_group_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,row_cause,last_modified_date,last_modified_by_id,is_deleted +src02.crm_account_share +org02.crm_account_share +CRM_AccountShare_ex.sql + diff --git a/s3/data/crm/settings/CRM_AccountShare_ex.sql b/s3/data/crm/settings/CRM_AccountShare_ex.sql new file mode 100644 index 00000000..d18e68db --- /dev/null +++ b/s3/data/crm/settings/CRM_AccountShare_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_account_share', 'last_modified_date'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Account_Territory_Loader_vod__c.txt b/s3/data/crm/settings/CRM_Account_Territory_Loader_vod__c.txt new file mode 100644 index 00000000..676e8b39 --- /dev/null +++ b/s3/data/crm/settings/CRM_Account_Territory_Loader_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +19 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,External_ID_vod__c,Territory_vod__c,Mobile_ID_vod__c,Territory_To_Add_vod__c,Territory_to_Drop_vod__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,external_id_vod__c,territory_vod__c,mobile_id_vod__c,territory_to_add_vod__c,territory_to_drop_vod__c +src02.crm_account_territory_loader_vod__c +org02.crm_account_territory_loader_vod__c + + diff --git a/s3/data/crm/settings/CRM_Account_ex.sql b/s3/data/crm/settings/CRM_Account_ex.sql new file mode 100644 index 00000000..792d7a28 --- /dev/null +++ b/s3/data/crm/settings/CRM_Account_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_account', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Approved_Document_vod__c.txt b/s3/data/crm/settings/CRM_Approved_Document_vod__c.txt new file mode 100644 index 00000000..23683a8c --- /dev/null +++ b/s3/data/crm/settings/CRM_Approved_Document_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +53 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Detail_Group_vod__c,Document_Description_vod__c,Document_Host_URL_vod__c,Document_ID_vod__c,Document_Last_Mod_DateTime_vod__c,Email_Allows_Documents_vod__c,Email_Domain_vod__c,Email_Fragment_HTML_vod__c,Email_From_Address_vod__c,Email_From_Name_vod__c,Email_HTML_1_vod__c,Email_HTML_2_vod__c,Email_ReplyTo_Address_vod__c,Email_ReplyTo_Name_vod__c,Email_Subject_vod__c,Email_Template_Fragment_Document_ID_vod__c,Email_Template_Fragment_HTML_vod__c,ISI_Document_ID_vod__c,Language_vod__c,Other_Document_ID_List_vod__c,PI_Document_ID_vod__c,Piece_Document_ID_vod__c,Product_vod__c,Status_vod__c,Territory_vod__c,Vault_Instance_ID_vod__c,Allow_Any_Product_Fragment_vod__c,Allowed_Document_IDs_vod__c,Engage_Document_Id_vod__c,Vault_Document_ID_vod__c,Key_Message_vod__c,Events_Management_Subtype_vod__c,Survey_vod__c,Content_Type_vod__c,Bcc_vod__c,Audience_vod__c,WeChat_Template_ID_vod__c,Check_Consent_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,detail_group_vod__c,document_description_vod__c,document_host_url_vod__c,document_id_vod__c,document_last_mod_date_time_vod__c,email_allows_documents_vod__c,email_domain_vod__c,email_fragment_html_vod__c,email_from_address_vod__c,email_from_name_vod__c,email_html_1_vod__c,email_html_2_vod__c,email_reply_to_address_vod__c,email_reply_to_name_vod__c,email_subject_vod__c,email_template_fragment_document_id_vod__c,email_template_fragment_html_vod__c,isi_document_id_vod__c,language_vod__c,other_document_id_list_vod__c,pi_document_id_vod__c,piece_document_id_vod__c,product_vod__c,status_vod__c,territory_vod__c,vault_instance_id_vod__c,allow_any_product_fragment_vod__c,allowed_document_ids_vod__c,engage_document_id_vod__c,vault_document_id_vod__c,key_message_vod__c,events_management_subtype_vod__c,survey_vod__c,content_type_vod__c,bcc_vod__c,audience_vod__c,we_chat_template_id_vod__c,check_consent_vod__c +src02.crm_approved_document_vod__c +org02.crm_approved_document_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call2_Detail_vod__c.txt b/s3/data/crm/settings/CRM_Call2_Detail_vod__c.txt new file mode 100644 index 00000000..67a53029 --- /dev/null +++ b/s3/data/crm/settings/CRM_Call2_Detail_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +17 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Is_Parent_Call_vod__c,Call2_vod__c,Product_vod__c,Detail_Priority_vod__c,Mobile_ID_vod__c,Override_Lock_vod__c,Type_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,is_parent_call_vod__c,call2_vod__c,product_vod__c,detail_priority_vod__c,mobile_id_vod__c,override_lock_vod__c,type_vod__c +src02.crm_call2_detail_vod__c +org02.crm_call2_detail_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call2_Discussion_vod__c.txt b/s3/data/crm/settings/CRM_Call2_Discussion_vod__c.txt new file mode 100644 index 00000000..c86c840c --- /dev/null +++ b/s3/data/crm/settings/CRM_Call2_Discussion_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +44 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Account_vod__c,Call2_vod__c,Activity__c,Comments__c,Contact_vod__c,Call_Date_vod__c,Product_Strategy_vod__c,Product_Tactic_vod__c,Restricted_Comments__c,Product_vod__c,Presentation__c,Discussion_Topics__c,Slides__c,User_vod__c,Indication__c,Mobile_ID_vod__c,Medical_Event_vod__c,Is_Parent_Call_vod__c,Override_Lock_vod__c,zvod_Product_Map_vod__c,Attendee_Type_vod__c,Entity_Reference_Id_vod__c,Account_Tactic_vod__c,MSJ_Material_Type__c,MSJ_Discussion_Contents__c,MSJ_IST_Minutes__c,MSJ_Off_Label_Minutes__c,MSJ_Discussion_Objectives__c,MSJ_Insight__c,EMDS_Materials__c,EMDS_Topic__c,MSJ_Visit_Purpose__c,MSJ_Insight_Count__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,account_vod__c,call2_vod__c,activity__c,comments__c,contact_vod__c,call_date_vod__c,product_strategy_vod__c,product_tactic_vod__c,restricted_comments__c,product_vod__c,presentation__c,discussion_topics__c,slides__c,user_vod__c,indication__c,mobile_id_vod__c,medical_event_vod__c,is_parent_call_vod__c,override_lock_vod__c,zvod_product_map_vod__c,attendee_type_vod__c,entity_reference_id_vod__c,account_tactic_vod__c,msj_material_type__c,msj_discussion_contents__c,msj_ist_minutes__c,msj_off_label_minutes__c,msj_discussion_objectives__c,msj_insight__c,emds_materials__c,emds_topic__c,msj_visit_purpose__c,msj_insight_count__c +src02.crm_call2_discussion_vod__c +org02.crm_call2_discussion_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call2_Key_Message_vod__c.txt b/s3/data/crm/settings/CRM_Call2_Key_Message_vod__c.txt new file mode 100644 index 00000000..40267bd0 --- /dev/null +++ b/s3/data/crm/settings/CRM_Call2_Key_Message_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +35 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Account_vod__c,Call2_vod__c,Reaction_vod__c,Product_vod__c,Key_Message_vod__c,Mobile_ID_vod__c,Contact_vod__c,Call_Date_vod__c,User_vod__c,Category_vod__c,Vehicle_vod__c,Is_Parent_Call_vod__c,Override_Lock_vod__c,CLM_ID_vod__c,Slide_Version_vod__c,Duration_vod__c,Presentation_ID_vod__c,Start_Time_vod__c,Attendee_Type_vod__c,Entity_Reference_Id_vod__c,Segment_vod__c,Display_Order_vod__c,Clm_Presentation_Name_vod__c,Clm_Presentation_Version_vod__c,Clm_Presentation_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,account_vod__c,call2_vod__c,reaction_vod__c,product_vod__c,key_message_vod__c,mobile_id_vod__c,contact_vod__c,call_date_vod__c,user_vod__c,category_vod__c,vehicle_vod__c,is_parent_call_vod__c,override_lock_vod__c,clm_id_vod__c,slide_version_vod__c,duration_vod__c,presentation_id_vod__c,start_time_vod__c,attendee_type_vod__c,entity_reference_id_vod__c,segment_vod__c,display_order_vod__c,clm_presentation_name_vod__c,clm_presentation_version_vod__c,clm_presentation_vod__c +src02.crm_call2_key_message_vod__c +org02.crm_call2_key_message_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call2_vod__c.txt b/s3/data/crm/settings/CRM_Call2_vod__c.txt new file mode 100644 index 00000000..6b2c330e --- /dev/null +++ b/s3/data/crm/settings/CRM_Call2_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +205 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Call_Comments_vod__c,Sample_Card_vod__c,Add_Detail_vod__c,Property_vod__c,Account_vod__c,zvod_Product_Discussion_vod__c,Status_vod__c,Parent_Address_vod__c,Account_Plan_vod__c,zvod_SaveNew_vod__c,Next_Call_Notes_vod__c,Pre_Call_Notes_vod__c,Mobile_ID_vod__c,zvod_Account_Credentials_vod_c_vod__c,zvod_Account_Preferred_Name_vod_c_vod__c,zvod_Account_Sample_Status_vod_c_vod__c,zvod_Attendees_vod__c,zvod_Key_Messages_vod__c,zvod_Detailing_vod__c,zvod_Expenses_vod__c,zvod_Followup_vod__c,zvod_Samples_vod__c,zvod_Save_vod__c,zvod_Submit_vod__c,zvod_Delete_vod__c,Activity_Type__c,Significant_Event__c,Location_vod__c,Subject_vod__c,Unlock_vod__c,Call_Datetime_vod__c,Disbursed_To_vod__c,Disclaimer_vod__c,Request_Receipt_vod__c,Signature_Date_vod__c,Signature_vod__c,Territory_vod__c,Submitted_By_Mobile_vod__c,Call_Type_vod__c,Add_Key_Message_vod__c,Address_vod__c,Attendees_vod__c,Attendee_Type_vod__c,Call_Date_vod__c,Detailed_Products_vod__c,No_Disbursement_vod__c,Parent_Call_vod__c,User_vod__c,Contact_vod__c,zvod_Entity_vod__c,Medical_Event_vod__c,Mobile_Created_Datetime_vod__c,Mobile_Last_Modified_Datetime_vod__c,License_vod__c,Is_Parent_Call_vod__c,Entity_Display_Name_vod__c,Override_Lock_vod__c,Last_Device_vod__c,Ship_Address_Line_1_vod__c,Ship_Address_Line_2_vod__c,Ship_City_vod__c,Ship_Country_vod__c,Ship_License_Expiration_Date_vod__c,Ship_License_Status_vod__c,Ship_License_vod__c,Ship_State_vod__c,Ship_To_Address_vod__c,Ship_Zip_vod__c,Ship_To_Address_Text_vod__c,CLM_vod__c,zvod_CLMDetails_vod__c,Is_Sampled_Call_vod__c,zvod_Surveys_vod__c,Presentations_vod__c,Entity_Reference_Id_vod__c,Error_Reference_Call_vod__c,Duration_vod__c,Color_vod__c,Allowed_Products_vod__c,zvod_Attachments_vod__c,Sample_Card_Reason_vod__c,ASSMCA_vod__c,Address_Line_1_vod__c,Address_Line_2_vod__c,City_vod__c,DEA_Address_Line_1_vod__c,DEA_Address_Line_2_vod__c,DEA_Address_vod__c,DEA_City_vod__c,DEA_Expiration_Date_vod__c,DEA_State_vod__c,DEA_Zip_4_vod__c,DEA_Zip_vod__c,DEA_vod__c,Ship_Zip_4_vod__c,State_vod__c,Zip_4_vod__c,Zip_vod__c,Sample_Send_Card_vod__c,zvod_Address_vod_c_DEA_Status_vod_c_vod__c,Signature_Page_Image_vod__c,Credentials_vod__c,Salutation_vod__c,zvod_Account_Call_Reminder_vod_c_vod__c,MSJ_Meeting_Duration__c,MSJ_Double_Visit_AM__c,zvod_Business_Account_vod__c,Product_Priority_1_vod__c,Product_Priority_2_vod__c,Product_Priority_3_vod__c,Product_Priority_4_vod__c,Product_Priority_5_vod__c,zvod_More_Actions_vod__c,zvod_Call_Conflict_Status_vod__c,Signature_Timestamp_vod__c,Expense_Amount_vod__c,Total_Expense_Attendees_Count_vod__c,Attendee_list_vod__c,Expense_Post_Status_vod__c,Attendee_Post_Status_vod__c,Expense_System_External_ID_vod__c,Incurred_Expense_vod__c,Assigner_vod__c,Assignment_Datetime_vod__c,zvod_Call_Objective_vod__c,Signature_Location_Longitude_vod__c,Signature_Location_Latitude_vod__c,Location_Services_Status_vod__c,MSJ_Double_Visit_Other__c,MSJ_Comment__c,MSJ_For_Reporting__c,MSJ_Number_of_Attendees__c,MSJ_Main_Dept__c,Planned_Type_vjh__c,Cobrowse_URL_Participant_vod__c,MSJ_Activity_Method_Text__c,MSJ_Activity_Method__c,MSJ_Classification__c,MSJ_Double_Visit_MSL__c,MSJ_MSL_Comment_for_MR__c,MSJ_APD__c,Medical_Inquiry_vod__c,MSJ_Call_Type_MSJ__c,MSJ_Prescription_Request__c,MSJ_Patient_Follow__c,Child_Account_Id_vod__c,Child_Account_vod__c,Location_Id_vod__c,Location_Name_vod__c,MSJ_Comments_about_technology__c,Remote_Meeting_vod__c,Veeva_Remote_Meeting_Id_vod__c,MSJ_Activity_Type_Report__c,MSJ_Activity_Type__c,MSJ_Activity__c,MSJ_Comments__c,MSJ_Therapy__c,MSJ_Time_Hrs__c,EMDS_CO_Reference__c,EMDS_Call_Sub_Type__c,EMDS_Call_Type__c,EMDS_Call_Unsuccessful__c,EMDS_Congress_Type__c,EMDS_Date_of_Service__c,EMDS_Fertility_DisInterest__c,EMDS_Fertility_Interest__c,EMDS_Installed_Equipment__c,EMDS_Pipeline_Stage_Value__c,EMDS_Pipeline_Stage__c,EMDS_Pipeline__c,EMDS_Reason_for_Call__c,EMDS_Training_Completed__c,MSJ_BrainStorming__c,MSJ_SIPAGL_1A__c,MSJ_SIPAGL_1B__c,MSJ_SIPAGL_2__c,MSJ_SIPAGL_3__c,MSJ_SIPAGL_4A__c,MSJ_SIPAGL_5A__c,MSJ_SIPAGL_comment__c,MSJ_SIPAGL_4B__c,MSJ_SIPAGL_5B__c,Location_Text_vod__c,Call_Channel_vod__c,MSJ_Scientific_Interaction__c,MSJ_Activity_Email_Reply__c,MSJ_Interaction_Duration__c,MSJ_SIPAGL_1A_date__c,MSJ_CoPromotion__c,Call_Channel_Formula_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,call_comments_vod__c,sample_card_vod__c,add_detail_vod__c,property_vod__c,account_vod__c,zvod_product_discussion_vod__c,status_vod__c,parent_address_vod__c,account_plan_vod__c,zvod_save_new_vod__c,next_call_notes_vod__c,pre_call_notes_vod__c,mobile_id_vod__c,zvod_account_credentials_vod_c_vod__c,zvod_account_preferred_name_vod_c_vod__c,zvod_account_sample_status_vod_c_vod__c,zvod_attendees_vod__c,zvod_key_messages_vod__c,zvod_detailing_vod__c,zvod_expenses_vod__c,zvod_followup_vod__c,zvod_samples_vod__c,zvod_save_vod__c,zvod_submit_vod__c,zvod_delete_vod__c,activity_type__c,significant_event__c,location_vod__c,subject_vod__c,unlock_vod__c,call_datetime_vod__c,disbursed_to_vod__c,disclaimer_vod__c,request_receipt_vod__c,signature_date_vod__c,signature_vod__c,territory_vod__c,submitted_by_mobile_vod__c,call_type_vod__c,add_key_message_vod__c,address_vod__c,attendees_vod__c,attendee_type_vod__c,call_date_vod__c,detailed_products_vod__c,no_disbursement_vod__c,parent_call_vod__c,user_vod__c,contact_vod__c,zvod_entity_vod__c,medical_event_vod__c,mobile_created_datetime_vod__c,mobile_last_modified_datetime_vod__c,license_vod__c,is_parent_call_vod__c,entity_display_name_vod__c,override_lock_vod__c,last_device_vod__c,ship_address_line_1_vod__c,ship_address_line_2_vod__c,ship_city_vod__c,ship_country_vod__c,ship_license_expiration_date_vod__c,ship_license_status_vod__c,ship_license_vod__c,ship_state_vod__c,ship_to_address_vod__c,ship_zip_vod__c,ship_to_address_text_vod__c,clm_vod__c,zvod_clmdetails_vod__c,is_sampled_call_vod__c,zvod_surveys_vod__c,presentations_vod__c,entity_reference_id_vod__c,error_reference_call_vod__c,duration_vod__c,color_vod__c,allowed_products_vod__c,zvod_attachments_vod__c,sample_card_reason_vod__c,assmca_vod__c,address_line_1_vod__c,address_line_2_vod__c,city_vod__c,dea_address_line_1_vod__c,dea_address_line_2_vod__c,dea_address_vod__c,dea_city_vod__c,dea_expiration_date_vod__c,dea_state_vod__c,dea_zip_4_vod__c,dea_zip_vod__c,dea_vod__c,ship_zip_4_vod__c,state_vod__c,zip_4_vod__c,zip_vod__c,sample_send_card_vod__c,zvod_address_vod_c_dea_status_vod_c_vod__c,signature_page_image_vod__c,credentials_vod__c,salutation_vod__c,zvod_account_call_reminder_vod_c_vod__c,msj_meeting_duration__c,msj_double_visit_am__c,zvod_business_account_vod__c,product_priority_1_vod__c,product_priority_2_vod__c,product_priority_3_vod__c,product_priority_4_vod__c,product_priority_5_vod__c,zvod_more_actions_vod__c,zvod_call_conflict_status_vod__c,signature_timestamp_vod__c,expense_amount_vod__c,total_expense_attendees_count_vod__c,attendee_list_vod__c,expense_post_status_vod__c,attendee_post_status_vod__c,expense_system_external_id_vod__c,incurred_expense_vod__c,assigner_vod__c,assignment_datetime_vod__c,zvod_call_objective_vod__c,signature_location_longitude_vod__c,signature_location_latitude_vod__c,location_services_status_vod__c,msj_double_visit_other__c,msj_comment__c,msj_for_reporting__c,msj_number_of_attendees__c,msj_main_dept__c,planned_type_vjh__c,cobrowse_url_participant_vod__c,msj_activity_method_text__c,msj_activity_method__c,msj_classification__c,msj_double_visit_msl__c,msj_msl_comment_for_mr__c,msj_apd__c,medical_inquiry_vod__c,msj_call_type_msj__c,msj_prescription_request__c,msj_patient_follow__c,child_account_id_vod__c,child_account_vod__c,location_id_vod__c,location_name_vod__c,msj_comments_about_technology__c,remote_meeting_vod__c,veeva_remote_meeting_id_vod__c,msj_activity_type_report__c,msj_activity_type__c,msj_activity__c,msj_comments__c,msj_therapy__c,msj_time_hrs__c,emds_co_reference__c,emds_call_sub_type__c,emds_call_type__c,emds_call_unsuccessful__c,emds_congress_type__c,emds_date_of_service__c,emds_fertility_dis_interest__c,emds_fertility_interest__c,emds_installed_equipment__c,emds_pipeline_stage_value__c,emds_pipeline_stage__c,emds_pipeline__c,emds_reason_for_call__c,emds_training_completed__c,msj_brain_storming__c,msj_sipagl_1_a__c,msj_sipagl_1_b__c,msj_sipagl_2__c,msj_sipagl_3__c,msj_sipagl_4_a__c,msj_sipagl_5_a__c,msj_sipagl_comment__c,msj_sipagl_4_b__c,msj_sipagl_5_b__c,location_text_vod__c,call_channel_vod__c,msj_scientific_interaction__c,msj_activity_email_reply__c,msj_interaction_duration__c,msj_sipagl_1_a_date__c,msj_co_promotion__c,call_channel_formula_vod__c +src02.crm_call2_vod__c +org02.crm_call2_vod__c + + diff --git a/s3/data/crm/settings/CRM_Call_Clickstream_vod__c.txt b/s3/data/crm/settings/CRM_Call_Clickstream_vod__c.txt new file mode 100644 index 00000000..e2b56535 --- /dev/null +++ b/s3/data/crm/settings/CRM_Call_Clickstream_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +34 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Answer_vod__c,Call_vod__c,Key_Message_vod__c,Mobile_ID_vod__c,Popup_Opened_vod__c,Possible_Answers_vod__c,Presentation_ID_vod__c,Product_vod__c,Range_Value_vod__c,Rollover_Entered_vod__c,Selected_Items_vod__c,CLM_ID_vod__c,Question_vod__c,Survey_Type_vod__c,Text_Entered_vod__c,Toggle_Button_On_vod__c,Track_Element_Description_vod__c,Track_Element_Id_vod__c,Track_Element_Type_vod__c,Usage_Duration_vod__c,Usage_Start_Time_vod__c,AuxillaryId_vod__c,ParentId_vod__c,Revision_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,answer_vod__c,call_vod__c,key_message_vod__c,mobile_id_vod__c,popup_opened_vod__c,possible_answers_vod__c,presentation_id_vod__c,product_vod__c,range_value_vod__c,rollover_entered_vod__c,selected_items_vod__c,clm_id_vod__c,question_vod__c,survey_type_vod__c,text_entered_vod__c,toggle_button_on_vod__c,track_element_description_vod__c,track_element_id_vod__c,track_element_type_vod__c,usage_duration_vod__c,usage_start_time_vod__c,auxillary_id_vod__c,parent_id_vod__c,revision_vod__c +src02.crm_call_clickstream_vod__c +org02.crm_call_clickstream_vod__c + + diff --git a/s3/data/crm/settings/CRM_Child_Account_vod__c.txt b/s3/data/crm/settings/CRM_Child_Account_vod__c.txt new file mode 100644 index 00000000..876bef05 --- /dev/null +++ b/s3/data/crm/settings/CRM_Child_Account_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +57 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Parent_Account_vod__c,Child_Account_vod__c,External_ID_vod__c,Mobile_ID_vod__c,Primary_vod__c,Copy_Address_vod__c,Child_Name_vod__c,Parent_Name_vod__c,Parent_Child_Name_vod__c,Account_Code__c,Child_Department__c,Child_Role__c,Child_Title__c,Child_Remark__c,MSJ_1C_segment__c,MSJ_BU_FE__c,MSJ_BU_ONC__c,MSJ_BVC_Segment__c,MSJ_CE_segment__c,MSJ_Child_Account_Link__c,MSJ_DCF_DR_Code__c,MSJ_DCF_HP_Code__c,MSJ_DR_Change_Log__c,MSJ_Delete__c,MSJ_Department__c,MSJ_EB_CRC_Segment__c,MSJ_EB_HN_Segment__c,MSJ_EB_H_N_LA_Segment__c,MSJ_EB_H_N_RM_Segment__c,MSJ_External_ID__c,MSJ_Fax__c,MSJ_GF2_segment__c,MSJ_GF_segment__c,MSJ_KOL_LOL__c,MSJ_KOL__c,MSJ_ONC_HP_Segment__c,MSJ_OPTIN_target__c,MSJ_OV_segment__c,MSJ_Parent_Child_Name__c,MSJ_Phone__c,MSJ_Remark__c,MSJ_Target_Call_Num__c,MSJ_Tech_segment__c,MSJ_Title__c,MSJ_XLK_Segment__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,parent_account_vod__c,child_account_vod__c,external_id_vod__c,mobile_id_vod__c,primary_vod__c,copy_address_vod__c,child_name_vod__c,parent_name_vod__c,parent_child_name_vod__c,account_code__c,child_department__c,child_role__c,child_title__c,child_remark__c,msj_1_c_segment__c,msj_bu_fe__c,msj_bu_onc__c,msj_bvc_segment__c,msj_ce_segment__c,msj_child_account_link__c,msj_dcf_dr_code__c,msj_dcf_hp_code__c,msj_dr_change_log__c,msj_delete__c,msj_department__c,msj_eb_crc_segment__c,msj_eb_hn_segment__c,msj_eb_h_n_la_segment__c,msj_eb_h_n_rm_segment__c,msj_external_id__c,msj_fax__c,msj_gf2_segment__c,msj_gf_segment__c,msj_kol_lol__c,msj_kol__c,msj_onc_hp_segment__c,msj_optin_target__c,msj_ov_segment__c,msj_parent_child_name__c,msj_phone__c,msj_remark__c,msj_target_call_num__c,msj_tech_segment__c,msj_title__c,msj_xlk_segment__c +src02.crm_child_account_vod__c +org02.crm_child_account_vod__c +CRM_Child_Account_vod__c_ex.sql + diff --git a/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql b/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql new file mode 100644 index 00000000..cac33f47 --- /dev/null +++ b/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_child_account_vod__c', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Clm_Presentation_Slide_vod__c.txt b/s3/data/crm/settings/CRM_Clm_Presentation_Slide_vod__c.txt new file mode 100644 index 00000000..422a6926 --- /dev/null +++ b/s3/data/crm/settings/CRM_Clm_Presentation_Slide_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +17 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Clm_Presentation_vod__c,Key_Message_vod__c,Display_Order_vod__c,Sub_Presentation_vod__c,Mobile_ID_vod__c,External_ID_vod__c,VExternal_Id_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,clm_presentation_vod__c,key_message_vod__c,display_order_vod__c,sub_presentation_vod__c,mobile_id_vod__c,external_id_vod__c,vexternal_id_vod__c +src02.crm_clm_presentation_slide_vod__c +org02.crm_clm_presentation_slide_vod__c + + diff --git a/s3/data/crm/settings/CRM_Clm_Presentation_vod__c.txt b/s3/data/crm/settings/CRM_Clm_Presentation_vod__c.txt new file mode 100644 index 00000000..364a308b --- /dev/null +++ b/s3/data/crm/settings/CRM_Clm_Presentation_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +46 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Mobile_ID_vod__c,Presentation_Id_vod__c,Product_vod__c,Default_Presentation_vod__c,Training_vod__c,ParentId_vod__c,Hidden_vod__c,Type_vod__c,Approved_vod__c,Copied_From_vod__c,Copy_Date_vod__c,Survey_vod__c,Original_Record_ID_vod__c,Directory_vod__c,End_Date_vod__c,Start_Date_vod__c,Status_vod__c,VExternal_Id_vod__c,Vault_DNS_vod__c,Vault_Doc_Id_vod__c,Vault_External_Id_vod__c,Vault_GUID_vod__c,Vault_Last_Modified_Date_Time_vod__c,Version_vod__c,Enable_Survey_Overlay_vod__c,Description_vod__c,Keywords_vod__c,Content_Channel_vod__c,original_material_approved_in_veritas__c,keywords__c,trade_team__c,ewizard_link__c,business_function__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,mobile_id_vod__c,presentation_id_vod__c,product_vod__c,default_presentation_vod__c,training_vod__c,parent_id_vod__c,hidden_vod__c,type_vod__c,approved_vod__c,copied_from_vod__c,copy_date_vod__c,survey_vod__c,original_record_id_vod__c,directory_vod__c,end_date_vod__c,start_date_vod__c,status_vod__c,vexternal_id_vod__c,vault_dns_vod__c,vault_doc_id_vod__c,vault_external_id_vod__c,vault_guid_vod__c,vault_last_modified_date_time_vod__c,version_vod__c,enable_survey_overlay_vod__c,description_vod__c,keywords_vod__c,content_channel_vod__c,original_material_approved_in_veritas__c,keywords__c,trade_team__c,ewizard_link__c,business_function__c +src02.crm_clm_presentation_vod__c +org02.crm_clm_presentation_vod__c + + diff --git a/s3/data/crm/settings/CRM_Coaching_Report_vod__c.txt b/s3/data/crm/settings/CRM_Coaching_Report_vod__c.txt new file mode 100644 index 00000000..fd64a473 --- /dev/null +++ b/s3/data/crm/settings/CRM_Coaching_Report_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +144 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Mobile_ID_vod__c,Manager_vod__c,Employee_vod__c,Review_Date__c,Review_Period__c,Status__c,Comments__c,Strategic_Planning__c,Customer_Focus__c,Knowledge_Expertise__c,Business_Account_Planning__c,Call_Productivity__c,Overall_Rating__c,MSJ_A01__c,MSJ_A02__c,MSJ_A03__c,MSJ_AM_Memo__c,MSJ_Aid_Total__c,MSJ_C0_GC__c,MSJ_C1_GC__c,MSJ_C2_GC__c,MSJ_Countermeasure__c,MSJ_Deadline__c,MSJ_Double_Visit_Time__c,MSJ_Hospital__c,MSJ_K01_FE__c,MSJ_K01_ONC__c,MSJ_K02_FE__c,MSJ_K02_ONC__c,MSJ_K03_FE__c,MSJ_K03_ONC__c,MSJ_K04_FE__c,MSJ_K04_ONC__c,MSJ_K05_FE__c,MSJ_K05_ONC__c,MSJ_K06_FE__c,MSJ_K06_ONC__c,MSJ_K0_GC__c,MSJ_K1_GC__c,MSJ_K2_GC__c,MSJ_Knowledge_Total__c,MSJ_L0_GC__c,MSJ_L1_GC__c,MSJ_L2_GC__c,MSJ_MR_GC__c,MSJ_MR_Problems__c,MSJ_N0_GC__c,MSJ_N1_GC__c,MSJ_N2_GC__c,MSJ_Num_of_DTL__c,MSJ_P01__c,MSJ_P02__c,MSJ_P03__c,MSJ_P04__c,MSJ_P05__c,MSJ_P0_GC__c,MSJ_P1_GC__c,MSJ_P2_GC__c,MSJ_PlanningTotal__c,MSJ_R0_GC__c,MSJ_R1_GC__c,MSJ_R2_GC__c,MSJ_S01__c,MSJ_S02__c,MSJ_S03__c,MSJ_S04__c,MSJ_S05__c,MSJ_S06__c,MSJ_S07__c,MSJ_S08__c,MSJ_S09__c,MSJ_S10__c,MSJ_S11__c,MSJ_S12__c,MSJ_Skill_Total__c,MSJ_After_Call_01__c,MSJ_After_Call_02__c,MSJ_After_Call_03__c,MSJ_After_Call_04__c,MSJ_Closing__c,MSJ_Comment_by_MR__c,MSJ_Confirmed_by_MR__c,MSJ_Createdby__c,MSJ_FT_AM_Name__c,MSJ_Interview_Preparation__c,MSJ_Interview_Reflection__c,MSJ_Notify_To_MR__c,MSJ_Opening__c,MSJ_Others_01_Result__c,MSJ_Others_01__c,MSJ_Others_02_Result__c,MSJ_Others_02__c,MSJ_Patient_Thinking__c,MSJ_Probing__c,MSJ_Supporting__c,MSJ_Patient_Thinking_for_FE__c,MSJ_After_Call_05__c,MSJ_After_Call_06__c,MSJ_After_Call_07__c,MSJ_After_Call_08__c,MSJ_Createdby_FE__c,MSJ_Createdby_ONC__c,MSJ_Development_Level__c,MSJ_Interview_Prep_01__c,MSJ_Interview_Prep_02__c,MSJ_Leadership_Style__c,MSJ_Overcome_01__c,MSJ_Overcome_02__c,MSJ_Overcome_03__c,MSJ_Overcome_04__c,MSJ_Review_01__c,MSJ_Review_02__c,MSJ_SK_01__c,MSJ_SK_02__c,MSJ_SK_03__c,MSJ_SK_04__c,MSJ_SK_05__c,MSJ_SK_06__c,MSJ_SK_07__c,MSJ_SK_08__c,MSJ_SK_09__c,MSJ_SK_10__c,MSJ_Specific_Action__c,MSJ_Training_Point__c,MSJ_Efforts_of_Year__c,MSJ_Efforts_of_Month__c,MSJ_Skill_Task__c,MSJ_Action_of_This_Month__c,MSJ_Achievement_of_This_Month__c,MSJ_Comment_from_AM__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,mobile_id_vod__c,manager_vod__c,employee_vod__c,review_date__c,review_period__c,status__c,comments__c,strategic_planning__c,customer_focus__c,knowledge_expertise__c,business_account_planning__c,call_productivity__c,overall_rating__c,msj_a01__c,msj_a02__c,msj_a03__c,msj_am_memo__c,msj_aid_total__c,msj_c0_gc__c,msj_c1_gc__c,msj_c2_gc__c,msj_countermeasure__c,msj_deadline__c,msj_double_visit_time__c,msj_hospital__c,msj_k01_fe__c,msj_k01_onc__c,msj_k02_fe__c,msj_k02_onc__c,msj_k03_fe__c,msj_k03_onc__c,msj_k04_fe__c,msj_k04_onc__c,msj_k05_fe__c,msj_k05_onc__c,msj_k06_fe__c,msj_k06_onc__c,msj_k0_gc__c,msj_k1_gc__c,msj_k2_gc__c,msj_knowledge_total__c,msj_l0_gc__c,msj_l1_gc__c,msj_l2_gc__c,msj_mr_gc__c,msj_mr_problems__c,msj_n0_gc__c,msj_n1_gc__c,msj_n2_gc__c,msj_num_of_dtl__c,msj_p01__c,msj_p02__c,msj_p03__c,msj_p04__c,msj_p05__c,msj_p0_gc__c,msj_p1_gc__c,msj_p2_gc__c,msj_planning_total__c,msj_r0_gc__c,msj_r1_gc__c,msj_r2_gc__c,msj_s01__c,msj_s02__c,msj_s03__c,msj_s04__c,msj_s05__c,msj_s06__c,msj_s07__c,msj_s08__c,msj_s09__c,msj_s10__c,msj_s11__c,msj_s12__c,msj_skill_total__c,msj_after_call_01__c,msj_after_call_02__c,msj_after_call_03__c,msj_after_call_04__c,msj_closing__c,msj_comment_by_mr__c,msj_confirmed_by_mr__c,msj_createdby__c,msj_ft_am_name__c,msj_interview_preparation__c,msj_interview_reflection__c,msj_notify_to_mr__c,msj_opening__c,msj_others_01_result__c,msj_others_01__c,msj_others_02_result__c,msj_others_02__c,msj_patient_thinking__c,msj_probing__c,msj_supporting__c,msj_patient_thinking_for_fe__c,msj_after_call_05__c,msj_after_call_06__c,msj_after_call_07__c,msj_after_call_08__c,msj_createdby_fe__c,msj_createdby_onc__c,msj_development_level__c,msj_interview_prep_01__c,msj_interview_prep_02__c,msj_leadership_style__c,msj_overcome_01__c,msj_overcome_02__c,msj_overcome_03__c,msj_overcome_04__c,msj_review_01__c,msj_review_02__c,msj_sk_01__c,msj_sk_02__c,msj_sk_03__c,msj_sk_04__c,msj_sk_05__c,msj_sk_06__c,msj_sk_07__c,msj_sk_08__c,msj_sk_09__c,msj_sk_10__c,msj_specific_action__c,msj_training_point__c,msj_efforts_of_year__c,msj_efforts_of_month__c,msj_skill_task__c,msj_action_of_this_month__c,msj_achievement_of_this_month__c,msj_comment_from_am__c +src02.crm_coaching_report_vod__c +org02.crm_coaching_report_vod__c + + diff --git a/s3/data/crm/settings/CRM_Consent_Header_vod__c.txt b/s3/data/crm/settings/CRM_Consent_Header_vod__c.txt new file mode 100644 index 00000000..729c3452 --- /dev/null +++ b/s3/data/crm/settings/CRM_Consent_Header_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +22 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consent_Header_Help_Text_vod__c,Country_vod__c,Inactive_Datetime_vod__c,Language_vod__c,Status_vod__c,Signature_Required_On_Opt_Out_vod__c,Request_Receipt_vod__c,Subscription_Option_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consent_header_help_text_vod__c,country_vod__c,inactive_datetime_vod__c,language_vod__c,status_vod__c,signature_required_on_opt_out_vod__c,request_receipt_vod__c,subscription_option_vod__c +src02.crm_consent_header_vod__c +org02.crm_consent_header_vod__c + + diff --git a/s3/data/crm/settings/CRM_Consent_Line_vod__c.txt b/s3/data/crm/settings/CRM_Consent_Line_vod__c.txt new file mode 100644 index 00000000..3cc95e31 --- /dev/null +++ b/s3/data/crm/settings/CRM_Consent_Line_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +25 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consent_Type_vod__c,Detail_Group_Display_Name_vod__c,Detail_Group_vod__c,Display_Order_vod__c,End_Date_vod__c,Group_By_vod__c,Product_Display_Name_vod__c,Product_vod__c,Start_Date_vod__c,Sub_Channel_Description_vod__c,Sub_Channel_Display_Name_vod__c,Sub_Channel_Key_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consent_type_vod__c,detail_group_display_name_vod__c,detail_group_vod__c,display_order_vod__c,end_date_vod__c,group_by_vod__c,product_display_name_vod__c,product_vod__c,start_date_vod__c,sub_channel_description_vod__c,sub_channel_display_name_vod__c,sub_channel_key_vod__c +src02.crm_consent_line_vod__c +org02.crm_consent_line_vod__c + + diff --git a/s3/data/crm/settings/CRM_Consent_Type_vod__c.txt b/s3/data/crm/settings/CRM_Consent_Type_vod__c.txt new file mode 100644 index 00000000..2a3451a0 --- /dev/null +++ b/s3/data/crm/settings/CRM_Consent_Type_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +26 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consent_Header_vod__c,Channel_Label_vod__c,Channel_Source_vod__c,Consent_Expires_In_vod__c,Default_Consent_Type_vod__c,Disclaimer_Text_vod__c,Display_Order_vod__c,Product_Preference_vod__c,zvod_Consent_Default_Consent_Text_vod__c,zvod_Consent_Line_vod__c,zvod_Signature_Capture_vod__c,Double_Opt_In_vod__c,zvod_Consent_Activity_Tracking_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consent_header_vod__c,channel_label_vod__c,channel_source_vod__c,consent_expires_in_vod__c,default_consent_type_vod__c,disclaimer_text_vod__c,display_order_vod__c,product_preference_vod__c,zvod_consent_default_consent_text_vod__c,zvod_consent_line_vod__c,zvod_signature_capture_vod__c,double_opt_in_vod__c,zvod_consent_activity_tracking_vod__c +src02.crm_consent_type_vod__c +org02.crm_consent_type_vod__c + + diff --git a/s3/data/crm/settings/CRM_Contact.txt b/s3/data/crm/settings/CRM_Contact.txt new file mode 100644 index 00000000..21a9c5dd --- /dev/null +++ b/s3/data/crm/settings/CRM_Contact.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +71 +Id,IsDeleted,MasterRecordId,AccountId,IsPersonAccount,LastName,FirstName,Salutation,Name,OtherStreet,OtherCity,OtherState,OtherPostalCode,OtherCountry,OtherLatitude,OtherLongitude,OtherGeocodeAccuracy,OtherAddress,MailingStreet,MailingCity,MailingState,MailingPostalCode,MailingCountry,MailingLatitude,MailingLongitude,MailingGeocodeAccuracy,MailingAddress,Phone,Fax,MobilePhone,HomePhone,OtherPhone,AssistantPhone,ReportsToId,Email,Title,Department,AssistantName,Birthdate,Description,OwnerId,HasOptedOutOfEmail,HasOptedOutOfFax,DoNotCall,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,LastCURequestDate,LastCUUpdateDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,EmailBouncedReason,EmailBouncedDate,IsEmailBounced,PhotoUrl,Jigsaw,JigsawContactId,IndividualId,Mobile_ID_vod__c,H1Insights__H1_NPI_Value_for_Testing__c,H1Insights__H1_Person_ID__c,H1Insights__H1_Request_Status__c,H1Insights__H1_URL__c,H1Insights__NPI_Number__c,H1Insights__NPI_Number_for_H1_Insights__c,MSJ_Marketing_Cloud_Integration__c +id,is_deleted,master_record_id,account_id,is_person_account,last_name,first_name,salutation,name,other_street,other_city,other_state,other_postal_code,other_country,other_latitude,other_longitude,other_geocode_accuracy,other_address,mailing_street,mailing_city,mailing_state,mailing_postal_code,mailing_country,mailing_latitude,mailing_longitude,mailing_geocode_accuracy,mailing_address,phone,fax,mobile_phone,home_phone,other_phone,assistant_phone,reports_to_id,email,title,department,assistant_name,birthdate,description,owner_id,has_opted_out_of_email,has_opted_out_of_fax,do_not_call,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,last_curequest_date,last_cuupdate_date,may_edit,is_locked,last_viewed_date,last_referenced_date,email_bounced_reason,email_bounced_date,is_email_bounced,photo_url,jigsaw,jigsaw_contact_id,individual_id,mobile_id_vod__c,h1_insights__h1_npi_value_for_testing__c,h1_insights__h1_person_id__c,h1_insights__h1_request_status__c,h1_insights__h1_url__c,h1_insights__npi_number__c,h1_insights__npi_number_for_h1_insights__c,msj_marketing_cloud_integration__c +src02.crm_contact +org02.crm_contact +CRM_Contact_ex.sql + diff --git a/s3/data/crm/settings/CRM_Contact_ex.sql b/s3/data/crm/settings/CRM_Contact_ex.sql new file mode 100644 index 00000000..4b916e23 --- /dev/null +++ b/s3/data/crm/settings/CRM_Contact_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_contact', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Dynamic_Attribute_Configuration_vod__c.txt b/s3/data/crm/settings/CRM_Dynamic_Attribute_Configuration_vod__c.txt new file mode 100644 index 00000000..d48828a7 --- /dev/null +++ b/s3/data/crm/settings/CRM_Dynamic_Attribute_Configuration_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +29 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Applies_To_vod__c,Attribute_Label_vod__c,Attribute_Name_vod__c,Available_Values_vod__c,Description_vod__c,Detail_Group_vod__c,Display_Order_vod__c,External_ID_vod__c,Help_Text_vod__c,Product_vod__c,Read_Only_vod__c,Section_Name_vod__c,Sharing_Group_vod__c,Status_vod__c,Track_Changes_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,applies_to_vod__c,attribute_label_vod__c,attribute_name_vod__c,available_values_vod__c,description_vod__c,detail_group_vod__c,display_order_vod__c,external_id_vod__c,help_text_vod__c,product_vod__c,read_only_vod__c,section_name_vod__c,sharing_group_vod__c,status_vod__c,track_changes_vod__c +src02.crm_dynamic_attribute_configuration_vod__c +org02.crm_dynamic_attribute_configuration_vod__c + + diff --git a/s3/data/crm/settings/CRM_Dynamic_Attribute_vod__c.txt b/s3/data/crm/settings/CRM_Dynamic_Attribute_vod__c.txt new file mode 100644 index 00000000..adea9267 --- /dev/null +++ b/s3/data/crm/settings/CRM_Dynamic_Attribute_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +26 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Account_vod__c,Active_vod__c,Dynamic_Attribute_Configuration_vod__c,Dynamic_Attribute_Description_vod__c,Dynamic_Attribute_Help_Text_vod__c,Dynamic_Attribute_Label_vod__c,Dynamic_Attribute_Name_vod__c,Dynamic_Attribute_Record_Type_vod__c,Dynamic_Attribute_Value_Checkbox_vod__c,Dynamic_Attribute_Value_Date_Time_vod__c,Dynamic_Attribute_Value_Date_vod__c,Dynamic_Attribute_Value_Number_vod__c,Dynamic_Attribute_Value_Text_Area_vod__c,Dynamic_Attribute_Value_Text_vod__c,Mobile_ID_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,account_vod__c,active_vod__c,dynamic_attribute_configuration_vod__c,dynamic_attribute_description_vod__c,dynamic_attribute_help_text_vod__c,dynamic_attribute_label_vod__c,dynamic_attribute_name_vod__c,dynamic_attribute_record_type_vod__c,dynamic_attribute_value_checkbox_vod__c,dynamic_attribute_value_date_time_vod__c,dynamic_attribute_value_date_vod__c,dynamic_attribute_value_number_vod__c,dynamic_attribute_value_text_area_vod__c,dynamic_attribute_value_text_vod__c,mobile_id_vod__c +src02.crm_dynamic_attribute_vod__c +org02.crm_dynamic_attribute_vod__c + + diff --git a/s3/data/crm/settings/CRM_Email_Activity_vod__c.txt b/s3/data/crm/settings/CRM_Email_Activity_vod__c.txt new file mode 100644 index 00000000..2c801e06 --- /dev/null +++ b/s3/data/crm/settings/CRM_Email_Activity_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +36 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,Sent_Email_vod__c,Activity_DateTime_vod__c,City_vod__c,Click_URL_vod__c,Client_Name_vod__c,Client_OS_vod__c,Client_Type_vod__c,Country_vod__c,Device_Type_vod__c,Event_Msg_vod__c,Event_type_vod__c,IP_Address_vod__c,Region_vod__c,User_Agent_vod__c,Vault_Doc_ID_vod__c,Vault_Doc_Name_vod__c,Vault_Document_Major_Version_vod__c,Vault_Document_Minor_Version_vod__c,Vault_Document_Number_vod__c,Vault_Document_Title_vod__c,Vault_Instance_ID_vod__c,Preference_Modification_vod__c,Approved_Document_vod__c,Link_Name_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,sent_email_vod__c,activity_date_time_vod__c,city_vod__c,click_url_vod__c,client_name_vod__c,client_os_vod__c,client_type_vod__c,country_vod__c,device_type_vod__c,event_msg_vod__c,event_type_vod__c,ip_address_vod__c,region_vod__c,user_agent_vod__c,vault_doc_id_vod__c,vault_doc_name_vod__c,vault_document_major_version_vod__c,vault_document_minor_version_vod__c,vault_document_number_vod__c,vault_document_title_vod__c,vault_instance_id_vod__c,preference_modification_vod__c,approved_document_vod__c,link_name_vod__c +src02.crm_email_activity_vod__c +org02.crm_email_activity_vod__c + + diff --git a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt new file mode 100644 index 00000000..d5d6c0dc --- /dev/null +++ b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +31 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c +src02.crm_event_attendee_vod__c +org02.crm_event_attendee_vod__c + + diff --git a/s3/data/crm/settings/CRM_Group.txt b/s3/data/crm/settings/CRM_Group.txt new file mode 100644 index 00000000..94f4d0ca --- /dev/null +++ b/s3/data/crm/settings/CRM_Group.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +14 +Id,Name,DeveloperName,RelatedId,Type,Email,OwnerId,DoesSendEmailToMembers,DoesIncludeBosses,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp +id,name,developer_name,related_id,type,email,owner_id,does_send_email_to_members,does_include_bosses,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp +src02.crm_group +org02.crm_group +CRM_Group_ex.sql + diff --git a/s3/data/crm/settings/CRM_Group_ex.sql b/s3/data/crm/settings/CRM_Group_ex.sql new file mode 100644 index 00000000..5e50dab6 --- /dev/null +++ b/s3/data/crm/settings/CRM_Group_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_group', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Key_Message_vod__c.txt b/s3/data/crm/settings/CRM_Key_Message_vod__c.txt new file mode 100644 index 00000000..1c1f1df0 --- /dev/null +++ b/s3/data/crm/settings/CRM_Key_Message_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +48 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Description_vod__c,Product_vod__c,Product_Strategy_vod__c,Display_Order_vod__c,Active_vod__c,Category_vod__c,Vehicle_vod__c,CLM_ID_vod__c,Custom_Reaction_vod__c,Slide_Version_vod__c,Language_vod__c,Media_File_CRC_vod__c,Media_File_Name_vod__c,Media_File_Size_vod__c,Segment_vod__c,Disable_Actions_vod__c,VExternal_Id_vod__c,CDN_Path_vod__c,Status_vod__c,Vault_DNS_vod__c,Vault_Doc_Id_vod__c,Vault_External_Id_vod__c,Vault_GUID_vod__c,Vault_Last_Modified_Date_Time_vod__c,Is_Shared_Resource_vod__c,iOS_Viewer_vod__c,iOS_Resolution_vod__c,approved_for_distribution_date__c,approved_for_use_date__c,ewizard_link__c,expiration_date__c,keywords__c,trade_team__c,business_function__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,description_vod__c,product_vod__c,product_strategy_vod__c,display_order_vod__c,active_vod__c,category_vod__c,vehicle_vod__c,clm_id_vod__c,custom_reaction_vod__c,slide_version_vod__c,language_vod__c,media_file_crc_vod__c,media_file_name_vod__c,media_file_size_vod__c,segment_vod__c,disable_actions_vod__c,vexternal_id_vod__c,cdn_path_vod__c,status_vod__c,vault_dns_vod__c,vault_doc_id_vod__c,vault_external_id_vod__c,vault_guid_vod__c,vault_last_modified_date_time_vod__c,is_shared_resource_vod__c,i_os_viewer_vod__c,i_os_resolution_vod__c,approved_for_distribution_date__c,approved_for_use_date__c,ewizard_link__c,expiration_date__c,keywords__c,trade_team__c,business_function__c +src02.crm_key_message_vod__c +org02.crm_key_message_vod__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Hospital_Medical_Regimen__c.txt b/s3/data/crm/settings/CRM_MSJ_Hospital_Medical_Regimen__c.txt new file mode 100644 index 00000000..c295fc5e --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Hospital_Medical_Regimen__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +18 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,MSJ_Account_Name__c,MSJ_Delete_Date__c,MSJ_Delete_Flag__c,MSJ_Indication__c,MSJ_Line__c,MSJ_Medical_Regimen__c,Mobile_ID_vod__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,msj_account_name__c,msj_delete_date__c,msj_delete_flag__c,msj_indication__c,msj_line__c,msj_medical_regimen__c,mobile_id_vod__c +src02.crm_msj_hospital_medical_regimen__c +org02.crm_msj_hospital_medical_regimen__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Inquiry_Assignment__c.txt b/s3/data/crm/settings/CRM_MSJ_Inquiry_Assignment__c.txt new file mode 100644 index 00000000..a50bf1f3 --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Inquiry_Assignment__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +47 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,MSJ_Medical_Inquiry__c,MSJ_Close__c,MSJ_Doctor_Name__c,MSJ_Hospital_Name__c,MSJ_Indication__c,MSJ_Inquiry_Text__c,MSJ_MEC_User__c,MSJ_MSL_Manager__c,MSJ_MSL_User__c,MSJ_Notice_to_MR__c,MSJ_Product_for_MEC__c,MSJ_Product_for_MR__c,MSJ_Reply_Date__c,MSJ_Reply__c,MSJ_AE_Infomation__c,MSJ_Cancel__c,MSJ_FAQ_number_c__c,MSJ_Return_Call__c,MSJ_Inquiry_Origin__c,First_Response__c,Inquiry_Created_Date__c,Inquiry_Type_1__c,Inquiry_Type_2__c,MSJ_First_User__c,MSJ_MEC_Comment__c,MSJ_Send_Email__c,MSJ_Temp_Aggregated_Info__c,MSJ_AE_Report__c,MSJ_Background__c,MSJ_Inquiry_Date__c,MSJ_MSL_Support__c,MSJ_Handover_Comment__c,MSJ_Handover_Email__c,MSJ_Material_Requirement__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,msj_medical_inquiry__c,msj_close__c,msj_doctor_name__c,msj_hospital_name__c,msj_indication__c,msj_inquiry_text__c,msj_mec_user__c,msj_msl_manager__c,msj_msl_user__c,msj_notice_to_mr__c,msj_product_for_mec__c,msj_product_for_mr__c,msj_reply_date__c,msj_reply__c,msj_ae_infomation__c,msj_cancel__c,msj_faq_number_c__c,msj_return_call__c,msj_inquiry_origin__c,first_response__c,inquiry_created_date__c,inquiry_type_1__c,inquiry_type_2__c,msj_first_user__c,msj_mec_comment__c,msj_send_email__c,msj_temp_aggregated_info__c,msj_ae_report__c,msj_background__c,msj_inquiry_date__c,msj_msl_support__c,msj_handover_comment__c,msj_handover_email__c,msj_material_requirement__c +src02.crm_msj_inquiry_assignment__c +org02.crm_msj_inquiry_assignment__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_MR_Weekly_Report__c.txt b/s3/data/crm/settings/CRM_MSJ_MR_Weekly_Report__c.txt new file mode 100644 index 00000000..89df0b85 --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_MR_Weekly_Report__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +35 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,MSJ_Account_Name__c,MSJ_Activity_Results_Summary__c,MSJ_MUID__c,MSJ_Next_Week_Action_What__c,MSJ_Next_Week_Action_When__c,MSJ_Next_Week_Action_Where__c,MSJ_Next_Week_Action_Who__c,MSJ_Report_Week__c,MSJ_Target_Patient_Count__c,Mobile_ID_vod__c,MSJ_Activity_Results_Summary_HN__c,MSJ_Next_Week_Action_Where_HN__c,MSJ_Next_Week_Action_Who_HN__c,MSJ_Next_Week_Action_What_HN__c,MSJ_Next_Week_Action_When_HN__c,MSJ_Target_Patient_Count_HN__c,MSJ_Activity_Results_Summary_MCC__c,MSJ_Next_Week_Action_Where_MCC__c,MSJ_Next_Week_Action_Who_MCC__c,MSJ_Next_Week_Action_What_MCC__c,MSJ_Next_Week_Action_When_MCC__c,MSJ_Target_Patient_Count_MCC__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,msj_account_name__c,msj_activity_results_summary__c,msj_muid__c,msj_next_week_action_what__c,msj_next_week_action_when__c,msj_next_week_action_where__c,msj_next_week_action_who__c,msj_report_week__c,msj_target_patient_count__c,mobile_id_vod__c,msj_activity_results_summary_hn__c,msj_next_week_action_where_hn__c,msj_next_week_action_who_hn__c,msj_next_week_action_what_hn__c,msj_next_week_action_when_hn__c,msj_target_patient_count_hn__c,msj_activity_results_summary_mcc__c,msj_next_week_action_where_mcc__c,msj_next_week_action_who_mcc__c,msj_next_week_action_what_mcc__c,msj_next_week_action_when_mcc__c,msj_target_patient_count_mcc__c +src02.crm_msj_mr_weekly_report__c +org02.crm_msj_mr_weekly_report__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Medical_Event_Evaluation__c.txt b/s3/data/crm/settings/CRM_MSJ_Medical_Event_Evaluation__c.txt new file mode 100644 index 00000000..5f1546f2 --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Medical_Event_Evaluation__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +14 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,MSJ_Medical_Event__c,MSJ_Evaluation_Comment__c,MSJ_Evaluation__c,Mobile_ID_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,msj_medical_event__c,msj_evaluation_comment__c,msj_evaluation__c,mobile_id_vod__c +src02.crm_msj_medical_event_evaluation__c +org02.crm_msj_medical_event_evaluation__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Medical_Regimen__c.txt b/s3/data/crm/settings/CRM_MSJ_Medical_Regimen__c.txt new file mode 100644 index 00000000..321b8a5e --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Medical_Regimen__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +17 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,MSJ_Delete_Date__c,MSJ_Delete_Flag__c,MSJ_Indication__c,MSJ_Remark__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,msj_delete_date__c,msj_delete_flag__c,msj_indication__c,msj_remark__c +src02.crm_msj_medical_regimen__c +org02.crm_msj_medical_regimen__c + + diff --git a/s3/data/crm/settings/CRM_MSJ_Patient__c.txt b/s3/data/crm/settings/CRM_MSJ_Patient__c.txt new file mode 100644 index 00000000..8b1bac96 --- /dev/null +++ b/s3/data/crm/settings/CRM_MSJ_Patient__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +37 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,MSJ_Account_Name__c,MSJ_CRC_Group__c,MSJ_Casus_or_Transfer_Point__c,MSJ_Entry_Date__c,MSJ_IST_Name__c,MSJ_Indication__c,MSJ_Line__c,MSJ_MR_Comments__c,MSJ_MUID__c,MSJ_Medical_Regimen__c,MSJ_Month__c,MSJ_Report_Comments__c,MSJ_Start_Date_Of_Administration__c,MSJ_Year__c,Mobile_ID_vod__c,MSJ_CRC_RAS_KRAS__c,MSJ_End_Date_Of_Administration__c,MSJ_End_Date_of_Stop_Administration__c,MSJ_HN_Hospitalized_Type__c,MSJ_Start_Date_of_Stop_Administration__c,MSJ_Patient_Status__c,MSJ_Patient_TA__c,MSJ_Child_Account_Name__c,MSJ_Child_Account__c,MSJ_Parent_Account_Name__c,MSJ_Parent_Child_Name__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,msj_account_name__c,msj_crc_group__c,msj_casus_or_transfer_point__c,msj_entry_date__c,msj_ist_name__c,msj_indication__c,msj_line__c,msj_mr_comments__c,msj_muid__c,msj_medical_regimen__c,msj_month__c,msj_report_comments__c,msj_start_date_of_administration__c,msj_year__c,mobile_id_vod__c,msj_crc_ras_kras__c,msj_end_date_of_administration__c,msj_end_date_of_stop_administration__c,msj_hn_hospitalized_type__c,msj_start_date_of_stop_administration__c,msj_patient_status__c,msj_patient_ta__c,msj_child_account_name__c,msj_child_account__c,msj_parent_account_name__c,msj_parent_child_name__c +src02.crm_msj_patient__c +org02.crm_msj_patient__c + + diff --git a/s3/data/crm/settings/CRM_Medical_Event_vod__c.txt b/s3/data/crm/settings/CRM_Medical_Event_vod__c.txt new file mode 100644 index 00000000..159a475b --- /dev/null +++ b/s3/data/crm/settings/CRM_Medical_Event_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +52 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Primary_Product__c,Description_vod__c,Start_Date_vod__c,Location__c,End_Date_vod__c,Secondary_Product__c,Website__c,Active_vod__c,Event_Type__c,MSJ_Area__c,MSJ_Business_Unit__c,MSJ_Comment__c,MSJ_Company__c,MSJ_Expense_App_No__c,MSJ_Form__c,MSJ_HQ_Area__c,MSJ_Location__c,MSJ_MR__c,MSJ_Number_of_Attendee__c,MSJ_Product__c,MSJ_Site__c,MSJ_Type__c,MSJ_Number_of_Attendee_Auto_Calc__c,MSJ_Number_of_Attendee_Invited__c,Account_vod__c,MSJ_MUID__c,Country_Name_vod__c,MSJ_CE_SIPAGL_Updater__c,MSJ_CE_SIPAGL_1A__c,MSJ_CE_SIPAGL_1B__c,MSJ_CE_SIPAGL_2__c,MSJ_CE_SIPAGL_3__c,MSJ_CE_SIPAGL_4__c,MSJ_CE_SIPAGL_5__c,MSJ_CE_SIPAGL_Comment__c,MSJ_CE_SIPAGL_1A_date__c,MSJ_CE_SIPAGL_6__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,primary_product__c,description_vod__c,start_date_vod__c,location__c,end_date_vod__c,secondary_product__c,website__c,active_vod__c,event_type__c,msj_area__c,msj_business_unit__c,msj_comment__c,msj_company__c,msj_expense_app_no__c,msj_form__c,msj_hq_area__c,msj_location__c,msj_mr__c,msj_number_of_attendee__c,msj_product__c,msj_site__c,msj_type__c,msj_number_of_attendee_auto_calc__c,msj_number_of_attendee_invited__c,account_vod__c,msj_muid__c,country_name_vod__c,msj_ce_sipagl_updater__c,msj_ce_sipagl_1_a__c,msj_ce_sipagl_1_b__c,msj_ce_sipagl_2__c,msj_ce_sipagl_3__c,msj_ce_sipagl_4__c,msj_ce_sipagl_5__c,msj_ce_sipagl_comment__c,msj_ce_sipagl_1_a_date__c,msj_ce_sipagl_6__c +src02.crm_medical_event_vod__c +org02.crm_medical_event_vod__c + + diff --git a/s3/data/crm/settings/CRM_Medical_Inquiry_vod__c.txt b/s3/data/crm/settings/CRM_Medical_Inquiry_vod__c.txt new file mode 100644 index 00000000..cafca6d9 --- /dev/null +++ b/s3/data/crm/settings/CRM_Medical_Inquiry_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +67 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,Address_Line_1_vod__c,Address_Line_2_vod__c,City_vod__c,Delivery_Method_vod__c,Email_vod__c,Fax_Number_vod__c,Inquiry_Text__c,Lock_vod__c,Mobile_ID_vod__c,Phone_Number_vod__c,Product__c,Rush_Delivery__c,Signature_Date_vod__c,Signature_vod__c,State_vod__c,Status_vod__c,Zip_vod__c,zvod_Delivery_Method_vod__c,zvod_Disclaimer_vod__c,Submitted_By_Mobile_vod__c,Disclaimer_vod__c,Entity_Reference_Id_vod__c,Call2_vod__c,Country_vod__c,Override_Lock_vod__c,MSJ_Department__c,MSJ_Doctor_Name__c,MSJ_Hospital_Name__c,MSJ_Indication__c,MSJ_Inquiry_Assignment__c,MSJ_Inquiry_Date__c,MSJ_Inquiry_Input_Manager__c,MSJ_Inquiry_Input_User__c,MSJ_MSL_Manager__c,MSJ_Notice_to_MR__c,MSJ_Person_in_charge_1__c,MSJ_Person_in_charge_2__c,MSJ_Product_for_MEC__c,MSJ_Product_for_MR__c,MSJ_Reply_Date__c,MSJ_Reply_User__c,MSJ_Reply__c,MSJ_Title__c,MSJ_AE_Infomation__c,MSJ_FAQ_Number_Report__c,MSJ_Return_Call_Report__c,MSJ_Inquiry_Origin_Report__c,MSJ_AE_Report__c,MSJ_Background__c,MSJ_MSL_Support__c,MSJ_Material_Requirement__c,MSJ_Hospital_Name_Disp__c,MSJ_Hospital__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,address_line_1_vod__c,address_line_2_vod__c,city_vod__c,delivery_method_vod__c,email_vod__c,fax_number_vod__c,inquiry_text__c,lock_vod__c,mobile_id_vod__c,phone_number_vod__c,product__c,rush_delivery__c,signature_date_vod__c,signature_vod__c,state_vod__c,status_vod__c,zip_vod__c,zvod_delivery_method_vod__c,zvod_disclaimer_vod__c,submitted_by_mobile_vod__c,disclaimer_vod__c,entity_reference_id_vod__c,call2_vod__c,country_vod__c,override_lock_vod__c,msj_department__c,msj_doctor_name__c,msj_hospital_name__c,msj_indication__c,msj_inquiry_assignment__c,msj_inquiry_date__c,msj_inquiry_input_manager__c,msj_inquiry_input_user__c,msj_msl_manager__c,msj_notice_to_mr__c,msj_person_in_charge_1__c,msj_person_in_charge_2__c,msj_product_for_mec__c,msj_product_for_mr__c,msj_reply_date__c,msj_reply_user__c,msj_reply__c,msj_title__c,msj_ae_infomation__c,msj_faq_number_report__c,msj_return_call_report__c,msj_inquiry_origin_report__c,msj_ae_report__c,msj_background__c,msj_msl_support__c,msj_material_requirement__c,msj_hospital_name_disp__c,msj_hospital__c +src02.crm_medical_inquiry_vod__c +org02.crm_medical_inquiry_vod__c + + diff --git a/s3/data/crm/settings/CRM_Medical_Insight_vod__c.txt b/s3/data/crm/settings/CRM_Medical_Insight_vod__c.txt new file mode 100644 index 00000000..0c477d5d --- /dev/null +++ b/s3/data/crm/settings/CRM_Medical_Insight_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +54 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,Clinical_Trial_vod__c,Date_vod__c,Description_vod__c,Entity_Reference_Id_vod__c,Interaction_vod__c,Medical_Event_vod__c,Mobile_ID_vod__c,Other_Source_vod__c,Override_Lock_vod__c,Publication_vod__c,Status_vod__c,Summary_vod__c,Unlock_vod__c,Commercial_Medical__c,MSJ_Level_1A__c,MSJ_Level_1B__c,MSJ_Level_2A__c,MSJ_Level_2B__c,MSJ_Level_3A__c,MSJ_Level_3B__c,MSJ_Level_4A__c,MSJ_Level_4B__c,MSJ_SubStatus__c,MSJ_Type_A__c,MSJ_Type_B__c,MSJ_Description_Backup__c,MSJ_Country__c,MSJ_Received_at_Boomi__c,MSJ_Level_1A_Value__c,MSJ_Level_1B_Value__c,MSJ_Level_2A_Value__c,MSJ_Level_2B_Value__c,MSJ_Level_3A_Value__c,MSJ_Level_3B_Value__c,MSJ_Level_4A_Value__c,MSJ_Level_4B_Value__c,MSJ_Hospital_ID__c,MSJ_Hospital_Name__c,MSJ_Hospital__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,clinical_trial_vod__c,date_vod__c,description_vod__c,entity_reference_id_vod__c,interaction_vod__c,medical_event_vod__c,mobile_id_vod__c,other_source_vod__c,override_lock_vod__c,publication_vod__c,status_vod__c,summary_vod__c,unlock_vod__c,commercial_medical__c,msj_level_1_a__c,msj_level_1_b__c,msj_level_2_a__c,msj_level_2_b__c,msj_level_3_a__c,msj_level_3_b__c,msj_level_4_a__c,msj_level_4_b__c,msj_sub_status__c,msj_type_a__c,msj_type_b__c,msj_description_backup__c,msj_country__c,msj_received_at_boomi__c,msj_level_1_a_value__c,msj_level_1_b_value__c,msj_level_2_a_value__c,msj_level_2_b_value__c,msj_level_3_a_value__c,msj_level_3_b_value__c,msj_level_4_a_value__c,msj_level_4_b_value__c,msj_hospital_id__c,msj_hospital_name__c,msj_hospital__c +src02.crm_medical_insight_vod__c +org02.crm_medical_insight_vod__c + + diff --git a/s3/data/crm/settings/CRM_Multichannel_Activity_Line_vod__c.txt b/s3/data/crm/settings/CRM_Multichannel_Activity_Line_vod__c.txt new file mode 100644 index 00000000..1b4ed66a --- /dev/null +++ b/s3/data/crm/settings/CRM_Multichannel_Activity_Line_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +35 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Multichannel_Activity_vod__c,Call_vod__c,Custom_vod__c,DateTime_vod__c,Debug_vod__c,Detail_Group_VExternal_Id_vod__c,Detail_Group_vod__c,Duration_vod__c,Event_Subtype_vod__c,Event_Type_vod__c,Key_Message_VExternal_Id_vod__c,Key_Message_vod__c,Multichannel_Content_Asset_Id_vod__c,Multichannel_Content_Asset_Version_vod__c,Multichannel_Content_Asset_vod__c,Multichannel_Content_vod__c,Product_VExternal_Id_vod__c,Product_vod__c,Sent_Email_vod__c,VExternal_Id_vod__c,Video_Last_Viewed_Time_vod__c,Video_Length_vod__c,Video_Total_Time_Spent_vod__c,View_Order_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,multichannel_activity_vod__c,call_vod__c,custom_vod__c,date_time_vod__c,debug_vod__c,detail_group_vexternal_id_vod__c,detail_group_vod__c,duration_vod__c,event_subtype_vod__c,event_type_vod__c,key_message_vexternal_id_vod__c,key_message_vod__c,multichannel_content_asset_id_vod__c,multichannel_content_asset_version_vod__c,multichannel_content_asset_vod__c,multichannel_content_vod__c,product_vexternal_id_vod__c,product_vod__c,sent_email_vod__c,vexternal_id_vod__c,video_last_viewed_time_vod__c,video_length_vod__c,video_total_time_spent_vod__c,view_order_vod__c +src02.crm_multichannel_activity_line_vod__c +org02.crm_multichannel_activity_line_vod__c + + diff --git a/s3/data/crm/settings/CRM_Multichannel_Activity_vod__c.txt b/s3/data/crm/settings/CRM_Multichannel_Activity_vod__c.txt new file mode 100644 index 00000000..9072d9e4 --- /dev/null +++ b/s3/data/crm/settings/CRM_Multichannel_Activity_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +47 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_External_ID_Map_vod__c,Account_vod__c,Call_vod__c,City_vod__c,Client_Name_vod__c,Client_OS_vod__c,Client_Type_vod__c,Country_vod__c,Debug_vod__c,Device_vod__c,IP_Address_vod__c,Multichannel_Activity_vod__c,Referring_Site_vod__c,Region_vod__c,Sent_Email_vod__c,Session_Id_vod__c,Site_vod__c,Start_DateTime_vod__c,Total_Duration_vod__c,URL_vod__c,User_Agent_vod__c,VExternal_Id_vod__c,Viewport_Height_vod__c,Viewport_Width_vod__c,Color_vod__c,Icon_vod__c,MCD_Primary_Key_vod__c,Record_Type_Name_vod__c,MSJ_Date_Opened__c,MSJ_Sent_Date__c,MSJ_Email_Subject__c,MSJ_Opens__c,MSJ_Email_Status__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_external_id_map_vod__c,account_vod__c,call_vod__c,city_vod__c,client_name_vod__c,client_os_vod__c,client_type_vod__c,country_vod__c,debug_vod__c,device_vod__c,ip_address_vod__c,multichannel_activity_vod__c,referring_site_vod__c,region_vod__c,sent_email_vod__c,session_id_vod__c,site_vod__c,start_date_time_vod__c,total_duration_vod__c,url_vod__c,user_agent_vod__c,vexternal_id_vod__c,viewport_height_vod__c,viewport_width_vod__c,color_vod__c,icon_vod__c,mcd_primary_key_vod__c,record_type_name_vod__c,msj_date_opened__c,msj_sent_date__c,msj_email_subject__c,msj_opens__c,msj_email_status__c +src02.crm_multichannel_activity_vod__c +org02.crm_multichannel_activity_vod__c + + diff --git a/s3/data/crm/settings/CRM_Multichannel_Consent_vod__c.txt b/s3/data/crm/settings/CRM_Multichannel_Consent_vod__c.txt new file mode 100644 index 00000000..be8d8b57 --- /dev/null +++ b/s3/data/crm/settings/CRM_Multichannel_Consent_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +48 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,Capture_Datetime_vod__c,Channel_Value_vod__c,Detail_Group_vod__c,External_ID_vod__c,Last_Device_vod__c,Mobile_ID_vod__c,Opt_Expiration_Date_vod__c,Opt_Type_vod__c,Optout_Event_Type_vod__c,Product_vod__c,Signature_Datetime_vod__c,Signature_ID_vod__c,Signature_vod__c,Sample_Consent_Template_Data_vod__c,Sample_Consent_Template_vod__c,Consent_Line_vod__c,Consent_Type_vod__c,Default_Consent_Text_vod__c,Disclaimer_Text_vod__c,Sub_Channel_Key_vod__c,Consent_Confirm_Datetime_vod__c,Related_Transaction_Id_vod__c,Sent_Email_vod__c,Content_Type_vod__c,Receipt_Email_vod__c,Receipt_Sent_Email_Transaction_Id_vod__c,Receipt_Sent_Email_vod__c,Captured_By_vod__c,Opt_Out_Disclaimer_Text_vod__c,Channel_Source_vod__c,Union_Id_vod__c,User_Last_Notified_vod__c,Sub_Channel_Display_Name__c,MSJ_Consent_Source__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,capture_datetime_vod__c,channel_value_vod__c,detail_group_vod__c,external_id_vod__c,last_device_vod__c,mobile_id_vod__c,opt_expiration_date_vod__c,opt_type_vod__c,optout_event_type_vod__c,product_vod__c,signature_datetime_vod__c,signature_id_vod__c,signature_vod__c,sample_consent_template_data_vod__c,sample_consent_template_vod__c,consent_line_vod__c,consent_type_vod__c,default_consent_text_vod__c,disclaimer_text_vod__c,sub_channel_key_vod__c,consent_confirm_datetime_vod__c,related_transaction_id_vod__c,sent_email_vod__c,content_type_vod__c,receipt_email_vod__c,receipt_sent_email_transaction_id_vod__c,receipt_sent_email_vod__c,captured_by_vod__c,opt_out_disclaimer_text_vod__c,channel_source_vod__c,union_id_vod__c,user_last_notified_vod__c,sub_channel_display_name__c,msj_consent_source__c +src02.crm_multichannel_consent_vod__c +org02.crm_multichannel_consent_vod__c + + diff --git a/s3/data/crm/settings/CRM_My_Setup_Products_vod__c.txt b/s3/data/crm/settings/CRM_My_Setup_Products_vod__c.txt new file mode 100644 index 00000000..47c71f53 --- /dev/null +++ b/s3/data/crm/settings/CRM_My_Setup_Products_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +12 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Product_vod__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,product_vod__c +src02.crm_my_setup_products_vod__c +org02.crm_my_setup_products_vod__c + + diff --git a/s3/data/crm/settings/CRM_ObjectTerritory2Association.txt b/s3/data/crm/settings/CRM_ObjectTerritory2Association.txt new file mode 100644 index 00000000..fff7b033 --- /dev/null +++ b/s3/data/crm/settings/CRM_ObjectTerritory2Association.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +9 +Id,ObjectId,Territory2Id,AssociationCause,SobjectType,IsDeleted,LastModifiedDate,LastModifiedById,SystemModstamp +id,object_id,territory2_id,association_cause,sobject_type,is_deleted,last_modified_date,last_modified_by_id,system_modstamp +src02.crm_object_territory2_association +org02.crm_object_territory2_association +CRM_ObjectTerritory2Association_ex.sql + diff --git a/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql new file mode 100644 index 00000000..d142c071 --- /dev/null +++ b/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_object_territory2_association', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Product_Group_vod__c.txt b/s3/data/crm/settings/CRM_Product_Group_vod__c.txt new file mode 100644 index 00000000..7b6c6cff --- /dev/null +++ b/s3/data/crm/settings/CRM_Product_Group_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +17 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Description_vod__c,Product_vod__c,Product_Catalog_vod__c,Start_Date_vod__c,End_Date_vod__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,description_vod__c,product_vod__c,product_catalog_vod__c,start_date_vod__c,end_date_vod__c +src02.crm_product_group_vod__c +org02.crm_product_group_vod__c + + diff --git a/s3/data/crm/settings/CRM_Product_Metrics_vod__c.txt b/s3/data/crm/settings/CRM_Product_Metrics_vod__c.txt new file mode 100644 index 00000000..4629f0eb --- /dev/null +++ b/s3/data/crm/settings/CRM_Product_Metrics_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +52 +Id,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_vod__c,Awareness__c,Selling_Stage__c,Formulary_Status__c,Movement__c,Products_vod__c,Segment__c,X12_mo_trx_chg__c,Speaker_Skills__c,Investigator_Readiness__c,Engagements__c,Mobile_ID_vod__c,External_ID_vod__c,MSJ_Patient__c,Detail_Group_vod__c,MSJ_EB_1st_Line_Liver_Meta__c,MSJ_EB_1st_Line_Multi_Meta__c,MSJ_EB_2nd_Line_Mono__c,MSJ_EB_2nd_Line_Combination__c,MSJ_EB_3rd_Line_Mono__c,MSJ_EB_3rd_Line_Combination__c,EMDS_Ability__c,EMDS_Brand_Loyalty__c,EMDS_Decision_Maker__c,EMDS_Early_Tech_Adopter__c,EMDS_Influence__c,EMDS_Main_Driver__c,EMDS_Priority__c,EMDS_Willingness__c,MSJ_KTL_Type__c,MSJ_KTL_Tier__c,MSJ_Publications__c,MSJ_Clinical_Trials__c,MSJ_Speaker_for_Medical_Events__c,MSJ_Advisor_to_Medical_Affairs__c,MSJ_Guidelines_Treatment_Standards__c,MSJ_Therapeutic_Area_Expertise__c,MSJ_MAP_GAP__c,MSJ_Associations__c,MSJ_Tier_Score__c +id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_vod__c,awareness__c,selling_stage__c,formulary_status__c,movement__c,products_vod__c,segment__c,x12_mo_trx_chg__c,speaker_skills__c,investigator_readiness__c,engagements__c,mobile_id_vod__c,external_id_vod__c,msj_patient__c,detail_group_vod__c,msj_eb_1st_line_liver_meta__c,msj_eb_1st_line_multi_meta__c,msj_eb_2nd_line_mono__c,msj_eb_2nd_line_combination__c,msj_eb_3rd_line_mono__c,msj_eb_3rd_line_combination__c,emds_ability__c,emds_brand_loyalty__c,emds_decision_maker__c,emds_early_tech_adopter__c,emds_influence__c,emds_main_driver__c,emds_priority__c,emds_willingness__c,msj_ktl_type__c,msj_ktl_tier__c,msj_publications__c,msj_clinical_trials__c,msj_speaker_for_medical_events__c,msj_advisor_to_medical_affairs__c,msj_guidelines_treatment_standards__c,msj_therapeutic_area_expertise__c,msj_map_gap__c,msj_associations__c,msj_tier_score__c +src02.crm_product_metrics_vod__c +org02.crm_product_metrics_vod__c +CRM_Product_Metrics_vod__c_ex.sql + diff --git a/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql b/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql new file mode 100644 index 00000000..af2e678e --- /dev/null +++ b/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_product_metrics_vod__c', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Product_vod__c.txt b/s3/data/crm/settings/CRM_Product_vod__c.txt new file mode 100644 index 00000000..843cec84 --- /dev/null +++ b/s3/data/crm/settings/CRM_Product_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +46 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consumer_site__c,Product_info__c,Therapeutic_Class_vod__c,Parent_Product_vod__c,Therapeutic_Area_vod__c,Product_Type_vod__c,Require_Key_Message_vod__c,Cost_vod__c,External_ID_vod__c,Manufacturer_vod__c,Company_Product_vod__c,Controlled_Substance_vod__c,Description_vod__c,Sample_Quantity_Picklist_vod__c,Display_Order_vod__c,No_Metrics_vod__c,Distributor_vod__c,Sample_Quantity_Bound_vod__c,Sample_U_M_vod__c,No_Details_vod__c,Quantity_Per_Case_vod__c,Schedule_vod__c,Restricted_vod__c,Pricing_Rule_Quantity_Bound_vod__c,No_Promo_Items_vod__c,User_Aligned_vod__c,Restricted_States_vod__c,Sort_Code_vod__c,No_Cycle_Plans_vod__c,Inventory_Order_UOM_vod__c,Inventory_Quantity_Per_Case_vod__c,VExternal_Id_vod__c,Country__c,MSJ_Product_Classification__c,MSJ_Indication__c,MSJ_Therapeutic_Area__c,MSJ_Global_Brand__c,MSJ_Global_Business_Unit__c,MSJ_Molecules__c,MSJ_SBU__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consumer_site__c,product_info__c,therapeutic_class_vod__c,parent_product_vod__c,therapeutic_area_vod__c,product_type_vod__c,require_key_message_vod__c,cost_vod__c,external_id_vod__c,manufacturer_vod__c,company_product_vod__c,controlled_substance_vod__c,description_vod__c,sample_quantity_picklist_vod__c,display_order_vod__c,no_metrics_vod__c,distributor_vod__c,sample_quantity_bound_vod__c,sample_u_m_vod__c,no_details_vod__c,quantity_per_case_vod__c,schedule_vod__c,restricted_vod__c,pricing_rule_quantity_bound_vod__c,no_promo_items_vod__c,user_aligned_vod__c,restricted_states_vod__c,sort_code_vod__c,no_cycle_plans_vod__c,inventory_order_uom_vod__c,inventory_quantity_per_case_vod__c,vexternal_id_vod__c,country__c,msj_product_classification__c,msj_indication__c,msj_therapeutic_area__c,msj_global_brand__c,msj_global_business_unit__c,msj_molecules__c,msj_sbu__c +src02.crm_product_vod__c +org02.crm_product_vod__c + + diff --git a/s3/data/crm/settings/CRM_Profile.txt b/s3/data/crm/settings/CRM_Profile.txt new file mode 100644 index 00000000..558ce629 --- /dev/null +++ b/s3/data/crm/settings/CRM_Profile.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +237 +Id,Name,PermissionsEmailSingle,PermissionsEmailMass,PermissionsEditTask,PermissionsEditEvent,PermissionsExportReport,PermissionsImportPersonal,PermissionsDataExport,PermissionsManageUsers,PermissionsEditPublicFilters,PermissionsEditPublicTemplates,PermissionsModifyAllData,PermissionsManageCases,PermissionsManageSolutions,PermissionsCustomizeApplication,PermissionsEditReadonlyFields,PermissionsRunReports,PermissionsViewSetup,PermissionsTransferAnyEntity,PermissionsNewReportBuilder,PermissionsManageSelfService,PermissionsManageCssUsers,PermissionsActivateContract,PermissionsApproveContract,PermissionsImportLeads,PermissionsManageLeads,PermissionsTransferAnyLead,PermissionsViewAllData,PermissionsEditPublicDocuments,PermissionsViewEncryptedData,PermissionsEditBrandTemplates,PermissionsEditHtmlTemplates,PermissionsManageTranslation,PermissionsDeleteActivatedContract,PermissionsSendSitRequests,PermissionsApiUserOnly,PermissionsManageRemoteAccess,PermissionsCanUseNewDashboardBuilder,PermissionsManageCategories,PermissionsConvertLeads,PermissionsTestInstanceCreate,PermissionsPasswordNeverExpires,PermissionsUseTeamReassignWizards,PermissionsInstallMultiforce,PermissionsPublishMultiforce,PermissionsEditOppLineItemUnitPrice,PermissionsManageTerritories,PermissionsCreateMultiforce,PermissionsBulkApiHardDelete,PermissionsInboundMigrationToolsUser,PermissionsSolutionImport,PermissionsManageCallCenters,PermissionsManageSynonyms,PermissionsOutboundMigrationToolsUser,PermissionsViewContent,PermissionsManageEmailClientConfig,PermissionsEnableNotifications,PermissionsManageDataIntegrations,PermissionsDistributeFromPersWksp,PermissionsViewDataCategories,PermissionsManageDataCategories,PermissionsAuthorApex,PermissionsManageMobile,PermissionsApiEnabled,PermissionsManageCustomReportTypes,PermissionsEditCaseComments,PermissionsTransferAnyCase,PermissionsContentAdministrator,PermissionsCreateWorkspaces,PermissionsManageContentPermissions,PermissionsManageContentProperties,PermissionsManageContentTypes,PermissionsScheduleJob,PermissionsManageExchangeConfig,PermissionsManageAnalyticSnapshots,PermissionsScheduleReports,PermissionsManageBusinessHourHolidays,PermissionsManageDynamicDashboards,PermissionsManageInteraction,PermissionsViewMyTeamsDashboards,PermissionsResetPasswords,PermissionsFlowUFLRequired,PermissionsActivitiesAccess,PermissionsEmailTemplateManagement,PermissionsEmailAdministration,PermissionsChatterFileLink,PermissionsForceTwoFactor,PermissionsViewEventLogFiles,PermissionsManageNetworks,PermissionsManageAuthProviders,PermissionsRunFlow,PermissionsCreateCustomizeDashboards,PermissionsCreateDashboardFolders,PermissionsViewPublicDashboards,PermissionsManageDashbdsInPubFolders,PermissionsCreateCustomizeReports,PermissionsCreateReportFolders,PermissionsViewPublicReports,PermissionsManageReportsInPubFolders,PermissionsEditMyDashboards,PermissionsEditMyReports,PermissionsViewAllUsers,PermissionsConnectOrgToEnvironmentHub,PermissionsCreateCustomizeFilters,PermissionsContentHubUser,PermissionsGovernNetworks,PermissionsSalesConsole,PermissionsTwoFactorApi,PermissionsDeleteTopics,PermissionsEditTopics,PermissionsCreateTopics,PermissionsAssignTopics,PermissionsIdentityEnabled,PermissionsIdentityConnect,PermissionsContentWorkspaces,PermissionsCustomMobileAppsAccess,PermissionsViewHelpLink,PermissionsManageProfilesPermissionsets,PermissionsAssignPermissionSets,PermissionsManageRoles,PermissionsManageIpAddresses,PermissionsManageSharing,PermissionsManageInternalUsers,PermissionsManagePasswordPolicies,PermissionsManageLoginAccessPolicies,PermissionsManageCustomPermissions,PermissionsStdAutomaticActivityCapture,PermissionsManageTwoFactor,PermissionsDebugApex,PermissionsLightningExperienceUser,PermissionsConfigCustomRecs,PermissionsSubmitMacrosAllowed,PermissionsBulkMacrosAllowed,PermissionsManageSessionPermissionSets,PermissionsCreateAuditFields,PermissionsUpdateWithInactiveOwner,PermissionsManageSandboxes,PermissionsAutomaticActivityCapture,PermissionsImportCustomObjects,PermissionsDelegatedTwoFactor,PermissionsSelectFilesFromSalesforce,PermissionsModerateNetworkUsers,PermissionsMergeTopics,PermissionsSubscribeToLightningReports,PermissionsManagePvtRptsAndDashbds,PermissionsAllowLightningLogin,PermissionsCampaignInfluence2,PermissionsViewDataAssessment,PermissionsCanApproveFeedPost,PermissionsAllowViewEditConvertedLeads,PermissionsShowCompanyNameAsUserBadge,PermissionsAccessCMC,PermissionsViewHealthCheck,PermissionsManageHealthCheck,PermissionsPackaging2,PermissionsManageCertificates,PermissionsCreateReportInLightning,PermissionsPreventClassicExperience,PermissionsListEmailSend,PermissionsChangeDashboardColors,PermissionsManageRecommendationStrategies,PermissionsManagePropositions,PermissionsSubscribeReportRolesGrps,PermissionsSubscribeDashboardRolesGrps,PermissionsUseWebLink,PermissionsHasUnlimitedNBAExecutions,PermissionsViewOnlyEmbeddedAppUser,PermissionsViewAllActivities,PermissionsSubscribeReportToOtherUsers,PermissionsLightningConsoleAllowedForUser,PermissionsSubscribeReportsRunAsUser,PermissionsSubscribeToLightningDashboards,PermissionsSubscribeDashboardToOtherUsers,PermissionsCreateLtngTempInPub,PermissionsTransactionalEmailSend,PermissionsViewPrivateStaticResources,PermissionsCreateLtngTempFolder,PermissionsApexRestServices,PermissionsEnableCommunityAppLauncher,PermissionsGiveRecognitionBadge,PermissionsUseMySearch,PermissionsLtngPromoReserved01UserPerm,PermissionsManageSubscriptions,PermissionsManageSurveys,PermissionsUseAssistantDialog,PermissionsUseQuerySuggestions,PermissionsViewRoles,PermissionsLMOutboundMessagingUserPerm,PermissionsModifyDataClassification,PermissionsPrivacyDataAccess,PermissionsQueryAllFiles,PermissionsModifyMetadata,PermissionsManageCMS,PermissionsSandboxTestingInCommunityApp,PermissionsCanEditPrompts,PermissionsViewUserPII,PermissionsManageHubConnections,PermissionsB2BMarketingAnalyticsUser,PermissionsTraceXdsQueries,PermissionsViewAllCustomSettings,PermissionsViewAllForeignKeyNames,PermissionsHeadlessCMSAccess,PermissionsLMEndMessagingSessionUserPerm,PermissionsConsentApiUpdate,PermissionsAccessContentBuilder,PermissionsAccountSwitcherUser,PermissionsManageC360AConnections,PermissionsManageReleaseUpdates,PermissionsViewAllProfiles,PermissionsSkipIdentityConfirmation,PermissionsSendCustomNotifications,PermissionsPackaging2Delete,PermissionsFSCComprehensiveUserAccess,PermissionsManageTrustMeasures,PermissionsViewTrustMeasures,PermissionsIsotopeCToCUser,PermissionsIsotopeAccess,PermissionsIsotopeLEX,PermissionsQuipMetricsAccess,PermissionsQuipUserEngagementMetrics,PermissionsManageExternalConnections,PermissionsAIViewInsightObjects,PermissionsAICreateInsightObjects,PermissionsNativeWebviewScrolling,PermissionsViewDeveloperName,Type,UserLicenseId,UserType,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,Description,LastViewedDate,LastReferencedDate +id,name,permissions_email_single,permissions_email_mass,permissions_edit_task,permissions_edit_event,permissions_export_report,permissions_import_personal,permissions_data_export,permissions_manage_users,permissions_edit_public_filters,permissions_edit_public_templates,permissions_modify_all_data,permissions_manage_cases,permissions_manage_solutions,permissions_customize_application,permissions_edit_readonly_fields,permissions_run_reports,permissions_view_setup,permissions_transfer_any_entity,permissions_new_report_builder,permissions_manage_self_service,permissions_manage_css_users,permissions_activate_contract,permissions_approve_contract,permissions_import_leads,permissions_manage_leads,permissions_transfer_any_lead,permissions_view_all_data,permissions_edit_public_documents,permissions_view_encrypted_data,permissions_edit_brand_templates,permissions_edit_html_templates,permissions_manage_translation,permissions_delete_activated_contract,permissions_send_sit_requests,permissions_api_user_only,permissions_manage_remote_access,permissions_can_use_new_dashboard_builder,permissions_manage_categories,permissions_convert_leads,permissions_test_instance_create,permissions_password_never_expires,permissions_use_team_reassign_wizards,permissions_install_multiforce,permissions_publish_multiforce,permissions_edit_opp_line_item_unit_price,permissions_manage_territories,permissions_create_multiforce,permissions_bulk_api_hard_delete,permissions_inbound_migration_tools_user,permissions_solution_import,permissions_manage_call_centers,permissions_manage_synonyms,permissions_outbound_migration_tools_user,permissions_view_content,permissions_manage_email_client_config,permissions_enable_notifications,permissions_manage_data_integrations,permissions_distribute_from_pers_wksp,permissions_view_data_categories,permissions_manage_data_categories,permissions_author_apex,permissions_manage_mobile,permissions_api_enabled,permissions_manage_custom_report_types,permissions_edit_case_comments,permissions_transfer_any_case,permissions_content_administrator,permissions_create_workspaces,permissions_manage_content_permissions,permissions_manage_content_properties,permissions_manage_content_types,permissions_schedule_job,permissions_manage_exchange_config,permissions_manage_analytic_snapshots,permissions_schedule_reports,permissions_manage_business_hour_holidays,permissions_manage_dynamic_dashboards,permissions_manage_interaction,permissions_view_my_teams_dashboards,permissions_reset_passwords,permissions_flow_uflrequired,permissions_activities_access,permissions_email_template_management,permissions_email_administration,permissions_chatter_file_link,permissions_force_two_factor,permissions_view_event_log_files,permissions_manage_networks,permissions_manage_auth_providers,permissions_run_flow,permissions_create_customize_dashboards,permissions_create_dashboard_folders,permissions_view_public_dashboards,permissions_manage_dashbds_in_pub_folders,permissions_create_customize_reports,permissions_create_report_folders,permissions_view_public_reports,permissions_manage_reports_in_pub_folders,permissions_edit_my_dashboards,permissions_edit_my_reports,permissions_view_all_users,permissions_connect_org_to_environment_hub,permissions_create_customize_filters,permissions_content_hub_user,permissions_govern_networks,permissions_sales_console,permissions_two_factor_api,permissions_delete_topics,permissions_edit_topics,permissions_create_topics,permissions_assign_topics,permissions_identity_enabled,permissions_identity_connect,permissions_content_workspaces,permissions_custom_mobile_apps_access,permissions_view_help_link,permissions_manage_profiles_permissionsets,permissions_assign_permission_sets,permissions_manage_roles,permissions_manage_ip_addresses,permissions_manage_sharing,permissions_manage_internal_users,permissions_manage_password_policies,permissions_manage_login_access_policies,permissions_manage_custom_permissions,permissions_std_automatic_activity_capture,permissions_manage_two_factor,permissions_debug_apex,permissions_lightning_experience_user,permissions_config_custom_recs,permissions_submit_macros_allowed,permissions_bulk_macros_allowed,permissions_manage_session_permission_sets,permissions_create_audit_fields,permissions_update_with_inactive_owner,permissions_manage_sandboxes,permissions_automatic_activity_capture,permissions_import_custom_objects,permissions_delegated_two_factor,permissions_select_files_from_salesforce,permissions_moderate_network_users,permissions_merge_topics,permissions_subscribe_to_lightning_reports,permissions_manage_pvt_rpts_and_dashbds,permissions_allow_lightning_login,permissions_campaign_influence2,permissions_view_data_assessment,permissions_can_approve_feed_post,permissions_allow_view_edit_converted_leads,permissions_show_company_name_as_user_badge,permissions_access_cmc,permissions_view_health_check,permissions_manage_health_check,permissions_packaging2,permissions_manage_certificates,permissions_create_report_in_lightning,permissions_prevent_classic_experience,permissions_list_email_send,permissions_change_dashboard_colors,permissions_manage_recommendation_strategies,permissions_manage_propositions,permissions_subscribe_report_roles_grps,permissions_subscribe_dashboard_roles_grps,permissions_use_web_link,permissions_has_unlimited_nbaexecutions,permissions_view_only_embedded_app_user,permissions_view_all_activities,permissions_subscribe_report_to_other_users,permissions_lightning_console_allowed_for_user,permissions_subscribe_reports_run_as_user,permissions_subscribe_to_lightning_dashboards,permissions_subscribe_dashboard_to_other_users,permissions_create_ltng_temp_in_pub,permissions_transactional_email_send,permissions_view_private_static_resources,permissions_create_ltng_temp_folder,permissions_apex_rest_services,permissions_enable_community_app_launcher,permissions_give_recognition_badge,permissions_use_my_search,permissions_ltng_promo_reserved01_user_perm,permissions_manage_subscriptions,permissions_manage_surveys,permissions_use_assistant_dialog,permissions_use_query_suggestions,permissions_view_roles,permissions_lmoutbound_messaging_user_perm,permissions_modify_data_classification,permissions_privacy_data_access,permissions_query_all_files,permissions_modify_metadata,permissions_manage_cms,permissions_sandbox_testing_in_community_app,permissions_can_edit_prompts,permissions_view_user_pii,permissions_manage_hub_connections,permissions_b2_bmarketing_analytics_user,permissions_trace_xds_queries,permissions_view_all_custom_settings,permissions_view_all_foreign_key_names,permissions_headless_cmsaccess,permissions_lmend_messaging_session_user_perm,permissions_consent_api_update,permissions_access_content_builder,permissions_account_switcher_user,permissions_manage_c360_aconnections,permissions_manage_release_updates,permissions_view_all_profiles,permissions_skip_identity_confirmation,permissions_send_custom_notifications,permissions_packaging2_delete,permissions_fsccomprehensive_user_access,permissions_manage_trust_measures,permissions_view_trust_measures,permissions_isotope_cto_cuser,permissions_isotope_access,permissions_isotope_lex,permissions_quip_metrics_access,permissions_quip_user_engagement_metrics,permissions_manage_external_connections,permissions_aiview_insight_objects,permissions_aicreate_insight_objects,permissions_native_webview_scrolling,permissions_view_developer_name,type,user_license_id,user_type,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,description,last_viewed_date,last_referenced_date +src02.crm_profile +org02.crm_profile +CRM_Profile_ex.sql + diff --git a/s3/data/crm/settings/CRM_Profile_ex.sql b/s3/data/crm/settings/CRM_Profile_ex.sql new file mode 100644 index 00000000..b858ab02 --- /dev/null +++ b/s3/data/crm/settings/CRM_Profile_ex.sql @@ -0,0 +1 @@ +CALL crm_history('src02.crm_profile', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Question_Response_vod__c.txt b/s3/data/crm/settings/CRM_Question_Response_vod__c.txt new file mode 100644 index 00000000..a994ab4a --- /dev/null +++ b/s3/data/crm/settings/CRM_Question_Response_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +30 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Survey_Target_vod__c,Answer_Choice_vod__c,Date_vod__c,Datetime_vod__c,External_ID_vod__c,Mobile_ID_vod__c,Number_vod__c,Order_vod__c,Question_Text_vod__c,Required_vod__c,Response_Hash_vod__c,Response_vod__c,Score_vod__c,Survey_Question_vod__c,Text_vod__c,Type_vod__c,Condition_vod__c,Inactive_Condition_vod__c,Source_ID_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,survey_target_vod__c,answer_choice_vod__c,date_vod__c,datetime_vod__c,external_id_vod__c,mobile_id_vod__c,number_vod__c,order_vod__c,question_text_vod__c,required_vod__c,response_hash_vod__c,response_vod__c,score_vod__c,survey_question_vod__c,text_vod__c,type_vod__c,condition_vod__c,inactive_condition_vod__c,source_id_vod__c +src02.crm_question_response_vod__c +org02.crm_question_response_vod__c + + diff --git a/s3/data/crm/settings/CRM_RecordType.txt b/s3/data/crm/settings/CRM_RecordType.txt new file mode 100644 index 00000000..78960693 --- /dev/null +++ b/s3/data/crm/settings/CRM_RecordType.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +14 +Id,Name,DeveloperName,NamespacePrefix,Description,BusinessProcessId,SobjectType,IsActive,IsPersonType,CreatedById,CreatedDate,LastModifiedById,LastModifiedDate,SystemModstamp +id,name,developer_name,namespace_prefix,description,business_process_id,sobject_type,is_active,is_person_type,created_by_id,created_date,last_modified_by_id,last_modified_date,system_modstamp +src02.crm_record_type +org02.crm_record_type + + diff --git a/s3/data/crm/settings/CRM_Remote_Meeting_vod__c.txt b/s3/data/crm/settings/CRM_Remote_Meeting_vod__c.txt new file mode 100644 index 00000000..1b4827c9 --- /dev/null +++ b/s3/data/crm/settings/CRM_Remote_Meeting_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +24 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Meeting_Id_vod__c,Meeting_Name_vod__c,Mobile_ID_vod__c,Scheduled_DateTime_vod__c,Scheduled_vod__c,Attendance_Report_Process_Status_vod__c,Latest_Meeting_Start_Datetime_vod__c,Meeting_Password_vod__c,Meeting_Outcome_Status_vod__c,Allow_for_Joining_via_Zoom_vod__c,Zoom_Join_Token_vod__c,VExternal_Id_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,meeting_id_vod__c,meeting_name_vod__c,mobile_id_vod__c,scheduled_date_time_vod__c,scheduled_vod__c,attendance_report_process_status_vod__c,latest_meeting_start_datetime_vod__c,meeting_password_vod__c,meeting_outcome_status_vod__c,allow_for_joining_via_zoom_vod__c,zoom_join_token_vod__c,vexternal_id_vod__c +src02.crm_remote_meeting_vod__c +org02.crm_remote_meeting_vod__c + + diff --git a/s3/data/crm/settings/CRM_Sent_Email_vod__c.txt b/s3/data/crm/settings/CRM_Sent_Email_vod__c.txt new file mode 100644 index 00000000..148ed757 --- /dev/null +++ b/s3/data/crm/settings/CRM_Sent_Email_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +65 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,Account_Email_vod__c,Account_vod__c,Approved_Email_Template_vod__c,Capture_Datetime_vod__c,Detail_Group_vod__c,Email_Config_Values_vod__c,Email_Content2_vod__c,Email_Content_vod__c,Email_Fragments_vod__c,Email_Sent_Date_vod__c,Failure_Msg_vod__c,Last_Activity_Date_vod__c,Last_Device_vod__c,MC_Capture_Datetime_vod__c,Mobile_ID_vod__c,Opened_vod__c,Product_Display_vod__c,Product_vod__c,Sender_Email_vod__c,Status_vod__c,Valid_Consent_Exists_vod__c,Approved_Document_Views_vod__c,Click_Count_vod__c,Last_Click_Date_vod__c,Last_Open_Date_vod__c,Open_Count_vod__c,Receipt_Entity_Type_vod__c,Receipt_Record_Id_vod__c,Territory_vod__c,Call2_vod__c,Medical_Inquiry_vod__c,Parent_Email_vod__c,Related_Transaction_ID_vod__c,Case_vod__c,Key_Message_vod__c,Suggestion_vod__c,EM_Attendee_vod__c,EM_Event_Speaker_vod__c,EM_Event_Team_Member_vod__c,Event_Attendee_vod__c,Event_vod__c,Medical_Event_vod__c,Scheduled_Send_Datetime_vod__c,User_vod__c,Content_Type_vod__c,Bcc_vod__c,Event_Attendee_Mobile_Id_vod__c,Event_Mobile_Id_vod__c,Activity_Tracking_Mode_vod__c,Email_Source_vod__c,Subject_vod__c,User_Input_Text_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,account_email_vod__c,account_vod__c,approved_email_template_vod__c,capture_datetime_vod__c,detail_group_vod__c,email_config_values_vod__c,email_content2_vod__c,email_content_vod__c,email_fragments_vod__c,email_sent_date_vod__c,failure_msg_vod__c,last_activity_date_vod__c,last_device_vod__c,mc_capture_datetime_vod__c,mobile_id_vod__c,opened_vod__c,product_display_vod__c,product_vod__c,sender_email_vod__c,status_vod__c,valid_consent_exists_vod__c,approved_document_views_vod__c,click_count_vod__c,last_click_date_vod__c,last_open_date_vod__c,open_count_vod__c,receipt_entity_type_vod__c,receipt_record_id_vod__c,territory_vod__c,call2_vod__c,medical_inquiry_vod__c,parent_email_vod__c,related_transaction_id_vod__c,case_vod__c,key_message_vod__c,suggestion_vod__c,em_attendee_vod__c,em_event_speaker_vod__c,em_event_team_member_vod__c,event_attendee_vod__c,event_vod__c,medical_event_vod__c,scheduled_send_datetime_vod__c,user_vod__c,content_type_vod__c,bcc_vod__c,event_attendee_mobile_id_vod__c,event_mobile_id_vod__c,activity_tracking_mode_vod__c,email_source_vod__c,subject_vod__c,user_input_text_vod__c +src02.crm_sent_email_vod__c +org02.crm_sent_email_vod__c + + diff --git a/s3/data/crm/settings/CRM_Sent_Fragment_vod__c.txt b/s3/data/crm/settings/CRM_Sent_Fragment_vod__c.txt new file mode 100644 index 00000000..c93f9f9e --- /dev/null +++ b/s3/data/crm/settings/CRM_Sent_Fragment_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +16 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,Sent_Email_vod__c,Account_vod__c,Email_Template_vod__c,Sent_Fragment_vod__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,sent_email_vod__c,account_vod__c,email_template_vod__c,sent_fragment_vod__c +src02.crm_sent_fragment_vod__c +org02.crm_sent_fragment_vod__c + + diff --git a/s3/data/crm/settings/CRM_Survey_Question_vod__c.txt b/s3/data/crm/settings/CRM_Survey_Question_vod__c.txt new file mode 100644 index 00000000..3a636f0a --- /dev/null +++ b/s3/data/crm/settings/CRM_Survey_Question_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +23 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Survey_vod__c,Answer_Choice_vod__c,External_ID_vod__c,Max_Score_vod__c,Min_Score_vod__c,Order_vod__c,Question_vod__c,Required_vod__c,Text_vod__c,Condition_vod__c,Source_ID_vod__c,MSJ_External_ID__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,survey_vod__c,answer_choice_vod__c,external_id_vod__c,max_score_vod__c,min_score_vod__c,order_vod__c,question_vod__c,required_vod__c,text_vod__c,condition_vod__c,source_id_vod__c,msj_external_id__c +src02.crm_survey_question_vod__c +org02.crm_survey_question_vod__c + + diff --git a/s3/data/crm/settings/CRM_Survey_Target_vod__c.txt b/s3/data/crm/settings/CRM_Survey_Target_vod__c.txt new file mode 100644 index 00000000..f9f80626 --- /dev/null +++ b/s3/data/crm/settings/CRM_Survey_Target_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +39 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Account_Display_Name_vod__c,Account_vod__c,Channels_vod__c,End_Date_vod__c,Entity_Reference_Id_vod__c,External_ID_vod__c,Language_vod__c,Lock_vod__c,Mobile_ID_vod__c,No_Autoassign_vod__c,Not_Completed_vod__c,Region_vod__c,Segment_vod__c,Start_Date_vod__c,Status_vod__c,Survey_vod__c,Territory_vod__c,zvod_Address_vod__c,zvod_Specialty_vod__c,Score_vod__c,User_vod__c,Child_Account_vod__c,Location_Entity_Reference_Id_vod__c,Location_vod__c,Target_Type_vod__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,account_display_name_vod__c,account_vod__c,channels_vod__c,end_date_vod__c,entity_reference_id_vod__c,external_id_vod__c,language_vod__c,lock_vod__c,mobile_id_vod__c,no_autoassign_vod__c,not_completed_vod__c,region_vod__c,segment_vod__c,start_date_vod__c,status_vod__c,survey_vod__c,territory_vod__c,zvod_address_vod__c,zvod_specialty_vod__c,score_vod__c,user_vod__c,child_account_vod__c,location_entity_reference_id_vod__c,location_vod__c,target_type_vod__c +src02.crm_survey_target_vod__c +org02.crm_survey_target_vod__c + + diff --git a/s3/data/crm/settings/CRM_Survey_vod__c.txt b/s3/data/crm/settings/CRM_Survey_vod__c.txt new file mode 100644 index 00000000..bfd04a2e --- /dev/null +++ b/s3/data/crm/settings/CRM_Survey_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +37 +Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Assignment_Type_vod__c,Channels_vod__c,End_Date_vod__c,Expired_vod__c,External_ID_vod__c,Language_vod__c,Lock_vod__c,Open_vod__c,Product_vod__c,Region_vod__c,Segment_vod__c,Start_Date_vod__c,Status_vod__c,Territory_vod__c,zvod_Questions_vod__c,zvod_Segments_vod__c,zvod_Targets_vod__c,Max_Score_vod__c,Min_Score_vod__c,Autotarget_vod__c,Territories_vod__c,Target_Type_vod__c,MSJ_External_ID__c +id,owner_id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,assignment_type_vod__c,channels_vod__c,end_date_vod__c,expired_vod__c,external_id_vod__c,language_vod__c,lock_vod__c,open_vod__c,product_vod__c,region_vod__c,segment_vod__c,start_date_vod__c,status_vod__c,territory_vod__c,zvod_questions_vod__c,zvod_segments_vod__c,zvod_targets_vod__c,max_score_vod__c,min_score_vod__c,autotarget_vod__c,territories_vod__c,target_type_vod__c,msj_external_id__c +src02.crm_survey_vod__c +org02.crm_survey_vod__c + + diff --git a/s3/data/crm/settings/CRM_Territory2.txt b/s3/data/crm/settings/CRM_Territory2.txt new file mode 100644 index 00000000..5486b657 --- /dev/null +++ b/s3/data/crm/settings/CRM_Territory2.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +15 +Id,Name,Territory2TypeId,Territory2ModelId,ParentTerritory2Id,Description,ForecastUserId,AccountAccessLevel,OpportunityAccessLevel,CaseAccessLevel,ContactAccessLevel,LastModifiedDate,LastModifiedById,SystemModstamp,DeveloperName,MSJ_Territory_Type__c,MSJ_Level__c +id,name,territory2_type_id,territory2_model_id,parent_territory2_id,description,forecast_user_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,last_modified_date,last_modified_by_id,system_modstamp,developer_name,msj_territory_type__c,msj_level__c +src02.crm_territory2 +org02.crm_territory2 +CRM_Territory2_ex.sql + diff --git a/s3/data/crm/settings/CRM_Territory2_ALL.txt b/s3/data/crm/settings/CRM_Territory2_ALL.txt new file mode 100644 index 00000000..525e1796 --- /dev/null +++ b/s3/data/crm/settings/CRM_Territory2_ALL.txt @@ -0,0 +1,14 @@ +CRM +, +utf-8 +" +CRLF +1 +15 +Id,Name,Territory2TypeId,Territory2ModelId,ParentTerritory2Id,Description,ForecastUserId,AccountAccessLevel,OpportunityAccessLevel,CaseAccessLevel,ContactAccessLevel,LastModifiedDate,LastModifiedById,SystemModstamp,DeveloperName +id,name,territory2_type_id,territory2_model_id,parent_territory2_id,description,forecast_user_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,last_modified_date,last_modified_by_id,system_modstamp,developer_name +src02.crm_territory2_all +org02.crm_territory2_all + + +truncate_src_table:src02.crm_territory2_all diff --git a/s3/data/crm/settings/CRM_Territory2_ex.sql b/s3/data/crm/settings/CRM_Territory2_ex.sql new file mode 100644 index 00000000..a30ccd91 --- /dev/null +++ b/s3/data/crm/settings/CRM_Territory2_ex.sql @@ -0,0 +1,2 @@ +CALL crm_data_sync('src02.crm_territory2', 'src02.crm_territory2_all', 'system_modstamp'); +CALL crm_history('src02.crm_territory2', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Time_Off_Territory_vod__c.txt b/s3/data/crm/settings/CRM_Time_Off_Territory_vod__c.txt new file mode 100644 index 00000000..e2b1b9b6 --- /dev/null +++ b/s3/data/crm/settings/CRM_Time_Off_Territory_vod__c.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +25 +Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,LastActivityDate,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Reason_vod__c,Territory_vod__c,Date_vod__c,Status_vod__c,Time_vod__c,Hours_vod__c,Mobile_ID_vod__c,Hours_off_vod__c,Start_Time_vod__c,MSJ_Day__c,MSJ_Comment__c +id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,last_activity_date,may_edit,is_locked,last_viewed_date,last_referenced_date,reason_vod__c,territory_vod__c,date_vod__c,status_vod__c,time_vod__c,hours_vod__c,mobile_id_vod__c,hours_off_vod__c,start_time_vod__c,msj_day__c,msj_comment__c +src02.crm_time_off_territory_vod__c +org02.crm_time_off_territory_vod__c + + diff --git a/s3/data/crm/settings/CRM_User.txt b/s3/data/crm/settings/CRM_User.txt new file mode 100644 index 00000000..317a5e14 --- /dev/null +++ b/s3/data/crm/settings/CRM_User.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +197 +Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,LastPasswordChangeDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,NumberOfFailedLogins,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesNativeEmailClient,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c +id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,last_password_change_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,number_of_failed_logins,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_native_email_client,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c +src02.crm_user +org02.crm_user + + diff --git a/s3/data/crm/settings/CRM_UserRole.txt b/s3/data/crm/settings/CRM_UserRole.txt new file mode 100644 index 00000000..4b10b6c2 --- /dev/null +++ b/s3/data/crm/settings/CRM_UserRole.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +16 +Id,Name,ParentRoleId,RollupDescription,OpportunityAccessForAccountOwner,CaseAccessForAccountOwner,ContactAccessForAccountOwner,ForecastUserId,MayForecastManagerShare,LastModifiedDate,LastModifiedById,SystemModstamp,DeveloperName,PortalAccountId,PortalType,PortalAccountOwnerId +id,name,parent_role_id,rollup_description,opportunity_access_for_account_owner,case_access_for_account_owner,contact_access_for_account_owner,forecast_user_id,may_forecast_manager_share,last_modified_date,last_modified_by_id,system_modstamp,developer_name,portal_account_id,portal_type,portal_account_owner_id +src02.crm_user_role +org02.crm_user_role + + diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association.txt b/s3/data/crm/settings/CRM_UserTerritory2Association.txt new file mode 100644 index 00000000..039757b9 --- /dev/null +++ b/s3/data/crm/settings/CRM_UserTerritory2Association.txt @@ -0,0 +1,13 @@ +CRM +, +utf-8 +" +CRLF +1 +8 +Id,UserId,Territory2Id,IsActive,RoleInTerritory2,LastModifiedDate,LastModifiedById,SystemModstamp +id,user_id,territory2_id,is_active,role_in_territory2,last_modified_date,last_modified_by_id,system_modstamp +src02.crm_user_territory2_association +org02.crm_user_territory2_association +CRM_UserTerritory2Association_ex.sql + diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt b/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt new file mode 100644 index 00000000..4617829d --- /dev/null +++ b/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt @@ -0,0 +1,14 @@ +CRM +, +utf-8 +" +CRLF +1 +8 +Id,UserId,Territory2Id,IsActive,RoleInTerritory2,LastModifiedDate,LastModifiedById,SystemModstamp +id,user_id,territory2_id,is_active,role_in_territory2,last_modified_date,last_modified_by_id,system_modstamp +src02.crm_user_territory2_association_all +org02.crm_user_territory2_association_all + + +truncate_src_table:src02.crm_user_territory2_association_all diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql new file mode 100644 index 00000000..6a9aaece --- /dev/null +++ b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql @@ -0,0 +1,2 @@ +CALL crm_data_sync('src02.crm_user_territory2_association', 'src02.crm_user_territory2_association_all', 'system_modstamp'); +CALL crm_history('src02.crm_user_territory2_association', 'system_modstamp'); diff --git a/s3/data/crm/settings/configmap.config b/s3/data/crm/settings/configmap.config new file mode 100644 index 00000000..89993b8b --- /dev/null +++ b/s3/data/crm/settings/configmap.config @@ -0,0 +1,58 @@ +/* 【CRMデータ 差分連携】 */ +CRM_Clm_Presentation_vod__c_[0-9]{14}\.(CSV|csv) CRM_Clm_Presentation_vod__c.txt +CRM_Clm_Presentation_Slide_vod__c_[0-9]{14}\.(CSV|csv) CRM_Clm_Presentation_Slide_vod__c.txt +CRM_Medical_Insight_vod__c_[0-9]{14}\.(CSV|csv) CRM_Medical_Insight_vod__c.txt +CRM_MSJ_MR_Weekly_Report__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_MR_Weekly_Report__c.txt +CRM_Account_Territory_Loader_vod__c_[0-9]{14}\.(CSV|csv) CRM_Account_Territory_Loader_vod__c.txt +CRM_Event_Attendee_vod__c_[0-9]{14}\.(CSV|csv) CRM_Event_Attendee_vod__c.txt +CRM_ObjectTerritory2Association_[0-9]{14}\.(CSV|csv) CRM_ObjectTerritory2Association.txt +CRM_Key_Message_vod__c_[0-9]{14}\.(CSV|csv) CRM_Key_Message_vod__c.txt +CRM_Group_[0-9]{14}\.(CSV|csv) CRM_Group.txt +CRM_Medical_Event_vod__c_[0-9]{14}\.(CSV|csv) CRM_Medical_Event_vod__c.txt +CRM_MSJ_Medical_Event_Evaluation__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Medical_Event_Evaluation__c.txt +CRM_Coaching_Report_vod__c_[0-9]{14}\.(CSV|csv) CRM_Coaching_Report_vod__c.txt +CRM_Call2_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call2_vod__c.txt +CRM_Call2_Detail_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call2_Detail_vod__c.txt +CRM_Call2_Key_Message_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call2_Key_Message_vod__c.txt +CRM_Call_Clickstream_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call_Clickstream_vod__c.txt +CRM_Call2_Discussion_vod__c_[0-9]{14}\.(CSV|csv) CRM_Call2_Discussion_vod__c.txt +CRM_Time_Off_Territory_vod__c_[0-9]{14}\.(CSV|csv) CRM_Time_Off_Territory_vod__c.txt +CRM_Dynamic_Attribute_vod__c_[0-9]{14}\.(CSV|csv) CRM_Dynamic_Attribute_vod__c.txt +CRM_Dynamic_Attribute_Configuration_vod__c_[0-9]{14}\.(CSV|csv) CRM_Dynamic_Attribute_Configuration_vod__c.txt +CRM_Territory2_[0-9]{14}\.(CSV|csv) CRM_Territory2.txt +CRM_Profile_[0-9]{14}\.(CSV|csv) CRM_Profile.txt +CRM_My_Setup_Products_vod__c_[0-9]{14}\.(CSV|csv) CRM_My_Setup_Products_vod__c.txt +CRM_Multichannel_Activity_vod__c_[0-9]{14}\.(CSV|csv) CRM_Multichannel_Activity_vod__c.txt +CRM_Multichannel_Activity_Line_vod__c_[0-9]{14}\.(CSV|csv) CRM_Multichannel_Activity_Line_vod__c.txt +CRM_Multichannel_Consent_vod__c_[0-9]{14}\.(CSV|csv) CRM_Multichannel_Consent_vod__c.txt +CRM_Medical_Inquiry_vod__c_[0-9]{14}\.(CSV|csv) CRM_Medical_Inquiry_vod__c.txt +CRM_Email_Activity_vod__c_[0-9]{14}\.(CSV|csv) CRM_Email_Activity_vod__c.txt +CRM_User_[0-9]{14}\.(CSV|csv) CRM_User.txt +CRM_UserTerritory2Association_[0-9]{14}\.(CSV|csv) CRM_UserTerritory2Association.txt +CRM_Remote_Meeting_vod__c_[0-9]{14}\.(CSV|csv) CRM_Remote_Meeting_vod__c.txt +CRM_RecordType_[0-9]{14}\.(CSV|csv) CRM_RecordType.txt +CRM_UserRole_[0-9]{14}\.(CSV|csv) CRM_UserRole.txt +CRM_Account_[0-9]{14}\.(CSV|csv) CRM_Account.txt +CRM_AccountShare_[0-9]{14}\.(CSV|csv) CRM_AccountShare.txt +CRM_Contact_[0-9]{14}\.(CSV|csv) CRM_Contact.txt +CRM_Consent_Type_vod__c_[0-9]{14}\.(CSV|csv) CRM_Consent_Type_vod__c.txt +CRM_Consent_Header_vod__c_[0-9]{14}\.(CSV|csv) CRM_Consent_Header_vod__c.txt +CRM_Consent_Line_vod__c_[0-9]{14}\.(CSV|csv) CRM_Consent_Line_vod__c.txt +CRM_MSJ_Inquiry_Assignment__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Inquiry_Assignment__c.txt +CRM_Approved_Document_vod__c_[0-9]{14}\.(CSV|csv) CRM_Approved_Document_vod__c.txt +CRM_Child_Account_vod__c_[0-9]{14}\.(CSV|csv) CRM_Child_Account_vod__c.txt +CRM_MSJ_Hospital_Medical_Regimen__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Hospital_Medical_Regimen__c.txt +CRM_MSJ_Medical_Regimen__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Medical_Regimen__c.txt +CRM_MSJ_Patient__c_[0-9]{14}\.(CSV|csv) CRM_MSJ_Patient__c.txt +CRM_Product_vod__c_[0-9]{14}\.(CSV|csv) CRM_Product_vod__c.txt +CRM_Product_Group_vod__c_[0-9]{14}\.(CSV|csv) CRM_Product_Group_vod__c.txt +CRM_Product_Metrics_vod__c_[0-9]{14}\.(CSV|csv) CRM_Product_Metrics_vod__c.txt +CRM_Survey_vod__c_[0-9]{14}\.(CSV|csv) CRM_Survey_vod__c.txt +CRM_Survey_Target_vod__c_[0-9]{14}\.(CSV|csv) CRM_Survey_Target_vod__c.txt +CRM_Survey_Question_vod__c_[0-9]{14}\.(CSV|csv) CRM_Survey_Question_vod__c.txt +CRM_Question_Response_vod__c_[0-9]{14}\.(CSV|csv) CRM_Question_Response_vod__c.txt +CRM_Sent_Fragment_vod__c_[0-9]{14}\.(CSV|csv) CRM_Sent_Fragment_vod__c.txt +CRM_Sent_Email_vod__c_[0-9]{14}\.(CSV|csv) CRM_Sent_Email_vod__c.txt +/* 【CRMデータ 全件連携】 */ +CRM_Territory2_ALL_[0-9]{14}\.(CSV|csv) CRM_Territory2_ALL.txt +CRM_UserTerritory2Association_ALL_[0-9]{14}\.(CSV|csv) CRM_UserTerritory2Association_ALL.txt From 07652d8ad17b56f659d320021f2f2e1c56614e14 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 101/275] =?UTF-8?q?feat:=20S3=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E3=80=81=E9=9B=9B=E5=BD=A2?= =?UTF-8?q?=E3=82=92=E4=BD=9C=E3=81=A3=E3=81=A6=E3=81=BF=E3=81=9F=E3=80=82?= =?UTF-8?q?=E5=8B=95=E4=BD=9C=E3=81=97=E3=81=A6=E3=81=84=E3=82=8B=E3=81=93?= =?UTF-8?q?=E3=81=A8=E3=81=AF=E7=A2=BA=E8=AA=8D=E3=81=A7=E3=81=8D=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 1 + ecs/crm-datafetch/Pipfile | 7 + ecs/crm-datafetch/Pipfile.lock | 471 ++++++++++++++++++++++-- ecs/crm-datafetch/tests/__init__.py | 0 ecs/crm-datafetch/tests/aws/__init__.py | 0 ecs/crm-datafetch/tests/aws/test_s3.py | 20 + ecs/crm-datafetch/tests/conftest.py | 21 ++ 7 files changed, 488 insertions(+), 32 deletions(-) create mode 100644 ecs/crm-datafetch/tests/__init__.py create mode 100644 ecs/crm-datafetch/tests/aws/__init__.py create mode 100644 ecs/crm-datafetch/tests/aws/test_s3.py create mode 100644 ecs/crm-datafetch/tests/conftest.py diff --git a/.gitignore b/.gitignore index 0e397fc3..88a052ff 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* __pycache__/ .env **/.vscode/settings.json +.coverage diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 33501d98..3d7988e7 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -3,6 +3,10 @@ url = "https://pypi.org/simple" verify_ssl = true name = "pypi" +[scripts] +test = "pytest tests/" +"test:cov" = "pytest --cov=src tests/" + [packages] boto3 = "*" simple-salesforce = "*" @@ -11,6 +15,9 @@ tenacity = "*" [dev-packages] autopep8 = "*" flake8 = "*" +pytest = "*" +pytest-cov = "*" +moto = "*" [requires] python_version = "3.8" diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index c8c60540..7c13f0c9 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "ec1d83143aff859500979be73f67196dcfe2298ad3553a7d81ad0605a277d672" + "sha256": "f1433a55f486f24bb14d6447713a0cdeeb704542a695103debd9514face495cc" }, "pipfile-spec": 6, "requires": { @@ -18,11 +18,11 @@ "default": { "attrs": { "hashes": [ - "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", - "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6", + "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==21.4.0" + "markers": "python_version >= '3.5'", + "version": "==22.1.0" }, "authlib": { "hashes": [ @@ -33,19 +33,19 @@ }, "boto3": { "hashes": [ - "sha256:5c775dcb12ca5d6be3f5aa3c49d77783faa64eb30fd3f4af93ff116bb42f9ffb", - "sha256:5d9bcc355cf6edd7f3849fedac4252e12a0aa2b436cdbc0d4371b16a0f852a30" + "sha256:aec404d06690a0ca806592efc6bbe30a9ac88fa2ad73b6d907f7794a8b3b1459", + "sha256:df3d6ef02304bd7c3711090936c092d5db735dda109decac1e236c3ef7fdb7af" ], "index": "pypi", - "version": "==1.24.34" + "version": "==1.24.42" }, "botocore": { "hashes": [ - "sha256:0d824a5315f5f5c3bea53c14107a69695ef43190edf647f1281bac8f172ca77c", - "sha256:9c695d47f1f1212f3e306e51f7bacdf67e58055194ddcf7d8296660b124cf135" + "sha256:38a180a6666c5a9b069a75ec3cf374ff2a64c3e90c9f24a916858bcdeb04456d", + "sha256:f8e6c2f69a9d577fb9c69e4e74f49f4315a48decee0e7dc88b6e470772110884" ], "markers": "python_version >= '3.7'", - "version": "==1.27.34" + "version": "==1.27.42" }, "cached-property": { "hashes": [ @@ -59,7 +59,7 @@ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2022.6.15" }, "cffi": { @@ -136,7 +136,7 @@ "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==2.1.0" }, "cryptography": { @@ -164,7 +164,7 @@ "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==37.0.4" }, "idna": { @@ -352,22 +352,30 @@ }, "urllib3": { "hashes": [ - "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", - "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" + "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc", + "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", - "version": "==1.26.10" + "version": "==1.26.11" }, "zeep": { "hashes": [ "sha256:5867f2eadd6b028d9751f4155af590d3aaf9280e3a0ed5e15a53343921c956e5", "sha256:81c491092b71f5b276de8c63dfd452be3f322622c48a54f3a497cf913bdfb2f4" ], - "markers": "python_version >= '3.6'", + "markers": "python_full_version >= '3.6.0'", "version": "==4.1.0" } }, "develop": { + "attrs": { + "hashes": [ + "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6", + "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c" + ], + "markers": "python_version >= '3.5'", + "version": "==22.1.0" + }, "autopep8": { "hashes": [ "sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979", @@ -376,36 +384,403 @@ "index": "pypi", "version": "==1.6.0" }, - "flake8": { + "boto3": { "hashes": [ - "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", - "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" + "sha256:aec404d06690a0ca806592efc6bbe30a9ac88fa2ad73b6d907f7794a8b3b1459", + "sha256:df3d6ef02304bd7c3711090936c092d5db735dda109decac1e236c3ef7fdb7af" ], "index": "pypi", - "version": "==4.0.1" + "version": "==1.24.42" + }, + "botocore": { + "hashes": [ + "sha256:38a180a6666c5a9b069a75ec3cf374ff2a64c3e90c9f24a916858bcdeb04456d", + "sha256:f8e6c2f69a9d577fb9c69e4e74f49f4315a48decee0e7dc88b6e470772110884" + ], + "markers": "python_version >= '3.7'", + "version": "==1.27.42" + }, + "certifi": { + "hashes": [ + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==2022.6.15" + }, + "cffi": { + "hashes": [ + "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", + "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", + "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", + "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", + "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", + "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", + "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", + "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", + "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", + "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", + "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", + "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", + "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", + "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", + "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", + "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", + "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", + "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", + "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", + "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", + "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", + "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", + "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", + "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", + "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", + "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", + "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", + "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", + "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", + "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", + "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", + "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", + "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", + "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", + "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", + "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", + "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", + "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", + "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", + "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", + "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", + "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", + "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", + "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", + "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", + "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", + "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", + "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", + "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", + "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", + "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", + "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", + "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", + "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", + "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", + "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", + "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", + "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", + "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", + "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", + "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", + "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", + "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", + "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" + ], + "version": "==1.15.1" + }, + "charset-normalizer": { + "hashes": [ + "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", + "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==2.1.0" + }, + "coverage": { + "extras": [ + "toml" + ], + "hashes": [ + "sha256:0895ea6e6f7f9939166cc835df8fa4599e2d9b759b02d1521b574e13b859ac32", + "sha256:0f211df2cba951ffcae210ee00e54921ab42e2b64e0bf2c0befc977377fb09b7", + "sha256:147605e1702d996279bb3cc3b164f408698850011210d133a2cb96a73a2f7996", + "sha256:24b04d305ea172ccb21bee5bacd559383cba2c6fcdef85b7701cf2de4188aa55", + "sha256:25b7ec944f114f70803d6529394b64f8749e93cbfac0fe6c5ea1b7e6c14e8a46", + "sha256:2b20286c2b726f94e766e86a3fddb7b7e37af5d0c635bdfa7e4399bc523563de", + "sha256:2dff52b3e7f76ada36f82124703f4953186d9029d00d6287f17c68a75e2e6039", + "sha256:2f8553878a24b00d5ab04b7a92a2af50409247ca5c4b7a2bf4eabe94ed20d3ee", + "sha256:3def6791adf580d66f025223078dc84c64696a26f174131059ce8e91452584e1", + "sha256:422fa44070b42fef9fb8dabd5af03861708cdd6deb69463adc2130b7bf81332f", + "sha256:4f89d8e03c8a3757aae65570d14033e8edf192ee9298303db15955cadcff0c63", + "sha256:5336e0352c0b12c7e72727d50ff02557005f79a0b8dcad9219c7c4940a930083", + "sha256:54d8d0e073a7f238f0666d3c7c0d37469b2aa43311e4024c925ee14f5d5a1cbe", + "sha256:5ef42e1db047ca42827a85e34abe973971c635f83aed49611b7f3ab49d0130f0", + "sha256:5f65e5d3ff2d895dab76b1faca4586b970a99b5d4b24e9aafffc0ce94a6022d6", + "sha256:6c3ccfe89c36f3e5b9837b9ee507472310164f352c9fe332120b764c9d60adbe", + "sha256:6d0b48aff8e9720bdec315d67723f0babd936a7211dc5df453ddf76f89c59933", + "sha256:6fe75dcfcb889b6800f072f2af5a331342d63d0c1b3d2bf0f7b4f6c353e8c9c0", + "sha256:79419370d6a637cb18553ecb25228893966bd7935a9120fa454e7076f13b627c", + "sha256:7bb00521ab4f99fdce2d5c05a91bddc0280f0afaee0e0a00425e28e209d4af07", + "sha256:80db4a47a199c4563d4a25919ff29c97c87569130375beca3483b41ad5f698e8", + "sha256:866ebf42b4c5dbafd64455b0a1cd5aa7b4837a894809413b930026c91e18090b", + "sha256:8af6c26ba8df6338e57bedbf916d76bdae6308e57fc8f14397f03b5da8622b4e", + "sha256:a13772c19619118903d65a91f1d5fea84be494d12fd406d06c849b00d31bf120", + "sha256:a697977157adc052284a7160569b36a8bbec09db3c3220642e6323b47cec090f", + "sha256:a9032f9b7d38bdf882ac9f66ebde3afb8145f0d4c24b2e600bc4c6304aafb87e", + "sha256:b5e28db9199dd3833cc8a07fa6cf429a01227b5d429facb56eccd765050c26cd", + "sha256:c77943ef768276b61c96a3eb854eba55633c7a3fddf0a79f82805f232326d33f", + "sha256:d230d333b0be8042ac34808ad722eabba30036232e7a6fb3e317c49f61c93386", + "sha256:d4548be38a1c810d79e097a38107b6bf2ff42151900e47d49635be69943763d8", + "sha256:d4e7ced84a11c10160c0697a6cc0b214a5d7ab21dfec1cd46e89fbf77cc66fae", + "sha256:d56f105592188ce7a797b2bd94b4a8cb2e36d5d9b0d8a1d2060ff2a71e6b9bbc", + "sha256:d714af0bdba67739598849c9f18efdcc5a0412f4993914a0ec5ce0f1e864d783", + "sha256:d774d9e97007b018a651eadc1b3970ed20237395527e22cbeb743d8e73e0563d", + "sha256:e0524adb49c716ca763dbc1d27bedce36b14f33e6b8af6dba56886476b42957c", + "sha256:e2618cb2cf5a7cc8d698306e42ebcacd02fb7ef8cfc18485c59394152c70be97", + "sha256:e36750fbbc422c1c46c9d13b937ab437138b998fe74a635ec88989afb57a3978", + "sha256:edfdabe7aa4f97ed2b9dd5dde52d2bb29cb466993bb9d612ddd10d0085a683cf", + "sha256:f22325010d8824594820d6ce84fa830838f581a7fd86a9235f0d2ed6deb61e29", + "sha256:f23876b018dfa5d3e98e96f5644b109090f16a4acb22064e0f06933663005d39", + "sha256:f7bd0ffbcd03dc39490a1f40b2669cc414fae0c4e16b77bb26806a4d0b7d1452" + ], + "markers": "python_version >= '3.7'", + "version": "==6.4.2" + }, + "cryptography": { + "hashes": [ + "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", + "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596", + "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", + "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", + "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab", + "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", + "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", + "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b", + "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441", + "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa", + "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d", + "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", + "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a", + "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", + "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", + "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", + "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282", + "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67", + "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", + "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", + "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", + "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==37.0.4" + }, + "flake8": { + "hashes": [ + "sha256:44e3ecd719bba1cb2ae65d1b54212cc9df4f5db15ac271f8856e5e6c2eebefed", + "sha256:9c51d3d1426379fd444d3b79eabbeb887849368bd053039066439523d8486961" + ], + "index": "pypi", + "version": "==5.0.1" + }, + "idna": { + "hashes": [ + "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + ], + "markers": "python_version >= '3.5'", + "version": "==3.3" + }, + "iniconfig": { + "hashes": [ + "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", + "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" + ], + "version": "==1.1.1" + }, + "jinja2": { + "hashes": [ + "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852", + "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61" + ], + "markers": "python_version >= '3.7'", + "version": "==3.1.2" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "markupsafe": { + "hashes": [ + "sha256:0212a68688482dc52b2d45013df70d169f542b7394fc744c02a57374a4207003", + "sha256:089cf3dbf0cd6c100f02945abeb18484bd1ee57a079aefd52cffd17fba910b88", + "sha256:10c1bfff05d95783da83491be968e8fe789263689c02724e0c691933c52994f5", + "sha256:33b74d289bd2f5e527beadcaa3f401e0df0a89927c1559c8566c066fa4248ab7", + "sha256:3799351e2336dc91ea70b034983ee71cf2f9533cdff7c14c90ea126bfd95d65a", + "sha256:3ce11ee3f23f79dbd06fb3d63e2f6af7b12db1d46932fe7bd8afa259a5996603", + "sha256:421be9fbf0ffe9ffd7a378aafebbf6f4602d564d34be190fc19a193232fd12b1", + "sha256:43093fb83d8343aac0b1baa75516da6092f58f41200907ef92448ecab8825135", + "sha256:46d00d6cfecdde84d40e572d63735ef81423ad31184100411e6e3388d405e247", + "sha256:4a33dea2b688b3190ee12bd7cfa29d39c9ed176bda40bfa11099a3ce5d3a7ac6", + "sha256:4b9fe39a2ccc108a4accc2676e77da025ce383c108593d65cc909add5c3bd601", + "sha256:56442863ed2b06d19c37f94d999035e15ee982988920e12a5b4ba29b62ad1f77", + "sha256:671cd1187ed5e62818414afe79ed29da836dde67166a9fac6d435873c44fdd02", + "sha256:694deca8d702d5db21ec83983ce0bb4b26a578e71fbdbd4fdcd387daa90e4d5e", + "sha256:6a074d34ee7a5ce3effbc526b7083ec9731bb3cbf921bbe1d3005d4d2bdb3a63", + "sha256:6d0072fea50feec76a4c418096652f2c3238eaa014b2f94aeb1d56a66b41403f", + "sha256:6fbf47b5d3728c6aea2abb0589b5d30459e369baa772e0f37a0320185e87c980", + "sha256:7f91197cc9e48f989d12e4e6fbc46495c446636dfc81b9ccf50bb0ec74b91d4b", + "sha256:86b1f75c4e7c2ac2ccdaec2b9022845dbb81880ca318bb7a0a01fbf7813e3812", + "sha256:8dc1c72a69aa7e082593c4a203dcf94ddb74bb5c8a731e4e1eb68d031e8498ff", + "sha256:8e3dcf21f367459434c18e71b2a9532d96547aef8a871872a5bd69a715c15f96", + "sha256:8e576a51ad59e4bfaac456023a78f6b5e6e7651dcd383bcc3e18d06f9b55d6d1", + "sha256:96e37a3dc86e80bf81758c152fe66dbf60ed5eca3d26305edf01892257049925", + "sha256:97a68e6ada378df82bc9f16b800ab77cbf4b2fada0081794318520138c088e4a", + "sha256:99a2a507ed3ac881b975a2976d59f38c19386d128e7a9a18b7df6fff1fd4c1d6", + "sha256:a49907dd8420c5685cfa064a1335b6754b74541bbb3706c259c02ed65b644b3e", + "sha256:b09bf97215625a311f669476f44b8b318b075847b49316d3e28c08e41a7a573f", + "sha256:b7bd98b796e2b6553da7225aeb61f447f80a1ca64f41d83612e6139ca5213aa4", + "sha256:b87db4360013327109564f0e591bd2a3b318547bcef31b468a92ee504d07ae4f", + "sha256:bcb3ed405ed3222f9904899563d6fc492ff75cce56cba05e32eff40e6acbeaa3", + "sha256:d4306c36ca495956b6d568d276ac11fdd9c30a36f1b6eb928070dc5360b22e1c", + "sha256:d5ee4f386140395a2c818d149221149c54849dfcfcb9f1debfe07a8b8bd63f9a", + "sha256:dda30ba7e87fbbb7eab1ec9f58678558fd9a6b8b853530e176eabd064da81417", + "sha256:e04e26803c9c3851c931eac40c695602c6295b8d432cbe78609649ad9bd2da8a", + "sha256:e1c0b87e09fa55a220f058d1d49d3fb8df88fbfab58558f1198e08c1e1de842a", + "sha256:e72591e9ecd94d7feb70c1cbd7be7b3ebea3f548870aa91e2732960fa4d57a37", + "sha256:e8c843bbcda3a2f1e3c2ab25913c80a3c5376cd00c6e8c4a86a89a28c8dc5452", + "sha256:efc1913fd2ca4f334418481c7e595c00aad186563bbc1ec76067848c7ca0a933", + "sha256:f121a1420d4e173a5d96e47e9a0c0dcff965afdf1626d28de1460815f7c4ee7a", + "sha256:fc7b548b17d238737688817ab67deebb30e8073c95749d55538ed473130ec0c7" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.1" }, "mccabe": { "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" ], - "version": "==0.6.1" + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "moto": { + "hashes": [ + "sha256:8bb8e267d9b948509d4739d81d995615a193d2c459f5c0a979aaeb0d3bd4b381", + "sha256:cbe8ad8a949f519771e5d25b670738604757fb67cd474d75d14c20677582e81f" + ], + "index": "pypi", + "version": "==3.1.16" + }, + "packaging": { + "hashes": [ + "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", + "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" + ], + "markers": "python_version >= '3.6'", + "version": "==21.3" + }, + "pluggy": { + "hashes": [ + "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", + "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" + ], + "markers": "python_version >= '3.6'", + "version": "==1.0.0" + }, + "py": { + "hashes": [ + "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719", + "sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==1.11.0" }, "pycodestyle": { "hashes": [ - "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", - "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" + "sha256:289cdc0969d589d90752582bef6dff57c5fbc6949ee8b013ad6d6449a8ae9437", + "sha256:beaba44501f89d785be791c9462553f06958a221d166c64e1f107320f839acc2" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==2.8.0" + "markers": "python_version >= '3.6'", + "version": "==2.9.0" + }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" }, "pyflakes": { "hashes": [ - "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", - "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" + "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2", + "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3" + ], + "markers": "python_version >= '3.6'", + "version": "==2.5.0" + }, + "pyparsing": { + "hashes": [ + "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", + "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" + ], + "markers": "python_full_version >= '3.6.8'", + "version": "==3.0.9" + }, + "pytest": { + "hashes": [ + "sha256:13d0e3ccfc2b6e26be000cb6568c832ba67ba32e719443bfe725814d3c42433c", + "sha256:a06a0425453864a270bc45e71f783330a7428defb4230fb5e6a731fde06ecd45" + ], + "index": "pypi", + "version": "==7.1.2" + }, + "pytest-cov": { + "hashes": [ + "sha256:578d5d15ac4a25e5f961c938b85a05b09fdaae9deef3bb6de9a6e766622ca7a6", + "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470" + ], + "index": "pypi", + "version": "==3.0.0" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.4.0" + "version": "==2.8.2" + }, + "pytz": { + "hashes": [ + "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", + "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" + ], + "version": "==2022.1" + }, + "requests": { + "hashes": [ + "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" + ], + "markers": "python_version >= '3.7' and python_version < '4'", + "version": "==2.28.1" + }, + "responses": { + "hashes": [ + "sha256:2dcc863ba63963c0c3d9ee3fa9507cbe36b7d7b0fccb4f0bdfd9e96c539b1487", + "sha256:b82502eb5f09a0289d8e209e7bad71ef3978334f56d09b444253d5ad67bf5253" + ], + "markers": "python_version >= '3.7'", + "version": "==0.21.0" + }, + "s3transfer": { + "hashes": [ + "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", + "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" + ], + "markers": "python_version >= '3.7'", + "version": "==0.6.0" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" }, "toml": { "hashes": [ @@ -414,6 +789,38 @@ ], "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.10.2" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version >= '3.7'", + "version": "==2.0.1" + }, + "urllib3": { + "hashes": [ + "sha256:c33ccba33c819596124764c23a97d25f32b28433ba0dedeb77d873a38722c9bc", + "sha256:ea6e8fb210b19d950fab93b60c9009226c63a28808bc8386e05301e25883ac0a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.11" + }, + "werkzeug": { + "hashes": [ + "sha256:4d7013ef96fd197d1cdeb03e066c6c5a491ccb44758a5b2b91137319383e5a5a", + "sha256:7e1db6a5ba6b9a8be061e47e900456355b8714c0f238b0313f53afce1a55a79a" + ], + "markers": "python_version >= '3.7'", + "version": "==2.2.1" + }, + "xmltodict": { + "hashes": [ + "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56", + "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852" + ], + "markers": "python_version >= '3.4'", + "version": "==0.13.0" } } } diff --git a/ecs/crm-datafetch/tests/__init__.py b/ecs/crm-datafetch/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/tests/aws/__init__.py b/ecs/crm-datafetch/tests/aws/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py new file mode 100644 index 00000000..61c3d352 --- /dev/null +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -0,0 +1,20 @@ +import pytest +from src.aws.s3 import S3Resource + + +@pytest.fixture +def bucket_name(): + return 'test-bucket' + + +@pytest.fixture +def s3_test(s3_client, bucket_name): + s3_client.create_bucket(Bucket=bucket_name) + yield + + +class TestS3Resource: + + def test_put_object(self, s3_test, s3_client, bucket_name): + s3_resource = S3Resource(bucket_name) + s3_resource.put_object('hogehoge', 'aaaaaaaaaaaaaaa') diff --git a/ecs/crm-datafetch/tests/conftest.py b/ecs/crm-datafetch/tests/conftest.py new file mode 100644 index 00000000..1120ced6 --- /dev/null +++ b/ecs/crm-datafetch/tests/conftest.py @@ -0,0 +1,21 @@ +import os + +import boto3 +import pytest +from moto import mock_s3 + + +@pytest.fixture +def aws_credentials(): + """Mocked AWS Credentials for moto.""" + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + + +@pytest.fixture +def s3_client(aws_credentials): + with mock_s3(): + conn = boto3.client("s3", region_name="us-east-1") + yield conn From 801f1d23f4ba35a640236489e4b566403705a9c7 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 102/275] =?UTF-8?q?feat:=20S3=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/aws/test_s3.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index 61c3d352..f341cd2c 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -15,6 +15,28 @@ def s3_test(s3_client, bucket_name): class TestS3Resource: + def test_get_object(self, s3_test, s3_client, bucket_name): + s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') + s3_resource = S3Resource(bucket_name) + actual = s3_resource.get_object('hogehoge/test.txt') + assert actual == 'aaaaaaaaaaaaaaa' + def test_put_object(self, s3_test, s3_client, bucket_name): s3_resource = S3Resource(bucket_name) - s3_resource.put_object('hogehoge', 'aaaaaaaaaaaaaaa') + s3_resource.put_object('hogehoge/test.txt', 'aaaaaaaaaaaaaaa') + actual = s3_client.get_object(Bucket=bucket_name, Key='hogehoge/test.txt') + assert actual['Body'].read() == b'aaaaaaaaaaaaaaa' + + def test_copy(self, s3_test, s3_client, bucket_name): + for_copy_bucket = 'for-copy-bucket' + s3_client.create_bucket(Bucket=for_copy_bucket) + s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') + s3_resource = S3Resource(bucket_name) + s3_resource.copy(bucket_name, 'hogehoge/test.txt', for_copy_bucket, 'test.txt') + actual = s3_client.get_object(Bucket=for_copy_bucket, Key='test.txt') + assert actual['Body'].read() == b'aaaaaaaaaaaaaaa' + + def test_init_raise_no_provide_bucket_name(self): + with pytest.raises(Exception) as e: + S3Resource() + assert e.value.args[0] == "__init__() missing 1 required positional argument: 'bucket_name'" From f2618831a29479ae17c888a89624a74d53e29ef2 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 103/275] =?UTF-8?q?feat:=20S3=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E8=BF=BD=E5=8A=A0=20Config=E3=83=90=E3=82=B1=E3=83=83?= =?UTF-8?q?=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/aws/test_s3.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index f341cd2c..db4f1c09 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -1,5 +1,7 @@ import pytest -from src.aws.s3 import S3Resource +from src.aws.s3 import ConfigBucket, S3Resource +from src.system_var import environments +from src.system_var.constants import OBJECT_INFO_FILENAME, OBJECT_INFO_FOLDER @pytest.fixture @@ -40,3 +42,18 @@ class TestS3Resource: with pytest.raises(Exception) as e: S3Resource() assert e.value.args[0] == "__init__() missing 1 required positional argument: 'bucket_name'" + + +class TestConfigBucket: + + def test_get_object_info_file(self, s3_test, s3_client, bucket_name, monkeypatch): + # = bucket_name + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', 'crm') + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'objects.json') + s3_client.put_object(Bucket=bucket_name, Key=f'crm/objects.json', Body=b'aaaaaaaaaaaaaaa') + config_bucket = ConfigBucket() + print('*' * 50, str(config_bucket), '*' * 50) + actual = config_bucket.get_object_info_file() + print(actual) + assert actual == 'aaaaaaaaaaaaaaa' From ea6ebbc0a1f972bea5ef904c9e8f66f6f5e89c46 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 104/275] =?UTF-8?q?feat:=20=E3=82=B3=E3=83=B3=E3=83=95?= =?UTF-8?q?=E3=82=A3=E3=82=B0=E3=83=90=E3=82=B1=E3=83=83=E3=83=88=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/aws/test_s3.py | 29 ++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index db4f1c09..db97825a 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -1,7 +1,5 @@ import pytest from src.aws.s3 import ConfigBucket, S3Resource -from src.system_var import environments -from src.system_var.constants import OBJECT_INFO_FILENAME, OBJECT_INFO_FOLDER @pytest.fixture @@ -27,7 +25,7 @@ class TestS3Resource: s3_resource = S3Resource(bucket_name) s3_resource.put_object('hogehoge/test.txt', 'aaaaaaaaaaaaaaa') actual = s3_client.get_object(Bucket=bucket_name, Key='hogehoge/test.txt') - assert actual['Body'].read() == b'aaaaaaaaaaaaaaa' + assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' def test_copy(self, s3_test, s3_client, bucket_name): for_copy_bucket = 'for-copy-bucket' @@ -47,7 +45,6 @@ class TestS3Resource: class TestConfigBucket: def test_get_object_info_file(self, s3_test, s3_client, bucket_name, monkeypatch): - # = bucket_name monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', 'crm') monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'objects.json') @@ -57,3 +54,27 @@ class TestConfigBucket: actual = config_bucket.get_object_info_file() print(actual) assert actual == 'aaaaaaaaaaaaaaa' + + def test_get_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') + s3_client.put_object(Bucket=bucket_name, Key=f'crm/Object.json', Body=b'aaaaaaaaaaaaaaa') + config_bucket = ConfigBucket() + print('*' * 50, str(config_bucket), '*' * 50) + actual = config_bucket.get_last_fetch_datetime_file('Object.json') + print(actual) + assert actual == 'aaaaaaaaaaaaaaa' + + def test_put_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') + config_bucket = ConfigBucket() + print('*' * 50, str(config_bucket), '*' * 50) + config_bucket.put_last_fetch_datetime_file('Object.json', 'aaaaaaaaaaaaaaa') + actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/Object.json') + assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' + + def test_config_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + config_bucket = ConfigBucket() + assert str(config_bucket) == bucket_name From 1151c015ca85aa462846ec46c78ac6cc5f1d3e8b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 105/275] =?UTF-8?q?feat:=20=E3=83=89=E3=82=AD=E3=83=A5?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=81=8B=E3=82=89=E3=83=AC=E3=83=9D=E3=83=BC=E3=83=88=E3=82=92?= =?UTF-8?q?=E7=94=9F=E6=88=90=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 + .../.vscode/python.code-snippets | 17 ++++ ecs/crm-datafetch/Pipfile | 2 + ecs/crm-datafetch/Pipfile.lock | 32 +++++-- ecs/crm-datafetch/tests/aws/test_s3.py | 89 +++++++++++++++++-- ecs/crm-datafetch/tests/conftest.py | 41 +++++++++ ecs/crm-datafetch/tests/docstring_parser.py | 32 +++++++ 7 files changed, 203 insertions(+), 13 deletions(-) create mode 100644 ecs/crm-datafetch/.vscode/python.code-snippets create mode 100644 ecs/crm-datafetch/tests/docstring_parser.py diff --git a/.gitignore b/.gitignore index 88a052ff..16a48619 100644 --- a/.gitignore +++ b/.gitignore @@ -5,4 +5,7 @@ lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* __pycache__/ .env **/.vscode/settings.json + +# python test .coverage +.report/ \ No newline at end of file diff --git a/ecs/crm-datafetch/.vscode/python.code-snippets b/ecs/crm-datafetch/.vscode/python.code-snippets new file mode 100644 index 00000000..9f01c623 --- /dev/null +++ b/ecs/crm-datafetch/.vscode/python.code-snippets @@ -0,0 +1,17 @@ +{ + "Generate Test docstring": { + "scope": "python", + "prefix": "\"\"\"\"\"\"", + "body": [ + "\"\"\"", + "Tests:", + " $1", + "Arranges:", + " $2", + "Expects:", + " $3", + "\"\"\"" + ], + "description": "Test docstring (User Snipets)" + } +} diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 3d7988e7..990beb4b 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -6,6 +6,7 @@ name = "pypi" [scripts] test = "pytest tests/" "test:cov" = "pytest --cov=src tests/" +"test:report" = "pytest --cov=src --html=.report/test_result.html tests/" [packages] boto3 = "*" @@ -17,6 +18,7 @@ autopep8 = "*" flake8 = "*" pytest = "*" pytest-cov = "*" +pytest-html = "*" moto = "*" [requires] diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index 7c13f0c9..cfd16ed2 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f1433a55f486f24bb14d6447713a0cdeeb704542a695103debd9514face495cc" + "sha256": "7006de596d6123ecd56760b584ab75430fa6bcfc0ecd3fdf49f08368ff53477d" }, "pipfile-spec": 6, "requires": { @@ -59,7 +59,7 @@ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2022.6.15" }, "cffi": { @@ -136,7 +136,7 @@ "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2.1.0" }, "cryptography": { @@ -164,7 +164,7 @@ "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==37.0.4" }, "idna": { @@ -363,7 +363,7 @@ "sha256:5867f2eadd6b028d9751f4155af590d3aaf9280e3a0ed5e15a53343921c956e5", "sha256:81c491092b71f5b276de8c63dfd452be3f322622c48a54f3a497cf913bdfb2f4" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==4.1.0" } }, @@ -405,7 +405,7 @@ "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2022.6.15" }, "cffi": { @@ -482,7 +482,7 @@ "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==2.1.0" }, "coverage": { @@ -560,7 +560,7 @@ "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" ], - "markers": "python_full_version >= '3.6.0'", + "markers": "python_version >= '3.6'", "version": "==37.0.4" }, "flake8": { @@ -735,6 +735,22 @@ "index": "pypi", "version": "==3.0.0" }, + "pytest-html": { + "hashes": [ + "sha256:3ee1cf319c913d19fe53aeb0bc400e7b0bc2dbeb477553733db1dad12eb75ee3", + "sha256:b7f82f123936a3f4d2950bc993c2c1ca09ce262c9ae12f9ac763a2401380b455" + ], + "index": "pypi", + "version": "==3.1.1" + }, + "pytest-metadata": { + "hashes": [ + "sha256:39261ee0086f17649b180baf2a8633e1922a4c4b6fcc28a2de7d8127a82541bf", + "sha256:fcd2f416f15be295943527b3c8ba16a44ae5a7141939c90c3dc5ce9d167cf2a5" + ], + "markers": "python_version >= '3.7' and python_version < '4'", + "version": "==2.0.2" + }, "python-dateutil": { "hashes": [ "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index db97825a..b6016b35 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -16,27 +16,71 @@ def s3_test(s3_client, bucket_name): class TestS3Resource: def test_get_object(self, s3_test, s3_client, bucket_name): + """ + Cases: + S3からオブジェクトが取得できるか + Arranges: + - S3をモック化する + - 期待値となるファイルを配置する + Expects: + オブジェクトが取得でき、期待値と正しいこと + """ + # Arrange s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') + + # ActAssert s3_resource = S3Resource(bucket_name) actual = s3_resource.get_object('hogehoge/test.txt') + + # Assert assert actual == 'aaaaaaaaaaaaaaa' def test_put_object(self, s3_test, s3_client, bucket_name): + """ + Cases: + S3にオブジェクトをPUTできるか + Arranges: + - S3をモック化する + Expects: + オブジェクトがPUTできること + """ s3_resource = S3Resource(bucket_name) + s3_resource.put_object('hogehoge/test.txt', 'aaaaaaaaaaaaaaa') actual = s3_client.get_object(Bucket=bucket_name, Key='hogehoge/test.txt') + assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' def test_copy(self, s3_test, s3_client, bucket_name): + """ + Cases: + S3内のオブジェクトを別バケットにコピーできるか + Arranges: + - S3をモック化する + - 期待値となるファイルをコピー元バケットに配置する + Expects: + - コピーされたファイルが存在する + - コピーされたファイルの内容が期待値と一致する + """ for_copy_bucket = 'for-copy-bucket' s3_client.create_bucket(Bucket=for_copy_bucket) s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') + s3_resource = S3Resource(bucket_name) s3_resource.copy(bucket_name, 'hogehoge/test.txt', for_copy_bucket, 'test.txt') + actual = s3_client.get_object(Bucket=for_copy_bucket, Key='test.txt') assert actual['Body'].read() == b'aaaaaaaaaaaaaaa' def test_init_raise_no_provide_bucket_name(self): + """ + Cases: + バケット名を指定しない場合、例外となること + Arranges: + + Expects: + 例外が発生すること + """ with pytest.raises(Exception) as e: S3Resource() assert e.value.args[0] == "__init__() missing 1 required positional argument: 'bucket_name'" @@ -45,36 +89,71 @@ class TestS3Resource: class TestConfigBucket: def test_get_object_info_file(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + オブジェクト情報ファイルが取得できること + Arranges: + オブジェクト情報ファイルを配置する + Expects: + オブジェクト情報ファイルが文字列として取得でき、期待値と一致する + """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', 'crm') monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'objects.json') s3_client.put_object(Bucket=bucket_name, Key=f'crm/objects.json', Body=b'aaaaaaaaaaaaaaa') + config_bucket = ConfigBucket() - print('*' * 50, str(config_bucket), '*' * 50) actual = config_bucket.get_object_info_file() - print(actual) + assert actual == 'aaaaaaaaaaaaaaa' def test_get_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + オブジェクト最終更新日時ファイルが取得できること + Arranges: + オブジェクト最終更新日時ファイルを配置する + Expects: + オブジェクト最終更新日時ファイルが文字列として取得でき、期待値と一致する + """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') s3_client.put_object(Bucket=bucket_name, Key=f'crm/Object.json', Body=b'aaaaaaaaaaaaaaa') + config_bucket = ConfigBucket() - print('*' * 50, str(config_bucket), '*' * 50) actual = config_bucket.get_last_fetch_datetime_file('Object.json') - print(actual) + assert actual == 'aaaaaaaaaaaaaaa' def test_put_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + オブジェクト最終更新日時ファイルをPUTできること + Arranges: + + Expects: + オブジェクト最終更新日時ファイルが存在する + """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') + config_bucket = ConfigBucket() - print('*' * 50, str(config_bucket), '*' * 50) config_bucket.put_last_fetch_datetime_file('Object.json', 'aaaaaaaaaaaaaaa') + actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/Object.json') assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' def test_config_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + 設定ファイル配置バケットを文字列化したとき、バケット名が取得できること + Arranges: + + Expects: + 環境変数の設定ファイル配置バケット名と一致する + """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + config_bucket = ConfigBucket() + assert str(config_bucket) == bucket_name diff --git a/ecs/crm-datafetch/tests/conftest.py b/ecs/crm-datafetch/tests/conftest.py index 1120ced6..e46746bb 100644 --- a/ecs/crm-datafetch/tests/conftest.py +++ b/ecs/crm-datafetch/tests/conftest.py @@ -1,8 +1,12 @@ import os +from datetime import datetime import boto3 import pytest from moto import mock_s3 +from py.xml import html # type: ignore + +from . import docstring_parser @pytest.fixture @@ -19,3 +23,40 @@ def s3_client(aws_credentials): with mock_s3(): conn = boto3.client("s3", region_name="us-east-1") yield conn + + +# 以下、レポート出力用の設定 + +def pytest_html_report_title(report): + # レポートタイトル + report.title = "CRMデータ連携 CRMデータ取得機能 単体機能テスト結果報告書" + + +# # def pytest_configure(config): +# # config._metadata["結果確認者"] = "" # Version情報を追加 + + +def pytest_html_results_table_header(cells): + del cells[2:] + cells.insert(3, html.th("Cases")) + cells.insert(4, html.th("Arranges")) + cells.insert(5, html.th("Expects")) + cells.append(html.th("Time", class_="sortable time", col="time")) + + +def pytest_html_results_table_row(report, cells): + del cells[2:] + cells.insert(3, html.td(html.pre(report.cases))) # 「テスト内容」をレポートに出力 + cells.insert(4, html.td(html.pre(report.arranges))) # 「期待結果」をレポートに出力 + cells.insert(5, html.td(html.pre(report.expects))) # 「期待結果」をレポートに出力 + cells.append(html.td(datetime.now(), class_="col-time")) # ついでに「時間」もレポートに出力 + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + docstring = docstring_parser.parse(str(item.function.__doc__)) + report.cases = docstring.get("Cases", '') # 「テスト内容」を`report`に追加 + report.arranges = docstring.get("Arranges", '') # 「準備作業」を`report`に追加 + report.expects = docstring.get("Expects", '') # 「期待結果」を`report`に追加 diff --git a/ecs/crm-datafetch/tests/docstring_parser.py b/ecs/crm-datafetch/tests/docstring_parser.py new file mode 100644 index 00000000..32ac1579 --- /dev/null +++ b/ecs/crm-datafetch/tests/docstring_parser.py @@ -0,0 +1,32 @@ +import re +from itertools import takewhile + +_section_rgx = re.compile(r"^\s*[a-zA-Z]+:\s*$") +_lspace_rgx = re.compile(r"^\s*") + + +def _parse_section(lines: list) -> list: + matches = map(lambda x: _section_rgx.match(x), lines) + indexes = [i for i, x in enumerate(matches) if x is not None] + return list(map(lambda x: (x, lines[x].strip()[: -1]), indexes)) + + +def _count_lspace(s: str) -> int: + rgx = _lspace_rgx.match(s) + if rgx is not None: + return rgx.end() + return 0 + + +def _parse_content(index: int, lines: list) -> str: + lspace = _count_lspace(lines[index]) + i = index + 1 + contents = takewhile(lambda x: _count_lspace(x) > lspace, lines[i:]) + return "\n".join(map(lambda x: x.strip(), contents)) + + +def parse(docstring: str) -> dict: + """🚧sloppy docstring parser🚧""" + lines = docstring.splitlines() + sections = _parse_section(lines) + return dict(map(lambda x: (x[1], _parse_content(x[0], lines)), sections)) From a003f0b96b5e6b1f3cf76b0dea1d5031498f51f8 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 16:19:32 +0900 Subject: [PATCH 106/275] =?UTF-8?q?docs:=20README=E3=81=AB=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=81=AB=E3=81=A4=E3=81=84=E3=81=A6=E8=BF=BD?= =?UTF-8?q?=E8=A8=98=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.env.example | 17 +++++ .../.vscode/python.code-snippets | 2 +- ecs/crm-datafetch/README.md | 72 +++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 ecs/crm-datafetch/.env.example diff --git a/ecs/crm-datafetch/.env.example b/ecs/crm-datafetch/.env.example new file mode 100644 index 00000000..aa6a3d18 --- /dev/null +++ b/ecs/crm-datafetch/.env.example @@ -0,0 +1,17 @@ +CRM_AUTH_DOMAIN=test +CRM_USER_NAME=test +CRM_USER_PASSWORD=test +CRM_USER_SECURITY_TOKEN=test +CRM_CONFIG_BUCKET=test +CRM_BACKUP_BUCKET=test +IMPORT_DATA_BUCKET=test +OBJECT_INFO_FOLDER=test +OBJECT_INFO_FILENAME=test +PROCESS_RESULT_FOLDER=test +PROCESS_RESULT_FILENAME=test +LAST_FETCH_DATE_FOLDER=test +CRM_IMPORT_DATA_FOLDER=test +LAST_FETCH_DATE_BACKUP_FOLDER=test +RESPONSE_JSON_BACKUP_FOLDER=test +CRM_IMPORT_DATA_BACKUP_FOLDER=test +TZ=Asia/Tokyo diff --git a/ecs/crm-datafetch/.vscode/python.code-snippets b/ecs/crm-datafetch/.vscode/python.code-snippets index 9f01c623..aa7b3501 100644 --- a/ecs/crm-datafetch/.vscode/python.code-snippets +++ b/ecs/crm-datafetch/.vscode/python.code-snippets @@ -12,6 +12,6 @@ " $3", "\"\"\"" ], - "description": "Test docstring (User Snipets)" + "description": "Test docstring (User Snippets)" } } diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md index 5a23e966..f0188f5f 100644 --- a/ecs/crm-datafetch/README.md +++ b/ecs/crm-datafetch/README.md @@ -67,3 +67,75 @@ - 環境変数が必要な場合、直接設定するか、上記JSONの`"envFile"`に設定されたパスに`.env`ファイルを作成し、環境変数を入力する - キーボードの「F5」キーを押して起動する - デバッグモードで実行されるため、適当なところにブレークポイントを置いてデバッグすることができる + +## 単体テストについて + +### 前提 + +- Pytestを使用する + - +- カバレッジも取得したいため、pytest-covも使う + - +- レポートを出力するため、pytest-htmlを使う + - +- S3をモック化したいため、motoをつかう + - +- CRMはテスト用の環境を使いたいため、newdwh_opeのアドレスでDeveloper組織を登録する + +### テスト環境構築 + +- Pipenvの仮想環境下で、以下のコマンドを実行する + +```sh +pipenv install --dev +``` + +- `.env.example`をコピーし、同じ階層に`.env`を作成する +- `.env`の以下に示す環境変数の値をDeveloper組織のものに書き換える + - CRM_AUTH_DOMAIN + - CRM_USER_NAME + - CRM_USER_PASSWORD + - CRM_USER_SECURITY_TOKEN +- 以下のコマンドを実行して単体テストを起動する + +```sh +pipenv run test:cov +``` + +#### 各コマンドの説明 + +- `pipenv run test` + - pytestを使用してテストを実行する + - `tests`フォルダに配置されているテストモジュールを対象に、単体テストを実行する +- `pipenv run test:cov` + - pytestのテスト終了時にカバレッジを収集する + - 標準出力とカバレッジファイル(`.coverage`)に出力される +- `pipenv run test:report` + - pytestのテスト終了時にテスト結果をHTMLで出力する + - `.report/test_result.html`が出力される + +## 単体テストの追加方法 + +- `tests`フォルダ内に、`src`フォルダの構成と同じようにフォルダを作り、`test_<テスト対象のモジュール名.py>`というファイルを作成する + - 例：`src/aws/s3.py`をテストする場合は`tests/aws/test_s3.py`というファイル名にする +- テスト関数はクラスにまとめて、テストスイートとする +- テスト関数にはドキュメントコメントを付け、テスト観点・準備作業・期待値を記載すること + - 上記が出力されるスニペットを用意してある。 + - `""""""`と入力し、「Test docstring (User Snippets)」を選択し、ドキュメントコメントを挿入できる + +```python + +from src.aws.s3 import S3Resource +class TestS3Resource: + def test_get_object(self, s3_test, s3_): + """ + Cases: + S3からオブジェクトが取得できるか + Arranges: + - S3をモック化する + - 期待値となるファイルを配置する + Expects: + オブジェクトが取得でき、期待値と正しいこと + """ + # more code... +``` From bf59acf1525ce48471477b254193b196aa228166 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 17:01:32 +0900 Subject: [PATCH 107/275] =?UTF-8?q?refactor:=20=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E5=AF=BE=E8=B1=A1=E3=81=AE=E3=83=A2=E3=82=B8=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E3=83=AB=E5=A4=89=E6=95=B0=E3=82=92sut=E3=81=AB=E7=B5=B1?= =?UTF-8?q?=E4=B8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/aws/test_s3.py | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index b6016b35..7153d2a0 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -28,9 +28,9 @@ class TestS3Resource: # Arrange s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') - # ActAssert - s3_resource = S3Resource(bucket_name) - actual = s3_resource.get_object('hogehoge/test.txt') + # Act + sut = S3Resource(bucket_name) + actual = sut.get_object('hogehoge/test.txt') # Assert assert actual == 'aaaaaaaaaaaaaaa' @@ -44,9 +44,9 @@ class TestS3Resource: Expects: オブジェクトがPUTできること """ - s3_resource = S3Resource(bucket_name) + sut = S3Resource(bucket_name) - s3_resource.put_object('hogehoge/test.txt', 'aaaaaaaaaaaaaaa') + sut.put_object('hogehoge/test.txt', 'aaaaaaaaaaaaaaa') actual = s3_client.get_object(Bucket=bucket_name, Key='hogehoge/test.txt') assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' @@ -66,8 +66,8 @@ class TestS3Resource: s3_client.create_bucket(Bucket=for_copy_bucket) s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') - s3_resource = S3Resource(bucket_name) - s3_resource.copy(bucket_name, 'hogehoge/test.txt', for_copy_bucket, 'test.txt') + sut = S3Resource(bucket_name) + sut.copy(bucket_name, 'hogehoge/test.txt', for_copy_bucket, 'test.txt') actual = s3_client.get_object(Bucket=for_copy_bucket, Key='test.txt') assert actual['Body'].read() == b'aaaaaaaaaaaaaaa' @@ -102,8 +102,8 @@ class TestConfigBucket: monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'objects.json') s3_client.put_object(Bucket=bucket_name, Key=f'crm/objects.json', Body=b'aaaaaaaaaaaaaaa') - config_bucket = ConfigBucket() - actual = config_bucket.get_object_info_file() + sut = ConfigBucket() + actual = sut.get_object_info_file() assert actual == 'aaaaaaaaaaaaaaa' @@ -120,8 +120,8 @@ class TestConfigBucket: monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') s3_client.put_object(Bucket=bucket_name, Key=f'crm/Object.json', Body=b'aaaaaaaaaaaaaaa') - config_bucket = ConfigBucket() - actual = config_bucket.get_last_fetch_datetime_file('Object.json') + sut = ConfigBucket() + actual = sut.get_last_fetch_datetime_file('Object.json') assert actual == 'aaaaaaaaaaaaaaa' @@ -137,13 +137,13 @@ class TestConfigBucket: monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') - config_bucket = ConfigBucket() - config_bucket.put_last_fetch_datetime_file('Object.json', 'aaaaaaaaaaaaaaa') + sut = ConfigBucket() + sut.put_last_fetch_datetime_file('Object.json', 'aaaaaaaaaaaaaaa') actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/Object.json') assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' - def test_config_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): + def test_sut_str(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: 設定ファイル配置バケットを文字列化したとき、バケット名が取得できること @@ -154,6 +154,6 @@ class TestConfigBucket: """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) - config_bucket = ConfigBucket() + sut = ConfigBucket() - assert str(config_bucket) == bucket_name + assert str(sut) == bucket_name From 4d444e025eb43d218f3a6a48a37edb7029cee480 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 17:15:36 +0900 Subject: [PATCH 108/275] =?UTF-8?q?feat:=20DataBucket=E3=81=AE=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/aws/test_s3.py | 60 +++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index 7153d2a0..e30deaa7 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -1,5 +1,5 @@ import pytest -from src.aws.s3 import ConfigBucket, S3Resource +from src.aws.s3 import ConfigBucket, DataBucket, S3Resource @pytest.fixture @@ -157,3 +157,61 @@ class TestConfigBucket: sut = ConfigBucket() assert str(sut) == bucket_name + + +class TestDataBucket: + + def test_put_csv(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + 任意のファイルをPUTできること + Arranges: + + Expects: + PUTしたファイルが存在する + """ + monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', 'crm/target') + + sut = DataBucket() + actual = sut.put_csv('test.csv', 'test,test,test') + + actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/target/test.csv') + assert actual['Body'].read().decode('utf-8') == 'test,test,test' + + def test_put_csv_from(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + 他のバケットから任意のファイルをコピーできること + Arranges: + + Expects: + コピーしたファイルが存在する + """ + for_copy_bucket = 'for-copy-bucket' + s3_client.create_bucket(Bucket=for_copy_bucket) + s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.csv', Body=b'test,test,test') + + monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', for_copy_bucket) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', 'crm/target') + + sut = DataBucket() + sut.put_csv_from(bucket_name, 'hogehoge/test.csv') + actual = s3_client.get_object(Bucket=for_copy_bucket, Key=f'crm/target/test.csv') + + assert actual['Body'].read().decode('utf-8') == 'test,test,test' + + def test_data_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + 設定ファイル配置バケットを文字列化したとき、バケット名が取得できること + Arranges: + + Expects: + 環境変数の設定ファイル配置バケット名と一致する + """ + monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', bucket_name) + + sut = DataBucket() + + assert str(sut) == bucket_name From a1613087df469d3ce0563eb9b9908c49bbcf8678 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 17:21:46 +0900 Subject: [PATCH 109/275] =?UTF-8?q?refactor:=20=E3=83=89=E3=82=AD=E3=83=A5?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3=E3=80=81=E9=96=A2=E6=95=B0=E5=90=8D?= =?UTF-8?q?=E3=82=82=E3=81=A1=E3=82=87=E3=81=A3=E3=81=A8=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/aws/test_s3.py | 66 +++++++++++++++----------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index e30deaa7..44a4ee8c 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -18,12 +18,12 @@ class TestS3Resource: def test_get_object(self, s3_test, s3_client, bucket_name): """ Cases: - S3からオブジェクトが取得できるか + - S3からオブジェクトが取得できるか Arranges: - S3をモック化する - 期待値となるファイルを配置する Expects: - オブジェクトが取得でき、期待値と正しいこと + - オブジェクトが取得でき、期待値と正しいこと """ # Arrange s3_client.put_object(Bucket=bucket_name, Key='hogehoge/test.txt', Body=b'aaaaaaaaaaaaaaa') @@ -38,11 +38,12 @@ class TestS3Resource: def test_put_object(self, s3_test, s3_client, bucket_name): """ Cases: - S3にオブジェクトをPUTできるか + - S3にオブジェクトをPUTできるか Arranges: - S3をモック化する Expects: - オブジェクトがPUTできること + - PUTされたファイルが存在する + - PUTされたファイルの内容が期待値と一致する """ sut = S3Resource(bucket_name) @@ -54,9 +55,10 @@ class TestS3Resource: def test_copy(self, s3_test, s3_client, bucket_name): """ Cases: - S3内のオブジェクトを別バケットにコピーできるか + - S3内のオブジェクトを別バケットにコピーできるか Arranges: - S3をモック化する + - コピー先バケットを作成する - 期待値となるファイルをコピー元バケットに配置する Expects: - コピーされたファイルが存在する @@ -75,11 +77,11 @@ class TestS3Resource: def test_init_raise_no_provide_bucket_name(self): """ Cases: - バケット名を指定しない場合、例外となること + - バケット名を指定しない場合、例外となること Arranges: Expects: - 例外が発生すること + - インスタンス生成時に例外が発生すること """ with pytest.raises(Exception) as e: S3Resource() @@ -91,11 +93,11 @@ class TestConfigBucket: def test_get_object_info_file(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: - オブジェクト情報ファイルが取得できること + - オブジェクト情報ファイルが取得できること Arranges: - オブジェクト情報ファイルを配置する + - オブジェクト情報ファイルを配置する Expects: - オブジェクト情報ファイルが文字列として取得でき、期待値と一致する + - オブジェクト情報ファイルが文字列として取得でき、期待値と一致する """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', 'crm') @@ -110,11 +112,12 @@ class TestConfigBucket: def test_get_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: - オブジェクト最終更新日時ファイルが取得できること + - オブジェクト最終更新日時ファイルが取得できること Arranges: - オブジェクト最終更新日時ファイルを配置する + - S3をモック化する + - オブジェクト最終更新日時ファイルを配置する Expects: - オブジェクト最終更新日時ファイルが文字列として取得でき、期待値と一致する + - オブジェクト最終更新日時ファイルが文字列として取得でき、期待値と一致する """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') @@ -128,11 +131,12 @@ class TestConfigBucket: def test_put_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: - オブジェクト最終更新日時ファイルをPUTできること + - オブジェクト最終更新日時ファイルをPUTできること Arranges: - + - S3をモック化する + - 環境変数をテスト用の値に置き換える Expects: - オブジェクト最終更新日時ファイルが存在する + - オブジェクト最終更新日時ファイルが存在する """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') @@ -143,14 +147,14 @@ class TestConfigBucket: actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/Object.json') assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' - def test_sut_str(self, s3_test, s3_client, bucket_name, monkeypatch): + def test_config_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: - 設定ファイル配置バケットを文字列化したとき、バケット名が取得できること + - 設定ファイル配置バケットを文字列化したとき、バケット名が取得できること Arranges: - + - 環境変数をテスト用の値に置き換える Expects: - 環境変数の設定ファイル配置バケット名と一致する + - 環境変数の設定ファイル配置バケット名と一致する """ monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) @@ -164,11 +168,12 @@ class TestDataBucket: def test_put_csv(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: - 任意のファイルをPUTできること + - 任意のファイルをPUTできること Arranges: - + - S3をモック化する + - 環境変数をテスト用の値に置き換える Expects: - PUTしたファイルが存在する + - PUTしたファイルが存在する """ monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', bucket_name) monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', 'crm/target') @@ -182,11 +187,14 @@ class TestDataBucket: def test_put_csv_from(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: - 他のバケットから任意のファイルをコピーできること + - 他のバケットから任意のファイルをコピーできること Arranges: - + - S3をモック化する + - コピー先バケットを作成する + - 期待値となるファイルコピー元バケットに配置する + - 環境変数をテスト用の値に置き換える Expects: - コピーしたファイルが存在する + - コピーしたファイルが存在する """ for_copy_bucket = 'for-copy-bucket' s3_client.create_bucket(Bucket=for_copy_bucket) @@ -204,11 +212,11 @@ class TestDataBucket: def test_data_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: - 設定ファイル配置バケットを文字列化したとき、バケット名が取得できること + - 設定ファイル配置バケットを文字列化したとき、バケット名が取得できること Arranges: - + - 環境変数をテスト用の値に置き換える Expects: - 環境変数の設定ファイル配置バケット名と一致する + - 環境変数の設定ファイル配置バケット名と一致する """ monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', bucket_name) From dd9c2209533b8d10a26876da9798f1809486717b Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 2 Aug 2022 17:58:00 +0900 Subject: [PATCH 110/275] =?UTF-8?q?docs:=20=E3=82=BD=E3=83=BC=E3=82=B9?= =?UTF-8?q?=E5=86=85=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=81=AE=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/converter/convert_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 89ad2a2e..8664d7cd 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -49,7 +49,7 @@ class BooleanConvertStrategy: class DatetimeConvertStrategy: def convert_value(self, convert_value: str) -> str: - # データ登録処理がJSTとして登録するため、変換処理内で事前にJSTの日次文字列に変換する + # データ登録処理がJSTとして登録するため、変換処理内で事前にJSTの日時文字列に変換する return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS) From 45d3d3aae1c3f2a5d3f3489a86de43c8993e6c2b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 18:35:24 +0900 Subject: [PATCH 111/275] =?UTF-8?q?feat:=20Backup=E3=83=90=E3=82=B1?= =?UTF-8?q?=E3=83=83=E3=83=88=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/aws/test_s3.py | 87 ++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 6 deletions(-) diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index 44a4ee8c..1daab1e7 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -1,5 +1,5 @@ import pytest -from src.aws.s3 import ConfigBucket, DataBucket, S3Resource +from src.aws.s3 import BackupBucket, ConfigBucket, DataBucket, S3Resource @pytest.fixture @@ -168,7 +168,7 @@ class TestDataBucket: def test_put_csv(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: - - 任意のファイルをPUTできること + - CSVファイルをPUTできること Arranges: - S3をモック化する - 環境変数をテスト用の値に置き換える @@ -179,7 +179,7 @@ class TestDataBucket: monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', 'crm/target') sut = DataBucket() - actual = sut.put_csv('test.csv', 'test,test,test') + sut.put_csv('test.csv', 'test,test,test') actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/target/test.csv') assert actual['Body'].read().decode('utf-8') == 'test,test,test' @@ -187,7 +187,7 @@ class TestDataBucket: def test_put_csv_from(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: - - 他のバケットから任意のファイルをコピーできること + - 他のバケットからCSVファイルをコピーできること Arranges: - S3をモック化する - コピー先バケットを作成する @@ -212,14 +212,89 @@ class TestDataBucket: def test_data_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: - - 設定ファイル配置バケットを文字列化したとき、バケット名が取得できること + - データ登録バケットを文字列化したとき、バケット名が取得できること Arranges: - 環境変数をテスト用の値に置き換える Expects: - - 環境変数の設定ファイル配置バケット名と一致する + - 環境変数のデータ登録バケット名と一致する """ monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', bucket_name) sut = DataBucket() assert str(sut) == bucket_name + + +class TestBackupBucket: + + def test_put_csv(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + - CSVファイルをPUTできること + Arranges: + - S3をモック化する + - 環境変数をテスト用の値に置き換える + Expects: + - PUTしたファイルが存在する + """ + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', 'data_import') + + sut = BackupBucket() + sut.put_csv('test.csv', 'test,test,test') + + actual = s3_client.get_object(Bucket=bucket_name, Key=f'data_import/test.csv') + assert actual['Body'].read().decode('utf-8') == 'test,test,test' + + def test_put_response_json(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + - JSONファイルをPUTできること + Arranges: + - S3をモック化する + - 環境変数をテスト用の値に置き換える + Expects: + - PUTしたファイルが存在する + """ + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', 'response_json') + + sut = BackupBucket() + sut.put_response_json('test.json', {"test": "test"}) + + actual = s3_client.get_object(Bucket=bucket_name, Key=f'response_json/test.json') + assert actual['Body'].read().decode('utf-8') == '{"test": "test"}' + + def test_put_result_json(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + - JSONファイルをPUTできること + Arranges: + - S3をモック化する + - 環境変数をテスト用の値に置き換える + Expects: + - PUTしたファイルが存在する + """ + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', 'data_import') + + sut = BackupBucket() + sut.put_result_json('result.json', {"test": "test"}) + + actual = s3_client.get_object(Bucket=bucket_name, Key=f'data_import/result.json') + assert actual['Body'].read().decode('utf-8') == '{"test": "test"}' + + def test_backup_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + - CRMデータバックアップバケットを文字列化したとき、バケット名が取得できること + Arranges: + - 環境変数をテスト用の値に置き換える + Expects: + - 環境変数のCRMデータバックアップバケット名と一致する + """ + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name) + + sut = BackupBucket() + + assert str(sut) == bucket_name From c7c0ee8f973e6f60f5bfff51f88ef589cbfc1757 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 2 Aug 2022 18:43:41 +0900 Subject: [PATCH 112/275] =?UTF-8?q?refactor:=20=E4=B8=8D=E8=A6=81=E3=81=AA?= =?UTF-8?q?=E7=92=B0=E5=A2=83=E5=A4=89=E6=95=B0=E3=82=92example=E3=81=8B?= =?UTF-8?q?=E3=82=89=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.env.example | 1 - 1 file changed, 1 deletion(-) diff --git a/ecs/crm-datafetch/.env.example b/ecs/crm-datafetch/.env.example index aa6a3d18..e4c3b353 100644 --- a/ecs/crm-datafetch/.env.example +++ b/ecs/crm-datafetch/.env.example @@ -14,4 +14,3 @@ CRM_IMPORT_DATA_FOLDER=test LAST_FETCH_DATE_BACKUP_FOLDER=test RESPONSE_JSON_BACKUP_FOLDER=test CRM_IMPORT_DATA_BACKUP_FOLDER=test -TZ=Asia/Tokyo From 5c22cd56c5cb0943fb37787926aa2de9e6f31603 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 00:51:55 +0900 Subject: [PATCH 113/275] =?UTF-8?q?style:=20=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E7=94=A8=E3=83=89=E3=82=AD=E3=83=A5=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=81=AE=E3=82=B9=E3=83=8B?= =?UTF-8?q?=E3=83=9A=E3=83=83=E3=83=88=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.vscode/python.code-snippets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/.vscode/python.code-snippets b/ecs/crm-datafetch/.vscode/python.code-snippets index aa7b3501..3b658c8d 100644 --- a/ecs/crm-datafetch/.vscode/python.code-snippets +++ b/ecs/crm-datafetch/.vscode/python.code-snippets @@ -4,7 +4,7 @@ "prefix": "\"\"\"\"\"\"", "body": [ "\"\"\"", - "Tests:", + "Cases:", " $1", "Arranges:", " $2", From be8823d07fcf954ce8ecd5549204b013e46eef8a Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 00:52:17 +0900 Subject: [PATCH 114/275] =?UTF-8?q?feat:=20=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E3=81=ABLOG=5FLEVEL=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.env.example | 1 + 1 file changed, 1 insertion(+) diff --git a/ecs/crm-datafetch/.env.example b/ecs/crm-datafetch/.env.example index e4c3b353..4f105876 100644 --- a/ecs/crm-datafetch/.env.example +++ b/ecs/crm-datafetch/.env.example @@ -14,3 +14,4 @@ CRM_IMPORT_DATA_FOLDER=test LAST_FETCH_DATE_BACKUP_FOLDER=test RESPONSE_JSON_BACKUP_FOLDER=test CRM_IMPORT_DATA_BACKUP_FOLDER=test +LOG_LEVEL=INFO \ No newline at end of file From 5a1012c4e058572c17d6d6f9f411b4e6e2af6e42 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 11:50:08 +0900 Subject: [PATCH 115/275] =?UTF-8?q?docs:=20README=E3=81=AB=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB/=E3=83=95=E3=82=A9=E3=83=AB?= =?UTF-8?q?=E3=83=80=E6=A7=8B=E6=88=90=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/README.md | 43 +++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md index f0188f5f..9d580fa5 100644 --- a/ecs/crm-datafetch/README.md +++ b/ecs/crm-datafetch/README.md @@ -68,6 +68,49 @@ - キーボードの「F5」キーを押して起動する - デバッグモードで実行されるため、適当なところにブレークポイントを置いてデバッグすることができる +## ファイル/フォルダ構成 + +`[〇〇処理モジュール]`と記載されているファイルは、設計書に記載のシートと一致したPythonファイルです + +```text +. +├── Dockerfile -- Dokcerイメージを作成するためのファイル +├── Pipfile -- Pipenv(Pythonの仮想環境管理モジュール)で、依存関係を管理するためのファイル +├── Pipfile.lock -- Pipenvでインストールされた依存関係のバージョン固定ファイル +├── README.md -- README +├── main.py -- CRMデータ取得処理のエントリーポイント +├── src/ -- プロダクトコード置き場 +│ ├── aws/ -- AWSのリソース操作関連のモジュール置き場 +│ ├── backup_crm_csv_data_process.py -- [CSVバックアップ処理]モジュール +│ ├── backup_crm_data_process.py -- [CRM電文データバックアップ処理]モジュール +│ ├── check_object_info_process.py -- [オブジェクト情報形式チェック処理]モジュール +│ ├── config/ -- 設定ファイル関連のモジュール置き場 +│ ├── controller.py -- [コントロール処理]モジュール +│ ├── convert_crm_csv_data_process.py -- [CSV変換処理]モジュール +│ ├── converter/ -- CSV変換処理で実際に変換を行うモジュール置き場 +│ ├── copy_crm_csv_data_process.py -- [CSVアップロード処理]モジュール +│ ├── error/ -- 処理エラー発生時カスタム例外モジュール置き場 +│ ├── fetch_crm_data_process.py -- [CRMデータ取得処理]モジュール +│ ├── parser/ -- [JSON変換処理]モジュール置き場 +│ ├── prepare_data_fetch_process.py -- データ取得準備処理 +│ ├── salesforce/ -- SalesforceのAPIリクエストモジュール置き場 +│ ├── set_datetime_period_process.py -- [データ取得期間設定処理]モジュール +│ ├── system_var/ -- 環境変数と定数ファイル置き場 +│ ├── upload_last_fetch_datetime_process.py -- [前回取得日時ファイル更新処理]モジュール +│ ├── upload_result_data_process.py -- [取得処理実施結果アップロード処理]モジュール +│ └── util/ -- ユーティリティモジュール置き場 +│ ├── counter_object.py -- リトライ判定のためのカウントアップクラス +│ ├── dict_checker.py -- 辞書型値オブジェクトの設定値チェック用クラス +│ ├── execute_datetime.py -- 取得処理開始年月日時分秒の管理クラス +│ └── logger.py -- ログ管理クラス +│ +└── tests/ -- テストコード置き場 + ├── aws -- AWS操作モジュールのテスト + ├── ... -- src配下のモジュール構成と同じ階層にテストコードを追加していく + ├── conftest.py -- pytestのフィクスチャやフックを管理するファイル + └── docstring_parser.py -- pytest-htmlのレポート出力用のヘルパー +``` + ## 単体テストについて ### 前提 From 02c6b97e9265bc06a2f57c1cdf236ce40f49099e Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 11:58:00 +0900 Subject: [PATCH 116/275] =?UTF-8?q?docs:=20docstring=5Fparser=E3=81=AB?= =?UTF-8?q?=E3=83=89=E3=82=AD=E3=83=A5=E3=83=A1=E3=83=B3=E3=83=88=E3=82=B3?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=92=E8=BF=BD=E8=A8=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/docstring_parser.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ecs/crm-datafetch/tests/docstring_parser.py b/ecs/crm-datafetch/tests/docstring_parser.py index 32ac1579..040c9b5d 100644 --- a/ecs/crm-datafetch/tests/docstring_parser.py +++ b/ecs/crm-datafetch/tests/docstring_parser.py @@ -1,3 +1,16 @@ +"""pytest-htmlでレポート出力するため、各テスト関数のドキュメントコメントのセクションを抜き出して、辞書にする +Examples: + <セクション名>:となっている部分が対象になる + \"\"\" + Cases: + テストケース + Arranges: + 準備作業 + Expects: + 期待値 + \"\"\" +""" + import re from itertools import takewhile From 33a608a4b3cd608f844e9eb4bb3fa95e98f32b71 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 12:00:46 +0900 Subject: [PATCH 117/275] =?UTF-8?q?docs:=20conftest.py=E3=81=AB=E3=83=89?= =?UTF-8?q?=E3=82=AD=E3=83=A5=E3=83=A1=E3=83=B3=E3=83=88=E3=82=B3=E3=83=A1?= =?UTF-8?q?=E3=83=B3=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ecs/crm-datafetch/tests/conftest.py b/ecs/crm-datafetch/tests/conftest.py index e46746bb..5f2e0073 100644 --- a/ecs/crm-datafetch/tests/conftest.py +++ b/ecs/crm-datafetch/tests/conftest.py @@ -1,3 +1,4 @@ +"""pytestでフィクスチャやフック(テスト実行前後に差し込む処理)を管理するモジュール""" import os from datetime import datetime From ca9dc2863f65884778d654447ab0fff044e5a325 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 12:01:55 +0900 Subject: [PATCH 118/275] =?UTF-8?q?style:=20=E4=B8=8D=E8=A6=81=E3=81=AA?= =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/conftest.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ecs/crm-datafetch/tests/conftest.py b/ecs/crm-datafetch/tests/conftest.py index 5f2e0073..55ade88c 100644 --- a/ecs/crm-datafetch/tests/conftest.py +++ b/ecs/crm-datafetch/tests/conftest.py @@ -33,10 +33,6 @@ def pytest_html_report_title(report): report.title = "CRMデータ連携 CRMデータ取得機能 単体機能テスト結果報告書" -# # def pytest_configure(config): -# # config._metadata["結果確認者"] = "" # Version情報を追加 - - def pytest_html_results_table_header(cells): del cells[2:] cells.insert(3, html.th("Cases")) From 9012769e1f54a9e4033232c37f2b3bb0e95e4664 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 13:11:58 +0900 Subject: [PATCH 119/275] =?UTF-8?q?feat:=20=E3=82=B3=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=83=AD=E3=83=BC=E3=83=A9=E3=83=BC=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E8=BF=BD=E5=8A=A0=E3=80=82=E3=82=B3=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=83=AD=E3=83=BC=E3=83=A9=E3=83=BC=E3=81=A7exit()=E3=81=97?= =?UTF-8?q?=E3=81=A6=E3=81=97=E3=81=BE=E3=81=86=E3=81=A8=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E5=81=B4=E3=81=A7=E4=BE=8B?= =?UTF-8?q?=E5=A4=96=E3=81=A8=E3=81=97=E3=81=A6=E6=89=B1=E3=82=8F=E3=81=AA?= =?UTF-8?q?=E3=81=84=E3=81=A8=E3=81=84=E3=81=91=E3=81=AA=E3=81=8F=E3=81=AA?= =?UTF-8?q?=E3=82=8B=E3=81=9F=E3=82=81=E3=80=81=E3=82=A8=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=83=AA=E3=83=BC=E3=83=9D=E3=82=A4=E3=83=B3=E3=83=88=E5=81=B4?= =?UTF-8?q?=E3=81=A7=E5=88=B6=E5=BE=A1=E3=81=99=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E4=BF=AE=E6=AD=A3=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/main.py | 2 +- ecs/crm-datafetch/src/config/objects.py | 4 +- ecs/crm-datafetch/src/controller.py | 9 +- .../src/prepare_data_fetch_process.py | 2 +- ecs/crm-datafetch/tests/test_controller.py | 82 +++++++++++++++++++ 5 files changed, 89 insertions(+), 10 deletions(-) create mode 100644 ecs/crm-datafetch/tests/test_controller.py diff --git a/ecs/crm-datafetch/main.py b/ecs/crm-datafetch/main.py index dcae9825..1d6f0ab9 100644 --- a/ecs/crm-datafetch/main.py +++ b/ecs/crm-datafetch/main.py @@ -2,4 +2,4 @@ from src.controller import controller """CRMデータ取得処理のエントリーポイント""" if __name__ == '__main__': - controller() + exit(controller()) diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 41ca753b..23d00f5c 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -22,7 +22,7 @@ class FetchTargetObjects(): def __init__(self, object_info_file_dict) -> None: self.__objects = object_info_file_dict self.__dict_checker = DictChecker(self.__objects) - self.validate() + self.__validate() self.__i = 0 def __iter__(self): @@ -35,7 +35,7 @@ class FetchTargetObjects(): self.__i += 1 return value - def validate(self) -> None: + def __validate(self) -> None: self.__dict_checker.assert_key_exist(OBJECTS_KEY) self.__dict_checker.assert_data_type(OBJECTS_KEY, OBJECTS_TYPE) diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index 3d0537b4..f289c592 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -48,19 +48,16 @@ def controller() -> None: else: logger.info('I-CTRL-19 すべてのデータの取得に成功しました') - # ⑦ CRMデータ取得処理終了ログを出力する - logger.info('I-CTRL-20 CRMデータ取得処理を終了します') - - return exit(0) + return 0 except MeDaCaCRMDataFetchException as e: logger.error(f'E-ERR-01 [{e.func_name}]でエラーが発生したため、処理を終了します') logger.exception(f'{e.error_id} {e}') - return exit(0) + return 0 except Exception as e: logger.exception('E-ERR-02 予期せぬエラーが発生したため、処理を終了します', e) - return exit(0) + return 0 def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: ExecuteDateTime, process_result: dict): diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index 3488c71f..03427265 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -79,4 +79,4 @@ def prepare_data_fetch_process(): logger.info('I-PRE-09 データ取得準備処理を終了します') # ⑧ 次の処理へ移行する - return(fetch_target_objects, execute_datetime, process_result) + return (fetch_target_objects, execute_datetime, process_result) diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py new file mode 100644 index 00000000..b75eec23 --- /dev/null +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -0,0 +1,82 @@ +from unittest.mock import MagicMock, patch + +from src import controller +from src.config.objects import (FetchTargetObjects, LastFetchDatetime, + TargetObject) +from src.util.execute_datetime import ExecuteDateTime + + +class TestController: + def test_controller_call_all_processes(self, s3_client, monkeypatch): + """ + Cases: + コントローラー処理からすべてのプロセスが呼ばれること + Arranges: + - 各種プロセスメソッドと内部で使用している値オブジェクトをモック化する + Expects: + - データ取得準備処理が1回のみ実行されること + - オブジェクト情報形式チェック処理が複数回実行されること + - データ取得期間設定処理が複数回実行されること + - CRMデータ取得処理が複数回実行されること + - CRM電文データバックアップ処理が複数回実行されること + - CSV変換処理が複数回実行されること + - CSVバックアップ処理が複数回実行されること + - CSVアップロード処理が複数回実行されること + - 前回取得日次ファイル更新処理が複数回実行されること + - 取得処理実施結果アップロード処理が1回のみ実行されること + """ + monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + # monkeypatch.setattr(ExecuteDateTime, '__str__', lambda x: 'YYYY-MM-DDTHH:MM:SS.000Z') + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + mock_target_objects = FetchTargetObjects({ + 'objects': [ + { + 'object_name': 'Account' + }, + { + 'object_name': 'Contact' + }, + { + 'object_name': 'Call2_vod__c' + } + ] + }) + mock_target_object = TargetObject({'object_name': 'Account'}, mock_execute_datetime) + mock_last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' + }, mock_execute_datetime) + mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(return_value=mock_target_object) + mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_fetch_crm_data_process = MagicMock(return_value=[{}]) + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock() + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process),\ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + controller.controller() + + mock_prepare_data_fetch_process.assert_called_once() + assert mock_check_object_info_process.call_count == 3 + assert mock_set_datetime_period_process.call_count == 3 + assert mock_fetch_crm_data_process.call_count == 3 + assert mock_backup_crm_data_process.call_count == 3 + assert mock_convert_crm_csv_data_process.call_count == 3 + assert mock_backup_crm_csv_data_process.call_count == 3 + assert mock_copy_crm_csv_data_process.call_count == 3 + assert mock_upload_last_fetch_datetime_process.call_count == 3 + mock_upload_result_data_process.assert_called_once() From 0a00e58cb585a828b63afbe4b857c51b70e53038 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 13:11:58 +0900 Subject: [PATCH 120/275] =?UTF-8?q?feat:=20=E5=86=85=E9=83=A8=E9=96=A2?= =?UTF-8?q?=E6=95=B0=E3=81=AE=E9=A0=AD=E3=81=AB=5F=E3=82=92=E4=BB=98?= =?UTF-8?q?=E3=81=91=E3=81=9F(=E3=83=97=E3=83=A9=E3=82=A4=E3=83=99?= =?UTF-8?q?=E3=83=BC=E3=83=88=E5=8C=96)=E3=80=82=E7=B5=82=E4=BA=86?= =?UTF-8?q?=E3=83=AD=E3=82=B0=E3=81=AFfinally=E3=81=A7=E5=87=BA=E5=8A=9B?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=E3=80=82=E5=87=A6=E7=90=86=E7=B5=90=E6=9E=9C=E3=81=8C0?= =?UTF-8?q?=E4=BB=B6=E3=81=AE=E5=A0=B4=E5=90=88=E3=80=81=E3=82=A2=E3=83=83?= =?UTF-8?q?=E3=83=97=E3=83=AD=E3=83=BC=E3=83=89=E3=81=9B=E3=81=9A=E3=81=AB?= =?UTF-8?q?=E7=B5=82=E4=BA=86=E3=81=95=E3=81=9B=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/controller.py | 37 ++++++++++++++++----- ecs/crm-datafetch/tests/test_controller.py | 38 +++++++++++++++++++++- 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index f289c592..5309fa83 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -33,20 +33,22 @@ def controller() -> None: # ③ object_infoのobjectsキーの値の件数分ループする logger.info('I-CTRL-03 取得対象オブジェクトのループ処理開始') - process_result = fetch_crm_data(fetch_target_objects, execute_datetime, process_result) + process_result = _fetch_crm_data(fetch_target_objects, execute_datetime, process_result) # ④ すべてのオブジェクトの処理が完了したことと、オブジェクト毎の処理結果をログに出力する logger.info(f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result}]') + # 最終結果が0件(1件も処理されていない)の場合、ログ出力して処理を終了する + if len(process_result.keys()) == 0: + logger.info('I-CTRL-21 処理対象のデータが存在しませんでした') + return 0 + # ⑤ 取得処理実施結果アップロード処理を呼び出す logger.info('I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始') upload_result_data_process(process_result, execute_datetime) # ⑥ 最終結果をチェックし、チェック結果をログに出力 - if not all([v == 'success' for v in process_result.values()]): - logger.error('E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') - else: - logger.info('I-CTRL-19 すべてのデータの取得に成功しました') + _check_process_result(process_result) return 0 @@ -59,8 +61,12 @@ def controller() -> None: logger.exception('E-ERR-02 予期せぬエラーが発生したため、処理を終了します', e) return 0 + finally: + # ⑦ CRMデータ取得処理終了ログを出力する + logger.info('I-CTRL-20 CRMデータ取得処理を終了します') -def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: ExecuteDateTime, process_result: dict): + +def _fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: ExecuteDateTime, process_result: dict): """取得対象オブジェクト情報をループし、１オブジェクトごとのデータを取得する Args: @@ -76,7 +82,7 @@ def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: E try: process_result[object_info.get(OBJECT_NAME_KEY)] = 'fail' - fetch_crm_data_per_object(object_info, execute_datetime) + _fetch_crm_data_per_object(object_info, execute_datetime) process_result[object_info.get(OBJECT_NAME_KEY)] = 'success' @@ -94,7 +100,7 @@ def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: E return process_result -def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTime) -> None: +def _fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTime) -> None: """オブジェクトごとにCRMのデータを取得し、取込フォルダにアップロードする Args: @@ -171,3 +177,18 @@ def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTi logger.info(f'I-CTRL-16 [{target_object_name}] 処理正常終了') return + + +def _check_process_result(process_result: dict) -> None: + """取得処理結果がすべて成功か、一部失敗しているかを判定し、ログ出力する + + Args: + process_result (dict): 取得処理結果辞書オブジェクト + """ + if not all([v == 'success' for v in process_result.values()]): + logger.error('E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') + return + + logger.info('I-CTRL-19 すべてのデータの取得に成功しました') + + return diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index b75eec23..b5cfbcb6 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -26,7 +26,6 @@ class TestController: - 取得処理実施結果アップロード処理が1回のみ実行されること """ monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - # monkeypatch.setattr(ExecuteDateTime, '__str__', lambda x: 'YYYY-MM-DDTHH:MM:SS.000Z') monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) mock_execute_datetime = ExecuteDateTime() @@ -80,3 +79,40 @@ class TestController: assert mock_copy_crm_csv_data_process.call_count == 3 assert mock_upload_last_fetch_datetime_process.call_count == 3 mock_upload_result_data_process.assert_called_once() + + def test_controller_do_not_call_upload_process_result_process(self, s3_client, monkeypatch): + """ + Cases: + 処理対象オブジェクトが0件の場合、取得処理実施結果アップロード処理が実行されないこと + Arranges: + - データ取得準備処理で返される取得対象オブジェクト情報を0件にする + - 各種プロセスメソッドと内部で使用している値オブジェクトをモック化する + Expects: + - データ取得準備処理が1回のみ実行されること + - 取得処理実施結果アップロード処理が実行されないこと + """ + mock_execute_datetime = ExecuteDateTime() + mock_prepare_data_fetch_process = MagicMock(return_value=([], mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock() + mock_set_datetime_period_process = MagicMock() + mock_fetch_crm_data_process = MagicMock() + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock() + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process),\ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + controller.controller() + + mock_prepare_data_fetch_process.assert_called_once() + mock_upload_result_data_process.call_count == 0 From e00f336e216aef20458f7b94de4add97e5cce888 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 13:11:58 +0900 Subject: [PATCH 121/275] =?UTF-8?q?feat:=20=E5=85=A8=E5=91=BC=E3=81=B3?= =?UTF-8?q?=E5=87=BA=E3=81=97=E3=81=AE=E3=83=AD=E3=82=B0=E6=A4=9C=E6=9F=BB?= =?UTF-8?q?=E3=82=92=E8=A1=8C=E3=81=86=E3=81=9F=E3=82=81=E3=80=81=E3=82=B3?= =?UTF-8?q?=E3=83=B3=E3=83=88=E3=83=AD=E3=83=BC=E3=83=AB=E5=87=A6=E7=90=86?= =?UTF-8?q?=E3=82=92=E5=85=B1=E9=80=9A=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_controller.py | 182 +++++++++++++-------- 1 file changed, 114 insertions(+), 68 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index b5cfbcb6..4500a845 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -1,5 +1,7 @@ +import logging from unittest.mock import MagicMock, patch +import pytest from src import controller from src.config.objects import (FetchTargetObjects, LastFetchDatetime, TargetObject) @@ -7,80 +9,121 @@ from src.util.execute_datetime import ExecuteDateTime class TestController: - def test_controller_call_all_processes(self, s3_client, monkeypatch): + + @pytest.fixture() + def call_all_processes(self, s3_client, monkeypatch, caplog): + """ + コントローラー処理内ですべてのプロセス関数が呼ばれることのテストで使用するフィクスチャ + 各種プロセス関数をモック化し、正常終了させるように動く + + Yields: + dict: プロセス関数呼び出し後のモックの辞書 + """ + def _func(): + monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + mock_target_objects = FetchTargetObjects({ + 'objects': [ + { + 'object_name': 'Account' + }, + { + 'object_name': 'Contact' + }, + { + 'object_name': 'Call2_vod__c' + } + ] + }) + mock_target_object = TargetObject({'object_name': 'Account'}, mock_execute_datetime) + mock_last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' + }, mock_execute_datetime) + mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(return_value=mock_target_object) + mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_fetch_crm_data_process = MagicMock(return_value=[{}]) + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock() + should_call_processes = { + 'prepare_data_fetch_process': mock_prepare_data_fetch_process, + 'check_object_info_process': mock_check_object_info_process, + 'set_datetime_period_process': mock_set_datetime_period_process, + 'fetch_crm_data_process': mock_fetch_crm_data_process, + 'backup_crm_data_process': mock_backup_crm_data_process, + 'convert_crm_csv_data_process': mock_convert_crm_csv_data_process, + 'backup_crm_csv_data_process': mock_backup_crm_csv_data_process, + 'copy_crm_csv_data_process': mock_copy_crm_csv_data_process, + 'upload_last_fetch_datetime_process': mock_upload_last_fetch_datetime_process, + 'upload_result_data_process': mock_upload_result_data_process + } + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process),\ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + controller.controller() + + return should_call_processes + + yield _func + + def test_controller_call_all_processes(self, call_all_processes): """ Cases: コントローラー処理からすべてのプロセスが呼ばれること Arranges: - - 各種プロセスメソッドと内部で使用している値オブジェクトをモック化する + - call_all_processesフィクスチャで、コントローラー処理を実行しておく Expects: - - データ取得準備処理が1回のみ実行されること - - オブジェクト情報形式チェック処理が複数回実行されること - - データ取得期間設定処理が複数回実行されること - - CRMデータ取得処理が複数回実行されること - - CRM電文データバックアップ処理が複数回実行されること - - CSV変換処理が複数回実行されること - - CSVバックアップ処理が複数回実行されること - - CSVアップロード処理が複数回実行されること - - 前回取得日次ファイル更新処理が複数回実行されること - - 取得処理実施結果アップロード処理が1回のみ実行されること + - データ取得準備処理が1回のみ実行される + - オブジェクト情報形式チェック処理が複数回実行される + - データ取得期間設定処理が複数回実行される + - CRMデータ取得処理が複数回実行される + - CRM電文データバックアップ処理が複数回実行される + - CSV変換処理が複数回実行される + - CSVバックアップ処理が複数回実行される + - CSVアップロード処理が複数回実行される + - 前回取得日時ファイル更新処理が複数回実行される + - 取得処理実施結果アップロード処理が1回のみ実行される """ - monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() - mock_target_objects = FetchTargetObjects({ - 'objects': [ - { - 'object_name': 'Account' - }, - { - 'object_name': 'Contact' - }, - { - 'object_name': 'Call2_vod__c' - } - ] - }) - mock_target_object = TargetObject({'object_name': 'Account'}, mock_execute_datetime) - mock_last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', - 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' - }, mock_execute_datetime) - mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) - mock_check_object_info_process = MagicMock(return_value=mock_target_object) - mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) - mock_fetch_crm_data_process = MagicMock(return_value=[{}]) - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process),\ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - controller.controller() + processes = call_all_processes() + processes['prepare_data_fetch_process'].assert_called_once(), 'データ取得準備処理が1回のみ実行されること' + assert processes['check_object_info_process'].call_count == 3, 'オブジェクト情報形式チェック処理が複数回実行されること' + assert processes['set_datetime_period_process'].call_count == 3, 'データ取得期間設定処理が複数回実行されること' + assert processes['fetch_crm_data_process'].call_count == 3, 'CRMデータ取得処理が複数回実行されること' + assert processes['backup_crm_data_process'].call_count == 3, 'CRM電文データバックアップ処理が複数回実行されること' + assert processes['convert_crm_csv_data_process'].call_count == 3, 'CSV変換処理が複数回実行されること' + assert processes['backup_crm_csv_data_process'].call_count == 3, 'CSVバックアップ処理が複数回実行されること' + assert processes['copy_crm_csv_data_process'].call_count == 3, 'CSVアップロード処理が複数回実行されること' + assert processes['upload_last_fetch_datetime_process'].call_count == 3, '前回取得日次ファイル更新処理が複数回実行されること' + processes['upload_result_data_process'].assert_called_once(), '取得処理実施結果アップロード処理が1回のみ実行されること' - mock_prepare_data_fetch_process.assert_called_once() - assert mock_check_object_info_process.call_count == 3 - assert mock_set_datetime_period_process.call_count == 3 - assert mock_fetch_crm_data_process.call_count == 3 - assert mock_backup_crm_data_process.call_count == 3 - assert mock_convert_crm_csv_data_process.call_count == 3 - assert mock_backup_crm_csv_data_process.call_count == 3 - assert mock_copy_crm_csv_data_process.call_count == 3 - assert mock_upload_last_fetch_datetime_process.call_count == 3 - mock_upload_result_data_process.assert_called_once() + def test_controller_print_normal_logs(self, call_all_processes, caplog): + """ + Cases: + コントローラー処理の正常系ログがすべての出力されていること + Arranges: + - call_all_processesフィクスチャで、コントローラー処理を実行しておく + Expects: + - コントローラー処理の正常系ログがすべて出力されている + """ + call_all_processes() + # シングルトンなロガーを引き回しているので、rootロガーになる + assert ('root', logging.INFO, 'I-CTRL-01 CRMデータ取得処理を開始します') in caplog.record_tuples - def test_controller_do_not_call_upload_process_result_process(self, s3_client, monkeypatch): + def test_controller_do_not_call_upload_process_result_process(self, s3_client, monkeypatch, caplog): """ Cases: 処理対象オブジェクトが0件の場合、取得処理実施結果アップロード処理が実行されないこと @@ -90,6 +133,7 @@ class TestController: Expects: - データ取得準備処理が1回のみ実行されること - 取得処理実施結果アップロード処理が実行されないこと + - 処理対象が存在しない旨を示すログメッセージが出力されていること """ mock_execute_datetime = ExecuteDateTime() mock_prepare_data_fetch_process = MagicMock(return_value=([], mock_execute_datetime, {})) @@ -114,5 +158,7 @@ class TestController: patch('src.controller.upload_result_data_process', mock_upload_result_data_process): controller.controller() - mock_prepare_data_fetch_process.assert_called_once() - mock_upload_result_data_process.call_count == 0 + mock_prepare_data_fetch_process.assert_called_once(), 'データ取得準備処理が1回のみ実行されること' + mock_upload_result_data_process.call_count == 0, '取得処理実施結果アップロード処理が実行されないこと' + # シングルトンなロガーを引き回しているので、rootロガーになる + assert ('root', logging.INFO, 'I-CTRL-21 処理対象のデータが存在しませんでした') in caplog.record_tuples, '処理対象が存在しない旨を示すログメッセージが出力されていること' From ceba91e33a4ce1c982fdf08696e1bd59f2246064 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 13:11:58 +0900 Subject: [PATCH 122/275] =?UTF-8?q?style:=20=E3=82=B3=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=83=AD=E3=83=BC=E3=83=A9=E3=83=BC=E2=86=92=E3=82=B3=E3=83=B3?= =?UTF-8?q?=E3=83=88=E3=83=AD=E3=83=BC=E3=83=AB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_controller.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index 4500a845..6e47ba43 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -13,7 +13,7 @@ class TestController: @pytest.fixture() def call_all_processes(self, s3_client, monkeypatch, caplog): """ - コントローラー処理内ですべてのプロセス関数が呼ばれることのテストで使用するフィクスチャ + コントロール処理内ですべてのプロセス関数が呼ばれることのテストで使用するフィクスチャ 各種プロセス関数をモック化し、正常終了させるように動く Yields: @@ -83,9 +83,9 @@ class TestController: def test_controller_call_all_processes(self, call_all_processes): """ Cases: - コントローラー処理からすべてのプロセスが呼ばれること + コントロール処理からすべてのプロセスが呼ばれること Arranges: - - call_all_processesフィクスチャで、コントローラー処理を実行しておく + - call_all_processesフィクスチャで、コントロール処理を実行しておく Expects: - データ取得準備処理が1回のみ実行される - オブジェクト情報形式チェック処理が複数回実行される @@ -113,11 +113,11 @@ class TestController: def test_controller_print_normal_logs(self, call_all_processes, caplog): """ Cases: - コントローラー処理の正常系ログがすべての出力されていること + コントロール処理の正常系ログがすべての出力されていること Arranges: - - call_all_processesフィクスチャで、コントローラー処理を実行しておく + - call_all_processesフィクスチャで、コントロール処理を実行しておく Expects: - - コントローラー処理の正常系ログがすべて出力されている + - コントロール処理の正常系ログがすべて出力されている """ call_all_processes() # シングルトンなロガーを引き回しているので、rootロガーになる From b8fd8a477e68fcbe04fc2cde9a9c2222250a3edd Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 13:53:58 +0900 Subject: [PATCH 123/275] =?UTF-8?q?feat:=20=E3=83=AD=E3=82=B0=E3=83=A1?= =?UTF-8?q?=E3=83=83=E3=82=BB=E3=83=BC=E3=82=B8=E3=81=AE=E7=A2=BA=E8=AA=8D?= =?UTF-8?q?=E3=82=B1=E3=83=BC=E3=82=B9=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_controller.py | 72 ++++++++++++++----- .../tests/test_utils/log_message.py | 7 ++ 2 files changed, 61 insertions(+), 18 deletions(-) create mode 100644 ecs/crm-datafetch/tests/test_utils/log_message.py diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index 6e47ba43..6b91abb2 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -1,4 +1,3 @@ -import logging from unittest.mock import MagicMock, patch import pytest @@ -7,6 +6,31 @@ from src.config.objects import (FetchTargetObjects, LastFetchDatetime, TargetObject) from src.util.execute_datetime import ExecuteDateTime +from .test_utils.log_message import generate_log_message_tuple + +TEST_OBJECT_INFO_LIST = { + 'objects': [ + { + 'object_name': 'Account', + 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Contact', + 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Call2_vod__c', + 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + } + ] +} + +EXPECT_PROCESS_RESULT = { + 'Account': 'success', + 'Contact': 'success', + 'Call2_vod__c': 'success' +} + class TestController: @@ -24,26 +48,17 @@ class TestController: monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) mock_execute_datetime = ExecuteDateTime() - mock_target_objects = FetchTargetObjects({ - 'objects': [ - { - 'object_name': 'Account' - }, - { - 'object_name': 'Contact' - }, - { - 'object_name': 'Call2_vod__c' - } - ] - }) - mock_target_object = TargetObject({'object_name': 'Account'}, mock_execute_datetime) + mock_target_objects = FetchTargetObjects(TEST_OBJECT_INFO_LIST) + mock_target_object_first = TargetObject(TEST_OBJECT_INFO_LIST['objects'][0], mock_execute_datetime) + mock_target_object_second = TargetObject(TEST_OBJECT_INFO_LIST['objects'][1], mock_execute_datetime) + mock_target_object_third = TargetObject(TEST_OBJECT_INFO_LIST['objects'][2], mock_execute_datetime) + mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] mock_last_fetch_datetime = LastFetchDatetime({ 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' }, mock_execute_datetime) mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) - mock_check_object_info_process = MagicMock(return_value=mock_target_object) + mock_check_object_info_process = MagicMock(side_effect=mock_return_values) mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) mock_fetch_crm_data_process = MagicMock(return_value=[{}]) mock_backup_crm_data_process = MagicMock() @@ -121,7 +136,27 @@ class TestController: """ call_all_processes() # シングルトンなロガーを引き回しているので、rootロガーになる - assert ('root', logging.INFO, 'I-CTRL-01 CRMデータ取得処理を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-CTRL-01 CRMデータ取得処理を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-CTRL-02 データ取得準備処理呼び出し') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-CTRL-03 取得対象オブジェクトのループ処理開始') in caplog.record_tuples + for obj in TEST_OBJECT_INFO_LIST['objects']: + object_name = obj['object_name'] + upload_file_name = obj['upload_file_name'] + assert generate_log_message_tuple(log_message='I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-06 [{object_name}]のデータ取得を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-08 [{object_name}]のデータ取得期間設定処理呼び出し') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-09 [{object_name}]のデータ取得処理呼び出し') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-10 [{object_name}] の出力ファイル名は [{upload_file_name}] となります') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-11 [{object_name}] CRM電文データバックアップ処理呼び出し') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-12 [{object_name}] CSV変換処理呼び出し') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-13 [{object_name}] CSVデータバックアップ処理呼び出し') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-14 [{object_name}] CSVデータアップロード処理呼び出し') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-15 [{object_name}] 前回取得日時ファイル更新処理呼び出し') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-16 [{object_name}] 処理正常終了') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{EXPECT_PROCESS_RESULT}]') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-19 すべてのデータの取得に成功しました') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples def test_controller_do_not_call_upload_process_result_process(self, s3_client, monkeypatch, caplog): """ @@ -161,4 +196,5 @@ class TestController: mock_prepare_data_fetch_process.assert_called_once(), 'データ取得準備処理が1回のみ実行されること' mock_upload_result_data_process.call_count == 0, '取得処理実施結果アップロード処理が実行されないこと' # シングルトンなロガーを引き回しているので、rootロガーになる - assert ('root', logging.INFO, 'I-CTRL-21 処理対象のデータが存在しませんでした') in caplog.record_tuples, '処理対象が存在しない旨を示すログメッセージが出力されていること' + assert generate_log_message_tuple(log_message='I-CTRL-21 処理対象のデータが存在しませんでした') in caplog.record_tuples, '処理対象が存在しない旨を示すログメッセージが出力されていること' + assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples diff --git a/ecs/crm-datafetch/tests/test_utils/log_message.py b/ecs/crm-datafetch/tests/test_utils/log_message.py new file mode 100644 index 00000000..25ce6e11 --- /dev/null +++ b/ecs/crm-datafetch/tests/test_utils/log_message.py @@ -0,0 +1,7 @@ +"""ログメッセージに関連するテストヘルパー""" + +import logging + + +def generate_log_message_tuple(logger_name='root', log_level=logging.INFO, log_message=''): + return (logger_name, log_level, log_message) From 6ad684e38bbf722d723fbc15e9c9332c1f91177f Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 14:51:04 +0900 Subject: [PATCH 124/275] =?UTF-8?q?feat:=E5=87=A6=E7=90=86=E3=82=B9?= =?UTF-8?q?=E3=82=AD=E3=83=83=E3=83=97=E6=99=82=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_controller.py | 135 ++++++++++++++++++++- 1 file changed, 129 insertions(+), 6 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index 6b91abb2..c1435a7d 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -1,3 +1,4 @@ +import copy from unittest.mock import MagicMock, patch import pytest @@ -33,7 +34,6 @@ EXPECT_PROCESS_RESULT = { class TestController: - @pytest.fixture() def call_all_processes(self, s3_client, monkeypatch, caplog): """ @@ -95,7 +95,7 @@ class TestController: yield _func - def test_controller_call_all_processes(self, call_all_processes): + def test_call_all_processes(self, call_all_processes): """ Cases: コントロール処理からすべてのプロセスが呼ばれること @@ -125,7 +125,7 @@ class TestController: assert processes['upload_last_fetch_datetime_process'].call_count == 3, '前回取得日次ファイル更新処理が複数回実行されること' processes['upload_result_data_process'].assert_called_once(), '取得処理実施結果アップロード処理が1回のみ実行されること' - def test_controller_print_normal_logs(self, call_all_processes, caplog): + def test_print_normal_logs(self, call_all_processes, caplog): """ Cases: コントロール処理の正常系ログがすべての出力されていること @@ -135,7 +135,7 @@ class TestController: - コントロール処理の正常系ログがすべて出力されている """ call_all_processes() - # シングルトンなロガーを引き回しているので、rootロガーになる + assert generate_log_message_tuple(log_message='I-CTRL-01 CRMデータ取得処理を開始します') in caplog.record_tuples assert generate_log_message_tuple(log_message='I-CTRL-02 データ取得準備処理呼び出し') in caplog.record_tuples assert generate_log_message_tuple(log_message='I-CTRL-03 取得対象オブジェクトのループ処理開始') in caplog.record_tuples @@ -158,7 +158,7 @@ class TestController: assert generate_log_message_tuple(log_message=f'I-CTRL-19 すべてのデータの取得に成功しました') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples - def test_controller_do_not_call_upload_process_result_process(self, s3_client, monkeypatch, caplog): + def test_do_not_call_upload_process_result_process(self, s3_client, monkeypatch, caplog): """ Cases: 処理対象オブジェクトが0件の場合、取得処理実施結果アップロード処理が実行されないこと @@ -193,8 +193,131 @@ class TestController: patch('src.controller.upload_result_data_process', mock_upload_result_data_process): controller.controller() + # 実行回数の確認 mock_prepare_data_fetch_process.assert_called_once(), 'データ取得準備処理が1回のみ実行されること' mock_upload_result_data_process.call_count == 0, '取得処理実施結果アップロード処理が実行されないこと' - # シングルトンなロガーを引き回しているので、rootロガーになる + + # ログ出力の確認 assert generate_log_message_tuple(log_message='I-CTRL-21 処理対象のデータが存在しませんでした') in caplog.record_tuples, '処理対象が存在しない旨を示すログメッセージが出力されていること' assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples + + def test_do_not_call_upload_csv_process(self, s3_client, monkeypatch, caplog): + """ + Cases: + オブジェクト情報.is_skipがTrueの場合、CSVアップロード処理が実行されないこと + Arranges: + - データ取得準備処理で返される取得対象オブジェクト情報のis_skipをTrueにする + - 各種プロセスメソッドと内部で使用している値オブジェクトをモック化する + Expects: + - オブジェクト情報形式チェック処理以降のプロセスが実行されないこと + - 処理をスキップする旨を示すログメッセージが出力されていること + """ + mock_check_object_info = copy.deepcopy(TEST_OBJECT_INFO_LIST)['objects'][0] + mock_check_object_info['is_skip'] = True + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + mock_return_values = [TargetObject(mock_check_object_info, mock_execute_datetime)] + mock_prepare_data_fetch_process = MagicMock(return_value=([mock_check_object_info], mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + mock_set_datetime_period_process = MagicMock() + mock_fetch_crm_data_process = MagicMock() + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock() + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process),\ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + controller.controller() + + # 実行回数の確認 + mock_check_object_info_process.assert_called_once() + # オブジェクト情報形式チェック処理以降のプロセスが実行されないこと + mock_set_datetime_period_process.call_count == 0 + mock_fetch_crm_data_process.call_count == 0 + mock_backup_crm_data_process.call_count == 0 + mock_convert_crm_csv_data_process.call_count == 0 + mock_backup_crm_csv_data_process.call_count == 0 + mock_copy_crm_csv_data_process.call_count == 0 + mock_upload_last_fetch_datetime_process.call_count == 0 + mock_upload_result_data_process.call_count == 0 + + # ログ出力の確認 + assert generate_log_message_tuple(log_message='I-CTRL-07 [Account]のデータ取得処理をスキップします') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples + + def test_do_not_call_upload_csv_process_of_part(self, s3_client, monkeypatch, caplog): + """ + Cases: + オブジェクト情報.is_skipがTrueのものが含まれる場合、CSVアップロード処理が実行されないこと + Arranges: + - データ取得準備処理で返される取得対象オブジェクト情報のうち、２つ目をis_skipをTrueにする + - 各種プロセスメソッドと内部で使用している値オブジェクトをモック化する + Expects: + - オブジェクト情報形式チェック処理が2回実行されること + - 処理をスキップする旨を示すログメッセージが出力されていること + - オブジェクト情報.is_skipがFalseのものはCSVアップロード処理のログメッセージが出力されていること + """ + mock_check_object_infos = copy.deepcopy(TEST_OBJECT_INFO_LIST) + mock_check_object_infos['objects'][1]['is_skip'] = True + monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + FetchTargetObjects(mock_check_object_infos) + mock_return_values = [ + TargetObject(mock_check_object_infos['objects'][0], mock_execute_datetime), + TargetObject(mock_check_object_infos['objects'][1], mock_execute_datetime), + TargetObject(mock_check_object_infos['objects'][2], mock_execute_datetime), + ] + mock_prepare_data_fetch_process = MagicMock(return_value=(FetchTargetObjects(mock_check_object_infos), mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + mock_set_datetime_period_process = MagicMock() + mock_fetch_crm_data_process = MagicMock() + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock() + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process),\ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + controller.controller() + + # 実行回数の確認 + mock_check_object_info_process.call_count == 3 + # オブジェクト情報形式チェック処理以降のプロセスが実行されないこと + mock_set_datetime_period_process.call_count == 2 + mock_fetch_crm_data_process.call_count == 2 + mock_backup_crm_data_process.call_count == 2 + mock_convert_crm_csv_data_process.call_count == 2 + mock_backup_crm_csv_data_process.call_count == 2 + mock_copy_crm_csv_data_process.call_count == 2 + mock_upload_last_fetch_datetime_process.call_count == 2 + mock_upload_result_data_process.call_count == 2 + + # ログ出力の確認 + assert generate_log_message_tuple( + log_message='I-CTRL-14 [Account] CSVデータアップロード処理呼び出し') in caplog.record_tuples, 'オブジェクト情報.is_skipがFalseのものはCSVアップロード処理のログメッセージが出力されていること' + assert generate_log_message_tuple( + log_message='I-CTRL-07 [Contact]のデータ取得処理をスキップします') in caplog.record_tuples, '処理をスキップする旨を示すログメッセージが出力されていること' + assert generate_log_message_tuple( + log_message='I-CTRL-14 [Call2_vod__c] CSVデータアップロード処理呼び出し' + ) in caplog.record_tuples, 'オブジェクト情報.is_skipがFalseのものはCSVアップロード処理のログメッセージが出力されていること' + assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples From 3abef79cef000465af85e3e87c8c39dcb924d4e1 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 15:17:14 +0900 Subject: [PATCH 125/275] =?UTF-8?q?docs:=20Test=20Explorer=E3=81=AE?= =?UTF-8?q?=E5=B0=8E=E5=85=A5=E3=81=AB=E3=81=A4=E3=81=84=E3=81=A6=E8=BF=BD?= =?UTF-8?q?=E8=A8=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md index 9d580fa5..034cf2d4 100644 --- a/ecs/crm-datafetch/README.md +++ b/ecs/crm-datafetch/README.md @@ -145,6 +145,12 @@ pipenv install --dev pipenv run test:cov ``` +#### 拡張機能：Python Test Explorer UIを導入する場合 + +- VSCodeの拡張機能メニューから、「Python Test Explorer for Visual Studio Code」をインストール +- コマンドパレットから「Python Configure Tests」を選択、「Pytest」を選択 +- テストメニュー(フラスコのマーク)から、テストを実行することができるようになる + #### 各コマンドの説明 - `pipenv run test` From 9b941c1cb328f4e440ea5902e5d75ae3a3919f0e Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 15:58:45 +0900 Subject: [PATCH 126/275] =?UTF-8?q?feat:=20=E4=BE=8B=E5=A4=96=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E8=BF=BD=E5=8A=A0=E3=80=82=E4=BE=8B=E5=A4=96?= =?UTF-8?q?=E3=82=92=E9=80=81=E5=87=BA=E3=81=99=E3=82=8B=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E3=82=B3=E3=83=B3=E3=83=88=E3=83=AD=E3=83=BC=E3=83=AB?= =?UTF-8?q?=E5=87=A6=E7=90=86=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/main.py | 5 +- ecs/crm-datafetch/src/controller.py | 4 +- ecs/crm-datafetch/tests/test_controller.py | 56 ++++++++++++++++++++++ 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/ecs/crm-datafetch/main.py b/ecs/crm-datafetch/main.py index 1d6f0ab9..525e2367 100644 --- a/ecs/crm-datafetch/main.py +++ b/ecs/crm-datafetch/main.py @@ -2,4 +2,7 @@ from src.controller import controller """CRMデータ取得処理のエントリーポイント""" if __name__ == '__main__': - exit(controller()) + try: + exit(controller()) + except Exception: + exit(0) diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index 5309fa83..165c8b7e 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -55,11 +55,11 @@ def controller() -> None: except MeDaCaCRMDataFetchException as e: logger.error(f'E-ERR-01 [{e.func_name}]でエラーが発生したため、処理を終了します') logger.exception(f'{e.error_id} {e}') - return 0 + raise e except Exception as e: logger.exception('E-ERR-02 予期せぬエラーが発生したため、処理を終了します', e) - return 0 + raise e finally: # ⑦ CRMデータ取得処理終了ログを出力する diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index c1435a7d..2861a4d0 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -1,10 +1,13 @@ import copy +import logging from unittest.mock import MagicMock, patch import pytest from src import controller from src.config.objects import (FetchTargetObjects, LastFetchDatetime, TargetObject) +from src.error.exceptions import MeDaCaCRMDataFetchException +from src.system_var.constants import PRE_JP_NAME from src.util.execute_datetime import ExecuteDateTime from .test_utils.log_message import generate_log_message_tuple @@ -33,6 +36,10 @@ EXPECT_PROCESS_RESULT = { } +class ForTestControllerException(MeDaCaCRMDataFetchException): + pass + + class TestController: @pytest.fixture() def call_all_processes(self, s3_client, monkeypatch, caplog): @@ -321,3 +328,52 @@ class TestController: log_message='I-CTRL-14 [Call2_vod__c] CSVデータアップロード処理呼び出し' ) in caplog.record_tuples, 'オブジェクト情報.is_skipがFalseのものはCSVアップロード処理のログメッセージが出力されていること' assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples + + def test_raise_mock_prepare_data_fetch_process(self, s3_client, monkeypatch, caplog): + """ + Cases: + データ取得準備処理でシステム例外が発生した場合、エラーで終了すること + Arranges: + - データ取得準備処理でシステム例外が発生するようにする + Expects: + - データ取得準備処理で例外が発生すること + - データ取得準備処理で発生した例外のログメッセージが出力されていること + """ + # mock_check_object_infos = copy.deepcopy(TEST_OBJECT_INFO_LIST) + # mock_check_object_infos['objects'][1]['is_skip'] = True + # monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + # monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + # mock_execute_datetime = ExecuteDateTime() + # FetchTargetObjects(mock_check_object_infos) + # mock_return_values = [ + # TargetObject(mock_check_object_infos['objects'][0], mock_execute_datetime), + # TargetObject(mock_check_object_infos['objects'][1], mock_execute_datetime), + # TargetObject(mock_check_object_infos['objects'][2], mock_execute_datetime), + # ] + expect_exception = ForTestControllerException('E-CHK-01', PRE_JP_NAME, '例外発生') + mock_prepare_data_fetch_process = MagicMock(side_effect=expect_exception) + mock_check_object_info_process = MagicMock() + # mock_set_datetime_period_process = MagicMock() + # mock_fetch_crm_data_process = MagicMock() + # mock_backup_crm_data_process = MagicMock() + # mock_convert_crm_csv_data_process = MagicMock() + # mock_backup_crm_csv_data_process = MagicMock() + # mock_copy_crm_csv_data_process = MagicMock() + # mock_upload_last_fetch_datetime_process = MagicMock() + # mock_upload_result_data_process = MagicMock() + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process): + # patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + # patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + # patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + # patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + # patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + # patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + # patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + # patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + with pytest.raises(MeDaCaCRMDataFetchException): + controller.controller() + assert mock_check_object_info_process.called is False + assert generate_log_message_tuple( + log_level=logging.ERROR, + log_message=f'E-ERR-01 [{PRE_JP_NAME}]でエラーが発生したため、処理を終了します'), 'データ取得準備処理で発生した例外のログメッセージが出力されていること' From 2fe59b15e7f7366fe8320a4133fdfe4ef71d7c40 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 15:59:20 +0900 Subject: [PATCH 127/275] =?UTF-8?q?feat:=20=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=82=B3=E3=83=9E=E3=83=B3=E3=83=89=E3=81=AE=E3=82=AB=E3=83=90?= =?UTF-8?q?=E3=83=AC=E3=83=83=E3=82=B8=E5=8F=8E=E9=9B=86=E3=83=AC=E3=83=99?= =?UTF-8?q?=E3=83=AB=E3=82=92=E5=A4=89=E6=9B=B4(C1=E3=81=BE=E3=81=A7)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Pipfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 990beb4b..82796598 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -5,8 +5,8 @@ name = "pypi" [scripts] test = "pytest tests/" -"test:cov" = "pytest --cov=src tests/" -"test:report" = "pytest --cov=src --html=.report/test_result.html tests/" +"test:cov" = "pytest --cov=src --cov-branch --cov-report=term-missing tests/" +"test:report" = "pytest --cov=src --cov-branch --cov-report=term-missing --html=.report/test_result.html tests/" [packages] boto3 = "*" From d56a6c079d222b995bf05c4911c3946bb1e990d7 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 16:47:06 +0900 Subject: [PATCH 128/275] =?UTF-8?q?refactor:=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=83=87=E3=83=BC=E3=82=BF=E3=81=A8=E6=9C=9F=E5=BE=85=E5=80=A4?= =?UTF-8?q?=E3=81=AF=E3=81=AA=E3=82=8B=E3=81=B9=E3=81=8F=E3=83=99=E3=82=BF?= =?UTF-8?q?=E6=9B=B8=E3=81=8D=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_controller.py | 81 ++++++++++++++-------- 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index 2861a4d0..f6da35e3 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -1,4 +1,3 @@ -import copy import logging from unittest.mock import MagicMock, patch @@ -12,23 +11,6 @@ from src.util.execute_datetime import ExecuteDateTime from .test_utils.log_message import generate_log_message_tuple -TEST_OBJECT_INFO_LIST = { - 'objects': [ - { - 'object_name': 'Account', - 'upload_file_name': 'Account_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Contact', - 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Call2_vod__c', - 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' - } - ] -} - EXPECT_PROCESS_RESULT = { 'Account': 'success', 'Contact': 'success', @@ -55,10 +37,26 @@ class TestController: monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) mock_execute_datetime = ExecuteDateTime() - mock_target_objects = FetchTargetObjects(TEST_OBJECT_INFO_LIST) - mock_target_object_first = TargetObject(TEST_OBJECT_INFO_LIST['objects'][0], mock_execute_datetime) - mock_target_object_second = TargetObject(TEST_OBJECT_INFO_LIST['objects'][1], mock_execute_datetime) - mock_target_object_third = TargetObject(TEST_OBJECT_INFO_LIST['objects'][2], mock_execute_datetime) + objects = { + 'objects': [ + { + 'object_name': 'Account', + 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Contact', + 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Call2_vod__c', + 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + } + ] + } + mock_target_objects = FetchTargetObjects(objects) + mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) + mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) + mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] mock_last_fetch_datetime = LastFetchDatetime({ 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', @@ -146,9 +144,9 @@ class TestController: assert generate_log_message_tuple(log_message='I-CTRL-01 CRMデータ取得処理を開始します') in caplog.record_tuples assert generate_log_message_tuple(log_message='I-CTRL-02 データ取得準備処理呼び出し') in caplog.record_tuples assert generate_log_message_tuple(log_message='I-CTRL-03 取得対象オブジェクトのループ処理開始') in caplog.record_tuples - for obj in TEST_OBJECT_INFO_LIST['objects']: - object_name = obj['object_name'] - upload_file_name = obj['upload_file_name'] + for name in ['Account', 'Contact', 'Call2_vod__c']: + object_name = name + upload_file_name = f'{name}_YYYYMMDDHHMMSS' assert generate_log_message_tuple(log_message='I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-06 [{object_name}]のデータ取得を開始します') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-08 [{object_name}]のデータ取得期間設定処理呼び出し') in caplog.record_tuples @@ -160,7 +158,13 @@ class TestController: assert generate_log_message_tuple(log_message=f'I-CTRL-14 [{object_name}] CSVデータアップロード処理呼び出し') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-15 [{object_name}] 前回取得日時ファイル更新処理呼び出し') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-16 [{object_name}] 処理正常終了') in caplog.record_tuples - assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{EXPECT_PROCESS_RESULT}]') in caplog.record_tuples + + expect_process_result = { + 'Account': 'success', + 'Contact': 'success', + 'Call2_vod__c': 'success' + } + assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-19 すべてのデータの取得に成功しました') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples @@ -219,8 +223,11 @@ class TestController: - オブジェクト情報形式チェック処理以降のプロセスが実行されないこと - 処理をスキップする旨を示すログメッセージが出力されていること """ - mock_check_object_info = copy.deepcopy(TEST_OBJECT_INFO_LIST)['objects'][0] - mock_check_object_info['is_skip'] = True + mock_check_object_info = { + 'object_name': 'Account', + 'is_skip': True + } + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) mock_execute_datetime = ExecuteDateTime() mock_return_values = [TargetObject(mock_check_object_info, mock_execute_datetime)] @@ -274,8 +281,22 @@ class TestController: - 処理をスキップする旨を示すログメッセージが出力されていること - オブジェクト情報.is_skipがFalseのものはCSVアップロード処理のログメッセージが出力されていること """ - mock_check_object_infos = copy.deepcopy(TEST_OBJECT_INFO_LIST) - mock_check_object_infos['objects'][1]['is_skip'] = True + mock_check_object_infos = { + 'objects': [ + { + 'object_name': 'Account', + 'is_skip': False + }, + { + 'object_name': 'Contact', + 'is_skip': True + }, + { + 'object_name': 'Call2_vod__c', + 'is_skip': False + } + ] + } monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) mock_execute_datetime = ExecuteDateTime() From 255297036312c42f75cebc00d06c7a5456505d45 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 18:38:34 +0900 Subject: [PATCH 129/275] =?UTF-8?q?feat:=20=E3=83=AB=E3=83=BC=E3=83=97?= =?UTF-8?q?=E4=B8=AD=E3=81=AB=E7=99=BA=E7=94=9F=E3=81=99=E3=82=8B=E4=BE=8B?= =?UTF-8?q?=E5=A4=96=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0=EF=BC=88=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=83=BB=E6=9C=9F=E9=96=93=E8=A8=AD=E5=AE=9A=E5=87=A6?= =?UTF-8?q?=E7=90=86=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_controller.py | 204 +++++++++++++++++++-- 1 file changed, 189 insertions(+), 15 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index f6da35e3..b0938eee 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -6,7 +6,7 @@ from src import controller from src.config.objects import (FetchTargetObjects, LastFetchDatetime, TargetObject) from src.error.exceptions import MeDaCaCRMDataFetchException -from src.system_var.constants import PRE_JP_NAME +from src.system_var.constants import CHK_JP_NAME, DATE_JP_NAME, PRE_JP_NAME from src.util.execute_datetime import ExecuteDateTime from .test_utils.log_message import generate_log_message_tuple @@ -350,7 +350,7 @@ class TestController: ) in caplog.record_tuples, 'オブジェクト情報.is_skipがFalseのものはCSVアップロード処理のログメッセージが出力されていること' assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples - def test_raise_mock_prepare_data_fetch_process(self, s3_client, monkeypatch, caplog): + def test_raise_prepare_data_fetch_process(self, s3_client, monkeypatch, caplog): """ Cases: データ取得準備処理でシステム例外が発生した場合、エラーで終了すること @@ -360,28 +360,50 @@ class TestController: - データ取得準備処理で例外が発生すること - データ取得準備処理で発生した例外のログメッセージが出力されていること """ - # mock_check_object_infos = copy.deepcopy(TEST_OBJECT_INFO_LIST) - # mock_check_object_infos['objects'][1]['is_skip'] = True + # TODO: ケースのコピー用にコメントを残してある # monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) # monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + # monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) # mock_execute_datetime = ExecuteDateTime() - # FetchTargetObjects(mock_check_object_infos) - # mock_return_values = [ - # TargetObject(mock_check_object_infos['objects'][0], mock_execute_datetime), - # TargetObject(mock_check_object_infos['objects'][1], mock_execute_datetime), - # TargetObject(mock_check_object_infos['objects'][2], mock_execute_datetime), - # ] - expect_exception = ForTestControllerException('E-CHK-01', PRE_JP_NAME, '例外発生') - mock_prepare_data_fetch_process = MagicMock(side_effect=expect_exception) - mock_check_object_info_process = MagicMock() - # mock_set_datetime_period_process = MagicMock() - # mock_fetch_crm_data_process = MagicMock() + # objects = { + # 'objects': [ + # { + # 'object_name': 'Account', + # 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + # }, + # { + # 'object_name': 'Contact', + # 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + # }, + # { + # 'object_name': 'Call2_vod__c', + # 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + # } + # ] + # } + # mock_target_objects = FetchTargetObjects(objects) + # mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) + # mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) + # mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) + # mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] + # mock_last_fetch_datetime = LastFetchDatetime({ + # 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + # 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' + # }, mock_execute_datetime) + # mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + # mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + # mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + # mock_fetch_crm_data_process = MagicMock(return_value=[{}]) # mock_backup_crm_data_process = MagicMock() # mock_convert_crm_csv_data_process = MagicMock() # mock_backup_crm_csv_data_process = MagicMock() # mock_copy_crm_csv_data_process = MagicMock() # mock_upload_last_fetch_datetime_process = MagicMock() # mock_upload_result_data_process = MagicMock() + + expect_exception = ForTestControllerException('E-PRE-01', PRE_JP_NAME, '例外発生') + mock_prepare_data_fetch_process = MagicMock(side_effect=expect_exception) + mock_check_object_info_process = MagicMock() with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ patch('src.controller.check_object_info_process', mock_check_object_info_process): # patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ @@ -398,3 +420,155 @@ class TestController: assert generate_log_message_tuple( log_level=logging.ERROR, log_message=f'E-ERR-01 [{PRE_JP_NAME}]でエラーが発生したため、処理を終了します'), 'データ取得準備処理で発生した例外のログメッセージが出力されていること' + + def test_raise_check_object_info_process(self, s3_client, monkeypatch, caplog): + """ + Cases: + オブジェクト情報形式チェック処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること + Arranges: + - オブジェクト情報形式チェック処理の1回のみでシステム例外が発生するようにする + Expects: + - オブジェクト情報形式チェック処理で例外が発生する + - オブジェクト情報形式チェック処理で発生した例外のログメッセージが出力されている + - 例外が発生したオブジェクト以外は正常に終了する + """ + monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + objects = { + 'objects': [ + { + 'object_name': 'Account', + 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Contact', + 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Call2_vod__c', + 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + } + ] + } + mock_target_objects = FetchTargetObjects(objects) + mock_target_object_first = ForTestControllerException('E-PRE-01', CHK_JP_NAME, '例外発生') + mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) + mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) + mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] + mock_last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' + }, mock_execute_datetime) + mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_fetch_crm_data_process = MagicMock(return_value=[{}]) + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock() + + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process),\ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + # ループ中の例外処理なので、例外は潰される + # with pytest.raises(MeDaCaCRMDataFetchException): + controller.controller() + assert mock_check_object_info_process.call_count == 3 + expect_process_result = { + 'Account': 'fail', + 'Contact': 'success', + 'Call2_vod__c': 'success' + } + assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{CHK_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples + assert generate_log_message_tuple( + log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples + + def test_raise_set_fetch_date_period_process(self, s3_client, monkeypatch, caplog): + """ + Cases: + データ取得期間設定処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること + Arranges: + - データ取得期間設定チェック処理の1回のみでシステム例外が発生するようにする + Expects: + - データ取得期間設定チェック処理で例外が発生する + - データ取得期間設定チェック処理で発生した例外のログメッセージが出力されている + - 例外が発生したオブジェクト以外は正常に終了する + """ + monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + objects = { + 'objects': [ + { + 'object_name': 'Account', + 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Contact', + 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Call2_vod__c', + 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + } + ] + } + mock_target_objects = FetchTargetObjects(objects) + mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) + mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) + mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) + mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] + mock_last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' + }, mock_execute_datetime) + mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + # mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_set_datetime_period_process = MagicMock(side_effect=[ForTestControllerException( + 'E-DATE-01', DATE_JP_NAME, '例外発生'), mock_last_fetch_datetime, mock_last_fetch_datetime]) + mock_fetch_crm_data_process = MagicMock(return_value=[{}]) + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock() + + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process),\ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + # ループ中の例外処理なので、例外は潰される + # with pytest.raises(MeDaCaCRMDataFetchException): + controller.controller() + assert mock_set_datetime_period_process.call_count == 3 + expect_process_result = { + 'Account': 'fail', + 'Contact': 'success', + 'Call2_vod__c': 'success' + } + assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{DATE_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples + assert generate_log_message_tuple( + log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples From f18aeb8d258392c10989afc10cb1b258db601d29 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 19:41:42 +0900 Subject: [PATCH 130/275] =?UTF-8?q?feat:=E3=83=AB=E3=83=BC=E3=83=97?= =?UTF-8?q?=E5=87=A6=E7=90=86=E4=B8=AD=E3=81=AE=E4=BE=8B=E5=A4=96=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_controller.py | 513 ++++++++++++++++++--- 1 file changed, 461 insertions(+), 52 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index b0938eee..43dfed94 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -6,7 +6,9 @@ from src import controller from src.config.objects import (FetchTargetObjects, LastFetchDatetime, TargetObject) from src.error.exceptions import MeDaCaCRMDataFetchException -from src.system_var.constants import CHK_JP_NAME, DATE_JP_NAME, PRE_JP_NAME +from src.system_var.constants import (CHK_JP_NAME, CONV_JP_NAME, CSVBK_JP_NAME, + DATE_JP_NAME, FETCH_JP_NAME, PRE_JP_NAME, + RESBK_JP_NAME, UPD_JP_NAME, UPLD_JP_NAME) from src.util.execute_datetime import ExecuteDateTime from .test_utils.log_message import generate_log_message_tuple @@ -360,60 +362,11 @@ class TestController: - データ取得準備処理で例外が発生すること - データ取得準備処理で発生した例外のログメッセージが出力されていること """ - # TODO: ケースのコピー用にコメントを残してある - # monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - # monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - # monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) - # mock_execute_datetime = ExecuteDateTime() - # objects = { - # 'objects': [ - # { - # 'object_name': 'Account', - # 'upload_file_name': 'Account_YYYYMMDDHHMMSS' - # }, - # { - # 'object_name': 'Contact', - # 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' - # }, - # { - # 'object_name': 'Call2_vod__c', - # 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' - # } - # ] - # } - # mock_target_objects = FetchTargetObjects(objects) - # mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) - # mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) - # mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) - # mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] - # mock_last_fetch_datetime = LastFetchDatetime({ - # 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', - # 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' - # }, mock_execute_datetime) - # mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) - # mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - # mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) - # mock_fetch_crm_data_process = MagicMock(return_value=[{}]) - # mock_backup_crm_data_process = MagicMock() - # mock_convert_crm_csv_data_process = MagicMock() - # mock_backup_crm_csv_data_process = MagicMock() - # mock_copy_crm_csv_data_process = MagicMock() - # mock_upload_last_fetch_datetime_process = MagicMock() - # mock_upload_result_data_process = MagicMock() - expect_exception = ForTestControllerException('E-PRE-01', PRE_JP_NAME, '例外発生') mock_prepare_data_fetch_process = MagicMock(side_effect=expect_exception) mock_check_object_info_process = MagicMock() with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ patch('src.controller.check_object_info_process', mock_check_object_info_process): - # patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - # patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - # patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - # patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - # patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - # patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - # patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - # patch('src.controller.upload_result_data_process', mock_upload_result_data_process): with pytest.raises(MeDaCaCRMDataFetchException): controller.controller() assert mock_check_object_info_process.called is False @@ -486,6 +439,7 @@ class TestController: # with pytest.raises(MeDaCaCRMDataFetchException): controller.controller() assert mock_check_object_info_process.call_count == 3 + assert mock_set_datetime_period_process.call_count == 2 expect_process_result = { 'Account': 'fail', 'Contact': 'success', @@ -496,7 +450,7 @@ class TestController: assert generate_log_message_tuple( log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples - def test_raise_set_fetch_date_period_process(self, s3_client, monkeypatch, caplog): + def test_raise_set_datetime_period_process(self, s3_client, monkeypatch, caplog): """ Cases: データ取得期間設定処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること @@ -538,7 +492,6 @@ class TestController: }, mock_execute_datetime) mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - # mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) mock_set_datetime_period_process = MagicMock(side_effect=[ForTestControllerException( 'E-DATE-01', DATE_JP_NAME, '例外発生'), mock_last_fetch_datetime, mock_last_fetch_datetime]) mock_fetch_crm_data_process = MagicMock(return_value=[{}]) @@ -563,6 +516,7 @@ class TestController: # with pytest.raises(MeDaCaCRMDataFetchException): controller.controller() assert mock_set_datetime_period_process.call_count == 3 + assert mock_fetch_crm_data_process.call_count == 2 expect_process_result = { 'Account': 'fail', 'Contact': 'success', @@ -572,3 +526,458 @@ class TestController: assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples assert generate_log_message_tuple( log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples + + def test_raise_fetch_crm_data_process(self, s3_client, monkeypatch, caplog): + """ + Cases: + CRMデータ取得処理でシステム例外が発生した場合、エラーで終了すること + Arranges: + - CRMデータ取得処理でシステム例外が発生するようにする + Expects: + - CRMデータ取得処理で例外が発生すること + - CRMデータ取得処理で発生した例外のログメッセージが出力されていること + """ + monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + objects = { + 'objects': [ + { + 'object_name': 'Account', + 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Contact', + 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Call2_vod__c', + 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + } + ] + } + mock_target_objects = FetchTargetObjects(objects) + mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) + mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) + mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) + mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] + mock_last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' + }, mock_execute_datetime) + mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_fetch_crm_data_process = MagicMock(side_effect=[ForTestControllerException( + 'E-FETCH-01', FETCH_JP_NAME, '例外発生'), {}, {}]) + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock() + + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process), \ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + # ループ中の例外処理なので、例外は潰される + # with pytest.raises(MeDaCaCRMDataFetchException): + controller.controller() + assert mock_fetch_crm_data_process.call_count == 3 + assert mock_backup_crm_data_process.call_count == 2 + expect_process_result = { + 'Account': 'fail', + 'Contact': 'success', + 'Call2_vod__c': 'success' + } + assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{FETCH_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples + assert generate_log_message_tuple( + log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples + + def test_raise_backup_crm_data_process(self, s3_client, monkeypatch, caplog): + """ + Cases: + CRM電文データバックアップ処理でシステム例外が発生した場合、エラーで終了すること + Arranges: + - CRM電文データバックアップ処理でシステム例外が発生するようにする + Expects: + - CRM電文データバックアップ処理で例外が発生すること + - CRM電文データバックアップ処理で発生した例外のログメッセージが出力されていること + """ + monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + objects = { + 'objects': [ + { + 'object_name': 'Account', + 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Contact', + 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Call2_vod__c', + 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + } + ] + } + mock_target_objects = FetchTargetObjects(objects) + mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) + mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) + mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) + mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] + mock_last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' + }, mock_execute_datetime) + mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + mock_backup_crm_data_process = MagicMock(side_effect=[ForTestControllerException( + 'E-RESBK-01', RESBK_JP_NAME, '例外発生'), None, None]) + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock() + + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process), \ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + # ループ中の例外処理なので、例外は潰される + # with pytest.raises(MeDaCaCRMDataFetchException): + controller.controller() + assert mock_backup_crm_data_process.call_count == 3 + assert mock_convert_crm_csv_data_process.call_count == 2 + expect_process_result = { + 'Account': 'fail', + 'Contact': 'success', + 'Call2_vod__c': 'success' + } + assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{RESBK_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples + assert generate_log_message_tuple( + log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples + + def test_raise_convert_crm_csv_data_process(self, s3_client, monkeypatch, caplog): + """ + Cases: + CSV変換処理でシステム例外が発生した場合、エラーで終了すること + Arranges: + - CSV変換処理でシステム例外が発生するようにする + Expects: + - CSV変換処理で例外が発生すること + - CSV変換処理で発生した例外のログメッセージが出力されていること + """ + monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + objects = { + 'objects': [ + { + 'object_name': 'Account', + 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Contact', + 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Call2_vod__c', + 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + } + ] + } + mock_target_objects = FetchTargetObjects(objects) + mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) + mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) + mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) + mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] + mock_last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' + }, mock_execute_datetime) + mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock(side_effect=[ForTestControllerException( + 'E-CONV-01', CONV_JP_NAME, '例外発生'), None, None]) + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock() + + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process), \ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + # ループ中の例外処理なので、例外は潰される + # with pytest.raises(MeDaCaCRMDataFetchException): + controller.controller() + assert mock_convert_crm_csv_data_process.call_count == 3 + assert mock_backup_crm_csv_data_process.call_count == 2 + expect_process_result = { + 'Account': 'fail', + 'Contact': 'success', + 'Call2_vod__c': 'success' + } + assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{CONV_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples + assert generate_log_message_tuple( + log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples + + def test_raise_backup_crm_csv_data_process(self, s3_client, monkeypatch, caplog): + """ + Cases: + CSVバックアップ処理でシステム例外が発生した場合、エラーで終了すること + Arranges: + - CSVバックアップ処理でシステム例外が発生するようにする + Expects: + - CSVバックアップ処理で例外が発生すること + - CSVバックアップ処理で発生した例外のログメッセージが出力されていること + """ + monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + objects = { + 'objects': [ + { + 'object_name': 'Account', + 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Contact', + 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Call2_vod__c', + 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + } + ] + } + mock_target_objects = FetchTargetObjects(objects) + mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) + mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) + mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) + mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] + mock_last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' + }, mock_execute_datetime) + mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock(side_effect=[ForTestControllerException( + 'E-CSVBK-01', CSVBK_JP_NAME, '例外発生'), None, None]) + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock() + + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process), \ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + # ループ中の例外処理なので、例外は潰される + # with pytest.raises(MeDaCaCRMDataFetchException): + controller.controller() + assert mock_backup_crm_csv_data_process.call_count == 3 + assert mock_copy_crm_csv_data_process.call_count == 2 + expect_process_result = { + 'Account': 'fail', + 'Contact': 'success', + 'Call2_vod__c': 'success' + } + assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{CSVBK_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples + assert generate_log_message_tuple( + log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples + + def test_raise_copy_crm_csv_data_process(self, s3_client, monkeypatch, caplog): + """ + Cases: + CSVアップロード処理でシステム例外が発生した場合、エラーで終了すること + Arranges: + - CSVアップロード処理でシステム例外が発生するようにする + Expects: + - CSVアップロード処理で例外が発生すること + - CSVアップロード処理で発生した例外のログメッセージが出力されていること + """ + monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + objects = { + 'objects': [ + { + 'object_name': 'Account', + 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Contact', + 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Call2_vod__c', + 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + } + ] + } + mock_target_objects = FetchTargetObjects(objects) + mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) + mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) + mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) + mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] + mock_last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' + }, mock_execute_datetime) + mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock(side_effect=[ForTestControllerException( + 'E-UPLD-01', UPLD_JP_NAME, '例外発生'), None, None]) + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock() + + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process), \ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + # ループ中の例外処理なので、例外は潰される + # with pytest.raises(MeDaCaCRMDataFetchException): + controller.controller() + assert mock_copy_crm_csv_data_process.call_count == 3 + assert mock_upload_last_fetch_datetime_process.call_count == 2 + expect_process_result = { + 'Account': 'fail', + 'Contact': 'success', + 'Call2_vod__c': 'success' + } + assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{UPLD_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples + assert generate_log_message_tuple( + log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples + + def test_raise_upload_last_fetch_datetime_process(self, s3_client, monkeypatch, caplog): + """ + Cases: + 前回取得日時ファイル更新処理でシステム例外が発生した場合、エラーで終了すること + Arranges: + - 前回取得日時ファイル更新処理でシステム例外が発生するようにする + Expects: + - 前回取得日時ファイル更新処理で例外が発生すること + - 前回取得日時ファイル更新処理で発生した例外のログメッセージが出力されていること + """ + monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + objects = { + 'objects': [ + { + 'object_name': 'Account', + 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Contact', + 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Call2_vod__c', + 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + } + ] + } + mock_target_objects = FetchTargetObjects(objects) + mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) + mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) + mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) + mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] + mock_last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' + }, mock_execute_datetime) + mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock(side_effect=[ForTestControllerException( + 'E-UPD-01', UPD_JP_NAME, '例外発生'), None, None]) + mock_upload_result_data_process = MagicMock() + + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process), \ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + # ループ中の例外処理なので、例外は潰される + # with pytest.raises(MeDaCaCRMDataFetchException): + controller.controller() + assert mock_upload_last_fetch_datetime_process.call_count == 3 + expect_process_result = { + 'Account': 'fail', + 'Contact': 'success', + 'Call2_vod__c': 'success' + } + assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{UPD_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples + assert generate_log_message_tuple( + log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples From a9d8a54fe96e41f7347421ceb657bf7ce2c6dd28 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 23:40:26 +0900 Subject: [PATCH 131/275] =?UTF-8?q?feat:=20=E6=BA=96=E5=82=99=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=81=A7=E3=82=B7=E3=82=B9=E3=83=86=E3=83=A0=E4=BE=8B?= =?UTF-8?q?=E5=A4=96=E4=BB=A5=E5=A4=96=E3=81=AE=E4=BE=8B=E5=A4=96=E3=81=8C?= =?UTF-8?q?=E7=99=BA=E7=94=9F=E3=81=97=E3=81=9F=E5=A0=B4=E5=90=88=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0(parametr?= =?UTF-8?q?ize=E3=83=86=E3=82=B9=E3=83=88)=E3=80=82=E3=83=AD=E3=82=B0?= =?UTF-8?q?=E5=87=BA=E5=8A=9B=E6=99=82=E3=81=AB=E3=83=90=E3=82=B0=E3=81=A3?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=9F=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/controller.py | 2 +- ecs/crm-datafetch/tests/test_controller.py | 44 ++++++++++++++-------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index 165c8b7e..a7a6fe00 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -58,7 +58,7 @@ def controller() -> None: raise e except Exception as e: - logger.exception('E-ERR-02 予期せぬエラーが発生したため、処理を終了します', e) + logger.exception(f'E-ERR-02 予期せぬエラーが発生したため、処理を終了します エラー内容: [{e}]') raise e finally: diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index 43dfed94..d9d21112 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -20,8 +20,15 @@ EXPECT_PROCESS_RESULT = { } -class ForTestControllerException(MeDaCaCRMDataFetchException): - pass +class ForTestMeDaCaCRMDataFetchException(MeDaCaCRMDataFetchException): + def __init__(self, error_id: str, func_name: str, message: str) -> None: + super().__init__(error_id, func_name, message) + + +class ForTestException(Exception): + # カスタム例外とインタフェースを合わせるための引数 + def __init__(self, error_id: str, func_name: str, message: str) -> None: + super().__init__(message) class TestController: @@ -352,27 +359,34 @@ class TestController: ) in caplog.record_tuples, 'オブジェクト情報.is_skipがFalseのものはCSVアップロード処理のログメッセージが出力されていること' assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples - def test_raise_prepare_data_fetch_process(self, s3_client, monkeypatch, caplog): + @pytest.mark.parametrize( + 'exception, message', + [ + (ForTestMeDaCaCRMDataFetchException, f'E-ERR-01 [{PRE_JP_NAME}]でエラーが発生したため、処理を終了します'), + (ForTestException, 'E-ERR-02 予期せぬエラーが発生したため、処理を終了します エラー内容: [例外発生]') + ]) + def test_raise_prepare_data_fetch_process(self, s3_client, monkeypatch, caplog, exception, message): """ Cases: - データ取得準備処理でシステム例外が発生した場合、エラーで終了すること + 1. データ取得準備処理でシステム例外が発生した場合、エラーで終了すること + 2. データ取得準備処理で想定外の例外が発生した場合、エラーで終了すること Arranges: - データ取得準備処理でシステム例外が発生するようにする Expects: - データ取得準備処理で例外が発生すること - データ取得準備処理で発生した例外のログメッセージが出力されていること """ - expect_exception = ForTestControllerException('E-PRE-01', PRE_JP_NAME, '例外発生') + expect_exception = exception('E-PRE-01', PRE_JP_NAME, '例外発生') mock_prepare_data_fetch_process = MagicMock(side_effect=expect_exception) mock_check_object_info_process = MagicMock() with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ patch('src.controller.check_object_info_process', mock_check_object_info_process): - with pytest.raises(MeDaCaCRMDataFetchException): + with pytest.raises(exception): controller.controller() assert mock_check_object_info_process.called is False assert generate_log_message_tuple( log_level=logging.ERROR, - log_message=f'E-ERR-01 [{PRE_JP_NAME}]でエラーが発生したため、処理を終了します'), 'データ取得準備処理で発生した例外のログメッセージが出力されていること' + log_message=message.format()) in caplog.record_tuples, 'データ取得準備処理で発生した例外のログメッセージが出力されていること' def test_raise_check_object_info_process(self, s3_client, monkeypatch, caplog): """ @@ -406,7 +420,7 @@ class TestController: ] } mock_target_objects = FetchTargetObjects(objects) - mock_target_object_first = ForTestControllerException('E-PRE-01', CHK_JP_NAME, '例外発生') + mock_target_object_first = ForTestMeDaCaCRMDataFetchException('E-PRE-01', CHK_JP_NAME, '例外発生') mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] @@ -492,7 +506,7 @@ class TestController: }, mock_execute_datetime) mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(side_effect=[ForTestControllerException( + mock_set_datetime_period_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( 'E-DATE-01', DATE_JP_NAME, '例外発生'), mock_last_fetch_datetime, mock_last_fetch_datetime]) mock_fetch_crm_data_process = MagicMock(return_value=[{}]) mock_backup_crm_data_process = MagicMock() @@ -569,7 +583,7 @@ class TestController: mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) - mock_fetch_crm_data_process = MagicMock(side_effect=[ForTestControllerException( + mock_fetch_crm_data_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( 'E-FETCH-01', FETCH_JP_NAME, '例外発生'), {}, {}]) mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() @@ -646,7 +660,7 @@ class TestController: mock_check_object_info_process = MagicMock(side_effect=mock_return_values) mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) - mock_backup_crm_data_process = MagicMock(side_effect=[ForTestControllerException( + mock_backup_crm_data_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( 'E-RESBK-01', RESBK_JP_NAME, '例外発生'), None, None]) mock_convert_crm_csv_data_process = MagicMock() mock_backup_crm_csv_data_process = MagicMock() @@ -723,7 +737,7 @@ class TestController: mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock(side_effect=[ForTestControllerException( + mock_convert_crm_csv_data_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( 'E-CONV-01', CONV_JP_NAME, '例外発生'), None, None]) mock_backup_crm_csv_data_process = MagicMock() mock_copy_crm_csv_data_process = MagicMock() @@ -800,7 +814,7 @@ class TestController: mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock(side_effect=[ForTestControllerException( + mock_backup_crm_csv_data_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( 'E-CSVBK-01', CSVBK_JP_NAME, '例外発生'), None, None]) mock_copy_crm_csv_data_process = MagicMock() mock_upload_last_fetch_datetime_process = MagicMock() @@ -877,7 +891,7 @@ class TestController: mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock(side_effect=[ForTestControllerException( + mock_copy_crm_csv_data_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( 'E-UPLD-01', UPLD_JP_NAME, '例外発生'), None, None]) mock_upload_last_fetch_datetime_process = MagicMock() mock_upload_result_data_process = MagicMock() @@ -954,7 +968,7 @@ class TestController: mock_convert_crm_csv_data_process = MagicMock() mock_backup_crm_csv_data_process = MagicMock() mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock(side_effect=[ForTestControllerException( + mock_upload_last_fetch_datetime_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( 'E-UPD-01', UPD_JP_NAME, '例外発生'), None, None]) mock_upload_result_data_process = MagicMock() From 7bd1e72ffdc579c71ba9e4c632b24391c97feb28 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 3 Aug 2022 23:59:20 +0900 Subject: [PATCH 132/275] =?UTF-8?q?feat:=20=E3=83=AB=E3=83=BC=E3=83=97?= =?UTF-8?q?=E4=B8=AD=E3=81=AE=E6=83=B3=E5=AE=9A=E5=A4=96=E3=81=AE=E4=BE=8B?= =?UTF-8?q?=E5=A4=96=E5=87=A6=E7=90=86=E3=82=82=E8=BF=BD=E5=8A=A0=E3=80=82?= =?UTF-8?q?=E3=83=AD=E3=82=B0=E5=87=BA=E5=8A=9B=E6=99=82=E3=81=AB=E3=83=90?= =?UTF-8?q?=E3=82=B0=E3=81=A3=E3=81=A6=E3=81=84=E3=81=9F=E3=81=AE=E3=82=92?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/controller.py | 2 +- ecs/crm-datafetch/tests/test_controller.py | 149 +++++++++++++++------ 2 files changed, 108 insertions(+), 43 deletions(-) diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index a7a6fe00..dc8f784d 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -94,7 +94,7 @@ def _fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: except Exception as e: logger.info( - f'I-ERR-04 [{object_info.get(OBJECT_NAME_KEY)}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e, exc_info=True) + f'I-ERR-04 [{object_info.get(OBJECT_NAME_KEY)}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [{e}]', exc_info=True) continue return process_result diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index d9d21112..c7cf4ebd 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -371,7 +371,8 @@ class TestController: 1. データ取得準備処理でシステム例外が発生した場合、エラーで終了すること 2. データ取得準備処理で想定外の例外が発生した場合、エラーで終了すること Arranges: - - データ取得準備処理でシステム例外が発生するようにする + - パラメータ１：データ取得準備処理でシステム例外が発生するようにする + - パラメータ２：データ取得準備処理で想定外の例外が発生するようにする Expects: - データ取得準備処理で例外が発生すること - データ取得準備処理で発生した例外のログメッセージが出力されていること @@ -386,14 +387,22 @@ class TestController: assert mock_check_object_info_process.called is False assert generate_log_message_tuple( log_level=logging.ERROR, - log_message=message.format()) in caplog.record_tuples, 'データ取得準備処理で発生した例外のログメッセージが出力されていること' + log_message=message) in caplog.record_tuples, 'データ取得準備処理で発生した例外のログメッセージが出力されていること' - def test_raise_check_object_info_process(self, s3_client, monkeypatch, caplog): + @pytest.mark.parametrize( + 'exception, message', + [ + (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{CHK_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), + (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') + ]) + def test_raise_check_object_info_process(self, s3_client, monkeypatch, caplog, exception, message): """ Cases: - オブジェクト情報形式チェック処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること + 1. オブジェクト情報形式チェック処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること + 2. オブジェクト情報形式チェック処理で想定外の例外が発生した場合、エラーログが送出され、後続の終了に移行すること Arranges: - - オブジェクト情報形式チェック処理の1回のみでシステム例外が発生するようにする + - パラメータ１：オブジェクト情報形式チェック処理でシステム例外が発生するようにする + - パラメータ２：オブジェクト情報形式チェック処理で想定外の例外が発生するようにする Expects: - オブジェクト情報形式チェック処理で例外が発生する - オブジェクト情報形式チェック処理で発生した例外のログメッセージが出力されている @@ -420,7 +429,7 @@ class TestController: ] } mock_target_objects = FetchTargetObjects(objects) - mock_target_object_first = ForTestMeDaCaCRMDataFetchException('E-PRE-01', CHK_JP_NAME, '例外発生') + mock_target_object_first = exception('E-PRE-01', CHK_JP_NAME, '例外発生') mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] @@ -459,17 +468,25 @@ class TestController: 'Contact': 'success', 'Call2_vod__c': 'success' } - assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{CHK_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=message) in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples assert generate_log_message_tuple( log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples - def test_raise_set_datetime_period_process(self, s3_client, monkeypatch, caplog): + @pytest.mark.parametrize( + 'exception, message', + [ + (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{DATE_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), + (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') + ]) + def test_raise_set_datetime_period_process(self, s3_client, monkeypatch, caplog, exception, message): """ Cases: - データ取得期間設定処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること + 1. データ取得期間設定処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること + 2. データ取得期間設定処理で想定外の例外が発生した場合、エラーログが送出され、後続の終了に移行すること Arranges: - - データ取得期間設定チェック処理の1回のみでシステム例外が発生するようにする + - パラメータ１：データ取得期間設定処理でシステム例外が発生するようにする + - パラメータ２：データ取得期間設定処理で想定外の例外が発生するようにする Expects: - データ取得期間設定チェック処理で例外が発生する - データ取得期間設定チェック処理で発生した例外のログメッセージが出力されている @@ -506,7 +523,7 @@ class TestController: }, mock_execute_datetime) mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( + mock_set_datetime_period_process = MagicMock(side_effect=[exception( 'E-DATE-01', DATE_JP_NAME, '例外発生'), mock_last_fetch_datetime, mock_last_fetch_datetime]) mock_fetch_crm_data_process = MagicMock(return_value=[{}]) mock_backup_crm_data_process = MagicMock() @@ -536,17 +553,25 @@ class TestController: 'Contact': 'success', 'Call2_vod__c': 'success' } - assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{DATE_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=message) in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples assert generate_log_message_tuple( log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples - def test_raise_fetch_crm_data_process(self, s3_client, monkeypatch, caplog): + @pytest.mark.parametrize( + 'exception, message', + [ + (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{FETCH_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), + (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') + ]) + def test_raise_fetch_crm_data_process(self, s3_client, monkeypatch, caplog, exception, message): """ Cases: - CRMデータ取得処理でシステム例外が発生した場合、エラーで終了すること + 1. CRMデータ取得処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること + 2. CRMデータ取得処理で想定外の例外が発生した場合、エラーログが送出され、後続の終了に移行すること Arranges: - - CRMデータ取得処理でシステム例外が発生するようにする + - パラメータ１：CRMデータ取得処理でシステム例外が発生するようにする + - パラメータ２：CRMデータ取得処理で想定外の例外が発生するようにする Expects: - CRMデータ取得処理で例外が発生すること - CRMデータ取得処理で発生した例外のログメッセージが出力されていること @@ -583,7 +608,7 @@ class TestController: mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) - mock_fetch_crm_data_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( + mock_fetch_crm_data_process = MagicMock(side_effect=[exception( 'E-FETCH-01', FETCH_JP_NAME, '例外発生'), {}, {}]) mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() @@ -612,17 +637,25 @@ class TestController: 'Contact': 'success', 'Call2_vod__c': 'success' } - assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{FETCH_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=message) in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples assert generate_log_message_tuple( log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples - def test_raise_backup_crm_data_process(self, s3_client, monkeypatch, caplog): + @pytest.mark.parametrize( + 'exception, message', + [ + (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{RESBK_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), + (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') + ]) + def test_raise_backup_crm_data_process(self, s3_client, monkeypatch, caplog, exception, message): """ Cases: - CRM電文データバックアップ処理でシステム例外が発生した場合、エラーで終了すること + 1. CRM電文データバックアップ処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること + 2. CRM電文データバックアップ処理で想定外の例外が発生した場合、エラーログが送出され、後続の終了に移行すること Arranges: - - CRM電文データバックアップ処理でシステム例外が発生するようにする + - パラメータ１：CRM電文データバックアップ処理でシステム例外が発生するようにする + - パラメータ２：CRM電文データバックアップ処理で想定外の例外が発生するようにする Expects: - CRM電文データバックアップ処理で例外が発生すること - CRM電文データバックアップ処理で発生した例外のログメッセージが出力されていること @@ -660,7 +693,7 @@ class TestController: mock_check_object_info_process = MagicMock(side_effect=mock_return_values) mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) - mock_backup_crm_data_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( + mock_backup_crm_data_process = MagicMock(side_effect=[exception( 'E-RESBK-01', RESBK_JP_NAME, '例外発生'), None, None]) mock_convert_crm_csv_data_process = MagicMock() mock_backup_crm_csv_data_process = MagicMock() @@ -688,17 +721,25 @@ class TestController: 'Contact': 'success', 'Call2_vod__c': 'success' } - assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{RESBK_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=message) in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples assert generate_log_message_tuple( log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples - def test_raise_convert_crm_csv_data_process(self, s3_client, monkeypatch, caplog): + @pytest.mark.parametrize( + 'exception, message', + [ + (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{CONV_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), + (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') + ]) + def test_raise_convert_crm_csv_data_process(self, s3_client, monkeypatch, caplog, exception, message): """ Cases: - CSV変換処理でシステム例外が発生した場合、エラーで終了すること + 1. CSV変換処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること + 2. CSV変換処理で想定外の例外が発生した場合、エラーログが送出され、後続の終了に移行すること Arranges: - - CSV変換処理でシステム例外が発生するようにする + - パラメータ１：CSV変換処理でシステム例外が発生するようにする + - パラメータ２：CSV変換処理で想定外の例外が発生するようにする Expects: - CSV変換処理で例外が発生すること - CSV変換処理で発生した例外のログメッセージが出力されていること @@ -737,7 +778,7 @@ class TestController: mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( + mock_convert_crm_csv_data_process = MagicMock(side_effect=[exception( 'E-CONV-01', CONV_JP_NAME, '例外発生'), None, None]) mock_backup_crm_csv_data_process = MagicMock() mock_copy_crm_csv_data_process = MagicMock() @@ -764,17 +805,25 @@ class TestController: 'Contact': 'success', 'Call2_vod__c': 'success' } - assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{CONV_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=message) in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples assert generate_log_message_tuple( log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples - def test_raise_backup_crm_csv_data_process(self, s3_client, monkeypatch, caplog): + @pytest.mark.parametrize( + 'exception, message', + [ + (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{CSVBK_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), + (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') + ]) + def test_raise_backup_crm_csv_data_process(self, s3_client, monkeypatch, caplog, exception, message): """ Cases: - CSVバックアップ処理でシステム例外が発生した場合、エラーで終了すること + 1. CSVバックアップ処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること + 2. CSVバックアップ処理で想定外の例外が発生した場合、エラーログが送出され、後続の終了に移行すること Arranges: - - CSVバックアップ処理でシステム例外が発生するようにする + - パラメータ１：CSVバックアップ処理でシステム例外が発生するようにする + - パラメータ２：CSVバックアップ処理で想定外の例外が発生するようにする Expects: - CSVバックアップ処理で例外が発生すること - CSVバックアップ処理で発生した例外のログメッセージが出力されていること @@ -814,7 +863,7 @@ class TestController: mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( + mock_backup_crm_csv_data_process = MagicMock(side_effect=[exception( 'E-CSVBK-01', CSVBK_JP_NAME, '例外発生'), None, None]) mock_copy_crm_csv_data_process = MagicMock() mock_upload_last_fetch_datetime_process = MagicMock() @@ -840,17 +889,25 @@ class TestController: 'Contact': 'success', 'Call2_vod__c': 'success' } - assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{CSVBK_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=message) in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples assert generate_log_message_tuple( log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples - def test_raise_copy_crm_csv_data_process(self, s3_client, monkeypatch, caplog): + @pytest.mark.parametrize( + 'exception, message', + [ + (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{UPLD_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), + (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') + ]) + def test_raise_copy_crm_csv_data_process(self, s3_client, monkeypatch, caplog, exception, message): """ Cases: - CSVアップロード処理でシステム例外が発生した場合、エラーで終了すること + 1. CSVアップロード処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること + 2. CSVアップロード処理で想定外の例外が発生した場合、エラーログが送出され、後続の終了に移行すること Arranges: - - CSVアップロード処理でシステム例外が発生するようにする + - パラメータ１：CSVアップロード処理でシステム例外が発生するようにする + - パラメータ２：CSVアップロード処理で想定外の例外が発生するようにする Expects: - CSVアップロード処理で例外が発生すること - CSVアップロード処理で発生した例外のログメッセージが出力されていること @@ -891,7 +948,7 @@ class TestController: mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( + mock_copy_crm_csv_data_process = MagicMock(side_effect=[exception( 'E-UPLD-01', UPLD_JP_NAME, '例外発生'), None, None]) mock_upload_last_fetch_datetime_process = MagicMock() mock_upload_result_data_process = MagicMock() @@ -916,17 +973,25 @@ class TestController: 'Contact': 'success', 'Call2_vod__c': 'success' } - assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{UPLD_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=message) in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples assert generate_log_message_tuple( log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples - def test_raise_upload_last_fetch_datetime_process(self, s3_client, monkeypatch, caplog): + @pytest.mark.parametrize( + 'exception, message', + [ + (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{UPD_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), + (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') + ]) + def test_raise_upload_last_fetch_datetime_process(self, s3_client, monkeypatch, caplog, exception, message): """ Cases: - 前回取得日時ファイル更新処理でシステム例外が発生した場合、エラーで終了すること + 1. 前回取得日時ファイル更新処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること + 2. 前回取得日時ファイル更新処理で想定外の例外が発生した場合、エラーログが送出され、後続の終了に移行すること Arranges: - - 前回取得日時ファイル更新処理でシステム例外が発生するようにする + - パラメータ１：前回取得日時ファイル更新処理でシステム例外が発生するようにする + - パラメータ２：前回取得日時ファイル更新処理で想定外の例外が発生するようにする Expects: - 前回取得日時ファイル更新処理で例外が発生すること - 前回取得日時ファイル更新処理で発生した例外のログメッセージが出力されていること @@ -968,7 +1033,7 @@ class TestController: mock_convert_crm_csv_data_process = MagicMock() mock_backup_crm_csv_data_process = MagicMock() mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock(side_effect=[ForTestMeDaCaCRMDataFetchException( + mock_upload_last_fetch_datetime_process = MagicMock(side_effect=[exception( 'E-UPD-01', UPD_JP_NAME, '例外発生'), None, None]) mock_upload_result_data_process = MagicMock() @@ -991,7 +1056,7 @@ class TestController: 'Contact': 'success', 'Call2_vod__c': 'success' } - assert generate_log_message_tuple(log_message=f'I-ERR-03 [Account] の[{UPD_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=message) in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples assert generate_log_message_tuple( log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples From f3a1def88a3f03170f41c68027230a6fc0d7c325 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 4 Aug 2022 00:08:39 +0900 Subject: [PATCH 133/275] =?UTF-8?q?feat:=20=E7=B5=90=E6=9E=9C=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=A2=E3=83=83=E3=83=97=E3=83=AD?= =?UTF-8?q?=E3=83=BC=E3=83=89=E5=87=A6=E7=90=86=E3=81=AE=E4=BE=8B=E5=A4=96?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_controller.py | 80 +++++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index c7cf4ebd..4dcad4a3 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -7,8 +7,9 @@ from src.config.objects import (FetchTargetObjects, LastFetchDatetime, TargetObject) from src.error.exceptions import MeDaCaCRMDataFetchException from src.system_var.constants import (CHK_JP_NAME, CONV_JP_NAME, CSVBK_JP_NAME, - DATE_JP_NAME, FETCH_JP_NAME, PRE_JP_NAME, - RESBK_JP_NAME, UPD_JP_NAME, UPLD_JP_NAME) + DATE_JP_NAME, END_JP_NAME, FETCH_JP_NAME, + PRE_JP_NAME, RESBK_JP_NAME, UPD_JP_NAME, + UPLD_JP_NAME) from src.util.execute_datetime import ExecuteDateTime from .test_utils.log_message import generate_log_message_tuple @@ -1060,3 +1061,78 @@ class TestController: assert generate_log_message_tuple(log_message=f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{expect_process_result}]') in caplog.record_tuples assert generate_log_message_tuple( log_level=logging.ERROR, log_message=f'E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') in caplog.record_tuples + + @pytest.mark.parametrize( + 'exception, message', + [ + (ForTestMeDaCaCRMDataFetchException, f'E-ERR-01 [{END_JP_NAME}]でエラーが発生したため、処理を終了します'), + (ForTestException, 'E-ERR-02 予期せぬエラーが発生したため、処理を終了します エラー内容: [例外発生]') + ]) + def test_raise_upload_result_data_process(self, s3_client, monkeypatch, caplog, exception, message): + """ + Cases: + 1. 取得処理実施結果アップロード処理でシステム例外が発生した場合、エラーで終了すること + 2. 取得処理実施結果アップロード処理で想定外の例外が発生した場合、エラーで終了すること + Arranges: + - パラメータ１：取得処理実施結果アップロード処理でシステム例外が発生するようにする + - パラメータ２：取得処理実施結果アップロード処理で想定外の例外が発生するようにする + Expects: + - 取得処理実施結果アップロード処理で例外が発生すること + - 取得処理実施結果アップロード処理で発生した例外のログメッセージが出力されていること + """ + monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) + monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) + monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) + mock_execute_datetime = ExecuteDateTime() + objects = { + 'objects': [ + { + 'object_name': 'Account', + 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Contact', + 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Call2_vod__c', + 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + } + ] + } + mock_target_objects = FetchTargetObjects(objects) + mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) + mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) + mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) + mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] + mock_last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' + }, mock_execute_datetime) + mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + mock_backup_crm_data_process = MagicMock() + mock_convert_crm_csv_data_process = MagicMock() + mock_backup_crm_csv_data_process = MagicMock() + mock_copy_crm_csv_data_process = MagicMock() + mock_upload_last_fetch_datetime_process = MagicMock() + mock_upload_result_data_process = MagicMock(side_effect=[exception( + 'E-END-01', END_JP_NAME, '例外発生'), None, None]) + + with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', mock_check_object_info_process), \ + patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', mock_upload_result_data_process): + with pytest.raises(exception): + controller.controller() + assert generate_log_message_tuple( + log_level=logging.ERROR, + log_message=message) in caplog.record_tuples, '取得処理実施結果アップロード処理で発生した例外のログメッセージが出力されていること' From 548a4ad9c6c16ac7f9c3a136936fea906a20b323 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 4 Aug 2022 00:36:12 +0900 Subject: [PATCH 134/275] =?UTF-8?q?refactor:=20=E6=B5=81=E7=9F=B3=E3=81=AB?= =?UTF-8?q?=E8=A1=8C=E6=95=B0=E5=A4=A7=E6=9D=89=E3=81=AA=E3=81=AE=E3=81=A7?= =?UTF-8?q?=E3=80=81=E5=85=B1=E9=80=9A=E7=9A=84=E3=81=AB=E4=BD=BF=E3=81=88?= =?UTF-8?q?=E3=82=8B=E3=82=82=E3=81=AE=E3=81=AF=E5=AE=9A=E6=95=B0=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_controller.py | 377 ++++----------------- 1 file changed, 63 insertions(+), 314 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index 4dcad4a3..a8ed1c63 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -1,4 +1,5 @@ import logging +from copy import deepcopy from unittest.mock import MagicMock, patch import pytest @@ -14,12 +15,39 @@ from src.util.execute_datetime import ExecuteDateTime from .test_utils.log_message import generate_log_message_tuple -EXPECT_PROCESS_RESULT = { - 'Account': 'success', - 'Contact': 'success', - 'Call2_vod__c': 'success' +COMMON_OBJECT_INFOS = { + 'objects': [ + { + 'object_name': 'Account', + 'columns': [], + 'upload_file_name': 'Account_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Contact', + 'columns': [], + 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' + }, + { + 'object_name': 'Call2_vod__c', + 'columns': [], + 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' + } + ] } +COMMON_EXECUTE_DATETIME = ExecuteDateTime() + +COMMON_FETCH_TARGET_OBJECTS = FetchTargetObjects(COMMON_OBJECT_INFOS) + +COMMON_TARGET_OBJECTS_1 = TargetObject(COMMON_OBJECT_INFOS['objects'][0], COMMON_EXECUTE_DATETIME) +COMMON_TARGET_OBJECTS_2 = TargetObject(COMMON_OBJECT_INFOS['objects'][1], COMMON_EXECUTE_DATETIME) +COMMON_TARGET_OBJECTS_3 = TargetObject(COMMON_OBJECT_INFOS['objects'][2], COMMON_EXECUTE_DATETIME) + +COMMON_LAST_FETCH_DATETIME = LastFetchDatetime({ + 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '' +}, COMMON_EXECUTE_DATETIME) + class ForTestMeDaCaCRMDataFetchException(MeDaCaCRMDataFetchException): def __init__(self, error_id: str, func_name: str, message: str) -> None: @@ -43,38 +71,10 @@ class TestController: dict: プロセス関数呼び出し後のモックの辞書 """ def _func(): - monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() - objects = { - 'objects': [ - { - 'object_name': 'Account', - 'upload_file_name': 'Account_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Contact', - 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Call2_vod__c', - 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' - } - ] - } - mock_target_objects = FetchTargetObjects(objects) - mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) - mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) - mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) - mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] - mock_last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', - 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' - }, mock_execute_datetime) - mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] + mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) mock_fetch_crm_data_process = MagicMock(return_value=[{}]) mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() @@ -409,38 +409,11 @@ class TestController: - オブジェクト情報形式チェック処理で発生した例外のログメッセージが出力されている - 例外が発生したオブジェクト以外は正常に終了する """ - monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() - objects = { - 'objects': [ - { - 'object_name': 'Account', - 'upload_file_name': 'Account_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Contact', - 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Call2_vod__c', - 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' - } - ] - } - mock_target_objects = FetchTargetObjects(objects) - mock_target_object_first = exception('E-PRE-01', CHK_JP_NAME, '例外発生') - mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) - mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) - mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] - mock_last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', - 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' - }, mock_execute_datetime) - mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + raise_object = exception('E-PRE-01', CHK_JP_NAME, '例外発生') + mock_return_values = [raise_object, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_2] + mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) mock_fetch_crm_data_process = MagicMock(return_value=[{}]) mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() @@ -493,39 +466,11 @@ class TestController: - データ取得期間設定チェック処理で発生した例外のログメッセージが出力されている - 例外が発生したオブジェクト以外は正常に終了する """ - monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() - objects = { - 'objects': [ - { - 'object_name': 'Account', - 'upload_file_name': 'Account_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Contact', - 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Call2_vod__c', - 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' - } - ] - } - mock_target_objects = FetchTargetObjects(objects) - mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) - mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) - mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) - mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] - mock_last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', - 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' - }, mock_execute_datetime) - mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] + mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) mock_set_datetime_period_process = MagicMock(side_effect=[exception( - 'E-DATE-01', DATE_JP_NAME, '例外発生'), mock_last_fetch_datetime, mock_last_fetch_datetime]) + 'E-DATE-01', DATE_JP_NAME, '例外発生'), COMMON_LAST_FETCH_DATETIME, COMMON_LAST_FETCH_DATETIME]) mock_fetch_crm_data_process = MagicMock(return_value=[{}]) mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() @@ -577,38 +522,10 @@ class TestController: - CRMデータ取得処理で例外が発生すること - CRMデータ取得処理で発生した例外のログメッセージが出力されていること """ - monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() - objects = { - 'objects': [ - { - 'object_name': 'Account', - 'upload_file_name': 'Account_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Contact', - 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Call2_vod__c', - 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' - } - ] - } - mock_target_objects = FetchTargetObjects(objects) - mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) - mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) - mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) - mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] - mock_last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', - 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' - }, mock_execute_datetime) - mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] + mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) mock_fetch_crm_data_process = MagicMock(side_effect=[exception( 'E-FETCH-01', FETCH_JP_NAME, '例外発生'), {}, {}]) mock_backup_crm_data_process = MagicMock() @@ -661,38 +578,10 @@ class TestController: - CRM電文データバックアップ処理で例外が発生すること - CRM電文データバックアップ処理で発生した例外のログメッセージが出力されていること """ - monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() - objects = { - 'objects': [ - { - 'object_name': 'Account', - 'upload_file_name': 'Account_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Contact', - 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Call2_vod__c', - 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' - } - ] - } - mock_target_objects = FetchTargetObjects(objects) - mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) - mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) - mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) - mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] - mock_last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', - 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' - }, mock_execute_datetime) - mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] + mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) mock_backup_crm_data_process = MagicMock(side_effect=[exception( 'E-RESBK-01', RESBK_JP_NAME, '例外発生'), None, None]) @@ -745,38 +634,10 @@ class TestController: - CSV変換処理で例外が発生すること - CSV変換処理で発生した例外のログメッセージが出力されていること """ - monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() - objects = { - 'objects': [ - { - 'object_name': 'Account', - 'upload_file_name': 'Account_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Contact', - 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Call2_vod__c', - 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' - } - ] - } - mock_target_objects = FetchTargetObjects(objects) - mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) - mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) - mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) - mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] - mock_last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', - 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' - }, mock_execute_datetime) - mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] + mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock(side_effect=[exception( @@ -829,38 +690,10 @@ class TestController: - CSVバックアップ処理で例外が発生すること - CSVバックアップ処理で発生した例外のログメッセージが出力されていること """ - monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() - objects = { - 'objects': [ - { - 'object_name': 'Account', - 'upload_file_name': 'Account_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Contact', - 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Call2_vod__c', - 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' - } - ] - } - mock_target_objects = FetchTargetObjects(objects) - mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) - mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) - mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) - mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] - mock_last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', - 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' - }, mock_execute_datetime) - mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] + mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() @@ -913,38 +746,10 @@ class TestController: - CSVアップロード処理で例外が発生すること - CSVアップロード処理で発生した例外のログメッセージが出力されていること """ - monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() - objects = { - 'objects': [ - { - 'object_name': 'Account', - 'upload_file_name': 'Account_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Contact', - 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Call2_vod__c', - 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' - } - ] - } - mock_target_objects = FetchTargetObjects(objects) - mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) - mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) - mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) - mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] - mock_last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', - 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' - }, mock_execute_datetime) - mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] + mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() @@ -997,38 +802,10 @@ class TestController: - 前回取得日時ファイル更新処理で例外が発生すること - 前回取得日時ファイル更新処理で発生した例外のログメッセージが出力されていること """ - monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() - objects = { - 'objects': [ - { - 'object_name': 'Account', - 'upload_file_name': 'Account_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Contact', - 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Call2_vod__c', - 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' - } - ] - } - mock_target_objects = FetchTargetObjects(objects) - mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) - mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) - mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) - mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] - mock_last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', - 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' - }, mock_execute_datetime) - mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] + mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() @@ -1080,38 +857,10 @@ class TestController: - 取得処理実施結果アップロード処理で例外が発生すること - 取得処理実施結果アップロード処理で発生した例外のログメッセージが出力されていること """ - monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - monkeypatch.setattr(LastFetchDatetime, '_LastFetchDatetime__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() - objects = { - 'objects': [ - { - 'object_name': 'Account', - 'upload_file_name': 'Account_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Contact', - 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' - }, - { - 'object_name': 'Call2_vod__c', - 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' - } - ] - } - mock_target_objects = FetchTargetObjects(objects) - mock_target_object_first = TargetObject(objects['objects'][0], mock_execute_datetime) - mock_target_object_second = TargetObject(objects['objects'][1], mock_execute_datetime) - mock_target_object_third = TargetObject(objects['objects'][2], mock_execute_datetime) - mock_return_values = [mock_target_object_first, mock_target_object_second, mock_target_object_third] - mock_last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '1900-01-01T00:00:00.000Z', - 'last_fetch_datetime_to': '9999-12-31T59:59:59.000Z' - }, mock_execute_datetime) - mock_prepare_data_fetch_process = MagicMock(return_value=(mock_target_objects, mock_execute_datetime, {})) + mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] + mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=mock_last_fetch_datetime) + mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) mock_backup_crm_data_process = MagicMock() mock_convert_crm_csv_data_process = MagicMock() From 02086d26b3495ac61b49f9ebe91ea9233d6ca240 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 4 Aug 2022 01:34:20 +0900 Subject: [PATCH 135/275] =?UTF-8?q?refactor:=20=E5=A4=A7=E6=94=B9=E9=80=A0?= =?UTF-8?q?=E3=80=82=E8=A1=8C=E6=95=B0800=E8=BF=91=E3=81=8F=E3=81=BE?= =?UTF-8?q?=E3=81=A7=E6=B8=9B=E3=82=89=E3=81=9B=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_controller.py | 690 +++++++++------------ 1 file changed, 303 insertions(+), 387 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index a8ed1c63..51ff1987 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -61,8 +61,55 @@ class ForTestException(Exception): class TestController: + + @pytest.fixture(autouse=True) + def setup_teardown(self): + # setup + self.mock_prepare_data_fetch_process = MagicMock() + self.mock_check_object_info_process = MagicMock() + self.mock_set_datetime_period_process = MagicMock() + self.mock_fetch_crm_data_process = MagicMock() + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() + + # run test + yield + + # teardown + self.mock_prepare_data_fetch_process.reset_mock() + self.mock_check_object_info_process.reset_mock() + self.mock_set_datetime_period_process.reset_mock() + self.mock_fetch_crm_data_process.reset_mock() + self.mock_backup_crm_data_process.reset_mock() + self.mock_convert_crm_csv_data_process.reset_mock() + self.mock_backup_crm_csv_data_process.reset_mock() + self.mock_copy_crm_csv_data_process.reset_mock() + self.mock_upload_last_fetch_datetime_process.reset_mock() + self.mock_upload_result_data_process.reset_mock() + @pytest.fixture() - def call_all_processes(self, s3_client, monkeypatch, caplog): + def run_control_process(self): + + def _func(): + with patch('src.controller.prepare_data_fetch_process', self.mock_prepare_data_fetch_process),\ + patch('src.controller.check_object_info_process', self.mock_check_object_info_process),\ + patch('src.controller.set_datetime_period_process', self.mock_set_datetime_period_process),\ + patch('src.controller.fetch_crm_data_process', self.mock_fetch_crm_data_process),\ + patch('src.controller.backup_crm_data_process', self.mock_backup_crm_data_process),\ + patch('src.controller.convert_crm_csv_data_process', self.mock_convert_crm_csv_data_process),\ + patch('src.controller.backup_crm_csv_data_process', self.mock_backup_crm_csv_data_process),\ + patch('src.controller.copy_crm_csv_data_process', self.mock_copy_crm_csv_data_process),\ + patch('src.controller.upload_last_fetch_datetime_process', self.mock_upload_last_fetch_datetime_process),\ + patch('src.controller.upload_result_data_process', self.mock_upload_result_data_process): + controller.controller() + yield _func + + @pytest.fixture() + def call_all_processes(self, caplog, run_control_process): """ コントロール処理内ですべてのプロセス関数が呼ばれることのテストで使用するフィクスチャ 各種プロセス関数をモック化し、正常終了させるように動く @@ -72,41 +119,18 @@ class TestController: """ def _func(): mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] - mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) - mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - mock_fetch_crm_data_process = MagicMock(return_value=[{}]) - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock() - should_call_processes = { - 'prepare_data_fetch_process': mock_prepare_data_fetch_process, - 'check_object_info_process': mock_check_object_info_process, - 'set_datetime_period_process': mock_set_datetime_period_process, - 'fetch_crm_data_process': mock_fetch_crm_data_process, - 'backup_crm_data_process': mock_backup_crm_data_process, - 'convert_crm_csv_data_process': mock_convert_crm_csv_data_process, - 'backup_crm_csv_data_process': mock_backup_crm_csv_data_process, - 'copy_crm_csv_data_process': mock_copy_crm_csv_data_process, - 'upload_last_fetch_datetime_process': mock_upload_last_fetch_datetime_process, - 'upload_result_data_process': mock_upload_result_data_process - } - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process),\ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - controller.controller() + self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) + self.mock_fetch_crm_data_process = MagicMock(return_value=[{}]) + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() - return should_call_processes + run_control_process() yield _func @@ -128,17 +152,19 @@ class TestController: - 前回取得日時ファイル更新処理が複数回実行される - 取得処理実施結果アップロード処理が1回のみ実行される """ - processes = call_all_processes() - processes['prepare_data_fetch_process'].assert_called_once(), 'データ取得準備処理が1回のみ実行されること' - assert processes['check_object_info_process'].call_count == 3, 'オブジェクト情報形式チェック処理が複数回実行されること' - assert processes['set_datetime_period_process'].call_count == 3, 'データ取得期間設定処理が複数回実行されること' - assert processes['fetch_crm_data_process'].call_count == 3, 'CRMデータ取得処理が複数回実行されること' - assert processes['backup_crm_data_process'].call_count == 3, 'CRM電文データバックアップ処理が複数回実行されること' - assert processes['convert_crm_csv_data_process'].call_count == 3, 'CSV変換処理が複数回実行されること' - assert processes['backup_crm_csv_data_process'].call_count == 3, 'CSVバックアップ処理が複数回実行されること' - assert processes['copy_crm_csv_data_process'].call_count == 3, 'CSVアップロード処理が複数回実行されること' - assert processes['upload_last_fetch_datetime_process'].call_count == 3, '前回取得日次ファイル更新処理が複数回実行されること' - processes['upload_result_data_process'].assert_called_once(), '取得処理実施結果アップロード処理が1回のみ実行されること' + call_all_processes() + assert self.mock_prepare_data_fetch_process.called is True, 'データ取得準備処理が1回のみ実行されること' + assert self.mock_prepare_data_fetch_process.call_count == 1, 'データ取得準備処理が1回のみ実行されること' + assert self.mock_check_object_info_process.call_count == 3, 'オブジェクト情報形式チェック処理が複数回実行されること' + assert self.mock_set_datetime_period_process.call_count == 3, 'データ取得期間設定処理が複数回実行されること' + assert self.mock_fetch_crm_data_process.call_count == 3, 'CRMデータ取得処理が複数回実行されること' + assert self.mock_backup_crm_data_process.call_count == 3, 'CRM電文データバックアップ処理が複数回実行されること' + assert self.mock_convert_crm_csv_data_process.call_count == 3, 'CSV変換処理が複数回実行されること' + assert self.mock_backup_crm_csv_data_process.call_count == 3, 'CSVバックアップ処理が複数回実行されること' + assert self.mock_copy_crm_csv_data_process.call_count == 3, 'CSVアップロード処理が複数回実行されること' + assert self.mock_upload_last_fetch_datetime_process.call_count == 3, '前回取得日次ファイル更新処理が複数回実行されること' + assert self.mock_upload_result_data_process.called is True, '取得処理実施結果アップロード処理が1回のみ実行されること' + assert self.mock_upload_result_data_process.call_count == 1, '取得処理実施結果アップロード処理が1回のみ実行されること' def test_print_normal_logs(self, call_all_processes, caplog): """ @@ -179,7 +205,7 @@ class TestController: assert generate_log_message_tuple(log_message=f'I-CTRL-19 すべてのデータの取得に成功しました') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples - def test_do_not_call_upload_process_result_process(self, s3_client, monkeypatch, caplog): + def test_do_not_call_upload_process_result_process(self, caplog, run_control_process): """ Cases: 処理対象オブジェクトが0件の場合、取得処理実施結果アップロード処理が実行されないこと @@ -191,38 +217,31 @@ class TestController: - 取得処理実施結果アップロード処理が実行されないこと - 処理対象が存在しない旨を示すログメッセージが出力されていること """ - mock_execute_datetime = ExecuteDateTime() - mock_prepare_data_fetch_process = MagicMock(return_value=([], mock_execute_datetime, {})) - mock_check_object_info_process = MagicMock() - mock_set_datetime_period_process = MagicMock() - mock_fetch_crm_data_process = MagicMock() - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process),\ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - controller.controller() + + self.mock_prepare_data_fetch_process = MagicMock(return_value=([], COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock() + self.mock_set_datetime_period_process = MagicMock() + self.mock_fetch_crm_data_process = MagicMock() + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() + + run_control_process() # 実行回数の確認 - mock_prepare_data_fetch_process.assert_called_once(), 'データ取得準備処理が1回のみ実行されること' - mock_upload_result_data_process.call_count == 0, '取得処理実施結果アップロード処理が実行されないこと' + assert self.mock_prepare_data_fetch_process.called, 'データ取得準備処理が1回のみ実行されること' + assert self.mock_prepare_data_fetch_process.call_count == 1, 'データ取得準備処理が1回のみ実行されること' + assert self.mock_upload_result_data_process.called is False, '取得処理実施結果アップロード処理が実行されないこと' + assert self.mock_upload_result_data_process.call_count == 0, '取得処理実施結果アップロード処理が実行されないこと' # ログ出力の確認 assert generate_log_message_tuple(log_message='I-CTRL-21 処理対象のデータが存在しませんでした') in caplog.record_tuples, '処理対象が存在しない旨を示すログメッセージが出力されていること' assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples - def test_do_not_call_upload_csv_process(self, s3_client, monkeypatch, caplog): + def test_do_not_call_upload_csv_process(self, caplog, run_control_process): """ Cases: オブジェクト情報.is_skipがTrueの場合、CSVアップロード処理が実行されないこと @@ -235,51 +254,44 @@ class TestController: """ mock_check_object_info = { 'object_name': 'Account', + 'columns': [], 'is_skip': True } + mock_return_values = [TargetObject(mock_check_object_info, COMMON_EXECUTE_DATETIME)] + self.mock_prepare_data_fetch_process = MagicMock(return_value=([mock_check_object_info], COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock() + self.mock_fetch_crm_data_process = MagicMock() + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() - mock_return_values = [TargetObject(mock_check_object_info, mock_execute_datetime)] - mock_prepare_data_fetch_process = MagicMock(return_value=([mock_check_object_info], mock_execute_datetime, {})) - mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock() - mock_fetch_crm_data_process = MagicMock() - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process),\ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - controller.controller() + run_control_process() # 実行回数の確認 - mock_check_object_info_process.assert_called_once() + + assert self.mock_check_object_info_process.called is True + assert self.mock_check_object_info_process.call_count == 1 # オブジェクト情報形式チェック処理以降のプロセスが実行されないこと - mock_set_datetime_period_process.call_count == 0 - mock_fetch_crm_data_process.call_count == 0 - mock_backup_crm_data_process.call_count == 0 - mock_convert_crm_csv_data_process.call_count == 0 - mock_backup_crm_csv_data_process.call_count == 0 - mock_copy_crm_csv_data_process.call_count == 0 - mock_upload_last_fetch_datetime_process.call_count == 0 - mock_upload_result_data_process.call_count == 0 + assert self.mock_set_datetime_period_process.call_count == 0 + assert self.mock_fetch_crm_data_process.call_count == 0 + assert self.mock_backup_crm_data_process.call_count == 0 + assert self.mock_convert_crm_csv_data_process.call_count == 0 + assert self.mock_backup_crm_csv_data_process.call_count == 0 + assert self.mock_copy_crm_csv_data_process.call_count == 0 + assert self.mock_upload_last_fetch_datetime_process.call_count == 0 + # 結果ファイルの出力は行う + assert self.mock_upload_result_data_process.called is True + assert self.mock_upload_result_data_process.call_count == 1 # ログ出力の確認 assert generate_log_message_tuple(log_message='I-CTRL-07 [Account]のデータ取得処理をスキップします') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples - def test_do_not_call_upload_csv_process_of_part(self, s3_client, monkeypatch, caplog): + def test_do_not_call_upload_csv_process_of_part(self, caplog, run_control_process): """ Cases: オブジェクト情報.is_skipがTrueのものが含まれる場合、CSVアップロード処理が実行されないこと @@ -295,60 +307,53 @@ class TestController: 'objects': [ { 'object_name': 'Account', + 'columns': [], 'is_skip': False }, { 'object_name': 'Contact', + 'columns': [], 'is_skip': True }, { 'object_name': 'Call2_vod__c', + 'columns': [], 'is_skip': False } ] } - monkeypatch.setattr(FetchTargetObjects, '_FetchTargetObjects__validate', lambda x: None) - monkeypatch.setattr(TargetObject, '_TargetObject__validate', lambda x: None) - mock_execute_datetime = ExecuteDateTime() FetchTargetObjects(mock_check_object_infos) mock_return_values = [ - TargetObject(mock_check_object_infos['objects'][0], mock_execute_datetime), - TargetObject(mock_check_object_infos['objects'][1], mock_execute_datetime), - TargetObject(mock_check_object_infos['objects'][2], mock_execute_datetime), + TargetObject(mock_check_object_infos['objects'][0], COMMON_EXECUTE_DATETIME), + TargetObject(mock_check_object_infos['objects'][1], COMMON_EXECUTE_DATETIME), + TargetObject(mock_check_object_infos['objects'][2], COMMON_EXECUTE_DATETIME), ] - mock_prepare_data_fetch_process = MagicMock(return_value=(FetchTargetObjects(mock_check_object_infos), mock_execute_datetime, {})) - mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock() - mock_fetch_crm_data_process = MagicMock() - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process),\ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - controller.controller() + self.mock_prepare_data_fetch_process = MagicMock(return_value=(FetchTargetObjects(mock_check_object_infos), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock() + self.mock_fetch_crm_data_process = MagicMock() + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() + run_control_process() # 実行回数の確認 - mock_check_object_info_process.call_count == 3 + assert self.mock_prepare_data_fetch_process.called is True + assert self.mock_prepare_data_fetch_process.call_count == 1 + assert self.mock_check_object_info_process.call_count == 3 # オブジェクト情報形式チェック処理以降のプロセスが実行されないこと - mock_set_datetime_period_process.call_count == 2 - mock_fetch_crm_data_process.call_count == 2 - mock_backup_crm_data_process.call_count == 2 - mock_convert_crm_csv_data_process.call_count == 2 - mock_backup_crm_csv_data_process.call_count == 2 - mock_copy_crm_csv_data_process.call_count == 2 - mock_upload_last_fetch_datetime_process.call_count == 2 - mock_upload_result_data_process.call_count == 2 + assert self.mock_set_datetime_period_process.call_count == 2 + assert self.mock_fetch_crm_data_process.call_count == 2 + assert self.mock_backup_crm_data_process.call_count == 2 + assert self.mock_convert_crm_csv_data_process.call_count == 2 + assert self.mock_backup_crm_csv_data_process.call_count == 2 + assert self.mock_copy_crm_csv_data_process.call_count == 2 + assert self.mock_upload_last_fetch_datetime_process.call_count == 2 + assert self.mock_upload_result_data_process.called is True + assert self.mock_upload_result_data_process.call_count == 1 # ログ出力の確認 assert generate_log_message_tuple( @@ -366,7 +371,7 @@ class TestController: (ForTestMeDaCaCRMDataFetchException, f'E-ERR-01 [{PRE_JP_NAME}]でエラーが発生したため、処理を終了します'), (ForTestException, 'E-ERR-02 予期せぬエラーが発生したため、処理を終了します エラー内容: [例外発生]') ]) - def test_raise_prepare_data_fetch_process(self, s3_client, monkeypatch, caplog, exception, message): + def test_raise_prepare_data_fetch_process(self, caplog, exception, message, run_control_process): """ Cases: 1. データ取得準備処理でシステム例外が発生した場合、エラーで終了すること @@ -379,13 +384,11 @@ class TestController: - データ取得準備処理で発生した例外のログメッセージが出力されていること """ expect_exception = exception('E-PRE-01', PRE_JP_NAME, '例外発生') - mock_prepare_data_fetch_process = MagicMock(side_effect=expect_exception) - mock_check_object_info_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process): - with pytest.raises(exception): - controller.controller() - assert mock_check_object_info_process.called is False + self.mock_prepare_data_fetch_process = MagicMock(side_effect=expect_exception) + self.mock_check_object_info_process = MagicMock() + with pytest.raises(exception): + run_control_process() + assert self.mock_check_object_info_process.called is False assert generate_log_message_tuple( log_level=logging.ERROR, log_message=message) in caplog.record_tuples, 'データ取得準備処理で発生した例外のログメッセージが出力されていること' @@ -396,7 +399,7 @@ class TestController: (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{CHK_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') ]) - def test_raise_check_object_info_process(self, s3_client, monkeypatch, caplog, exception, message): + def test_raise_check_object_info_process(self, caplog, exception, message, run_control_process): """ Cases: 1. オブジェクト情報形式チェック処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること @@ -411,32 +414,21 @@ class TestController: """ raise_object = exception('E-PRE-01', CHK_JP_NAME, '例外発生') mock_return_values = [raise_object, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_2] - mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) - mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - mock_fetch_crm_data_process = MagicMock(return_value=[{}]) - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process),\ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - # ループ中の例外処理なので、例外は潰される - # with pytest.raises(MeDaCaCRMDataFetchException): - controller.controller() - assert mock_check_object_info_process.call_count == 3 - assert mock_set_datetime_period_process.call_count == 2 + self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) + self.mock_fetch_crm_data_process = MagicMock(return_value=[{}]) + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() + + run_control_process() + assert self.mock_check_object_info_process.call_count == 3 + assert self.mock_set_datetime_period_process.call_count == 2 expect_process_result = { 'Account': 'fail', 'Contact': 'success', @@ -453,7 +445,7 @@ class TestController: (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{DATE_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') ]) - def test_raise_set_datetime_period_process(self, s3_client, monkeypatch, caplog, exception, message): + def test_raise_set_datetime_period_process(self, caplog, exception, message, run_control_process): """ Cases: 1. データ取得期間設定処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること @@ -467,33 +459,23 @@ class TestController: - 例外が発生したオブジェクト以外は正常に終了する """ mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] - mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) - mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(side_effect=[exception( - 'E-DATE-01', DATE_JP_NAME, '例外発生'), COMMON_LAST_FETCH_DATETIME, COMMON_LAST_FETCH_DATETIME]) - mock_fetch_crm_data_process = MagicMock(return_value=[{}]) - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process),\ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - # ループ中の例外処理なので、例外は潰される - # with pytest.raises(MeDaCaCRMDataFetchException): - controller.controller() - assert mock_set_datetime_period_process.call_count == 3 - assert mock_fetch_crm_data_process.call_count == 2 + self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock(side_effect=[exception( + 'E-DATE-01', DATE_JP_NAME, '例外発生'), COMMON_LAST_FETCH_DATETIME, COMMON_LAST_FETCH_DATETIME]) + self.mock_fetch_crm_data_process = MagicMock(return_value=[{}]) + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() + + run_control_process() + + assert self.mock_set_datetime_period_process.call_count == 3 + assert self.mock_fetch_crm_data_process.call_count == 2 expect_process_result = { 'Account': 'fail', 'Contact': 'success', @@ -510,7 +492,7 @@ class TestController: (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{FETCH_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') ]) - def test_raise_fetch_crm_data_process(self, s3_client, monkeypatch, caplog, exception, message): + def test_raise_fetch_crm_data_process(self, caplog, exception, message, run_control_process): """ Cases: 1. CRMデータ取得処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること @@ -523,33 +505,23 @@ class TestController: - CRMデータ取得処理で発生した例外のログメッセージが出力されていること """ mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] - mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) - mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - mock_fetch_crm_data_process = MagicMock(side_effect=[exception( - 'E-FETCH-01', FETCH_JP_NAME, '例外発生'), {}, {}]) - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process), \ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - # ループ中の例外処理なので、例外は潰される - # with pytest.raises(MeDaCaCRMDataFetchException): - controller.controller() - assert mock_fetch_crm_data_process.call_count == 3 - assert mock_backup_crm_data_process.call_count == 2 + self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[exception( + 'E-FETCH-01', FETCH_JP_NAME, '例外発生'), {}, {}]) + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() + + run_control_process() + + assert self.mock_fetch_crm_data_process.call_count == 3 + assert self.mock_backup_crm_data_process.call_count == 2 expect_process_result = { 'Account': 'fail', 'Contact': 'success', @@ -566,7 +538,7 @@ class TestController: (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{RESBK_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') ]) - def test_raise_backup_crm_data_process(self, s3_client, monkeypatch, caplog, exception, message): + def test_raise_backup_crm_data_process(self, caplog, exception, message, run_control_process): """ Cases: 1. CRM電文データバックアップ処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること @@ -579,33 +551,23 @@ class TestController: - CRM電文データバックアップ処理で発生した例外のログメッセージが出力されていること """ mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] - mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) - mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) - mock_backup_crm_data_process = MagicMock(side_effect=[exception( - 'E-RESBK-01', RESBK_JP_NAME, '例外発生'), None, None]) - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process), \ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - # ループ中の例外処理なので、例外は潰される - # with pytest.raises(MeDaCaCRMDataFetchException): - controller.controller() - assert mock_backup_crm_data_process.call_count == 3 - assert mock_convert_crm_csv_data_process.call_count == 2 + self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + self.mock_backup_crm_data_process = MagicMock(side_effect=[exception( + 'E-RESBK-01', RESBK_JP_NAME, '例外発生'), None, None]) + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() + + run_control_process() + + assert self.mock_backup_crm_data_process.call_count == 3 + assert self.mock_convert_crm_csv_data_process.call_count == 2 expect_process_result = { 'Account': 'fail', 'Contact': 'success', @@ -622,7 +584,7 @@ class TestController: (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{CONV_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') ]) - def test_raise_convert_crm_csv_data_process(self, s3_client, monkeypatch, caplog, exception, message): + def test_raise_convert_crm_csv_data_process(self, caplog, exception, message, run_control_process): """ Cases: 1. CSV変換処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること @@ -635,33 +597,23 @@ class TestController: - CSV変換処理で発生した例外のログメッセージが出力されていること """ mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] - mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) - mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock(side_effect=[exception( - 'E-CONV-01', CONV_JP_NAME, '例外発生'), None, None]) - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process), \ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - # ループ中の例外処理なので、例外は潰される - # with pytest.raises(MeDaCaCRMDataFetchException): - controller.controller() - assert mock_convert_crm_csv_data_process.call_count == 3 - assert mock_backup_crm_csv_data_process.call_count == 2 + self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock(side_effect=[exception( + 'E-CONV-01', CONV_JP_NAME, '例外発生'), None, None]) + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() + + run_control_process() + + assert self.mock_convert_crm_csv_data_process.call_count == 3 + assert self.mock_backup_crm_csv_data_process.call_count == 2 expect_process_result = { 'Account': 'fail', 'Contact': 'success', @@ -678,7 +630,7 @@ class TestController: (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{CSVBK_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') ]) - def test_raise_backup_crm_csv_data_process(self, s3_client, monkeypatch, caplog, exception, message): + def test_raise_backup_crm_csv_data_process(self, caplog, exception, message, run_control_process): """ Cases: 1. CSVバックアップ処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること @@ -691,33 +643,23 @@ class TestController: - CSVバックアップ処理で発生した例外のログメッセージが出力されていること """ mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] - mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) - mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock(side_effect=[exception( - 'E-CSVBK-01', CSVBK_JP_NAME, '例外発生'), None, None]) - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process), \ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - # ループ中の例外処理なので、例外は潰される - # with pytest.raises(MeDaCaCRMDataFetchException): - controller.controller() - assert mock_backup_crm_csv_data_process.call_count == 3 - assert mock_copy_crm_csv_data_process.call_count == 2 + self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock(side_effect=[exception( + 'E-CSVBK-01', CSVBK_JP_NAME, '例外発生'), None, None]) + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() + + run_control_process() + + assert self.mock_backup_crm_csv_data_process.call_count == 3 + assert self.mock_copy_crm_csv_data_process.call_count == 2 expect_process_result = { 'Account': 'fail', 'Contact': 'success', @@ -734,7 +676,7 @@ class TestController: (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{UPLD_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') ]) - def test_raise_copy_crm_csv_data_process(self, s3_client, monkeypatch, caplog, exception, message): + def test_raise_copy_crm_csv_data_process(self, caplog, exception, message, run_control_process): """ Cases: 1. CSVアップロード処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること @@ -747,33 +689,23 @@ class TestController: - CSVアップロード処理で発生した例外のログメッセージが出力されていること """ mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] - mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) - mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock(side_effect=[exception( - 'E-UPLD-01', UPLD_JP_NAME, '例外発生'), None, None]) - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process), \ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - # ループ中の例外処理なので、例外は潰される - # with pytest.raises(MeDaCaCRMDataFetchException): - controller.controller() - assert mock_copy_crm_csv_data_process.call_count == 3 - assert mock_upload_last_fetch_datetime_process.call_count == 2 + self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock(side_effect=[exception( + 'E-UPLD-01', UPLD_JP_NAME, '例外発生'), None, None]) + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() + + run_control_process() + + assert self.mock_copy_crm_csv_data_process.call_count == 3 + assert self.mock_upload_last_fetch_datetime_process.call_count == 2 expect_process_result = { 'Account': 'fail', 'Contact': 'success', @@ -790,7 +722,7 @@ class TestController: (ForTestMeDaCaCRMDataFetchException, f'I-ERR-03 [Account] の[{UPD_JP_NAME}]でエラーが発生しました 次のオブジェクトの処理に移行します'), (ForTestException, 'I-ERR-04 [Account] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します エラー内容: [例外発生]') ]) - def test_raise_upload_last_fetch_datetime_process(self, s3_client, monkeypatch, caplog, exception, message): + def test_raise_upload_last_fetch_datetime_process(self, caplog, exception, message, run_control_process): """ Cases: 1. 前回取得日時ファイル更新処理でシステム例外が発生した場合、エラーログが送出され、後続の終了に移行すること @@ -803,32 +735,22 @@ class TestController: - 前回取得日時ファイル更新処理で発生した例外のログメッセージが出力されていること """ mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] - mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) - mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock(side_effect=[exception( - 'E-UPD-01', UPD_JP_NAME, '例外発生'), None, None]) - mock_upload_result_data_process = MagicMock() - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process), \ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - # ループ中の例外処理なので、例外は潰される - # with pytest.raises(MeDaCaCRMDataFetchException): - controller.controller() - assert mock_upload_last_fetch_datetime_process.call_count == 3 + self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock(side_effect=[exception( + 'E-UPD-01', UPD_JP_NAME, '例外発生'), None, None]) + self.mock_upload_result_data_process = MagicMock() + + run_control_process() + + assert self.mock_upload_last_fetch_datetime_process.call_count == 3 expect_process_result = { 'Account': 'fail', 'Contact': 'success', @@ -845,7 +767,7 @@ class TestController: (ForTestMeDaCaCRMDataFetchException, f'E-ERR-01 [{END_JP_NAME}]でエラーが発生したため、処理を終了します'), (ForTestException, 'E-ERR-02 予期せぬエラーが発生したため、処理を終了します エラー内容: [例外発生]') ]) - def test_raise_upload_result_data_process(self, s3_client, monkeypatch, caplog, exception, message): + def test_raise_upload_result_data_process(self, caplog, exception, message, run_control_process): """ Cases: 1. 取得処理実施結果アップロード処理でシステム例外が発生した場合、エラーで終了すること @@ -858,30 +780,24 @@ class TestController: - 取得処理実施結果アップロード処理で発生した例外のログメッセージが出力されていること """ mock_return_values = [COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3] - mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) - mock_check_object_info_process = MagicMock(side_effect=mock_return_values) - mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) - mock_backup_crm_data_process = MagicMock() - mock_convert_crm_csv_data_process = MagicMock() - mock_backup_crm_csv_data_process = MagicMock() - mock_copy_crm_csv_data_process = MagicMock() - mock_upload_last_fetch_datetime_process = MagicMock() - mock_upload_result_data_process = MagicMock(side_effect=[exception( + + self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock(side_effect=[exception( 'E-END-01', END_JP_NAME, '例外発生'), None, None]) - with patch('src.controller.prepare_data_fetch_process', mock_prepare_data_fetch_process),\ - patch('src.controller.check_object_info_process', mock_check_object_info_process), \ - patch('src.controller.set_datetime_period_process', mock_set_datetime_period_process),\ - patch('src.controller.fetch_crm_data_process', mock_fetch_crm_data_process),\ - patch('src.controller.backup_crm_data_process', mock_backup_crm_data_process),\ - patch('src.controller.convert_crm_csv_data_process', mock_convert_crm_csv_data_process),\ - patch('src.controller.backup_crm_csv_data_process', mock_backup_crm_csv_data_process),\ - patch('src.controller.copy_crm_csv_data_process', mock_copy_crm_csv_data_process),\ - patch('src.controller.upload_last_fetch_datetime_process', mock_upload_last_fetch_datetime_process),\ - patch('src.controller.upload_result_data_process', mock_upload_result_data_process): - with pytest.raises(exception): - controller.controller() + with pytest.raises(exception): + run_control_process() + + assert self.mock_upload_result_data_process.called is True + assert self.mock_upload_result_data_process.call_count == 1 assert generate_log_message_tuple( log_level=logging.ERROR, log_message=message) in caplog.record_tuples, '取得処理実施結果アップロード処理で発生した例外のログメッセージが出力されていること' From ed80499ce57e25f9901f68a5e0caa4a76765bb8c Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 4 Aug 2022 09:06:57 +0900 Subject: [PATCH 136/275] =?UTF-8?q?docs:=20README=E3=81=AE=E3=83=95?= =?UTF-8?q?=E3=82=A9=E3=83=AB=E3=83=80=E6=A7=8B=E6=88=90=E3=81=AB=E8=BF=BD?= =?UTF-8?q?=E8=A8=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md index 034cf2d4..dc992c81 100644 --- a/ecs/crm-datafetch/README.md +++ b/ecs/crm-datafetch/README.md @@ -105,7 +105,8 @@ │ └── logger.py -- ログ管理クラス │ └── tests/ -- テストコード置き場 - ├── aws -- AWS操作モジュールのテスト + ├── test_utils/ -- テストコードで共通的に使用できる関数群 + ├── aws/ -- AWS操作モジュールのテスト ├── ... -- src配下のモジュール構成と同じ階層にテストコードを追加していく ├── conftest.py -- pytestのフィクスチャやフックを管理するファイル └── docstring_parser.py -- pytest-htmlのレポート出力用のヘルパー From d5a35317fbe952b3d25d20c05ea8c720ae1e15ba Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 4 Aug 2022 10:08:57 +0900 Subject: [PATCH 137/275] =?UTF-8?q?docs:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=AF=BE=E5=BF=9C=20https://nds-tyo?= =?UTF-8?q?.backlog.com/git/NEWDWH2021/newsdwh2021/pullRequests/54#comment?= =?UTF-8?q?-1301153=20https://nds-tyo.backlog.com/git/NEWDWH2021/newsdwh20?= =?UTF-8?q?21/pullRequests/54#comment-1301166=20https://nds-tyo.backlog.co?= =?UTF-8?q?m/git/NEWDWH2021/newsdwh2021/pullRequests/54#comment-1301158?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md index 034cf2d4..243e33e5 100644 --- a/ecs/crm-datafetch/README.md +++ b/ecs/crm-datafetch/README.md @@ -70,7 +70,7 @@ ## ファイル/フォルダ構成 -`[〇〇処理モジュール]`と記載されているファイルは、設計書に記載のシートと一致したPythonファイルです +`[〇〇処理]モジュール`と記載されているファイルは、設計書に記載のシートと一致したPythonファイルです ```text . @@ -91,8 +91,8 @@ │ ├── copy_crm_csv_data_process.py -- [CSVアップロード処理]モジュール │ ├── error/ -- 処理エラー発生時カスタム例外モジュール置き場 │ ├── fetch_crm_data_process.py -- [CRMデータ取得処理]モジュール -│ ├── parser/ -- [JSON変換処理]モジュール置き場 -│ ├── prepare_data_fetch_process.py -- データ取得準備処理 +│ ├── parser/ -- JSON設定ファイル読み込み処理モジュール置き場 +│ ├── prepare_data_fetch_process.py -- [データ取得準備処理]モジュール │ ├── salesforce/ -- SalesforceのAPIリクエストモジュール置き場 │ ├── set_datetime_period_process.py -- [データ取得期間設定処理]モジュール │ ├── system_var/ -- 環境変数と定数ファイル置き場 From a1464a555992d21caa3397f7f5f8eb6299d54e41 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Thu, 4 Aug 2022 11:11:51 +0900 Subject: [PATCH 138/275] =?UTF-8?q?fix:=E9=A0=85=E7=9B=AE=E6=95=B0?= =?UTF-8?q?=E3=81=AE=E8=A8=AD=E5=AE=9A=E5=80=A4=E3=82=92=E7=8F=BE=E5=9C=A8?= =?UTF-8?q?=E3=81=AE=E3=82=82=E3=81=AE=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/crm/settings/CRM_Product_vod__c.txt | 2 +- s3/data/crm/settings/CRM_Territory2.txt | 2 +- s3/data/crm/settings/CRM_Territory2_ALL.txt | 6 +++--- s3/data/crm/settings/CRM_User.txt | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/s3/data/crm/settings/CRM_Product_vod__c.txt b/s3/data/crm/settings/CRM_Product_vod__c.txt index 843cec84..e9538c2e 100644 --- a/s3/data/crm/settings/CRM_Product_vod__c.txt +++ b/s3/data/crm/settings/CRM_Product_vod__c.txt @@ -4,7 +4,7 @@ utf-8 " CRLF 1 -46 +53 Id,OwnerId,IsDeleted,Name,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Consumer_site__c,Product_info__c,Therapeutic_Class_vod__c,Parent_Product_vod__c,Therapeutic_Area_vod__c,Product_Type_vod__c,Require_Key_Message_vod__c,Cost_vod__c,External_ID_vod__c,Manufacturer_vod__c,Company_Product_vod__c,Controlled_Substance_vod__c,Description_vod__c,Sample_Quantity_Picklist_vod__c,Display_Order_vod__c,No_Metrics_vod__c,Distributor_vod__c,Sample_Quantity_Bound_vod__c,Sample_U_M_vod__c,No_Details_vod__c,Quantity_Per_Case_vod__c,Schedule_vod__c,Restricted_vod__c,Pricing_Rule_Quantity_Bound_vod__c,No_Promo_Items_vod__c,User_Aligned_vod__c,Restricted_States_vod__c,Sort_Code_vod__c,No_Cycle_Plans_vod__c,Inventory_Order_UOM_vod__c,Inventory_Quantity_Per_Case_vod__c,VExternal_Id_vod__c,Country__c,MSJ_Product_Classification__c,MSJ_Indication__c,MSJ_Therapeutic_Area__c,MSJ_Global_Brand__c,MSJ_Global_Business_Unit__c,MSJ_Molecules__c,MSJ_SBU__c id,owner_id,is_deleted,name,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,last_viewed_date,last_referenced_date,consumer_site__c,product_info__c,therapeutic_class_vod__c,parent_product_vod__c,therapeutic_area_vod__c,product_type_vod__c,require_key_message_vod__c,cost_vod__c,external_id_vod__c,manufacturer_vod__c,company_product_vod__c,controlled_substance_vod__c,description_vod__c,sample_quantity_picklist_vod__c,display_order_vod__c,no_metrics_vod__c,distributor_vod__c,sample_quantity_bound_vod__c,sample_u_m_vod__c,no_details_vod__c,quantity_per_case_vod__c,schedule_vod__c,restricted_vod__c,pricing_rule_quantity_bound_vod__c,no_promo_items_vod__c,user_aligned_vod__c,restricted_states_vod__c,sort_code_vod__c,no_cycle_plans_vod__c,inventory_order_uom_vod__c,inventory_quantity_per_case_vod__c,vexternal_id_vod__c,country__c,msj_product_classification__c,msj_indication__c,msj_therapeutic_area__c,msj_global_brand__c,msj_global_business_unit__c,msj_molecules__c,msj_sbu__c src02.crm_product_vod__c diff --git a/s3/data/crm/settings/CRM_Territory2.txt b/s3/data/crm/settings/CRM_Territory2.txt index 5486b657..7751606d 100644 --- a/s3/data/crm/settings/CRM_Territory2.txt +++ b/s3/data/crm/settings/CRM_Territory2.txt @@ -4,7 +4,7 @@ utf-8 " CRLF 1 -15 +17 Id,Name,Territory2TypeId,Territory2ModelId,ParentTerritory2Id,Description,ForecastUserId,AccountAccessLevel,OpportunityAccessLevel,CaseAccessLevel,ContactAccessLevel,LastModifiedDate,LastModifiedById,SystemModstamp,DeveloperName,MSJ_Territory_Type__c,MSJ_Level__c id,name,territory2_type_id,territory2_model_id,parent_territory2_id,description,forecast_user_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,last_modified_date,last_modified_by_id,system_modstamp,developer_name,msj_territory_type__c,msj_level__c src02.crm_territory2 diff --git a/s3/data/crm/settings/CRM_Territory2_ALL.txt b/s3/data/crm/settings/CRM_Territory2_ALL.txt index 525e1796..fff3fc29 100644 --- a/s3/data/crm/settings/CRM_Territory2_ALL.txt +++ b/s3/data/crm/settings/CRM_Territory2_ALL.txt @@ -4,9 +4,9 @@ utf-8 " CRLF 1 -15 -Id,Name,Territory2TypeId,Territory2ModelId,ParentTerritory2Id,Description,ForecastUserId,AccountAccessLevel,OpportunityAccessLevel,CaseAccessLevel,ContactAccessLevel,LastModifiedDate,LastModifiedById,SystemModstamp,DeveloperName -id,name,territory2_type_id,territory2_model_id,parent_territory2_id,description,forecast_user_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,last_modified_date,last_modified_by_id,system_modstamp,developer_name +17 +Id,Name,Territory2TypeId,Territory2ModelId,ParentTerritory2Id,Description,ForecastUserId,AccountAccessLevel,OpportunityAccessLevel,CaseAccessLevel,ContactAccessLevel,LastModifiedDate,LastModifiedById,SystemModstamp,DeveloperName,MSJ_Territory_Type__c,MSJ_Level__c +id,name,territory2_type_id,territory2_model_id,parent_territory2_id,description,forecast_user_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,last_modified_date,last_modified_by_id,system_modstamp,developer_name,msj_territory_type__c,msj_level__c src02.crm_territory2_all org02.crm_territory2_all diff --git a/s3/data/crm/settings/CRM_User.txt b/s3/data/crm/settings/CRM_User.txt index 317a5e14..a0d5c519 100644 --- a/s3/data/crm/settings/CRM_User.txt +++ b/s3/data/crm/settings/CRM_User.txt @@ -4,7 +4,7 @@ utf-8 " CRLF 1 -197 +203 Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,LastPasswordChangeDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,NumberOfFailedLogins,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesNativeEmailClient,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,last_password_change_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,number_of_failed_logins,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_native_email_client,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c src02.crm_user From 81fc1ebeeeae37d4aaf39d2b04c0c7e8f32826b2 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 4 Aug 2022 11:16:54 +0900 Subject: [PATCH 139/275] =?UTF-8?q?feat:=20=E5=8F=96=E5=BE=97=E3=83=87?= =?UTF-8?q?=E3=83=BC=E3=82=BF=E4=BB=B6=E6=95=B0=E3=81=8C0=E4=BB=B6?= =?UTF-8?q?=E3=81=AE=E5=A0=B4=E5=90=88=E3=81=ABCSV=E5=87=BA=E5=8A=9B?= =?UTF-8?q?=E3=81=97=E3=81=AA=E3=81=84=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/controller.py | 6 + ecs/crm-datafetch/tests/test_controller.py | 130 ++++++++++++++++++--- 2 files changed, 122 insertions(+), 14 deletions(-) diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index dc8f784d..f9c87960 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -139,6 +139,12 @@ def _fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateT crm_data_response = fetch_crm_data_process(target_object, last_fetch_datetime) + # 取得件数が0件の場合、次のオブジェクトの処理に移行する + if len(crm_data_response) == 0: + logger.info( + f'I-CTRL-22 [{target_object_name}]のレコード件数が0件のため、ファイルアップロードをスキップします') + return + # 7. 出力ファイル名をログ出力する logger.info( f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{target_object.upload_file_name}] となります') diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index 51ff1987..8b9a464a 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -122,7 +122,7 @@ class TestController: self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - self.mock_fetch_crm_data_process = MagicMock(return_value=[{}]) + self.mock_fetch_crm_data_process = MagicMock(return_value=[{'Name': 'Test'}]) self.mock_backup_crm_data_process = MagicMock() self.mock_convert_crm_csv_data_process = MagicMock() self.mock_backup_crm_csv_data_process = MagicMock() @@ -241,7 +241,7 @@ class TestController: assert generate_log_message_tuple(log_message='I-CTRL-21 処理対象のデータが存在しませんでした') in caplog.record_tuples, '処理対象が存在しない旨を示すログメッセージが出力されていること' assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples - def test_do_not_call_upload_csv_process(self, caplog, run_control_process): + def test_do_not_call_upload_csv_process_cause_is_skip_true(self, caplog, run_control_process): """ Cases: オブジェクト情報.is_skipがTrueの場合、CSVアップロード処理が実行されないこと @@ -291,10 +291,10 @@ class TestController: assert generate_log_message_tuple(log_message='I-CTRL-07 [Account]のデータ取得処理をスキップします') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples - def test_do_not_call_upload_csv_process_of_part(self, caplog, run_control_process): + def test_do_not_call_upload_csv_process_cause_is_skip_true_in_loop(self, caplog, run_control_process): """ Cases: - オブジェクト情報.is_skipがTrueのものが含まれる場合、CSVアップロード処理が実行されないこと + オブジェクト情報.is_skipがTrueのものが含まれる場合、対象オブジェクトのCSVアップロード処理が実行されないこと Arranges: - データ取得準備処理で返される取得対象オブジェクト情報のうち、２つ目をis_skipをTrueにする - 各種プロセスメソッドと内部で使用している値オブジェクトをモック化する @@ -331,7 +331,7 @@ class TestController: self.mock_prepare_data_fetch_process = MagicMock(return_value=(FetchTargetObjects(mock_check_object_infos), COMMON_EXECUTE_DATETIME, {})) self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) self.mock_set_datetime_period_process = MagicMock() - self.mock_fetch_crm_data_process = MagicMock() + self.mock_fetch_crm_data_process = MagicMock(side_effect=[[{"Name": "Test"}], [{"Name": "Test"}]]) self.mock_backup_crm_data_process = MagicMock() self.mock_convert_crm_csv_data_process = MagicMock() self.mock_backup_crm_csv_data_process = MagicMock() @@ -365,6 +365,108 @@ class TestController: ) in caplog.record_tuples, 'オブジェクト情報.is_skipがFalseのものはCSVアップロード処理のログメッセージが出力されていること' assert generate_log_message_tuple(log_message=f'I-CTRL-20 CRMデータ取得処理を終了します') in caplog.record_tuples + def test_do_not_call_upload_csv_process_cause_crm_data_empty(self, caplog, run_control_process): + """ + Cases: + CRMデータ取得処理で取得できた件数が0件の場合、CSVアップロード処理が実行されないこと + Arranges: + - CRMデータ取得処理で返されるオブジェクトを空のリストにする + - 各種プロセスメソッドと内部で使用している値オブジェクトをモック化する + Expects: + - CRM電文データバックアップ処理以降のプロセスが実行されないこと + - 処理をスキップする旨を示すログメッセージが出力されていること + """ + mock_check_object_infos = { + 'objects': [ + { + 'object_name': 'Account', + 'columns': [], + 'is_skip': False + } + ] + } + + FetchTargetObjects(mock_check_object_infos) + mock_return_values = [ + TargetObject(mock_check_object_infos['objects'][0], COMMON_EXECUTE_DATETIME) + ] + self.mock_prepare_data_fetch_process = MagicMock(return_value=(FetchTargetObjects(mock_check_object_infos), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) + self.mock_set_datetime_period_process = MagicMock() + self.mock_fetch_crm_data_process = MagicMock(return_value=[]) + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() + + run_control_process() + # 実行回数の確認 + assert self.mock_prepare_data_fetch_process.called is True + assert self.mock_prepare_data_fetch_process.call_count == 1 + assert self.mock_check_object_info_process.call_count == 1 + assert self.mock_set_datetime_period_process.call_count == 1 + assert self.mock_fetch_crm_data_process.call_count == 1 + # CRM電文データバックアップ処理以降のプロセスが実行されないこと + assert self.mock_backup_crm_data_process.call_count == 0 + assert self.mock_convert_crm_csv_data_process.call_count == 0 + assert self.mock_backup_crm_csv_data_process.call_count == 0 + assert self.mock_copy_crm_csv_data_process.call_count == 0 + assert self.mock_upload_last_fetch_datetime_process.call_count == 0 + assert self.mock_upload_result_data_process.called is True + assert self.mock_upload_result_data_process.call_count == 1 + + # ログ出力の確認 + assert generate_log_message_tuple( + log_message='I-CTRL-22 [Account]のレコード件数が0件のため、ファイルアップロードをスキップします') in caplog.record_tuples, '処理をスキップする旨を示すログメッセージが出力されていること' + + def test_do_not_call_upload_csv_process_cause_crm_data_empty_in_loop(self, caplog, run_control_process): + """ + Cases: + CRMデータ取得処理で取得できた件数が0件のものが含まれる場合、対象オブジェクトのCSVアップロード処理が実行されないこと + Arranges: + - CRMデータ取得処理で返されるオブジェクトのうち、２つ目を空のリストにする + - 各種プロセスメソッドと内部で使用している値オブジェクトをモック化する + Expects: + - CRMデータ取得処理が2回実行されること + - 処理をスキップする旨を示すログメッセージが出力されていること + - 取得オブジェクトが1件以上取れているものはCSVアップロード処理のログメッセージが出力されていること + """ + self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) + self.mock_check_object_info_process = MagicMock(side_effect=[COMMON_TARGET_OBJECTS_1, COMMON_TARGET_OBJECTS_2, COMMON_TARGET_OBJECTS_3]) + self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[[{"Name": "Test"}], [], [{"Name": "Test"}]]) + self.mock_backup_crm_data_process = MagicMock() + self.mock_convert_crm_csv_data_process = MagicMock() + self.mock_backup_crm_csv_data_process = MagicMock() + self.mock_copy_crm_csv_data_process = MagicMock() + self.mock_upload_last_fetch_datetime_process = MagicMock() + self.mock_upload_result_data_process = MagicMock() + + run_control_process() + # 実行回数の確認 + assert self.mock_check_object_info_process.call_count == 3 + assert self.mock_set_datetime_period_process.call_count == 3 + assert self.mock_fetch_crm_data_process.call_count == 3 + # CRM電文データバックアップ処理以降のプロセスは件数があるもののみ実行されること + assert self.mock_backup_crm_data_process.call_count == 2 + assert self.mock_convert_crm_csv_data_process.call_count == 2 + assert self.mock_backup_crm_csv_data_process.call_count == 2 + assert self.mock_copy_crm_csv_data_process.call_count == 2 + assert self.mock_upload_last_fetch_datetime_process.call_count == 2 + assert self.mock_upload_result_data_process.called is True + assert self.mock_upload_result_data_process.call_count == 1 + + # ログ出力の確認 + assert generate_log_message_tuple( + log_message='I-CTRL-14 [Account] CSVデータアップロード処理呼び出し') in caplog.record_tuples, '取得オブジェクトが1件以上取れているものはCSVアップロード処理のログメッセージが出力されていること' + assert generate_log_message_tuple( + log_message='I-CTRL-22 [Contact]のレコード件数が0件のため、ファイルアップロードをスキップします') in caplog.record_tuples, '処理をスキップする旨を示すログメッセージが出力されていること' + assert generate_log_message_tuple( + log_message='I-CTRL-14 [Call2_vod__c] CSVデータアップロード処理呼び出し' + ) in caplog.record_tuples, '取得オブジェクトが1件以上取れているものはCSVアップロード処理のログメッセージが出力されていること' + @pytest.mark.parametrize( 'exception, message', [ @@ -418,7 +520,7 @@ class TestController: self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - self.mock_fetch_crm_data_process = MagicMock(return_value=[{}]) + self.mock_fetch_crm_data_process = MagicMock(return_value=[{'Name': 'Test'}]) self.mock_backup_crm_data_process = MagicMock() self.mock_convert_crm_csv_data_process = MagicMock() self.mock_backup_crm_csv_data_process = MagicMock() @@ -464,7 +566,7 @@ class TestController: self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) self.mock_set_datetime_period_process = MagicMock(side_effect=[exception( 'E-DATE-01', DATE_JP_NAME, '例外発生'), COMMON_LAST_FETCH_DATETIME, COMMON_LAST_FETCH_DATETIME]) - self.mock_fetch_crm_data_process = MagicMock(return_value=[{}]) + self.mock_fetch_crm_data_process = MagicMock(return_value=[{'Name': 'Test'}]) self.mock_backup_crm_data_process = MagicMock() self.mock_convert_crm_csv_data_process = MagicMock() self.mock_backup_crm_csv_data_process = MagicMock() @@ -510,7 +612,7 @@ class TestController: self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) self.mock_fetch_crm_data_process = MagicMock(side_effect=[exception( - 'E-FETCH-01', FETCH_JP_NAME, '例外発生'), {}, {}]) + 'E-FETCH-01', FETCH_JP_NAME, '例外発生'), [{'Name': 'Test'}], [{'Name': 'Test'}]]) self.mock_backup_crm_data_process = MagicMock() self.mock_convert_crm_csv_data_process = MagicMock() self.mock_backup_crm_csv_data_process = MagicMock() @@ -555,7 +657,7 @@ class TestController: self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - self.mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[[{'Name': 'Test'}], [{'Name': 'Test'}], [{'Name': 'Test'}]]) self.mock_backup_crm_data_process = MagicMock(side_effect=[exception( 'E-RESBK-01', RESBK_JP_NAME, '例外発生'), None, None]) self.mock_convert_crm_csv_data_process = MagicMock() @@ -601,7 +703,7 @@ class TestController: self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - self.mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[[{'Name': 'Test'}], [{'Name': 'Test'}], [{'Name': 'Test'}]]) self.mock_backup_crm_data_process = MagicMock() self.mock_convert_crm_csv_data_process = MagicMock(side_effect=[exception( 'E-CONV-01', CONV_JP_NAME, '例外発生'), None, None]) @@ -647,7 +749,7 @@ class TestController: self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - self.mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[[{'Name': 'Test'}], [{'Name': 'Test'}], [{'Name': 'Test'}]]) self.mock_backup_crm_data_process = MagicMock() self.mock_convert_crm_csv_data_process = MagicMock() self.mock_backup_crm_csv_data_process = MagicMock(side_effect=[exception( @@ -693,7 +795,7 @@ class TestController: self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - self.mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[[{'Name': 'Test'}], [{'Name': 'Test'}], [{'Name': 'Test'}]]) self.mock_backup_crm_data_process = MagicMock() self.mock_convert_crm_csv_data_process = MagicMock() self.mock_backup_crm_csv_data_process = MagicMock() @@ -739,7 +841,7 @@ class TestController: self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - self.mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[[{'Name': 'Test'}], [{'Name': 'Test'}], [{'Name': 'Test'}]]) self.mock_backup_crm_data_process = MagicMock() self.mock_convert_crm_csv_data_process = MagicMock() self.mock_backup_crm_csv_data_process = MagicMock() @@ -784,7 +886,7 @@ class TestController: self.mock_prepare_data_fetch_process = MagicMock(return_value=(deepcopy(COMMON_FETCH_TARGET_OBJECTS), COMMON_EXECUTE_DATETIME, {})) self.mock_check_object_info_process = MagicMock(side_effect=mock_return_values) self.mock_set_datetime_period_process = MagicMock(return_value=COMMON_LAST_FETCH_DATETIME) - self.mock_fetch_crm_data_process = MagicMock(side_effect=[{}, {}, {}]) + self.mock_fetch_crm_data_process = MagicMock(side_effect=[[{'Name': 'Test'}], [{'Name': 'Test'}], [{'Name': 'Test'}]]) self.mock_backup_crm_data_process = MagicMock() self.mock_convert_crm_csv_data_process = MagicMock() self.mock_backup_crm_csv_data_process = MagicMock() From a68027bb9f7c11e1be3970d8a6aa94002db7a157 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 4 Aug 2022 15:33:31 +0900 Subject: [PATCH 140/275] =?UTF-8?q?feat:=20json=5Fparser=E3=81=AE=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E5=87=A6=E7=90=86=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/parser/test_json_parse.py | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 ecs/crm-datafetch/tests/parser/test_json_parse.py diff --git a/ecs/crm-datafetch/tests/parser/test_json_parse.py b/ecs/crm-datafetch/tests/parser/test_json_parse.py new file mode 100644 index 00000000..fe2c96ea --- /dev/null +++ b/ecs/crm-datafetch/tests/parser/test_json_parse.py @@ -0,0 +1,43 @@ +from src.parser.json_parse import JsonParser + + +class TestJsonParser(): + + def test_parse(self) -> dict: + """ + Cases: + - コメントアウトが記載されているJSONからコメントを取り除き、辞書型を返すこと + Arranges: + - JSON文字列を準備する + Expects: + - json.loadsされたファイルの内容が期待値と一致する + """ + + # Arranges + json_string = """{ + "aaaa": "aaaa", + # これはコメントです + "#これはコメントではありません": "#これはコメントではありません", + "bbb": false, + "hogehoge": [ + "ccc", + /これはコメントです + "/これはコメントではありません" + ] + }""" + + sut = JsonParser(json_string) + actual = sut.parse() + + # Expects + expected_value = { + "aaaa": "aaaa", + "#これはコメントではありません": "#これはコメントではありません", + "bbb": False, + "hogehoge": [ + "ccc", + "/これはコメントではありません" + ] + } + + assert actual == expected_value From bf5fedbfa3c1fb34b6cb77286035a9cbadec4bd9 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 4 Aug 2022 18:17:47 +0900 Subject: [PATCH 141/275] =?UTF-8?q?feat:=20=E7=95=B0=E5=B8=B8=E7=B3=BB?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/parser/test_json_parse.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/ecs/crm-datafetch/tests/parser/test_json_parse.py b/ecs/crm-datafetch/tests/parser/test_json_parse.py index fe2c96ea..1991aa54 100644 --- a/ecs/crm-datafetch/tests/parser/test_json_parse.py +++ b/ecs/crm-datafetch/tests/parser/test_json_parse.py @@ -1,3 +1,4 @@ +import pytest from src.parser.json_parse import JsonParser @@ -26,6 +27,7 @@ class TestJsonParser(): ] }""" + # Act sut = JsonParser(json_string) actual = sut.parse() @@ -41,3 +43,35 @@ class TestJsonParser(): } assert actual == expected_value + + def test_raise_parse(self) -> dict: + """ + Cases: + - コメントアウト記号ではない文字をコメントアウトとしたときに、例外が発生すること + Arranges: + - JSON文字列を準備する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + json_string = """{ + "aaaa": "aaaa", + $ これはコメントです + "#これはコメントではありません": "#これはコメントではありません", + "bbb": false, + "hogehoge": [ + "ccc", + /これはコメントです + "/これはコメントではありません" + ] + }""" + + # Act + with pytest.raises(Exception) as e: + + sut = JsonParser(json_string) + sut.parse() + + # Expects + assert str(e.value) == "Expecting property name enclosed in double quotes: line 3 column 13 (char 42)" From a70d03b3660a12a11cbc99613eff7855b94c6734 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 4 Aug 2022 19:40:13 +0900 Subject: [PATCH 142/275] =?UTF-8?q?feat:=20couter=5Fobject.py=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/util/test_counter_object.py | 114 ++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 ecs/crm-datafetch/tests/util/test_counter_object.py diff --git a/ecs/crm-datafetch/tests/util/test_counter_object.py b/ecs/crm-datafetch/tests/util/test_counter_object.py new file mode 100644 index 00000000..47b9928a --- /dev/null +++ b/ecs/crm-datafetch/tests/util/test_counter_object.py @@ -0,0 +1,114 @@ +import pytest +from src.util.counter_object import CounterObject + + +class TestCounterObject: + + def test_describe(self) -> int: + """ + Cases: + カウンターオブジェクトにて保持した値を返すこと + Arranges: + なし + Expects: + 問い合わせた値が期待値と一致する + """ + + # Act + sut = CounterObject(5) + actual = sut.describe() + + # Expects + assert actual == 5 + + def test_raise_describe(self) -> int: + """ + Cases: + カウンターオブジェクトにて保持した値を返すこと + Arranges: + なし + Expects: + 問い合わせた値が期待値と一致する + """ + + # Act + with pytest.raises(Exception) as e: + sut = CounterObject("aaa") + sut.describe() + + # Expects + assert str(e.value) == '例外' + + def test_increment(self, num=1) -> int: + """ + Cases: + カウンターオブジェクトにて保持した値がインクリメントされていること + Arranges: + なし + Expects: + 戻り値が期待値と一致する + """ + + # Act + sut = CounterObject(5) + sut.increment() + actual = sut.increment() + + # Expects + assert actual == 7 + + def test_raise_increment(self, num=1) -> int: + """ + Cases: + 文字列を引数で渡すことで、例外が発生すること + Arranges: + なし + Expects: + 発生した例外が期待値と一致する + """ + + # Act + with pytest.raises(Exception) as e: + sut = CounterObject(5) + sut.increment('aaa') + sut.increment('aaa') + + # Expects + assert str(e.value) == "unsupported operand type(s) for +=: 'int' and 'str'" + + def test_decrement(self, num=1) -> int: + """ + Cases: + カウンターオブジェクトにて保持した値がデクリメントされていること + Arranges: + なし + Expects: + 戻り値が期待値と一致する + """ + + # Act + sut = CounterObject(5) + sut.decrement(2) + actual = sut.decrement(2) + + # Expects + assert actual == 1 + + def test_raise_decrement(self, num=1) -> int: + """ + Cases: + 文字列を引数で渡すことで、例外が発生すること + Arranges: + なし + Expects: + 発生した例外が期待値と一致する + """ + + # Act + with pytest.raises(Exception) as e: + sut = CounterObject(5) + sut.decrement('aaa') + sut.decrement('aaa') + + # Expects + assert str(e.value) == "unsupported operand type(s) for -=: 'int' and 'str'" From 36fb6105621e4a5c33cea956f3e63e6777ac19c3 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Fri, 5 Aug 2022 09:41:36 +0900 Subject: [PATCH 143/275] =?UTF-8?q?fix:=20=E6=AD=A3=E5=B8=B8=E7=B3=BB?= =?UTF-8?q?=E3=80=81=E7=95=B0=E5=B8=B8=E7=B3=BB=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E5=86=85=E5=AE=B9=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/util/test_counter_object.py | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ecs/crm-datafetch/tests/util/test_counter_object.py b/ecs/crm-datafetch/tests/util/test_counter_object.py index 47b9928a..8bf677a0 100644 --- a/ecs/crm-datafetch/tests/util/test_counter_object.py +++ b/ecs/crm-datafetch/tests/util/test_counter_object.py @@ -15,16 +15,16 @@ class TestCounterObject: """ # Act - sut = CounterObject(5) + sut = CounterObject() actual = sut.describe() # Expects - assert actual == 5 + assert actual == 1 def test_raise_describe(self) -> int: """ Cases: - カウンターオブジェクトにて保持した値を返すこと + カウンターオブジェクトの保持した値を問い合わせる際、引数を渡すと例外が発生すること Arranges: なし Expects: @@ -33,13 +33,13 @@ class TestCounterObject: # Act with pytest.raises(Exception) as e: - sut = CounterObject("aaa") - sut.describe() + sut = CounterObject() + sut.describe(1) # Expects - assert str(e.value) == '例外' + assert str(e.value) == 'describe() takes 1 positional argument but 2 were given' - def test_increment(self, num=1) -> int: + def test_increment(self) -> int: """ Cases: カウンターオブジェクトにて保持した値がインクリメントされていること @@ -57,7 +57,7 @@ class TestCounterObject: # Expects assert actual == 7 - def test_raise_increment(self, num=1) -> int: + def test_raise_increment(self) -> int: """ Cases: 文字列を引数で渡すことで、例外が発生すること @@ -76,7 +76,7 @@ class TestCounterObject: # Expects assert str(e.value) == "unsupported operand type(s) for +=: 'int' and 'str'" - def test_decrement(self, num=1) -> int: + def test_decrement(self) -> int: """ Cases: カウンターオブジェクトにて保持した値がデクリメントされていること @@ -94,7 +94,7 @@ class TestCounterObject: # Expects assert actual == 1 - def test_raise_decrement(self, num=1) -> int: + def test_raise_decrement(self) -> int: """ Cases: 文字列を引数で渡すことで、例外が発生すること From 9af0d9dd5cf895c16bf8c8600b36101fa4622645 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 4 Aug 2022 16:30:56 +0900 Subject: [PATCH 144/275] =?UTF-8?q?feat:=20SOQLBuilder=E3=81=AE=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/__init__.py | 0 .../tests/salesforce/test_soql_builder.py | 104 ++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 ecs/crm-datafetch/tests/salesforce/__init__.py create mode 100644 ecs/crm-datafetch/tests/salesforce/test_soql_builder.py diff --git a/ecs/crm-datafetch/tests/salesforce/__init__.py b/ecs/crm-datafetch/tests/salesforce/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py new file mode 100644 index 00000000..9a6f9a45 --- /dev/null +++ b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py @@ -0,0 +1,104 @@ + +from src.config.objects import ExecuteDateTime, LastFetchDatetime, TargetObject +from src.salesforce.soql_builder import SOQLBuilder + + +class TestSOQLBuilder: + + def test_create_count_soql(self): + test_target_object_json = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + } + + test_last_fetch_datetime_json = { + 'last_fetch_datetime_from': '1999-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2000-01-01T00:00:00.000Z', + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(test_target_object_json, execute_datetime) + test_last_fetch_datetime = LastFetchDatetime(test_last_fetch_datetime_json, execute_datetime) + + sut = SOQLBuilder(target_object, test_last_fetch_datetime) + actual = sut.create_count_soql() + + expect = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 1999-01-01T00:00:00.000Z AND + SystemModstamp <= 2000-01-01T00:00:00.000Z + """ + + assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') + + def test_create_fetch_soql(self): + test_target_object_json = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + } + + test_last_fetch_datetime_json = { + 'last_fetch_datetime_from': '1999-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2000-01-01T00:00:00.000Z', + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(test_target_object_json, execute_datetime) + test_last_fetch_datetime = LastFetchDatetime(test_last_fetch_datetime_json, execute_datetime) + + sut = SOQLBuilder(target_object, test_last_fetch_datetime) + actual = sut.create_fetch_soql() + + expect = """SELECT + Id, + Name, + SystemModstamp, + LastModifiedDate, + CustomItem1__c, + CustomItem2__c, + CustomItem3__c, + CustomItem4__c, + CustomItem5__c, + CustomItem6__c, + CustomItem7__c, + CustomItem8__c + FROM + Account + WHERE + SystemModstamp > 1999-01-01T00:00:00.000Z AND + SystemModstamp <= 2000-01-01T00:00:00.000Z + """ + + print('actual', actual) + print('expect', expect) + + assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') From e3cd0dbb101a4ec1749e737bcfa61278704af63c Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 08:14:09 +0900 Subject: [PATCH 145/275] =?UTF-8?q?feat:Salesforce=E9=96=A2=E9=80=A3?= =?UTF-8?q?=E3=81=AE=E6=9C=80=E4=BD=8E=E9=99=90=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/salesforce/salesforce_api.py | 8 +-- .../tests/salesforce/test_salesfoce.py | 52 +++++++++++++++++++ 2 files changed, 56 insertions(+), 4 deletions(-) create mode 100644 ecs/crm-datafetch/tests/salesforce/test_salesfoce.py diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index 4e4bb267..edd9aeb3 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -13,16 +13,16 @@ class SalesforceApiClient(): domain=CRM_AUTH_DOMAIN ) - def query(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + def __query(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): return self.__sf.query(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) - def query_all(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + def __query_all(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): return self.__sf.query_all(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) def fetch_sf_count(self, soql: str): - count_res = self.query(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) + count_res = self.__query(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) return count_res.get('records')[0].get('expr0') def fetch_sf_data(self, soql: str): - data_res = self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + data_res = self.__query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) return data_res.get('records') diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py new file mode 100644 index 00000000..413fae92 --- /dev/null +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -0,0 +1,52 @@ +import pytest +from src.salesforce.salesforce_api import SalesforceApiClient + + +class TestSalesforceApiClient: + + def test_fetch_sf_count(self): + + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-04T16:36:00.000Z + """ + sut = SalesforceApiClient() + + actual = sut.fetch_sf_count(soql) + assert actual >= 1 + + def test_fetch_sf_data_one_record(self): + + soql = """SELECT + Id, + Name, + SystemModstamp, + LastModifiedDate, + CustomItem1__c, + CustomItem2__c, + CustomItem3__c, + CustomItem4__c, + CustomItem5__c, + CustomItem6__c, + CustomItem7__c, + CustomItem8__c + FROM + Account + WHERE + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-04T16:36:00.000Z + """ + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + print(actual) + assert len(actual) >= 1 + + def test_raise_create_instance_cause_auth_failed(self, monkeypatch): + monkeypatch.setattr('src.salesforce.salesforce_api.CRM_USER_NAME', 'invalid_username') + with pytest.raises(Exception): + SalesforceApiClient() From 8373ccc456fdf30c60c0a9df605e0af7cf82ea62 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 20:08:03 +0900 Subject: [PATCH 146/275] =?UTF-8?q?feat:=E3=82=B1=E3=83=BC=E3=82=B9?= =?UTF-8?q?=E3=81=AE=E8=BF=BD=E5=8A=A0=E3=83=BB=E4=BF=AE=E6=AD=A3=E3=80=82?= =?UTF-8?q?=E3=83=89=E3=82=AD=E3=83=A5=E3=83=A1=E3=83=B3=E3=83=88=E3=82=B3?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesfoce.py | 51 ++++++++++++++++- .../tests/salesforce/test_soql_builder.py | 56 +++++++++++++++++++ 2 files changed, 104 insertions(+), 3 deletions(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py index 413fae92..4b3f8ae1 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -5,7 +5,14 @@ from src.salesforce.salesforce_api import SalesforceApiClient class TestSalesforceApiClient: def test_fetch_sf_count(self): - + """ + Cases: + Salesforceからオブジェクトの件数が取得できること + Arranges: + SalesforceのAccountオブジェクトに、レコードを作成する(手作業、コード上では行わない) + Expects: + 取得件数が1件以上になる + """ soql = """SELECT COUNT(Id) FROM @@ -20,7 +27,14 @@ class TestSalesforceApiClient: assert actual >= 1 def test_fetch_sf_data_one_record(self): - + """ + Cases: + Salesforceからオブジェクトが取得できること + Arranges: + SalesforceのAccountオブジェクトに、レコードを作成する(手作業、コード上では行わない) + Expects: + オブジェクトが取得でき、期待値と一致していること + """ soql = """SELECT Id, Name, @@ -42,11 +56,42 @@ class TestSalesforceApiClient: """ sut = SalesforceApiClient() + expect = { + 'Name': 'テスト取引先名1', + 'CustomItem1__c': 'テスト', + 'CustomItem2__c': 1.0, + 'CustomItem3__c': True, + 'CustomItem4__c': '01:15:00.000Z', + 'CustomItem5__c': '1;2;3;4', + 'CustomItem6__c': '改行ありの\r\nテスト\r\n項目です', + 'CustomItem7__c': '2022-08-04', + 'CustomItem8__c': '2022-08-04T03:00:00.000+0000', + } actual = sut.fetch_sf_data(soql) - print(actual) assert len(actual) >= 1 + # Id, SystemModstamp, LastModifiedDateは自動生成なので、キーの有無だけ確認する + # Attributesも + assert 'Id' in actual[0].keys() + assert 'SystemModstamp' in actual[0].keys() + assert 'LastModifiedDate' in actual[0].keys() + assert 'attributes' in actual[0].keys() + + del actual[0]['Id'] + del actual[0]['SystemModstamp'] + del actual[0]['LastModifiedDate'] + del actual[0]['attributes'] + + assert dict(actual[0]) == expect def test_raise_create_instance_cause_auth_failed(self, monkeypatch): + """ + Cases: + 存在しないユーザを指定した場合、エラーが発生すること + Arranges: + なし + Expects: + ユーザ認証でエラーが発生すること + """ monkeypatch.setattr('src.salesforce.salesforce_api.CRM_USER_NAME', 'invalid_username') with pytest.raises(Exception): SalesforceApiClient() diff --git a/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py index 9a6f9a45..da42d486 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py +++ b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py @@ -6,6 +6,14 @@ from src.salesforce.soql_builder import SOQLBuilder class TestSOQLBuilder: def test_create_count_soql(self): + """ + Cases: + 件数取得のSOQLが生成できること + Arranges: + SOQL生成用のパラメータを用意する + Expects: + 期待値通りのSOQLが生成されること + """ test_target_object_json = { 'object_name': 'Account', 'columns': [ @@ -48,6 +56,14 @@ class TestSOQLBuilder: assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') def test_create_fetch_soql(self): + """ + Cases: + データ取得用のSOQLが生成できること + Arranges: + SOQL生成用のパラメータを用意する + Expects: + 期待値通りのSOQLが生成されること + """ test_target_object_json = { 'object_name': 'Account', 'columns': [ @@ -102,3 +118,43 @@ class TestSOQLBuilder: print('expect', expect) assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') + + def test_create_fetch_soql_no_columns(self): + """ + Cases: + データ取得用のSOQLがカラムがない状態で生成されること + Arranges: + SOQL生成用のパラメータを用意する + Expects: + 取得対象のカラムがないSOQLが生成されること + """ + test_target_object_json = { + 'object_name': 'Account', + 'columns': [] + } + + test_last_fetch_datetime_json = { + 'last_fetch_datetime_from': '1999-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2000-01-01T00:00:00.000Z', + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(test_target_object_json, execute_datetime) + test_last_fetch_datetime = LastFetchDatetime(test_last_fetch_datetime_json, execute_datetime) + + sut = SOQLBuilder(target_object, test_last_fetch_datetime) + actual = sut.create_fetch_soql() + + # TargetObjectのバリデーションで、columnsが空の場合はエラーになるため、本来は発生しない + expect = """SELECT + FROM + Account + WHERE + SystemModstamp > 1999-01-01T00:00:00.000Z AND + SystemModstamp <= 2000-01-01T00:00:00.000Z + """ + + print('actual', actual) + print('expect', expect) + + assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') From c479c3bca316d2431e35ab68b671953f97c1dbd5 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 21:07:36 +0900 Subject: [PATCH 147/275] =?UTF-8?q?feat:=20SOQLBuilder=E3=81=8B=E3=82=89?= =?UTF-8?q?=E7=94=9F=E6=88=90=E3=81=97=E3=81=9FSOQL=E3=81=A7=E3=81=AE?= =?UTF-8?q?=E5=8F=96=E5=BE=97=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesfoce.py | 244 +++++++++++++++++- 1 file changed, 242 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py index 4b3f8ae1..3b58b49d 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -1,5 +1,19 @@ +""" +！！！！注意！！！！ +当テストコードはSalesforceのレコードに依存しています。 +Accountオブジェクトの下記SFIDのレコードはいじらないようにしてください + - 0015i00000LNywwAAD(テスト取引先1) + - 0015i00000LOClSAAX(テスト取引先2) + - 0015i00000LOCmGAAX(テスト取引先3) + +変更してしまった場合は各SOQLの取得日付とデータを修正してください +""" + import pytest +from src.config.objects import LastFetchDatetime, TargetObject from src.salesforce.salesforce_api import SalesforceApiClient +from src.salesforce.soql_builder import SOQLBuilder +from src.util.execute_datetime import ExecuteDateTime class TestSalesforceApiClient: @@ -19,13 +33,99 @@ class TestSalesforceApiClient: Account WHERE SystemModstamp > 2022-08-04T00:00:00.000Z AND - SystemModstamp <= 2022-08-04T16:36:00.000Z + SystemModstamp <= 2022-08-06T00:00:00.000Z """ sut = SalesforceApiClient() actual = sut.fetch_sf_count(soql) assert actual >= 1 + def test_fetch_sf_count_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること + - SystemModStampがFrom指定日付未満のレコードは取得できないこと + - SystemModStampのToが指定日付以上のレコードは取得できること + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromがSystemModstamp未満になるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstampピッタリになるように指定する（UTC指定） + Expects: + 取得件数が1になる + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_count_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_count(soql) + assert actual == 1 + + def test_fetch_sf_count_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること + - SystemModStampのFromが指定日付のより大きいレコードは取得できること + - SystemModStampのToが指定日付未満のレコードは取得できないこと + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定） + Expects: + 取得件数が1になる + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_count_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_count(soql) + assert actual == 1 + def test_fetch_sf_data_one_record(self): """ Cases: @@ -52,7 +152,7 @@ class TestSalesforceApiClient: Account WHERE SystemModstamp > 2022-08-04T00:00:00.000Z AND - SystemModstamp <= 2022-08-04T16:36:00.000Z + SystemModstamp <= 2022-08-06T00:00:00.000Z """ sut = SalesforceApiClient() @@ -69,6 +169,8 @@ class TestSalesforceApiClient: } actual = sut.fetch_sf_data(soql) assert len(actual) >= 1 + # 複数取れるが、アサーション対象は1つだけ + actual = [record for record in actual if record['Name'] == 'テスト取引先名1'] # Id, SystemModstamp, LastModifiedDateは自動生成なので、キーの有無だけ確認する # Attributesも assert 'Id' in actual[0].keys() @@ -83,6 +185,144 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect + def test_fetch_sf_data_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること + - SystemModStampがFrom指定日付未満のレコードは取得できないこと + - SystemModStampのToが指定日付以上のレコードは取得できること + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromがSystemModstamp未満になるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstampピッタリになるように指定する（UTC指定） + Expects: + 取得できたオブジェクト1件が期待値どおりであること + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_fetch_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) == 1 + + # Id, SystemModstamp, LastModifiedDateは自動生成なので、キーの有無だけ確認する + # Attributesも + assert 'Id' in actual[0].keys() + assert 'SystemModstamp' in actual[0].keys() + assert 'LastModifiedDate' in actual[0].keys() + assert 'attributes' in actual[0].keys() + + del actual[0]['Id'] + del actual[0]['SystemModstamp'] + del actual[0]['LastModifiedDate'] + del actual[0]['attributes'] + + expect = { + 'Name': 'テスト取引先名3', + 'CustomItem1__c': 'テスト3', + 'CustomItem2__c': 3.0, + 'CustomItem3__c': True, + 'CustomItem4__c': '00:45:00.000Z', + 'CustomItem5__c': '2;3', + 'CustomItem6__c': 'かいぎょ', + 'CustomItem7__c': '2022-08-06', + 'CustomItem8__c': '2022-08-06T00:00:00.000+0000', + } + + assert dict(actual[0]) == expect + + def test_fetch_sf_data_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること + - SystemModStampのFromが指定日付のより大きいレコードは取得できること + - SystemModStampのToが指定日付未満のレコードは取得できないこと + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定） + Expects: + 取が1になる + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_fetch_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) == 1 + + # Id, SystemModstamp, LastModifiedDateは自動生成なので、キーの有無だけ確認する + # Attributesも + assert 'Id' in actual[0].keys() + assert 'SystemModstamp' in actual[0].keys() + assert 'LastModifiedDate' in actual[0].keys() + assert 'attributes' in actual[0].keys() + + del actual[0]['Id'] + del actual[0]['SystemModstamp'] + del actual[0]['LastModifiedDate'] + del actual[0]['attributes'] + + expect = { + 'Name': 'テスト取引先名2', + 'CustomItem1__c': 'テスト2', + 'CustomItem2__c': 2.0, + 'CustomItem3__c': False, + 'CustomItem4__c': '00:30:00.000Z', + 'CustomItem5__c': '1;4', + 'CustomItem6__c': '改行あり項目', + 'CustomItem7__c': '2022-08-05', + 'CustomItem8__c': '2022-08-04T23:30:00.000+0000', + } + + assert dict(actual[0]) == expect + def test_raise_create_instance_cause_auth_failed(self, monkeypatch): """ Cases: From 03458d851a23ea1091f841f0bfea875e656de6ce Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 21:32:04 +0900 Subject: [PATCH 148/275] =?UTF-8?q?feat:=20=E3=83=AD=E3=82=B0=E3=83=A1?= =?UTF-8?q?=E3=83=83=E3=82=BB=E3=83=BC=E3=82=B8=E7=94=9F=E6=88=90=E3=83=A6?= =?UTF-8?q?=E3=83=BC=E3=83=86=E3=82=A3=E3=83=AA=E3=83=86=E3=82=A3=E9=96=A2?= =?UTF-8?q?=E6=95=B0=E8=BF=BD=E5=8A=A0=EF=BC=88=E4=BB=96=E3=83=96=E3=83=A9?= =?UTF-8?q?=E3=83=B3=E3=83=81=E3=81=8B=E3=82=89Cherry-Pick=E3=81=97?= =?UTF-8?q?=E3=81=9F=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_utils/log_message.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ecs/crm-datafetch/tests/test_utils/log_message.py diff --git a/ecs/crm-datafetch/tests/test_utils/log_message.py b/ecs/crm-datafetch/tests/test_utils/log_message.py new file mode 100644 index 00000000..25ce6e11 --- /dev/null +++ b/ecs/crm-datafetch/tests/test_utils/log_message.py @@ -0,0 +1,7 @@ +"""ログメッセージに関連するテストヘルパー""" + +import logging + + +def generate_log_message_tuple(logger_name='root', log_level=logging.INFO, log_message=''): + return (logger_name, log_level, log_message) From 604a9aa30bf015918c181ace2693c373282148bb Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 22:24:35 +0900 Subject: [PATCH 149/275] =?UTF-8?q?feat:=20=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E5=8F=96=E5=BE=97=E6=BA=96=E5=82=99=E5=87=A6=E7=90=86=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0=E3=80=82?= =?UTF-8?q?=E3=81=BE=E3=81=9A=E3=81=AF1=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_prepare_data_fetch_process.py | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py diff --git a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py new file mode 100644 index 00000000..8eff4b2a --- /dev/null +++ b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py @@ -0,0 +1,94 @@ +import json +import logging +from datetime import datetime + +import pytest +from src.config.objects import FetchTargetObjects +from src.error.exceptions import MeDaCaCRMDataFetchException +from src.prepare_data_fetch_process import prepare_data_fetch_process +from src.system_var.constants import YYYYMMDDTHHMMSSTZ + +from .test_utils.log_message import generate_log_message_tuple + + +class ForTestMeDaCaCRMDataFetchException(MeDaCaCRMDataFetchException): + def __init__(self, error_id: str, func_name: str, message: str) -> None: + super().__init__(error_id, func_name, message) + + +class ForTestException(Exception): + # カスタム例外とインタフェースを合わせるための引数 + def __init__(self, error_id: str, func_name: str, message: str) -> None: + super().__init__(message) + + +class TestPrepareDataFetchProcess: + + @pytest.fixture + def bucket_name(self): + return 'test-config-bucket' + + @pytest.fixture + def prepare_bucket(self, s3_client, bucket_name): + s3_client.create_bucket(Bucket=bucket_name) + yield + + def test_run_process_success(self, s3_client, prepare_bucket, bucket_name, monkeypatch, caplog): + """ + Cases: + データ取得準備処理が正常終了し、期待通りの結果が返ること + Arranges: + - prepare_bucketフィクスチャで、CRM取得オブジェクト情報ファイルを置くためのモックバケットを作る + - 作成したモックバケットを指すように環境変数を設定する + - CRM取得オブジェクト情報ファイルを置く + Expects: + - CRM取得オブジェクトクラスのインスタンスが返却される + - CRM取得オブジェクトクラスのインスタンスが期待値と一致 + - 実行日時クラスのインスタンスが返却される + - 実行日時クラスが、テスト実行後の日時よりも小さい + - 処理結果出力用の空のdictが返却される + - データ取得準備処理の仕様に沿った正常系ログが出力されること(デバッグログは除く) + """ + # Arrange + target_objects_dict = { + 'objects': [ + { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ] + }, + { + 'object_name': 'Contact', + 'columns': [ + 'Id', + 'Name' + ] + } + ] + } + + s3_client.put_object(Bucket=bucket_name, Key='crm/object_info/crm_object_list_diff.json', Body=json.dumps(target_objects_dict)) + # 環境変数を編集 + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', 'crm/object_info') + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'crm_object_list_diff.json') + + # Act + actual_fetch_target_objects, actual_execute_datetime, actual_process_result = \ + prepare_data_fetch_process() + + # Assert + + # 返り値の期待値チェック + expect_fetch_target_objects = FetchTargetObjects(target_objects_dict) + assert isinstance(actual_fetch_target_objects, FetchTargetObjects), 'CRM取得オブジェクトクラスのインスタンスが返却される' + assert all([a == e for a, e in zip(actual_fetch_target_objects, expect_fetch_target_objects)]), 'CRM取得オブジェクトクラスのインスタンスが期待値と一致' + assert datetime.strptime(str(actual_execute_datetime), YYYYMMDDTHHMMSSTZ) < datetime.now(), '実行日時がアサーション時点の日時よりも小さいこと' + assert actual_process_result == {}, '処理結果出力用の空のdictが返却される' + + # ログの確認 + assert generate_log_message_tuple(log_message='I-PRE-01 データ取得準備処理を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-PRE-02 データ取得処理開始日時：{str(actual_execute_datetime)}') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-PRE-09 データ取得準備処理を終了します') in caplog.record_tuples From fe9f6ef42ae73b91b4f7a3e7dda45d602221411b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 22:24:58 +0900 Subject: [PATCH 150/275] =?UTF-8?q?style:=20retrun=E3=81=AE=E9=9A=A3?= =?UTF-8?q?=E3=81=AB=E3=82=B9=E3=83=9A=E3=83=BC=E3=82=B9=E3=82=92=E6=8C=BF?= =?UTF-8?q?=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/prepare_data_fetch_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index 3488c71f..03427265 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -79,4 +79,4 @@ def prepare_data_fetch_process(): logger.info('I-PRE-09 データ取得準備処理を終了します') # ⑧ 次の処理へ移行する - return(fetch_target_objects, execute_datetime, process_result) + return (fetch_target_objects, execute_datetime, process_result) From a4c500331a7e5047753d257f4c2c61bfa5900a6d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 23:13:17 +0900 Subject: [PATCH 151/275] =?UTF-8?q?feat:=20=E4=BE=9D=E5=AD=98=E3=83=A2?= =?UTF-8?q?=E3=82=B8=E3=83=A5=E3=83=BC=E3=83=AB=E3=81=8C=E5=91=BC=E3=81=B0?= =?UTF-8?q?=E3=82=8C=E3=81=A6=E3=81=84=E3=82=8B=E3=81=8B=E3=81=A9=E3=81=86?= =?UTF-8?q?=E3=81=8B=E3=82=92=E7=A2=BA=E8=AA=8D=E3=81=99=E3=82=8B=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E8=BF=BD=E5=8A=A0=E3=80=81json=5Fparse=20?= =?UTF-8?q?=E2=86=92=20json=5Fparser?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../parser/{json_parse.py => json_parser.py} | 0 .../src/prepare_data_fetch_process.py | 2 +- .../src/set_datetime_period_process.py | 2 +- .../tests/test_prepare_data_fetch_process.py | 36 +++++++++++++++++++ 4 files changed, 38 insertions(+), 2 deletions(-) rename ecs/crm-datafetch/src/parser/{json_parse.py => json_parser.py} (100%) diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parser.py similarity index 100% rename from ecs/crm-datafetch/src/parser/json_parse.py rename to ecs/crm-datafetch/src/parser/json_parser.py diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index 03427265..f0ea9ab6 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -1,7 +1,7 @@ from src.aws.s3 import ConfigBucket from src.config.objects import FetchTargetObjects from src.error.exceptions import FileNotFoundException, InvalidConfigException -from src.parser.json_parse import JsonParser +from src.parser.json_parser import JsonParser from src.system_var.constants import PRE_JP_NAME from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, diff --git a/ecs/crm-datafetch/src/set_datetime_period_process.py b/ecs/crm-datafetch/src/set_datetime_period_process.py index bfebd5c2..062fac97 100644 --- a/ecs/crm-datafetch/src/set_datetime_period_process.py +++ b/ecs/crm-datafetch/src/set_datetime_period_process.py @@ -1,7 +1,7 @@ from src.aws.s3 import ConfigBucket from src.config.objects import LastFetchDatetime, TargetObject from src.error.exceptions import FileNotFoundException, InvalidConfigException -from src.parser.json_parse import JsonParser +from src.parser.json_parser import JsonParser from src.system_var.constants import DATE_JP_NAME from src.system_var.environments import (CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER) diff --git a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py index 8eff4b2a..298eb192 100644 --- a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py @@ -1,6 +1,7 @@ import json import logging from datetime import datetime +from unittest.mock import MagicMock, patch import pytest from src.config.objects import FetchTargetObjects @@ -92,3 +93,38 @@ class TestPrepareDataFetchProcess: assert generate_log_message_tuple(log_message='I-PRE-01 データ取得準備処理を開始します') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-PRE-02 データ取得処理開始日時：{str(actual_execute_datetime)}') in caplog.record_tuples assert generate_log_message_tuple(log_message='I-PRE-09 データ取得準備処理を終了します') in caplog.record_tuples + + def mock_get_object_info_file(): + return {} + + def mock_parse(): + return {} + + def mock_constructor(): + pass + + def test_call_depended_modules(self, s3_client, prepare_bucket, bucket_name, monkeypatch, caplog): + """ + Cases: + データ取得準備処理内で依存しているモジュールが正しく呼ばれていること + Arranges: + - データ取得準備処理の依存モジュールをモック化する + Expects: + - 依存しているモジュールが正しく呼ばれている + """ + + # Arrange + mock_config_bucket = MagicMock(return_value='') + mock_json_parser = MagicMock(return_value={}) + mock_fetch_target_objects = MagicMock(return_value=None) + # Act + with patch('src.aws.s3.ConfigBucket.get_object_info_file', mock_config_bucket), \ + patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ + patch('src.config.objects.FetchTargetObjects.__init__', mock_fetch_target_objects): + prepare_data_fetch_process() + + # Assert + + assert mock_config_bucket.called is True + assert mock_json_parser.called is True + assert mock_fetch_target_objects.called is True From 37f0c4f1f14a470088e33b8847cda6ceb5936fb3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 23:14:08 +0900 Subject: [PATCH 152/275] =?UTF-8?q?feat:=20=E3=81=84=E3=82=89=E3=81=AA?= =?UTF-8?q?=E3=81=84=E3=83=95=E3=82=A3=E3=82=AF=E3=82=B9=E3=83=81=E3=83=A3?= =?UTF-8?q?=E5=BC=95=E6=95=B0=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_prepare_data_fetch_process.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py index 298eb192..e5896058 100644 --- a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py @@ -94,16 +94,7 @@ class TestPrepareDataFetchProcess: assert generate_log_message_tuple(log_message=f'I-PRE-02 データ取得処理開始日時：{str(actual_execute_datetime)}') in caplog.record_tuples assert generate_log_message_tuple(log_message='I-PRE-09 データ取得準備処理を終了します') in caplog.record_tuples - def mock_get_object_info_file(): - return {} - - def mock_parse(): - return {} - - def mock_constructor(): - pass - - def test_call_depended_modules(self, s3_client, prepare_bucket, bucket_name, monkeypatch, caplog): + def test_call_depended_modules(self): """ Cases: データ取得準備処理内で依存しているモジュールが正しく呼ばれていること From dfcd17d78ad496cae232c21a762c8149e76ddf6a Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Fri, 5 Aug 2022 23:29:19 +0900 Subject: [PATCH 153/275] =?UTF-8?q?feat:=20objects=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/config/objects.py | 19 +- ecs/crm-datafetch/src/system_var/constants.py | 2 + ecs/crm-datafetch/src/util/dict_checker.py | 8 + .../tests/config/test_objects.py | 2506 +++++++++++++++++ 4 files changed, 2528 insertions(+), 7 deletions(-) create mode 100644 ecs/crm-datafetch/tests/config/test_objects.py diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 41ca753b..2979e0f0 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -9,7 +9,9 @@ from src.system_var.constants import (COLUMNS_KEY, COLUMNS_TYPE, LAST_FETCH_DATETIME_FILE_NAME_KEY, LAST_FETCH_DATETIME_FILE_NAME_TYPE, LAST_FETCH_DATETIME_FROM_KEY, + LAST_FETCH_DATETIME_FROM_TYPE, LAST_FETCH_DATETIME_TO_KEY, + LAST_FETCH_DATETIME_TO_TYPE, OBJECT_NAME_KEY, OBJECT_NAME_TYPE, OBJECTS_KEY, OBJECTS_TYPE, UPLOAD_FILE_NAME_KEY, @@ -22,7 +24,7 @@ class FetchTargetObjects(): def __init__(self, object_info_file_dict) -> None: self.__objects = object_info_file_dict self.__dict_checker = DictChecker(self.__objects) - self.validate() + self.__validate() self.__i = 0 def __iter__(self): @@ -35,7 +37,7 @@ class FetchTargetObjects(): self.__i += 1 return value - def validate(self) -> None: + def __validate(self) -> None: self.__dict_checker.assert_key_exist(OBJECTS_KEY) self.__dict_checker.assert_data_type(OBJECTS_KEY, OBJECTS_TYPE) @@ -58,6 +60,7 @@ class TargetObject(): self.__dict_checker.assert_data_type(OBJECT_NAME_KEY, OBJECT_NAME_TYPE) self.__dict_checker.assert_key_exist(COLUMNS_KEY) self.__dict_checker.assert_data_type(COLUMNS_KEY, COLUMNS_TYPE) + self.__dict_checker.assert_list_empty(COLUMNS_KEY) return @@ -95,7 +98,7 @@ class TargetObject(): def is_update_last_fetch_datetime(self) -> bool: if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY): return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] - return False + return True @property def last_fetch_datetime_file_name(self) -> str: @@ -107,7 +110,7 @@ class TargetObject(): def upload_file_name(self) -> str: if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime.format_date()) - return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.format_date()}' + return f'CRM_{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.format_date()}' @property def datetime_column(self) -> str: @@ -122,9 +125,11 @@ class LastFetchDatetime(): self.__validate() def __validate(self) -> None: - if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FROM_KEY): - self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) + self.__dict_checker.assert_key_exist(LAST_FETCH_DATETIME_FROM_KEY) + self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_FROM_KEY, LAST_FETCH_DATETIME_FROM_TYPE) + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): + self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_TO_KEY, LAST_FETCH_DATETIME_TO_TYPE) self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) return @@ -136,4 +141,4 @@ class LastFetchDatetime(): def last_fetch_datetime_to(self) -> str: if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY] - return self.__execute_datetime + return str(self.__execute_datetime) diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index bf65010c..0214e5ac 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -93,4 +93,6 @@ DATETIME_COLUMN_KEY = 'datetime_column' DATETIME_COLUMN_TYPE = str DATETIME_COLUMN_DEFAULT_VALUE = 'SystemModstamp' LAST_FETCH_DATETIME_TO_KEY = 'last_fetch_datetime_to' +LAST_FETCH_DATETIME_TO_TYPE = str LAST_FETCH_DATETIME_FROM_KEY = 'last_fetch_datetime_from' +LAST_FETCH_DATETIME_FROM_TYPE = str diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py index 39ebf9f0..52c4f1ae 100644 --- a/ecs/crm-datafetch/src/util/dict_checker.py +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -9,6 +9,10 @@ class DictChecker: """辞書型バリュー空文字チェック""" return self.__object_dict[check_key] != '' and self.__object_dict[check_key] is not None + def is_list_empty(self, check_key): + """list型データ存在チェック""" + return len(self.__object_dict[check_key]) != 0 + def check_key_exist(self, check_key: str) -> bool: """辞書型キー存在チェック""" return check_key in self.__object_dict and self.is_empty(check_key) @@ -41,3 +45,7 @@ class DictChecker: raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') return + + def assert_list_empty(self, check_key: str): + if not self.is_list_empty(check_key): + raise Exception(f'「{check_key}」キーのリストの値は必須です') diff --git a/ecs/crm-datafetch/tests/config/test_objects.py b/ecs/crm-datafetch/tests/config/test_objects.py new file mode 100644 index 00000000..51c31981 --- /dev/null +++ b/ecs/crm-datafetch/tests/config/test_objects.py @@ -0,0 +1,2506 @@ +import pytest +from src.config.objects import (FetchTargetObjects, LastFetchDatetime, + TargetObject) +from src.parser.json_parse import JsonParser +from src.util.execute_datetime import ExecuteDateTime + + +class TestFetchTargetObjects(): + + def test_constructor(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キーがあるかまた、キーの型が正しいかをチェック + Arranges: + - オブジェクト情報文字列を準備する + - オブジェクト情報を辞書型にパースする + Expects: + - 例外が発生しないこと + """ + + # Arranges + fetch_objects = '''{ + "objects": [ + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true, + "datetime_column": "LastModifiedDate" + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + } + ] + }''' + + json_parser = JsonParser(fetch_objects) + fetch_objects_dict = json_parser.parse() + + # Act + FetchTargetObjects(fetch_objects_dict) + + # Expects + None + + def test_constructor_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、必要なキーがない場合、例外が発生すること + Arranges: + - オブジェクト情報を辞書型にパースする + Expects: + - 例外が発生し期待値と一致する + """ + + fetch_objects = '''{ + "test_objects": [ + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true, + "datetime_column": "LastModifiedDate" + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + } + ] + }''' + + # Arranges + json_parser = JsonParser(fetch_objects) + fetch_objects_dict = json_parser.parse() + + # Act + with pytest.raises(Exception) as e: + FetchTargetObjects(fetch_objects_dict) + + # Expects + assert str(e.value) == '「objects」キーは必須です' + + def test_constructor_no_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキーの値の型が想定と違う場合、例外が発生すること + Arranges: + - オブジェクト情報を辞書型にパースする + Expects: + - 例外が発生し期待値と一致する + """ + + fetch_objects = '''{ + "objects": "test_value" + }''' + + # Arranges + json_parser = JsonParser(fetch_objects) + fetch_objects_dict = json_parser.parse() + + # Act + with pytest.raises(Exception) as e: + FetchTargetObjects(fetch_objects_dict) + + # Expects + assert str(e.value) == '「objects」キーの値は「」でなければなりません' + + +class TestTargetObject(): + + def test_constructor(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、バリデーションチェックを行いチェックが通ることを確認する + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + # object_name + + def test_raise_constructor_object_name_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、必要なキー(object_name)がない場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「object_name」キーは必須です' + + def test_raise_constructor_object_name_no_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(object_name)の値が空文字の場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「object_name」キーは必須です' + + def test_raise_constructor_object_name_none_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(object_name)の値がNoneの場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": None, + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「object_name」キーは必須です' + + def test_raise_constructor_object_name_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(object_name)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": 1, + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「object_name」キーの値は「」でなければなりません' + + # columns + + def test_raise_constructor_columns_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、必要なキー(columns)がない場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「columns」キーは必須です' + + def test_raise_constructor_columns_no_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(columns)の値が空文字の場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": "", + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「columns」キーは必須です' + + def test_raise_constructor_columns_none_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(columns)の値がNoneの場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": None, + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「columns」キーは必須です' + + def test_raise_constructor_columns_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(columns)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": False, + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「columns」キーの値は「」でなければなりません' + + def test_raise_constructor_columns_no_value_list(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(columns)の値がリスト型で空の場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「columns」キーのリストの値は必須です' + + # is_skip + + def test_raise_constructor_is_skip_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(is_skip)がない場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_is_skip_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(is_skip)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": "", + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_is_skip_none_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(is_skip)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": None, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_raise_constructor_is_skip_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(is_skip)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": "False", + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「is_skip」キーの値は「」でなければなりません' + + # is_update_last_fetch_datetime + + def test_raise_constructor_is_update_last_fetch_datetime_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(is_update_last_fetch_datetime)がない場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_is_update_last_fetch_datetime_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(is_update_last_fetch_datetime)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": "", + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_is_update_last_fetch_datetime_none_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(is_update_last_fetch_datetime)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": None, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_raise_constructor_is_update_last_fetch_datetime_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(is_update_last_fetch_datetime)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": "False", + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「is_update_last_fetch_datetime」キーの値は「」でなければなりません' + + # last_fetch_datetime_file_name + + def test_raise_constructor_last_fetch_datetime_file_name_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(last_fetch_datetime_file_name)がない場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_last_fetch_datetime_file_name_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(last_fetch_datetime_file_name)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_last_fetch_datetime_file_name_none_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(last_fetch_datetime_file_name)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": None, + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_raise_constructor_last_fetch_datetime_file_name_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_file_name)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": 1, + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_file_name」キーの値は「」でなければなりません' + + # upload_file_name + + def test_raise_constructor_upload_file_name_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(upload_file_name)がない場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_upload_file_name_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(upload_file_name)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_upload_file_name_none_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(upload_file_name)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": None, + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_raise_constructor_upload_file_name_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(upload_file_name)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": 1, + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「upload_file_name」キーの値は「」でなければなりません' + + # datetime_column + + def test_raise_constructor_datetime_column_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(datetime_column)がない場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_datetime_column_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(datetime_column)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_datetime_column_none_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(datetime_column)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": None + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_raise_constructor_datetime_column_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(datetime_column)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": 1 + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「datetime_column」キーの値は「」でなければなりません' + + # property + + def test_object_name(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.object_name + + # Expects + assert actual == 'AccountShare' + + def test_columns(self) -> list: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.columns + + # Expects + expected_value = [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ] + assert actual == expected_value + + def test_is_skip(self) -> bool: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.is_skip + + # Expects + assert actual is True + + def test_is_skip_default(self) -> bool: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": "", + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.is_skip + + # Expects + assert actual is False + + def test_is_update_last_fetch_datetime(self) -> bool: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.is_update_last_fetch_datetime + + # Expects + assert actual is False + + def test_is_update_last_fetch_datetime_default(self) -> bool: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": "", + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.is_update_last_fetch_datetime + + # Expects + assert actual is True + + def test_last_fetch_datetime_file_name(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare_Test.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_file_name + + # Expects + assert actual == 'AccountShare_Test.json' + + def test_last_fetch_datetime_file_name_default(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "", + "upload_file_name": "CRM_AccountShare", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_file_name + + # Expects + assert actual == 'AccountShare.json' + + def test_upload_file_name(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare_Test.json", + "upload_file_name": "CRM_AccountShare_Test_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.upload_file_name + + # Expects + assert actual == f'CRM_AccountShare_Test_{execute_datetime.format_date()}' + + def test_upload_file_name_default(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare_Test.json", + "upload_file_name": "", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.upload_file_name + + # Expects + assert actual == f'CRM_AccountShare_{execute_datetime.format_date()}' + + def test_datetime_column(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare_Test.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.datetime_column + + # Expects + assert actual == 'LastModifiedDate' + + def test_datetime_column_default(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare_Test.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.datetime_column + + # Expects + assert actual == 'SystemModstamp' + + +class TestLastFetchDatetime(): + def test_constructor(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、バリデーションチェックを行いチェックが通ることを確認する + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + None + + def test_raise_constructor_last_fetch_datetime_from_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、必要なキー(last_fetch_datetime_from)がない場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' + + def test_raise_constructor_last_fetch_datetime_from_no_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値が空文字の場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' + + def test_raise_constructor_last_fetch_datetime_from_none_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値がNoneの場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": None, + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' + + def test_raise_constructor_last_fetch_datetime_from_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": 1, + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーの値は「」でなければなりません' + + def test_raise_constructor_last_fetch_datetime_from_other_string(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値が正規表現と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "aaa", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str( + e.value) == '「last_fetch_datetime_from」キーの値の正規表現「[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z」チェックに失敗しました' + + def test_raise_constructor_last_fetch_datetime_to_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(last_fetch_datetime_to)がない場合、例外が発生しないこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + None + + def test_constructor_last_fetch_datetime_to_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + None + + def test_constructor_last_fetch_datetime_to_none__value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": None + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + None + + def test_raise_constructor_last_fetch_datetime_to_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": 1 + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_to」キーの値は「」でなければなりません' + + def test_raise_constructor_last_fetch_datetime_to_other_string(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値が正規表現と違う場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "aaa" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str( + e.value) == '「last_fetch_datetime_to」キーの値の正規表現「[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z」チェックに失敗しました' + + def test_last_fetch_datetime_from(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + - 前回取得日時インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_from + + # Expects + assert actual == '1900-01-01T00:00:00.000Z' + + def test_last_fetch_datetime_to(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + - 前回取得日時インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_to + + # Expects + assert actual == '2022-01-01T00:00:00.000Z' + + def test_last_fetch_datetime_to_default(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + - 前回取得日時インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" + } + + execute_datetime = ExecuteDateTime() + + sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_to + + # Expects + assert actual == str(execute_datetime) From a298e322049aed6317e0470e14bb6311d566f737 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 23:56:24 +0900 Subject: [PATCH 154/275] =?UTF-8?q?feat:=20=E4=BE=8B=E5=A4=96=E7=B3=BB?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=B1=E3=83=BC=E3=82=B9=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_prepare_data_fetch_process.py | 115 +++++++++++++++--- 1 file changed, 99 insertions(+), 16 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py index e5896058..9471946f 100644 --- a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py @@ -1,28 +1,16 @@ import json -import logging from datetime import datetime from unittest.mock import MagicMock, patch import pytest from src.config.objects import FetchTargetObjects -from src.error.exceptions import MeDaCaCRMDataFetchException +from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.prepare_data_fetch_process import prepare_data_fetch_process -from src.system_var.constants import YYYYMMDDTHHMMSSTZ +from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ from .test_utils.log_message import generate_log_message_tuple -class ForTestMeDaCaCRMDataFetchException(MeDaCaCRMDataFetchException): - def __init__(self, error_id: str, func_name: str, message: str) -> None: - super().__init__(error_id, func_name, message) - - -class ForTestException(Exception): - # カスタム例外とインタフェースを合わせるための引数 - def __init__(self, error_id: str, func_name: str, message: str) -> None: - super().__init__(message) - - class TestPrepareDataFetchProcess: @pytest.fixture @@ -39,9 +27,9 @@ class TestPrepareDataFetchProcess: Cases: データ取得準備処理が正常終了し、期待通りの結果が返ること Arranges: - - prepare_bucketフィクスチャで、CRM取得オブジェクト情報ファイルを置くためのモックバケットを作る + - prepare_bucketフィクスチャで、CRM_取得オブジェクト情報ファイルを置くためのモックバケットを作る - 作成したモックバケットを指すように環境変数を設定する - - CRM取得オブジェクト情報ファイルを置く + - CRM_取得オブジェクト情報ファイルを置く Expects: - CRM取得オブジェクトクラスのインスタンスが返却される - CRM取得オブジェクトクラスのインスタンスが期待値と一致 @@ -119,3 +107,98 @@ class TestPrepareDataFetchProcess: assert mock_config_bucket.called is True assert mock_json_parser.called is True assert mock_fetch_target_objects.called is True + + def test_raise_get_object_info(self, bucket_name, monkeypatch, caplog): + """ + Cases: + CRM_取得オブジェクト情報ファイルを取得できない場合、エラーが発生すること + Arranges: + - オブジェクト情報ファイル取得処理で例外が発生するようにする + Expects: + - 例外が発生する + - ファイルが読み込めないエラーが返却される + """ + + # Arrange + monkeypatch.setattr('src.prepare_data_fetch_process.OBJECT_INFO_FILENAME', 'crm_object_list_diff.json') + mock_config_bucket = MagicMock(side_effect=Exception('ファイル取得エラー')) + mock_json_parser = MagicMock(return_value={}) + mock_fetch_target_objects = MagicMock(return_value=None) + # Act + with patch('src.aws.s3.ConfigBucket.get_object_info_file', mock_config_bucket), \ + patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ + patch('src.config.objects.FetchTargetObjects.__init__', mock_fetch_target_objects): + with pytest.raises(FileNotFoundException) as e: + prepare_data_fetch_process() + + # Assert + + assert mock_config_bucket.called is True + assert mock_json_parser.called is False + assert mock_fetch_target_objects.called is False + assert e.value.error_id == 'E-PRE-01' + assert e.value.func_name == PRE_JP_NAME + assert e.value.args[0] == f'CRM_取得オブジェクト情報ファイルが存在しません ファイル名：[crm_object_list_diff.json] エラー内容：[ファイル取得エラー]' + + def test_raise_parse_json_str(self, bucket_name, monkeypatch, caplog): + """ + Cases: + CRM_取得オブジェクト情報ファイルをパースできない場合、エラーが発生すること + Arranges: + - オブジェクト情報ファイルパース処理で例外が発生するようにする + Expects: + - 例外が発生する + - パースが失敗した旨、エラーメッセージが表示される + """ + + # Arrange + mock_config_bucket = MagicMock(return_value='') + mock_json_parser = MagicMock(side_effect=Exception('JSONパースエラー')) + mock_fetch_target_objects = MagicMock(return_value=None) + # Act + with patch('src.aws.s3.ConfigBucket.get_object_info_file', mock_config_bucket), \ + patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ + patch('src.config.objects.FetchTargetObjects.__init__', mock_fetch_target_objects): + with pytest.raises(InvalidConfigException) as e: + prepare_data_fetch_process() + + # Assert + + assert mock_config_bucket.called is True + assert mock_json_parser.called is True + assert mock_fetch_target_objects.called is False + assert e.value.error_id == 'E-PRE-02' + assert e.value.func_name == PRE_JP_NAME + assert e.value.args[0] == f'CRM_取得オブジェクト情報ファイルのパースに失敗しました エラー内容：[JSONパースエラー]' + + def test_raise_check_objects_format(self, bucket_name, monkeypatch, caplog): + """ + Cases: + CRM_取得オブジェクト情報のキーobjectsの形式チェックが不正の場合、エラーが発生すること + Arranges: + - CRM_取得オブジェクト情報のキーobjectsの形式チェックで例外が発生するようにする + Expects: + - 例外が発生する + - 形式チェックが失敗した旨、エラーメッセージが表示される + """ + + # Arrange + monkeypatch.setattr('src.prepare_data_fetch_process.OBJECT_INFO_FILENAME', 'crm_object_list_diff.json') + mock_config_bucket = MagicMock(return_value='') + mock_json_parser = MagicMock(return_value={}) + mock_fetch_target_objects = MagicMock(side_effect=Exception('形式チェックエラー')) + # Act + with patch('src.aws.s3.ConfigBucket.get_object_info_file', mock_config_bucket), \ + patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ + patch('src.config.objects.FetchTargetObjects.__init__', mock_fetch_target_objects): + with pytest.raises(InvalidConfigException) as e: + prepare_data_fetch_process() + + # Assert + + assert mock_config_bucket.called is True + assert mock_json_parser.called is True + assert mock_fetch_target_objects.called is True + assert e.value.error_id == 'E-PRE-03' + assert e.value.func_name == PRE_JP_NAME + assert e.value.args[0] == 'CRM_取得オブジェクト情報ファイルの形式チェックに失敗しました ファイル名：[crm_object_list_diff.json] エラー内容：[形式チェックエラー]' From 3702a43c9dda36ad446d36881f83756f91116830 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Fri, 5 Aug 2022 23:58:09 +0900 Subject: [PATCH 155/275] =?UTF-8?q?refactor:=20=E7=99=BB=E9=8C=B2=E3=81=95?= =?UTF-8?q?=E3=82=8C=E3=81=A6=E3=81=97=E3=81=BE=E3=81=A3=E3=81=9F=E4=B8=8D?= =?UTF-8?q?=E8=A6=81=E3=81=AA=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=92?= =?UTF-8?q?=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/util/test_counter_object.py | 114 ------------------ 1 file changed, 114 deletions(-) delete mode 100644 ecs/crm-datafetch/tests/util/test_counter_object.py diff --git a/ecs/crm-datafetch/tests/util/test_counter_object.py b/ecs/crm-datafetch/tests/util/test_counter_object.py deleted file mode 100644 index 8bf677a0..00000000 --- a/ecs/crm-datafetch/tests/util/test_counter_object.py +++ /dev/null @@ -1,114 +0,0 @@ -import pytest -from src.util.counter_object import CounterObject - - -class TestCounterObject: - - def test_describe(self) -> int: - """ - Cases: - カウンターオブジェクトにて保持した値を返すこと - Arranges: - なし - Expects: - 問い合わせた値が期待値と一致する - """ - - # Act - sut = CounterObject() - actual = sut.describe() - - # Expects - assert actual == 1 - - def test_raise_describe(self) -> int: - """ - Cases: - カウンターオブジェクトの保持した値を問い合わせる際、引数を渡すと例外が発生すること - Arranges: - なし - Expects: - 問い合わせた値が期待値と一致する - """ - - # Act - with pytest.raises(Exception) as e: - sut = CounterObject() - sut.describe(1) - - # Expects - assert str(e.value) == 'describe() takes 1 positional argument but 2 were given' - - def test_increment(self) -> int: - """ - Cases: - カウンターオブジェクトにて保持した値がインクリメントされていること - Arranges: - なし - Expects: - 戻り値が期待値と一致する - """ - - # Act - sut = CounterObject(5) - sut.increment() - actual = sut.increment() - - # Expects - assert actual == 7 - - def test_raise_increment(self) -> int: - """ - Cases: - 文字列を引数で渡すことで、例外が発生すること - Arranges: - なし - Expects: - 発生した例外が期待値と一致する - """ - - # Act - with pytest.raises(Exception) as e: - sut = CounterObject(5) - sut.increment('aaa') - sut.increment('aaa') - - # Expects - assert str(e.value) == "unsupported operand type(s) for +=: 'int' and 'str'" - - def test_decrement(self) -> int: - """ - Cases: - カウンターオブジェクトにて保持した値がデクリメントされていること - Arranges: - なし - Expects: - 戻り値が期待値と一致する - """ - - # Act - sut = CounterObject(5) - sut.decrement(2) - actual = sut.decrement(2) - - # Expects - assert actual == 1 - - def test_raise_decrement(self) -> int: - """ - Cases: - 文字列を引数で渡すことで、例外が発生すること - Arranges: - なし - Expects: - 発生した例外が期待値と一致する - """ - - # Act - with pytest.raises(Exception) as e: - sut = CounterObject(5) - sut.decrement('aaa') - sut.decrement('aaa') - - # Expects - assert str(e.value) == "unsupported operand type(s) for -=: 'int' and 'str'" From 4e215335f7fea1f19a4b7d92b2ad4d1538786d5a Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Sat, 6 Aug 2022 00:13:54 +0900 Subject: [PATCH 156/275] =?UTF-8?q?feat:=20=E4=BB=96PR=E3=81=AE=E3=83=95?= =?UTF-8?q?=E3=82=A3=E3=83=BC=E3=83=89=E3=83=90=E3=83=83=E3=82=AF=20S3?= =?UTF-8?q?=E3=81=AE=E4=BE=8B=E5=A4=96=E3=83=86=E3=82=B9=E3=83=88=E3=82=92?= =?UTF-8?q?=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/aws/test_s3.py | 191 ++++++++++++++++++++++++- 1 file changed, 189 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/tests/aws/test_s3.py b/ecs/crm-datafetch/tests/aws/test_s3.py index 1daab1e7..943b1e56 100644 --- a/ecs/crm-datafetch/tests/aws/test_s3.py +++ b/ecs/crm-datafetch/tests/aws/test_s3.py @@ -35,6 +35,22 @@ class TestS3Resource: # Assert assert actual == 'aaaaaaaaaaaaaaa' + def test_raise_get_object(self, s3_test, s3_client, bucket_name): + """ + Cases: + - オブジェクトのキーが存在しない場合エラーになること + Arranges: + - S3をモック化する + Expects: + - オブジェクト取得時にエラーになること + """ + # Arrange + # Act + sut = S3Resource(bucket_name) + with pytest.raises(Exception): + # Assert + sut.get_object('hogehoge/test.txt') + def test_put_object(self, s3_test, s3_client, bucket_name): """ Cases: @@ -52,6 +68,23 @@ class TestS3Resource: assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' + def test_raise_put_object(self, s3_client, bucket_name): + """ + Cases: + - バケットが存在しない場合エラーになること + Arranges: + - S3をモック化する + - バケットを作成しない + Expects: + - オブジェクト登録時にエラーになること + """ + # Arrange + # Act + sut = S3Resource(bucket_name) + with pytest.raises(Exception): + # Assert + sut.put_object('hogehoge/test.txt', 'aaaaaaaaaaaaaaa') + def test_copy(self, s3_test, s3_client, bucket_name): """ Cases: @@ -74,7 +107,23 @@ class TestS3Resource: actual = s3_client.get_object(Bucket=for_copy_bucket, Key='test.txt') assert actual['Body'].read() == b'aaaaaaaaaaaaaaa' - def test_init_raise_no_provide_bucket_name(self): + def test_raise_copy(self, s3_client, s3_test, bucket_name): + """ + Cases: + - コピー対象のキーが存在しない場合エラーになること + Arranges: + - S3をモック化する + Expects: + - オブジェクト登録時にエラーになること + """ + # Arrange + # Act + sut = S3Resource(bucket_name) + with pytest.raises(Exception): + # Assert + sut.copy(bucket_name, 'hogehoge/test.txt', 'for_copy_bucket', 'test.txt') + + def test_raise_init_no_provide_bucket_name(self): """ Cases: - バケット名を指定しない場合、例外となること @@ -95,6 +144,7 @@ class TestConfigBucket: Cases: - オブジェクト情報ファイルが取得できること Arranges: + - 環境変数を置き換える - オブジェクト情報ファイルを配置する Expects: - オブジェクト情報ファイルが文字列として取得でき、期待値と一致する @@ -109,6 +159,24 @@ class TestConfigBucket: assert actual == 'aaaaaaaaaaaaaaa' + def test_raise_get_object_info_file(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + - オブジェクト情報ファイルが存在しない場合、エラーになること + Arranges: + - s3_testフィクスチャを引数に指定しない + - 環境変数を置き換える + Expects: + - オブジェクト情報ファイルが取得できず、例外が発生する + """ + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', 'crm') + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'objects.json') + + sut = ConfigBucket() + with pytest.raises(Exception): + sut.get_object_info_file() + def test_get_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: @@ -128,6 +196,23 @@ class TestConfigBucket: assert actual == 'aaaaaaaaaaaaaaa' + def test_raise_get_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): + """ + Cases: + - オブジェクト最終更新日時ファイルが存在しない場合、エラーになること + Arranges: + - s3_testフィクスチャを引数に指定しない + - 環境変数を置き換える + Expects: + - オブジェクト最終更新日時ファイルが取得できず、例外が発生する + """ + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') + + sut = ConfigBucket() + with pytest.raises(Exception): + sut.get_last_fetch_datetime_file('Object.json') + def test_put_last_fetch_datetime_file(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: @@ -147,6 +232,23 @@ class TestConfigBucket: actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/Object.json') assert actual['Body'].read().decode('utf-8') == 'aaaaaaaaaaaaaaa' + def test_raise_put_last_fetch_datetime_file(self, s3_client, bucket_name, monkeypatch): + """ + Cases: + - オブジェクト最終更新日時ファイルをPUTできない場合、エラーになること + Arranges: + - s3_testフィクスチャを引数に指定しない + - 環境変数を置き換える + Expects: + - オブジェクト最終更新日時ファイルをPUTできず、例外が発生する + """ + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm') + + sut = ConfigBucket() + with pytest.raises(Exception): + sut.put_last_fetch_datetime_file('Object.json', 'aaaaaaaaaaaaaaa') + def test_config_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: @@ -184,6 +286,23 @@ class TestDataBucket: actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/target/test.csv') assert actual['Body'].read().decode('utf-8') == 'test,test,test' + def test_raise_put_csv(self, s3_client, bucket_name, monkeypatch): + """ + Cases: + - CSVファイルをPUTできず、エラーになること + Arranges: + - s3_testフィクスチャを引数に指定しない + - 環境変数を置き換える + Expects: + - CSVファイルをPUTできず、例外が発生する + """ + monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', 'crm/target') + + sut = DataBucket() + with pytest.raises(Exception): + sut.put_csv('test.csv', 'test,test,test') + def test_put_csv_from(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: @@ -209,6 +328,23 @@ class TestDataBucket: assert actual['Body'].read().decode('utf-8') == 'test,test,test' + def test_raise_put_csv_from(self, s3_client, bucket_name, monkeypatch): + """ + Cases: + - 他のバケットからCSVファイルをコピーできず、エラーになること + Arranges: + - s3_testフィクスチャを引数に指定しない + - 環境変数を置き換える + Expects: + - コピーできず、例外が発生する + """ + monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', 'for_copy_bucket') + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', 'crm/target') + + sut = DataBucket() + with pytest.raises(Exception): + sut.put_csv_from(bucket_name, 'hogehoge/test.csv') + def test_data_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: @@ -246,6 +382,23 @@ class TestBackupBucket: actual = s3_client.get_object(Bucket=bucket_name, Key=f'data_import/test.csv') assert actual['Body'].read().decode('utf-8') == 'test,test,test' + def test_raise_put_csv_from(self, s3_client, bucket_name, monkeypatch): + """ + Cases: + - CSVファイルをPUTできず、エラーになること + Arranges: + - s3_testフィクスチャを引数に指定しない + - 環境変数を置き換える + Expects: + - CSVファイルがPUTできず、例外が発生する + """ + monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', 'crm/target') + + sut = BackupBucket() + with pytest.raises(Exception): + sut.put_csv('test.csv', 'test,test,test') + def test_put_response_json(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: @@ -265,10 +418,27 @@ class TestBackupBucket: actual = s3_client.get_object(Bucket=bucket_name, Key=f'response_json/test.json') assert actual['Body'].read().decode('utf-8') == '{"test": "test"}' + def test_raise_response_json(self, s3_client, bucket_name, monkeypatch): + """ + Cases: + - JSONファイルをPUTできず、エラーになること + Arranges: + - s3_testフィクスチャを引数に指定しない + - 環境変数を置き換える + Expects: + - JSONファイルをPUTできず、例外が発生する + """ + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', 'response_json') + + sut = BackupBucket() + with pytest.raises(Exception): + sut.put_response_json('test.json', {"test": "test"}) + def test_put_result_json(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: - - JSONファイルをPUTできること + - 結果のJSONファイルをPUTできること Arranges: - S3をモック化する - 環境変数をテスト用の値に置き換える @@ -284,6 +454,23 @@ class TestBackupBucket: actual = s3_client.get_object(Bucket=bucket_name, Key=f'data_import/result.json') assert actual['Body'].read().decode('utf-8') == '{"test": "test"}' + def test_raise_result_json(self, s3_client, bucket_name, monkeypatch): + """ + Cases: + - 結果のJSONファイルをPUTできず、エラーになること + Arranges: + - s3_testフィクスチャを引数に指定しない + - 環境変数を置き換える + Expects: + - 結果のJSONファイルをPUTできず、例外が発生する + """ + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', 'data_import') + + sut = BackupBucket() + with pytest.raises(Exception): + sut.put_result_json('result.json', {"test": "test"}) + def test_backup_bucket_str(self, s3_test, s3_client, bucket_name, monkeypatch): """ Cases: From e534090034c971fe2c766c73f92137e6d8621cfb Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 8 Aug 2022 13:24:03 +0900 Subject: [PATCH 157/275] =?UTF-8?q?fix:=20=E5=8D=98=E4=BD=93=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E6=8C=87=E6=91=98?= =?UTF-8?q?=E3=81=AB=E4=BC=B4=E3=81=86=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/config/objects.py | 32 +- ecs/crm-datafetch/src/system_var/constants.py | 2 + ecs/crm-datafetch/src/util/dict_checker.py | 12 +- .../test_objects_fetch_target_objects.py | 467 ++++++++++ .../test_objects_last_fetch_datetime.py | 380 ++++++++ ...jects.py => test_objects_target_object.py} | 822 ++---------------- 6 files changed, 926 insertions(+), 789 deletions(-) create mode 100644 ecs/crm-datafetch/tests/config/test_objects_fetch_target_objects.py create mode 100644 ecs/crm-datafetch/tests/config/test_objects_last_fetch_datetime.py rename ecs/crm-datafetch/tests/config/{test_objects.py => test_objects_target_object.py} (66%) diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 2979e0f0..70beb2eb 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,21 +1,13 @@ -from src.system_var.constants import (COLUMNS_KEY, COLUMNS_TYPE, - DATE_PATTERN_YYYYMMDDTHHMMSSTZ, - DATETIME_COLUMN_DEFAULT_VALUE, - DATETIME_COLUMN_KEY, - DATETIME_COLUMN_TYPE, IS_SKIP_KEY, - IS_SKIP_TYPE, - IS_UPDATE_LAST_FETCH_DATETIME_KEY, - IS_UPDATE_LAST_FETCH_DATETIME_TYPE, - LAST_FETCH_DATETIME_FILE_NAME_KEY, - LAST_FETCH_DATETIME_FILE_NAME_TYPE, - LAST_FETCH_DATETIME_FROM_KEY, - LAST_FETCH_DATETIME_FROM_TYPE, - LAST_FETCH_DATETIME_TO_KEY, - LAST_FETCH_DATETIME_TO_TYPE, - OBJECT_NAME_KEY, OBJECT_NAME_TYPE, - OBJECTS_KEY, OBJECTS_TYPE, - UPLOAD_FILE_NAME_KEY, - UPLOAD_FILE_NAME_TYPE) +from src.system_var.constants import ( + COLUMNS_KEY, COLUMNS_TYPE, DATE_PATTERN_YYYYMMDDTHHMMSSTZ, + DATETIME_COLUMN_DEFAULT_VALUE, DATETIME_COLUMN_KEY, DATETIME_COLUMN_TYPE, + DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ, IS_SKIP_KEY, IS_SKIP_TYPE, + IS_UPDATE_LAST_FETCH_DATETIME_KEY, IS_UPDATE_LAST_FETCH_DATETIME_TYPE, + LAST_FETCH_DATETIME_FILE_NAME_KEY, LAST_FETCH_DATETIME_FILE_NAME_TYPE, + LAST_FETCH_DATETIME_FROM_KEY, LAST_FETCH_DATETIME_FROM_TYPE, + LAST_FETCH_DATETIME_TO_KEY, LAST_FETCH_DATETIME_TO_TYPE, OBJECT_NAME_KEY, + OBJECT_NAME_TYPE, OBJECTS_KEY, OBJECTS_TYPE, UPLOAD_FILE_NAME_KEY, + UPLOAD_FILE_NAME_TYPE) from src.util.dict_checker import DictChecker from src.util.execute_datetime import ExecuteDateTime @@ -127,10 +119,10 @@ class LastFetchDatetime(): def __validate(self) -> None: self.__dict_checker.assert_key_exist(LAST_FETCH_DATETIME_FROM_KEY) self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_FROM_KEY, LAST_FETCH_DATETIME_FROM_TYPE) - self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ,DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_TO_KEY, LAST_FETCH_DATETIME_TO_TYPE) - self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ,DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ) return @property diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index 0214e5ac..9a8f6cd2 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -50,8 +50,10 @@ S3_CHAR_CODE = 'utf-8' # 正規表現チェック EXCLUDE_SYMBOL = ['#', '/'] DATE_PATTERN_YYYYMMDDTHHMMSSTZ = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' +DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ = 'YYYY-MM-DDTHH:MM:SS.000Z' DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.000\+0000' + # logger LOG_FORMAT = '[%(levelname)s]\t%(asctime)s\t%(message)s\n' LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py index 52c4f1ae..87e830e6 100644 --- a/ecs/crm-datafetch/src/util/dict_checker.py +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -7,15 +7,15 @@ class DictChecker: def is_empty(self, check_key): """辞書型バリュー空文字チェック""" - return self.__object_dict[check_key] != '' and self.__object_dict[check_key] is not None + return self.__object_dict[check_key] == '' or self.__object_dict[check_key] is None def is_list_empty(self, check_key): """list型データ存在チェック""" - return len(self.__object_dict[check_key]) != 0 + return len(self.__object_dict[check_key]) == 0 def check_key_exist(self, check_key: str) -> bool: """辞書型キー存在チェック""" - return check_key in self.__object_dict and self.is_empty(check_key) + return check_key in self.__object_dict and not self.is_empty(check_key) def check_data_type(self, check_key: str, check_type: type) -> bool: """辞書型バリュー型チェック""" @@ -39,13 +39,13 @@ class DictChecker: return - def assert_match_pattern(self, check_key: str, regex_str: str): + def assert_match_pattern(self, check_key: str, regex_str: str, expected_str: str): """正規表現検査""" if not self.check_match_pattern(regex_str, check_key): - raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') + raise Exception(f'「{check_key}」キーの値の正規表現チェックに失敗しました 「{expected_str}」形式である必要があります') return def assert_list_empty(self, check_key: str): - if not self.is_list_empty(check_key): + if self.is_list_empty(check_key): raise Exception(f'「{check_key}」キーのリストの値は必須です') diff --git a/ecs/crm-datafetch/tests/config/test_objects_fetch_target_objects.py b/ecs/crm-datafetch/tests/config/test_objects_fetch_target_objects.py new file mode 100644 index 00000000..3e7df076 --- /dev/null +++ b/ecs/crm-datafetch/tests/config/test_objects_fetch_target_objects.py @@ -0,0 +1,467 @@ +import pytest +from src.config.objects import FetchTargetObjects +from src.parser.json_parse import JsonParser + + +class TestFetchTargetObjects(): + + def test_constructor(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キーがあるかまた、キーの型が正しいかをチェック + Arranges: + - オブジェクト情報文字列を準備する + - オブジェクト情報を辞書型にパースする + Expects: + - 例外が発生しないこと + """ + + # Arranges + fetch_objects = '''{ + "objects": [ + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true, + "datetime_column": "LastModifiedDate" + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + } + ] + }''' + + json_parser = JsonParser(fetch_objects) + fetch_objects_dict = json_parser.parse() + + # Act + FetchTargetObjects(fetch_objects_dict) + + # Expects + pass + + def test_raise_constructor_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、必要なキーがない場合、例外が発生すること + Arranges: + - オブジェクト情報文字列を準備する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + fetch_objects_dict = { + "test_objects": [ + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "datetime_column": "LastModifiedDate" + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": False, + "is_update_last_fetch_datetime": True + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": False, + "is_update_last_fetch_datetime": True + } + ] + } + + # Act + with pytest.raises(Exception) as e: + FetchTargetObjects(fetch_objects_dict) + + # Expects + assert str(e.value) == '「objects」キーは必須です' + + def test_raise_constructor_no_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキーの値の型が想定と違う場合、例外が発生すること + Arranges: + - オブジェクト情報文字列を準備する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + fetch_objects_dict = { + "objects": "test_value" + } + + # Act + with pytest.raises(Exception) as e: + FetchTargetObjects(fetch_objects_dict) + + # Expects + assert str(e.value) == '「objects」キーの値は「」でなければなりません' + + def test_constructor_iterator(self) -> None: + """ + Cases: + インスタンス生成テスト + 登録されたオブジェクトリストをすべて取り出せること + Arranges: + - オブジェクト情報文字列を準備する + Expects: + - ループが最後まで回ること + """ + + fetch_objects_dict = { + "objects": [ + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "datetime_column": "LastModifiedDate" + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": False, + "is_update_last_fetch_datetime": True + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": False, + "is_update_last_fetch_datetime": True + } + ] + } + + # Act + sut = FetchTargetObjects(fetch_objects_dict) + for i, item in enumerate(sut, 1): + assert item is not None + + # Expects + assert i == 3 diff --git a/ecs/crm-datafetch/tests/config/test_objects_last_fetch_datetime.py b/ecs/crm-datafetch/tests/config/test_objects_last_fetch_datetime.py new file mode 100644 index 00000000..f3712372 --- /dev/null +++ b/ecs/crm-datafetch/tests/config/test_objects_last_fetch_datetime.py @@ -0,0 +1,380 @@ +import pytest +from src.config.objects import LastFetchDatetime +from src.util.execute_datetime import ExecuteDateTime + + +class TestLastFetchDatetime(): + def test_constructor(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、バリデーションチェックを行いチェックが通ることを確認する + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + pass + + def test_raise_constructor_last_fetch_datetime_from_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、必要なキー(last_fetch_datetime_from)がない場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' + + def test_raise_constructor_last_fetch_datetime_from_no_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値が空文字の場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' + + def test_raise_constructor_last_fetch_datetime_from_none_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値がNoneの場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": None, + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' + + def test_raise_constructor_last_fetch_datetime_from_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": 1, + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーの値は「」でなければなりません' + + def test_raise_constructor_last_fetch_datetime_from_other_string(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値が正規表現と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "aaa", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str( + e.value) == '「last_fetch_datetime_from」キーの値の正規表現チェックに失敗しました 「YYYY-MM-DDTHH:MM:SS.000Z」形式である必要があります' + + def test_raise_constructor_last_fetch_datetime_to_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(last_fetch_datetime_to)がない場合、例外が発生しないこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + pass + + def test_constructor_last_fetch_datetime_to_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + None + + def test_constructor_last_fetch_datetime_to_none__value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": None + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + pass + + def test_raise_constructor_last_fetch_datetime_to_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": 1 + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_to」キーの値は「」でなければなりません' + + def test_raise_constructor_last_fetch_datetime_to_other_string(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値が正規表現と違う場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "aaa" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str( + e.value) == '「last_fetch_datetime_to」キーの値の正規表現チェックに失敗しました 「YYYY-MM-DDTHH:MM:SS.000Z」形式である必要があります' + + def test_last_fetch_datetime_from(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + - 前回取得日時インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_from + + # Expects + assert actual == '1900-01-01T00:00:00.000Z' + + def test_last_fetch_datetime_to(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + - 前回取得日時インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_to + + # Expects + assert actual == '2022-01-01T00:00:00.000Z' + + def test_last_fetch_datetime_to_default(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + - 前回取得日時インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" + } + + execute_datetime = ExecuteDateTime() + + sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_to + + # Expects + assert actual == str(execute_datetime) diff --git a/ecs/crm-datafetch/tests/config/test_objects.py b/ecs/crm-datafetch/tests/config/test_objects_target_object.py similarity index 66% rename from ecs/crm-datafetch/tests/config/test_objects.py rename to ecs/crm-datafetch/tests/config/test_objects_target_object.py index 51c31981..19b89db7 100644 --- a/ecs/crm-datafetch/tests/config/test_objects.py +++ b/ecs/crm-datafetch/tests/config/test_objects_target_object.py @@ -1,335 +1,8 @@ import pytest -from src.config.objects import (FetchTargetObjects, LastFetchDatetime, - TargetObject) -from src.parser.json_parse import JsonParser +from src.config.objects import TargetObject from src.util.execute_datetime import ExecuteDateTime -class TestFetchTargetObjects(): - - def test_constructor(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータに対して、キーがあるかまた、キーの型が正しいかをチェック - Arranges: - - オブジェクト情報文字列を準備する - - オブジェクト情報を辞書型にパースする - Expects: - - 例外が発生しないこと - """ - - # Arranges - fetch_objects = '''{ - "objects": [ - { - "object_name": "AccountShare", - "columns": [ - "Id", - "AccountId", - "UserOrGroupId", - "AccountAccessLevel", - "OpportunityAccessLevel", - "CaseAccessLevel", - "ContactAccessLevel", - "RowCause", - "LastModifiedDate", - "LastModifiedById", - "IsDeleted" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true, - "datetime_column": "LastModifiedDate" - }, - { - "object_name": "Contact", - "columns": [ - "Id", - "IsDeleted", - "MasterRecordId", - "AccountId", - "IsPersonAccount", - "LastName", - "FirstName", - "Salutation", - "Name", - "OtherStreet", - "OtherCity", - "OtherState", - "OtherPostalCode", - "OtherCountry", - "OtherLatitude", - "OtherLongitude", - "OtherGeocodeAccuracy", - "OtherAddress", - "MailingStreet", - "MailingCity", - "MailingState", - "MailingPostalCode", - "MailingCountry", - "MailingLatitude", - "MailingLongitude", - "MailingGeocodeAccuracy", - "MailingAddress", - "Phone", - "Fax", - "MobilePhone", - "HomePhone", - "OtherPhone", - "AssistantPhone", - "ReportsToId", - "Email", - "Title", - "Department", - "AssistantName", - "Birthdate", - "Description", - "OwnerId", - "HasOptedOutOfEmail", - "HasOptedOutOfFax", - "DoNotCall", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "LastCURequestDate", - "LastCUUpdateDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "EmailBouncedReason", - "EmailBouncedDate", - "IsEmailBounced", - "PhotoUrl", - "Jigsaw", - "JigsawContactId", - "IndividualId", - "Mobile_ID_vod__c", - "H1Insights__H1_NPI_Value_for_Testing__c", - "H1Insights__H1_Person_ID__c", - "H1Insights__H1_Request_Status__c", - "H1Insights__H1_URL__c", - "H1Insights__NPI_Number__c", - "H1Insights__NPI_Number_for_H1_Insights__c", - "MSJ_Marketing_Cloud_Integration__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Territory2", - "columns": [ - "Id", - "Name", - "Territory2TypeId", - "Territory2ModelId", - "ParentTerritory2Id", - "Description", - "ForecastUserId", - "AccountAccessLevel", - "OpportunityAccessLevel", - "CaseAccessLevel", - "ContactAccessLevel", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "DeveloperName", - "MSJ_Territory_Type__c", - "MSJ_Level__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - } - ] - }''' - - json_parser = JsonParser(fetch_objects) - fetch_objects_dict = json_parser.parse() - - # Act - FetchTargetObjects(fetch_objects_dict) - - # Expects - None - - def test_constructor_no_key(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータに対して、必要なキーがない場合、例外が発生すること - Arranges: - - オブジェクト情報を辞書型にパースする - Expects: - - 例外が発生し期待値と一致する - """ - - fetch_objects = '''{ - "test_objects": [ - { - "object_name": "AccountShare", - "columns": [ - "Id", - "AccountId", - "UserOrGroupId", - "AccountAccessLevel", - "OpportunityAccessLevel", - "CaseAccessLevel", - "ContactAccessLevel", - "RowCause", - "LastModifiedDate", - "LastModifiedById", - "IsDeleted" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true, - "datetime_column": "LastModifiedDate" - }, - { - "object_name": "Contact", - "columns": [ - "Id", - "IsDeleted", - "MasterRecordId", - "AccountId", - "IsPersonAccount", - "LastName", - "FirstName", - "Salutation", - "Name", - "OtherStreet", - "OtherCity", - "OtherState", - "OtherPostalCode", - "OtherCountry", - "OtherLatitude", - "OtherLongitude", - "OtherGeocodeAccuracy", - "OtherAddress", - "MailingStreet", - "MailingCity", - "MailingState", - "MailingPostalCode", - "MailingCountry", - "MailingLatitude", - "MailingLongitude", - "MailingGeocodeAccuracy", - "MailingAddress", - "Phone", - "Fax", - "MobilePhone", - "HomePhone", - "OtherPhone", - "AssistantPhone", - "ReportsToId", - "Email", - "Title", - "Department", - "AssistantName", - "Birthdate", - "Description", - "OwnerId", - "HasOptedOutOfEmail", - "HasOptedOutOfFax", - "DoNotCall", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "LastCURequestDate", - "LastCUUpdateDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "EmailBouncedReason", - "EmailBouncedDate", - "IsEmailBounced", - "PhotoUrl", - "Jigsaw", - "JigsawContactId", - "IndividualId", - "Mobile_ID_vod__c", - "H1Insights__H1_NPI_Value_for_Testing__c", - "H1Insights__H1_Person_ID__c", - "H1Insights__H1_Request_Status__c", - "H1Insights__H1_URL__c", - "H1Insights__NPI_Number__c", - "H1Insights__NPI_Number_for_H1_Insights__c", - "MSJ_Marketing_Cloud_Integration__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Territory2", - "columns": [ - "Id", - "Name", - "Territory2TypeId", - "Territory2ModelId", - "ParentTerritory2Id", - "Description", - "ForecastUserId", - "AccountAccessLevel", - "OpportunityAccessLevel", - "CaseAccessLevel", - "ContactAccessLevel", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "DeveloperName", - "MSJ_Territory_Type__c", - "MSJ_Level__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - } - ] - }''' - - # Arranges - json_parser = JsonParser(fetch_objects) - fetch_objects_dict = json_parser.parse() - - # Act - with pytest.raises(Exception) as e: - FetchTargetObjects(fetch_objects_dict) - - # Expects - assert str(e.value) == '「objects」キーは必須です' - - def test_constructor_no_type(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキーの値の型が想定と違う場合、例外が発生すること - Arranges: - - オブジェクト情報を辞書型にパースする - Expects: - - 例外が発生し期待値と一致する - """ - - fetch_objects = '''{ - "objects": "test_value" - }''' - - # Arranges - json_parser = JsonParser(fetch_objects) - fetch_objects_dict = json_parser.parse() - - # Act - with pytest.raises(Exception) as e: - FetchTargetObjects(fetch_objects_dict) - - # Expects - assert str(e.value) == '「objects」キーの値は「」でなければなりません' - - class TestTargetObject(): def test_constructor(self) -> None: @@ -339,7 +12,7 @@ class TestTargetObject(): 辞書型のデータに対して、バリデーションチェックを行いチェックが通ることを確認する Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -373,7 +46,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass # object_name @@ -384,7 +57,7 @@ class TestTargetObject(): 辞書型のデータに対して、必要なキー(object_name)がない場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -427,7 +100,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(object_name)の値が空文字の場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -471,7 +144,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(object_name)の値がNoneの場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -515,7 +188,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(object_name)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -561,7 +234,7 @@ class TestTargetObject(): 辞書型のデータに対して、必要なキー(columns)がない場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -592,7 +265,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(columns)の値が空文字の場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -624,7 +297,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(columns)の値がNoneの場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -656,7 +329,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(columns)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -688,7 +361,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(columns)の値がリスト型で空の場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -722,7 +395,7 @@ class TestTargetObject(): 辞書型のデータに対して、キー(is_skip)がない場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -755,7 +428,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_is_skip_no_value(self) -> None: """ @@ -763,7 +436,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(is_skip)の値の型が空文字の場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -797,7 +470,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_is_skip_none_value(self) -> None: """ @@ -805,7 +478,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(is_skip)の値の型がNoneの場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -839,7 +512,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_raise_constructor_is_skip_other_type(self) -> None: """ @@ -848,7 +521,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(is_skip)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -894,7 +567,7 @@ class TestTargetObject(): 辞書型のデータに対して、キー(is_update_last_fetch_datetime)がない場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -927,7 +600,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_is_update_last_fetch_datetime_no_value(self) -> None: """ @@ -935,7 +608,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(is_update_last_fetch_datetime)の値の型が空文字の場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -969,7 +642,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_is_update_last_fetch_datetime_none_value(self) -> None: """ @@ -977,7 +650,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(is_update_last_fetch_datetime)の値の型がNoneの場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1011,7 +684,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_raise_constructor_is_update_last_fetch_datetime_other_type(self) -> None: """ @@ -1020,7 +693,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(is_update_last_fetch_datetime)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -1066,7 +739,7 @@ class TestTargetObject(): 辞書型のデータに対して、キー(last_fetch_datetime_file_name)がない場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1099,7 +772,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_last_fetch_datetime_file_name_no_value(self) -> None: """ @@ -1107,7 +780,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(last_fetch_datetime_file_name)の値の型が空文字の場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1141,7 +814,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_last_fetch_datetime_file_name_none_value(self) -> None: """ @@ -1149,7 +822,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(last_fetch_datetime_file_name)の値の型がNoneの場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1183,7 +856,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_raise_constructor_last_fetch_datetime_file_name_other_type(self) -> None: """ @@ -1192,7 +865,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(last_fetch_datetime_file_name)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -1238,7 +911,7 @@ class TestTargetObject(): 辞書型のデータに対して、キー(upload_file_name)がない場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1271,7 +944,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_upload_file_name_no_value(self) -> None: """ @@ -1279,7 +952,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(upload_file_name)の値の型が空文字の場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1313,7 +986,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_upload_file_name_none_value(self) -> None: """ @@ -1321,7 +994,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(upload_file_name)の値の型がNoneの場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1355,7 +1028,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_raise_constructor_upload_file_name_other_type(self) -> None: """ @@ -1364,7 +1037,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(upload_file_name)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -1410,7 +1083,7 @@ class TestTargetObject(): 辞書型のデータに対して、キー(datetime_column)がない場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1443,7 +1116,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_datetime_column_no_value(self) -> None: """ @@ -1451,7 +1124,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(datetime_column)の値の型が空文字の場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1485,7 +1158,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_datetime_column_none_value(self) -> None: """ @@ -1493,7 +1166,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(datetime_column)の値の型がNoneの場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1527,7 +1200,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_raise_constructor_datetime_column_other_type(self) -> None: """ @@ -1536,7 +1209,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(datetime_column)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -1581,7 +1254,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1626,7 +1299,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1684,7 +1357,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1729,7 +1402,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1774,7 +1447,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1819,7 +1492,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1864,7 +1537,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1909,7 +1582,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1954,7 +1627,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1999,7 +1672,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -2044,7 +1717,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -2089,7 +1762,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -2127,380 +1800,3 @@ class TestTargetObject(): # Expects assert actual == 'SystemModstamp' - - -class TestLastFetchDatetime(): - def test_constructor(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータに対して、バリデーションチェックを行いチェックが通ることを確認する - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生しないこと - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - None - - def test_raise_constructor_last_fetch_datetime_from_no_key(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータに対して、必要なキー(last_fetch_datetime_from)がない場合、例外が発生すること - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' - - def test_raise_constructor_last_fetch_datetime_from_no_value(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキー(last_fetch_datetime_from)の値が空文字の場合、例外が発生すること - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "", - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' - - def test_raise_constructor_last_fetch_datetime_from_none_value(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキー(last_fetch_datetime_from)の値がNoneの場合、例外が発生すること - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": None, - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' - - def test_raise_constructor_last_fetch_datetime_from_other_type(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキー(last_fetch_datetime_from)の値の型が想定と違う場合、例外が発生すること - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": 1, - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str(e.value) == '「last_fetch_datetime_from」キーの値は「」でなければなりません' - - def test_raise_constructor_last_fetch_datetime_from_other_string(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキー(last_fetch_datetime_from)の値が正規表現と違う場合、例外が発生すること - Arranges: - - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "aaa", - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str( - e.value) == '「last_fetch_datetime_from」キーの値の正規表現「[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z」チェックに失敗しました' - - def test_raise_constructor_last_fetch_datetime_to_no_key(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータに対して、キー(last_fetch_datetime_to)がない場合、例外が発生しないこと - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生しないこと - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - None - - def test_constructor_last_fetch_datetime_to_no_value(self) -> None: - """ - Cases: - 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型が空文字の場合、例外が発生しないこと - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生しないこと - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "" - } - - execute_datetime = ExecuteDateTime() - - # Act - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - None - - def test_constructor_last_fetch_datetime_to_none__value(self) -> None: - """ - Cases: - 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型がNoneの場合、例外が発生しないこと - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生しないこと - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": None - } - - execute_datetime = ExecuteDateTime() - - # Act - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - None - - def test_raise_constructor_last_fetch_datetime_to_other_type(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型が想定と違う場合、例外が発生すること - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": 1 - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str(e.value) == '「last_fetch_datetime_to」キーの値は「」でなければなりません' - - def test_raise_constructor_last_fetch_datetime_to_other_string(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキー(last_fetch_datetime_to)の値が正規表現と違う場合、例外が発生すること - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "aaa" - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str( - e.value) == '「last_fetch_datetime_to」キーの値の正規表現「[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z」チェックに失敗しました' - - def test_last_fetch_datetime_from(self) -> str: - """ - Cases: - オブジェクト情報から対象の値を返すこと - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - - 前回取得日時インスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Act - actual = sut.last_fetch_datetime_from - - # Expects - assert actual == '1900-01-01T00:00:00.000Z' - - def test_last_fetch_datetime_to(self) -> str: - """ - Cases: - オブジェクト情報から対象の値を返すこと - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - - 前回取得日時インスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Act - actual = sut.last_fetch_datetime_to - - # Expects - assert actual == '2022-01-01T00:00:00.000Z' - - def test_last_fetch_datetime_to_default(self) -> str: - """ - Cases: - オブジェクト情報から対象の値を返すこと - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - - 前回取得日時インスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "" - } - - execute_datetime = ExecuteDateTime() - - sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Act - actual = sut.last_fetch_datetime_to - - # Expects - assert actual == str(execute_datetime) From 2153eb5a203d64abe769f7931b2ec83bae89b347 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 8 Aug 2022 15:23:30 +0900 Subject: [PATCH 158/275] =?UTF-8?q?feat:=20execute=5Fdatetime=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/util/test_execute_datetime.py | 81 +++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 ecs/crm-datafetch/tests/util/test_execute_datetime.py diff --git a/ecs/crm-datafetch/tests/util/test_execute_datetime.py b/ecs/crm-datafetch/tests/util/test_execute_datetime.py new file mode 100644 index 00000000..cace2ff8 --- /dev/null +++ b/ecs/crm-datafetch/tests/util/test_execute_datetime.py @@ -0,0 +1,81 @@ +import re +from datetime import datetime, timezone + +from src.util.execute_datetime import ExecuteDateTime + + +class TestExecuteDateTime: + + def test_constructor(self): + """ + Cases: + インスタンス生成テスト + UTC時刻が文字列で設定されるかのチェック + Arranges: + - 現在時刻の保持 + - 期待される値の正規表現作成 + Expects: + - 作成されたクラスが保持している時刻が、期待値と一致すること + - 作成されたクラスが保持している時刻が、正規表現と一致すること + """ + + # Arranges + datetime_now = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.000Z') + + pattern = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' + + # Expects + sut = ExecuteDateTime() + + assert str(sut) == datetime_now + assert re.fullmatch(pattern, str(sut)) + + def test_to_path(self) -> str: + """ + Cases: + 生成されたパスがYYYYMMDD/HHMMSSの形式で出力されること + Arranges: + - 現在時刻をパスに変換する + - 期待される値の正規表現作成 + Expects: + - 戻り値が、期待値と一致する + - 戻り値が、正規表現と一致すること + """ + + # Arranges + datetime_now_path = datetime.now(timezone.utc).strftime( + '%Y-%m-%dT%H:%M:%S.000Z').rstrip('000Z').translate(str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) + + pattern = r'[12]\d{3}\/(0[1-9]|1[0-2])\/(0[1-9]|[12][0-9]|3[01])\/([01][0-9]|2[0-3])[0-5][0-9][0-5][0-9]' + + # Expects + sut = ExecuteDateTime() + actual = sut.to_path() + + assert actual == datetime_now_path + assert re.fullmatch(pattern, actual) + + def test_format_date(self) -> str: + """ + Cases: + 生成されたパスがYYYYMMDDHHMMSSの形式で出力されること + Arranges: + - 現在時刻をパスに変換する + - 期待される値の正規表現作成 + Expects: + - 戻り値が、期待値と一致する + - 戻り値が、正規表現と一致すること + """ + + # Arranges + datetime_now_path = datetime.now(timezone.utc).strftime( + '%Y-%m-%dT%H:%M:%S.000Z').rstrip('000Z').translate(str.maketrans({'-': None, 'T': None, ':': None, '.': None})) + + pattern = r'[12]\d{3}(0[1-9]|1[0-2])(0[1-9]|[12][0-9]|3[01])([01][0-9]|2[0-3])[0-5][0-9][0-5][0-9]' + + # Expects + sut = ExecuteDateTime() + actual = sut.format_date() + + assert actual == datetime_now_path + assert re.fullmatch(pattern, actual) From e10201c1f29de8b02c13442c126e5ef6acee9685 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 8 Aug 2022 17:43:51 +0900 Subject: [PATCH 159/275] =?UTF-8?q?feat:=20dict=5Fchecker=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/util/test_dict_checker.py | 580 ++++++++++++++++++ 1 file changed, 580 insertions(+) create mode 100644 ecs/crm-datafetch/tests/util/test_dict_checker.py diff --git a/ecs/crm-datafetch/tests/util/test_dict_checker.py b/ecs/crm-datafetch/tests/util/test_dict_checker.py new file mode 100644 index 00000000..6707b8be --- /dev/null +++ b/ecs/crm-datafetch/tests/util/test_dict_checker.py @@ -0,0 +1,580 @@ +import pytest +from src.util.dict_checker import DictChecker + + +class TestDictChecker: + + def test_is_empty_no_value(self): + """ + Cases: + 辞書型データの対象のキーの値が''(空文字)の場合、Trueが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": "" + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.is_empty("test_key") + + # Expects + assert actual is True + + def test_is_empty_none(self): + """ + Cases: + 辞書型データの対象のキーの値がNoneの場合、Trueが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": None + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.is_empty("test_key") + + # Expects + assert actual is True + + def test_is_empty_exist_value(self): + """ + Cases: + 辞書型データの対象のキーの値が存在する場合、Falseが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": "test_value" + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.is_empty("test_key") + + # Expects + assert actual is False + + def test_is_list_empty_no_value(self): + """ + Cases: + 辞書型のデータの対象のキーのリストの値が''(空文字)の場合、Falseが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": [ + "" + ] + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.is_list_empty("test_key") + + # Expects + assert actual is False + + def test_is_list_empty_none(self): + """ + Cases: + 辞書型データの対象のキーのリストの値がNoneの場合、Falseが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": [ + None + ] + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.is_list_empty("test_key") + + # Expects + assert actual is False + + def test_is_list_empty_blank(self): + """ + Cases: + 辞書型データの対象のキーのリストの値が[](空)の場合、Trueが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": [] + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.is_list_empty("test_key") + + # Expects + assert actual is True + + def test_is_list_empty_exist_string(self): + """ + Cases: + 辞書型データの対象のキーの値に文字列が存在する場合、Falseが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": "test_value" + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.is_list_empty("test_key") + + # Expects + assert actual is False + + def test_is_list_empty_exist_value(self): + """ + Cases: + 辞書型データの対象のキーのリストの値が存在する場合、Falseが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": [ + "test_value" + ] + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.is_list_empty("test_key") + + # Expects + assert actual is False + + def test_check_key_exist(self) -> bool: + """ + Cases: + 辞書型データの対象のキーが存在する、かつ値が存在する場合、Trueが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": "test_value" + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.check_key_exist("test_key") + + # Expects + assert actual is True + + def test_check_key_exist_no_key(self) -> bool: + """ + Cases: + 辞書型データの対象のキーが存在しない場合、Falseが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.check_key_exist("test_key") + + # Expects + assert actual is False + + def test_check_key_exist_no_value(self) -> bool: + """ + Cases: + 辞書型データの対象のキーの値が存在しない場合、Falseが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": "" + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.check_key_exist("test_key") + + # Expects + assert actual is False + + def test_check_data_type(self) -> bool: + """ + Cases: + 辞書型データの対象のキーの値の型が指定した型と一致した場合、Trueが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": "test_value" + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.check_data_type("test_key", str) + + # Expects + assert actual is True + + def test_check_data_type_other_type(self) -> bool: + """ + Cases: + 辞書型データの対象のキーの値の型が指定した型と一致しない場合、Falseが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": 1 + } + + sut = DictChecker(object_dict) + + # Act + actual = sut.check_data_type("test_key", str) + + # Expects + assert actual is False + + def test_check_match_pattern(self) -> bool: + """ + Cases: + 辞書型データの対象のキーの値の型が指定した正規表現と一致した場合、Trueが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + - 正規表現パターンの生成 + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": "2022-08-01T10:00:00.000Z" + } + + sut = DictChecker(object_dict) + + pattern = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' + + # Act + actual = sut.check_match_pattern(pattern, "test_key") + + # Expects + assert actual is True + + def test_check_match_pattern_not_match(self) -> bool: + """ + Cases: + 辞書型データの対象のキーの値の型が指定した正規表現と一致しない場合、Falseが返る + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + - 正規表現パターンの生成 + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_dict = { + "test_key": "test_value" + } + + sut = DictChecker(object_dict) + + pattern = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' + + # Act + actual = sut.check_match_pattern(pattern, "test_key") + + # Expects + assert actual is False + + def test_assert_key_exist(self) -> None: + """ + Cases: + 辞書型データの対象のキーの値が存在する場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_dict = { + "test_key": "test_value" + } + + sut = DictChecker(object_dict) + + # Act + sut.assert_key_exist("test_key") + + # Expects + pass + + def test_raise_assert_key_exist(self) -> None: + """ + Cases: + 辞書型データの対象のキーの値が存在しない場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 発生した例外が期待値と一致すること + """ + + # Arranges + object_dict = { + "test_key": "" + } + + sut = DictChecker(object_dict) + + # Act + with pytest.raises(Exception) as e: + sut.assert_key_exist("test_key") + + # Expects + assert str(e.value) == '「test_key」キーは必須です' + + def test_assert_data_type(self) -> None: + """ + Cases: + 辞書型データの対象のキーの値の型が指定した型と一致した場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_dict = { + "test_key": "test_value" + } + + sut = DictChecker(object_dict) + + # Act + sut.assert_data_type("test_key", str) + + # Expects + pass + + def test_raise_assert_data_type(self) -> None: + """ + Cases: + 辞書型データの対象のキーの値の型が指定した型と一致しない場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 発生した例外が期待値と一致すること + """ + + # Arranges + object_dict = { + "test_key": 1 + } + + sut = DictChecker(object_dict) + + # Act + with pytest.raises(Exception) as e: + sut.assert_data_type("test_key", str) + + # Expects + assert str(e.value) == "「test_key」キーの値は「」でなければなりません" + + def test_assert_match_pattern(self): + """ + Cases: + 辞書型データの対象のキーの値の型が指定した正規表現と一致した場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + - 正規表現パターンの生成 + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_dict = { + "test_key": "2022-08-01T10:00:00.000Z" + } + + sut = DictChecker(object_dict) + + pattern = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' + + # Act + sut.assert_match_pattern("test_key", pattern, 'YYYY-MM-DDTHH:MM:SS.000Z') + + # Expects + pass + + def test_raise_assert_match_pattern(self): + """ + Cases: + 辞書型データの対象のキーの値の型が指定した正規表現と一致しない場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + - 正規表現パターンの生成 + Expects: + - 発生した例外が期待値と一致すること + """ + + # Arranges + object_dict = { + "test_key": "test_value" + } + + sut = DictChecker(object_dict) + + pattern = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' + + # Act + with pytest.raises(Exception) as e: + sut.assert_match_pattern("test_key", pattern, 'YYYY-MM-DDTHH:MM:SS.000Z') + + assert str(e.value) == '「test_key」キーの値の正規表現チェックに失敗しました 「YYYY-MM-DDTHH:MM:SS.000Z」形式である必要があります' + + def test_assert_list_empty(self): + """ + Cases: + 辞書型データの対象のキーのリストの値が存在する場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_dict = { + "test_key": [ + "test_value" + ] + } + + sut = DictChecker(object_dict) + + # Act + sut.assert_list_empty("test_key") + + # Expects + pass + + def test_raise_assert_list_empty(self): + """ + Cases: + 辞書型データの対象のキーのリストの値が存在しない場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 辞書型チェックインスタンスを生成する + Expects: + - 発生した例外が期待値と一致すること + """ + + # Arranges + object_dict = { + "test_key": [ + ] + } + + sut = DictChecker(object_dict) + + # Act + with pytest.raises(Exception) as e: + sut.assert_list_empty("test_key") + + # Expects + assert str(e.value) == '「test_key」キーのリストの値は必須です' From a40bad20333ef1a2c5ded2c19901131cda007b6f Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 8 Aug 2022 20:26:05 +0900 Subject: [PATCH 160/275] =?UTF-8?q?feat:=20convert=5Fstrategy=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/converter/convert_strategy.py | 2 +- ecs/crm-datafetch/src/system_var/constants.py | 4 +- .../tests/converter/test_convert_strategy.py | 262 ++++++++++++++++++ 3 files changed, 265 insertions(+), 3 deletions(-) create mode 100644 ecs/crm-datafetch/tests/converter/test_convert_strategy.py diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 8664d7cd..de8c124a 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -59,5 +59,5 @@ class FloatConvertStrategy: class NonConvertStrategy: - def convert_value(self, convert_value: str): + def convert_value(self, convert_value): return convert_value diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index bf65010c..788c0409 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -71,8 +71,8 @@ UPD_JP_NAME = '前回取得日時ファイル更新' END_JP_NAME = '取得処理実施結果アップロード処理' # CSVチェック -CSV_TRUE_VALUE = '1' -CSV_FALSE_VALUE = '0' +CSV_TRUE_VALUE = 1 +CSV_FALSE_VALUE = 0 # オブジェクト変数 OBJECTS_KEY = 'objects' diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py new file mode 100644 index 00000000..1018db5c --- /dev/null +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -0,0 +1,262 @@ +from src.converter.convert_strategy import (BooleanConvertStrategy, + ConvertStrategyFactory, + DatetimeConvertStrategy, + FloatConvertStrategy, + NonConvertStrategy, + NoneValueConvertStrategy) + + +class TestConvertStrategyFactory: + + def test_create_none(self): + """ + Cases: + 引数にnull(None)を指定した場合、NoneValueConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(None) + + # Expects + assert type(actual).__name__ == "NoneValueConvertStrategy" + + def test_create_float(self): + """ + Cases: + 引数に指数表記を指定した場合、FloatConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(1.2345678E7) + + # Expects + assert type(actual).__name__ == "FloatConvertStrategy" + + def test_create_bool_true(self): + """ + Cases: + 引数に論理値(True)を指定した場合、BooleanConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(True) + + # Expects + assert type(actual).__name__ == "BooleanConvertStrategy" + + def test_create_bool_false(self): + """ + Cases: + 引数に論理値(False)を指定した場合、BooleanConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(False) + + # Expects + assert type(actual).__name__ == "BooleanConvertStrategy" + + def test_create_datetime(self): + """ + Cases: + 引数に文字列かつ日付文字列を指定した場合、DatetimeConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create('2022-06-13T10:15:32.000+0000') + + # Expects + assert type(actual).__name__ == "DatetimeConvertStrategy" + + def test_create_other_str(self): + """ + Cases: + 引数にSalesforce日付型以外の文字列を指定した場合、NonConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create('test_string') + + # Expects + assert type(actual).__name__ == "NonConvertStrategy" + + def test_create_other_int(self): + """ + Cases: + 引数に整数を指定した場合、NonConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(100) + + # Expects + assert type(actual).__name__ == "NonConvertStrategy" + + +class TestNoneValueConvertStrategy: + + def test_convert_value(self) -> str: + """ + Cases: + 引数にnull(None)を指定した場合、''(空文字)が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = NoneValueConvertStrategy() + actual = sut.convert_value(None) + + # Expects + assert actual == "" + + +class TestBooleanConvertStrategy: + + def test_convert_value_true(self) -> bool: + """ + Cases: + 引数に論理値(True)を指定した場合、数値(1)が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = BooleanConvertStrategy() + actual = sut.convert_value(True) + + # Expects + assert actual == 1 + + def test_convert_value_false(self) -> bool: + """ + Cases: + 引数に論理値(False)を指定した場合、数値(0)が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = BooleanConvertStrategy() + actual = sut.convert_value(False) + + # Expects + assert actual == 0 + + +class TestDatetimeConvertStrategy: + + def test_convert_value(self) -> str: + """ + Cases: + 引数に日付文字列を指定した場合、YYYY-MM-DD HH:MM:SSがJSTで返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DatetimeConvertStrategy() + actual = sut.convert_value('2022-06-13T20:15:32.000+0000') + + # Expects + assert actual == "2022-06-14 05:15:32" + + +class TestFloatConvertStrategy: + + def test_convert_value(self) -> int: + """ + Cases: + 引数に指数表記を指定した場合、整数で返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = FloatConvertStrategy() + actual = sut.convert_value(1.2345678E7) + + # Expects + assert actual == 12345678 + + +class TestNonConvertStrategy: + + def test_convert_value_str(self): + """ + Cases: + 引数に文字列を指定した場合、加工されず文字列が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = NonConvertStrategy() + actual = sut.convert_value('テストデータ') + + # Expects + assert actual == 'テストデータ' + + def test_convert_value_int(self): + """ + Cases: + 引数に整数を指定した場合、加工されず整数が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = NonConvertStrategy() + actual = sut.convert_value(100) + + # Expects + assert actual == 100 From 27e12dc079604ceba25892595c9f7135bc7efdec Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 9 Aug 2022 22:44:00 +0900 Subject: [PATCH 161/275] =?UTF-8?q?feat:=20converter=E3=81=AE=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/converter/converter.py | 18 +- .../tests/converter/test_converter.py | 391 ++++++++++++++++++ 2 files changed, 398 insertions(+), 11 deletions(-) create mode 100644 ecs/crm-datafetch/tests/converter/test_converter.py diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 761f550e..ace996ef 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -27,21 +27,17 @@ class CSVStringConverter: return extracted_sf_object_jsons except Exception as e: - raise Exception('必要なjsonのデータ抽出に失敗しました', e) + raise Exception('必要なjsonのデータ成形に失敗しました', e) def __extract_necessary_props_from(self, sf_object_json) -> dict: - try: - clone_sf_object = {**sf_object_json} + clone_sf_object = {**sf_object_json} - del clone_sf_object['attributes'] + del clone_sf_object['attributes'] - uppercase_key_sf_object = { - k.upper(): v for k, v in clone_sf_object.items()} + uppercase_key_sf_object = { + k.upper(): v for k, v in clone_sf_object.items()} - return uppercase_key_sf_object - - except Exception as e: - raise Exception('必要なjsonのデータ成形に失敗しました', e) + return uppercase_key_sf_object def __convert_to_csv(self, extracted_sf_object_jsons) -> list: try: @@ -76,4 +72,4 @@ class CSVStringConverter: return csv_value except Exception as e: - raise Exception('csvデータの取得に失敗しました', e) + raise Exception('CSVデータの取得に失敗しました', e) diff --git a/ecs/crm-datafetch/tests/converter/test_converter.py b/ecs/crm-datafetch/tests/converter/test_converter.py new file mode 100644 index 00000000..429a7510 --- /dev/null +++ b/ecs/crm-datafetch/tests/converter/test_converter.py @@ -0,0 +1,391 @@ +import textwrap +from collections import OrderedDict +from unittest import mock + +import pytest +from src.config.objects import TargetObject +from src.converter.converter import CSVStringConverter +from src.util.execute_datetime import ExecuteDateTime + + +class TestCSVStringConverter: + + def test_convert(self) -> str: + """ + Cases: + 入力データがCSV形式の文字列で出力されること + Arranges: + - オブジェクト情報の作成 + - データの作成 + - 実行日時取得インスタンスの生成 + - オブジェクト情報インスタンスの生成 + Expects: + 戻り値が期待値と一致すること + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + data = [ + OrderedDict([ + ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), + ('Id', 'TEST001'), + ('AccountId', 'test001'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 1), + ('OpportunityAccessLevel', 2), + ('CaseAccessLevel', 3), + ('ContactAccessLevel', 4), + ('RowCause', 'テストのため1'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('IsDeleted', False) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), + ('Id', 'TEST002'), + ('AccountId', 'test002'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 5), + ('OpportunityAccessLevel', 6), + ('CaseAccessLevel', 7), + ('ContactAccessLevel', 8), + ('RowCause', 'テストのため2'), + ('LastModifiedDate', '2022-06-02T16:30:30.000+0000'), + ('LastModifiedById', 2.234567E+6), + ('IsDeleted', True) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), + ('Id', 'TEST003'), + ('AccountId', 'test003'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 9), + ('OpportunityAccessLevel', 10), + ('CaseAccessLevel', 11), + ('ContactAccessLevel', 12), + ('RowCause', 'テストのため3'), + ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), + ('LastModifiedById', 3.234567E+6), + ('IsDeleted', False) + ]) + ] + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(object_info, execute_datetime) + + # Act + csv_string_converter = CSVStringConverter(target_object, data) + actual = csv_string_converter.convert() + + # Expects + expected_value = '''\ + "Id","AccountId","UserOrGroupId","AccountAccessLevel","OpportunityAccessLevel","CaseAccessLevel","ContactAccessLevel","RowCause","LastModifiedDate","LastModifiedById","IsDeleted"\r\n\ + "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567","0"\r\n\ + "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2234567","1"\r\n\ + "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3234567","0"\r\n\ + ''' + + assert actual == textwrap.dedent(expected_value) + + def test_raise_convert_extract_jsons(self) -> str: + """ + Cases: + 一部データのキー（attributes）の不足により、JSONの成形部分で例外が発生すること + Arranges: + - オブジェクト情報の作成 + - データの作成 + - 実行日時取得インスタンスの生成 + - オブジェクト情報インスタンスの生成 + Expects: + 例外が期待値と一致すること + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + data = [ + OrderedDict([ + ('Id', 'TEST001'), + ('AccountId', 'test001'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 1), + ('OpportunityAccessLevel', 2), + ('CaseAccessLevel', 3), + ('ContactAccessLevel', 4), + ('RowCause', 'テストのため1'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('IsDeleted', False) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), + ('Id', 'TEST002'), + ('AccountId', 'test002'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 5), + ('OpportunityAccessLevel', 6), + ('CaseAccessLevel', 7), + ('ContactAccessLevel', 8), + ('RowCause', 'テストのため2'), + ('LastModifiedDate', '2022-06-02T16:30:30.000+0000'), + ('LastModifiedById', 2.234567E+6), + ('IsDeleted', True) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), + ('Id', 'TEST003'), + ('AccountId', 'test003'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 9), + ('OpportunityAccessLevel', 10), + ('CaseAccessLevel', 11), + ('ContactAccessLevel', 12), + ('RowCause', 'テストのため3'), + ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), + ('LastModifiedById', 3.234567E+6), + ('IsDeleted', False) + ]) + ] + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(object_info, execute_datetime) + + # Act + csv_string_converter = CSVStringConverter(target_object, data) + + with pytest.raises(Exception) as e: + csv_string_converter.convert() + + # Expects + assert '必要なjsonのデータ成形に失敗しました' in str(e.value) + + def test_raise_convert_convert_csv(self) -> str: + """ + Cases: + 一部データのカラム（Id）の不足により、CSV変換で例外が発生すること + Arranges: + - オブジェクト情報の作成 + - データの作成 + - 実行日時取得インスタンスの生成 + - オブジェクト情報インスタンスの生成 + Expects: + 例外が期待値と一致すること + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + data = [ + OrderedDict([ + ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), + ('AccountId', 'test001'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 1), + ('OpportunityAccessLevel', 2), + ('CaseAccessLevel', 3), + ('ContactAccessLevel', 4), + ('RowCause', 'テストのため1'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('IsDeleted', False) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), + ('Id', 'TEST002'), + ('AccountId', 'test002'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 5), + ('OpportunityAccessLevel', 6), + ('CaseAccessLevel', 7), + ('ContactAccessLevel', 8), + ('RowCause', 'テストのため2'), + ('LastModifiedDate', '2022-06-02T16:30:30.000+0000'), + ('LastModifiedById', 2.234567E+6), + ('IsDeleted', True) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), + ('Id', 'TEST003'), + ('AccountId', 'test003'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 9), + ('OpportunityAccessLevel', 10), + ('CaseAccessLevel', 11), + ('ContactAccessLevel', 12), + ('RowCause', 'テストのため3'), + ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), + ('LastModifiedById', 3.234567E+6), + ('IsDeleted', False) + ]) + ] + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(object_info, execute_datetime) + + # Act + csv_string_converter = CSVStringConverter(target_object, data) + + with pytest.raises(Exception) as e: + csv_string_converter.convert() + + # Expects + assert 'CSV変換に失敗しました カラム名：[Id] 行番号: [1]' in str(e.value) + + @mock.patch('src.converter.converter.CSVStringConverter.create') + def test_raise_convert_csv_string(self, execute_mock) -> str: + """ + Cases: + 一部データのカラム（Id）の不足により、CSV変換で例外が発生すること + Arranges: + - オブジェクト情報の作成 + - データの作成 + - 実行日時取得インスタンスの生成 + - オブジェクト情報インスタンスの生成 + Expects: + 例外が期待値と一致すること + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + data = [ + OrderedDict([ + ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), + ('Id', 'TEST001'), + ('AccountId', 'test001'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 1), + ('OpportunityAccessLevel', 2), + ('CaseAccessLevel', 3), + ('ContactAccessLevel', 4), + ('RowCause', 'テストのため1'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('IsDeleted', False) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), + ('Id', 'TEST002'), + ('AccountId', 'test002'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 5), + ('OpportunityAccessLevel', 6), + ('CaseAccessLevel', 7), + ('ContactAccessLevel', 8), + ('RowCause', 'テストのため2'), + ('LastModifiedDate', '2022-06-02T16:30:30.000+0000'), + ('LastModifiedById', 2.234567E+6), + ('IsDeleted', True) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), + ('Id', 'TEST003'), + ('AccountId', 'test003'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 9), + ('OpportunityAccessLevel', 10), + ('CaseAccessLevel', 11), + ('ContactAccessLevel', 12), + ('RowCause', 'テストのため3'), + ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), + ('LastModifiedById', 3.234567E+6), + ('IsDeleted', False) + ]) + ] + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(object_info, execute_datetime) + + # Act + csv_string_converter = CSVStringConverter(target_object, data) + + execute_mock.side_effect = Exception + with pytest.raises(Exception) as e: + csv_string_converter.convert() + +# actual = csv_string_converter.convert() + + # Expects + assert 'CSVデータの取得に失敗しました' in str(e.value) +# assert actual == 'CSVデータの取得に失敗しました' From d935bc2c3d687227d16af35af6029a6b5dbb5c14 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Wed, 10 Aug 2022 09:25:00 +0900 Subject: [PATCH 162/275] =?UTF-8?q?fix:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=86=85=E5=AE=B9=E3=82=92=E5=8F=8D?= =?UTF-8?q?=E6=98=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/parser/test_json_parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/parser/test_json_parse.py b/ecs/crm-datafetch/tests/parser/test_json_parse.py index 1991aa54..f670eb5e 100644 --- a/ecs/crm-datafetch/tests/parser/test_json_parse.py +++ b/ecs/crm-datafetch/tests/parser/test_json_parse.py @@ -74,4 +74,4 @@ class TestJsonParser(): sut.parse() # Expects - assert str(e.value) == "Expecting property name enclosed in double quotes: line 3 column 13 (char 42)" + assert "Expecting property name enclosed in double quotes:" in str(e.value) From e3240e71796b348c69bf344f18af6c7a059bde5a Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Wed, 10 Aug 2022 10:08:49 +0900 Subject: [PATCH 163/275] =?UTF-8?q?fix:=20=E3=82=A4=E3=83=B3=E3=82=B9?= =?UTF-8?q?=E3=82=BF=E3=83=B3=E3=82=B9=E7=94=9F=E6=88=90=E6=99=82=E3=81=AB?= =?UTF-8?q?=E6=96=87=E5=AD=97=E5=88=97=E3=82=92=E6=B8=A1=E3=81=99=E3=81=A8?= =?UTF-8?q?=E4=BE=8B=E5=A4=96=E3=81=8C=E7=99=BA=E7=94=9F=E3=81=99=E3=82=8B?= =?UTF-8?q?=E5=87=A6=E7=90=86=E3=82=92=E8=BF=BD=E5=8A=A0=E3=80=82=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E3=81=AE=E3=83=AC?= =?UTF-8?q?=E3=83=93=E3=83=A5=E3=83=BC=E6=8C=87=E6=91=98=E5=86=85=E5=AE=B9?= =?UTF-8?q?=E3=81=AE=E5=8F=8D=E6=98=A0=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/util/counter_object.py | 2 +- .../tests/util/test_counter_object.py | 111 +++++++++++++++++- 2 files changed, 106 insertions(+), 7 deletions(-) diff --git a/ecs/crm-datafetch/src/util/counter_object.py b/ecs/crm-datafetch/src/util/counter_object.py index 23da4f82..6cf6b7c2 100644 --- a/ecs/crm-datafetch/src/util/counter_object.py +++ b/ecs/crm-datafetch/src/util/counter_object.py @@ -1,6 +1,6 @@ class CounterObject: def __init__(self, base_num=1) -> None: - self.__counter = base_num + self.__counter = int(base_num) def describe(self) -> int: return self.__counter diff --git a/ecs/crm-datafetch/tests/util/test_counter_object.py b/ecs/crm-datafetch/tests/util/test_counter_object.py index 8bf677a0..787099ba 100644 --- a/ecs/crm-datafetch/tests/util/test_counter_object.py +++ b/ecs/crm-datafetch/tests/util/test_counter_object.py @@ -4,10 +4,56 @@ from src.util.counter_object import CounterObject class TestCounterObject: + def test_constructor(self) -> int: + """ + Cases: + カウンターオブジェクト生成時に、数値を渡すと例外が発生しないこと + Arranges: + なし + Expects: + 例外が発生しないこと + """ + # Act + CounterObject(1) + + # Expects + pass + + def test_constructor_string_number(self) -> int: + """ + Cases: + カウンターオブジェクト生成時に、文字列型の数値を渡すと例外が発生しないこと + Arranges: + なし + Expects: + 例外が発生しないこと + """ + # Act + CounterObject("1") + + # Expects + pass + + def test_raise_constructor_string(self) -> int: + """ + Cases: + カウンターオブジェクト生成時に、文字列を渡すと例外が発生すること + Arranges: + なし + Expects: + 発生した例外が期待値と一致すること + """ + # Act + with pytest.raises(Exception) as e: + CounterObject("test1") + + # Expects + assert "invalid literal for int() with base 10:" in str(e.value) + def test_describe(self) -> int: """ Cases: - カウンターオブジェクトにて保持した値を返すこと + カウンターオブジェクトにて保持した値を返すこと(インスタンス生成時引数なし) Arranges: なし Expects: @@ -21,6 +67,23 @@ class TestCounterObject: # Expects assert actual == 1 + def test_describe_argument(self) -> int: + """ + Cases: + カウンターオブジェクトにて保持した値を返すこと(インスタンス生成時引数あり) + Arranges: + なし + Expects: + 問い合わせた値が期待値と一致する + """ + + # Act + sut = CounterObject(3) + actual = sut.describe() + + # Expects + assert actual == 3 + def test_raise_describe(self) -> int: """ Cases: @@ -42,7 +105,25 @@ class TestCounterObject: def test_increment(self) -> int: """ Cases: - カウンターオブジェクトにて保持した値がインクリメントされていること + カウンターオブジェクトにて保持した値がインクリメントされていること(引数なし) + Arranges: + なし + Expects: + 戻り値が期待値と一致する + """ + + # Act + sut = CounterObject() + sut.increment() + actual = sut.increment() + + # Expects + assert actual == 3 + + def test_increment_argument(self) -> int: + """ + Cases: + カウンターオブジェクトにて保持した値がインクリメントされていること(引数あり) Arranges: なし Expects: @@ -51,11 +132,11 @@ class TestCounterObject: # Act sut = CounterObject(5) - sut.increment() - actual = sut.increment() + sut.increment(2) + actual = sut.increment(2) # Expects - assert actual == 7 + assert actual == 9 def test_raise_increment(self) -> int: """ @@ -79,7 +160,25 @@ class TestCounterObject: def test_decrement(self) -> int: """ Cases: - カウンターオブジェクトにて保持した値がデクリメントされていること + カウンターオブジェクトにて保持した値がデクリメントされていること(引数なし) + Arranges: + なし + Expects: + 戻り値が期待値と一致する + """ + + # Act + sut = CounterObject() + sut.decrement() + actual = sut.decrement() + + # Expects + assert actual == -1 + + def test_decrement_argument(self) -> int: + """ + Cases: + カウンターオブジェクトにて保持した値がデクリメントされていること(引数あり) Arranges: なし Expects: From 69e06e1b42908c6bfaa752900c08b2c8629e1251 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Aug 2022 07:14:54 +0900 Subject: [PATCH 164/275] =?UTF-8?q?docs:=20README=E3=81=AEtypo=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md index 7f5e6216..f2928706 100644 --- a/ecs/crm-datafetch/README.md +++ b/ecs/crm-datafetch/README.md @@ -74,7 +74,7 @@ ```text . -├── Dockerfile -- Dokcerイメージを作成するためのファイル +├── Dockerfile -- Dockerイメージを作成するためのファイル ├── Pipfile -- Pipenv(Pythonの仮想環境管理モジュール)で、依存関係を管理するためのファイル ├── Pipfile.lock -- Pipenvでインストールされた依存関係のバージョン固定ファイル ├── README.md -- README From 7a029f4bb7c26ee44f7a4776c87b443ac33854b3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Aug 2022 07:30:59 +0900 Subject: [PATCH 165/275] =?UTF-8?q?feat;=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=8F=8D=E6=98=A0=20=E5=89=8D?= =?UTF-8?q?=E7=AF=84=E5=9B=B2=E5=8F=96=E5=BE=97=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0=20https://nds-tyo.backlog.?= =?UTF-8?q?com/git/NEWDWH2021/newsdwh2021/pullRequests/59#comment-1313618?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesfoce.py | 90 ++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py index 3b58b49d..623ead4f 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -10,6 +10,7 @@ Accountオブジェクトの下記SFIDのレコードはいじらないように """ import pytest +from requests.exceptions import ConnectTimeout, ReadTimeout from src.config.objects import LastFetchDatetime, TargetObject from src.salesforce.salesforce_api import SalesforceApiClient from src.salesforce.soql_builder import SOQLBuilder @@ -126,6 +127,49 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual == 1 + def test_fetch_sf_count_by_soql_builder_system_modstamp_all_range(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること + - SystemModStampのFromが2000年1月1日以降のレコードが取得できること + - SystemModStampのToが2100年12月31日未満のレコードが取得できること + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromに2000年1月1日を指定する + - LastFetchDatetimeのToに2100年12月31日を指定する + Expects: + 取得件数が16になる + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2000-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_count_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_count(soql) + assert actual == 16 + def test_fetch_sf_data_one_record(self): """ Cases: @@ -265,7 +309,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定） - LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定） Expects: - 取が1になる + 取得できたオブジェクト1件が期待値どおりであること """ execute_datetime = ExecuteDateTime() @@ -323,6 +367,50 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect + def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること + - SystemModStampのFromが2000年1月1日以降のレコードが取得できること + - SystemModStampのToが2100年12月31日未満のレコードが取得できること + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromに2000年1月1日を指定する + - LastFetchDatetimeのToに2100年12月31日を指定する + Expects: + 取得できたオブジェクト件数が16件になる + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2000-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_fetch_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) == 16 + # 内容の確認は別のケースで行っているため省略 + def test_raise_create_instance_cause_auth_failed(self, monkeypatch): """ Cases: From 61d7275c2b3142f327c163c460f33375dc7a0266 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Aug 2022 07:43:08 +0900 Subject: [PATCH 166/275] =?UTF-8?q?feat:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=8F=8D=E6=98=A0=20=E3=82=BF?= =?UTF-8?q?=E3=82=A4=E3=83=A0=E3=82=A2=E3=82=A6=E3=83=88=E4=BE=8B=E5=A4=96?= =?UTF-8?q?=E3=83=BBSOQL=E4=B8=8D=E6=AD=A3=E3=81=AE=E4=BE=8B=E5=A4=96?= =?UTF-8?q?=E3=82=B1=E3=83=BC=E3=82=B9=E3=82=92=E8=BF=BD=E5=8A=A0=20https:?= =?UTF-8?q?//nds-https://nds-tyo.backlog.com/git/NEWDWH2021/newsdwh2021/pu?= =?UTF-8?q?llRequests/59#comment-1313618?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesfoce.py | 118 +++++++++++++++++- 1 file changed, 117 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py index 623ead4f..aac7f109 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -416,10 +416,126 @@ class TestSalesforceApiClient: Cases: 存在しないユーザを指定した場合、エラーが発生すること Arranges: - なし + CRMのユーザ名を保持する環境変数に、存在しないユーザー名を指定する Expects: ユーザ認証でエラーが発生すること """ monkeypatch.setattr('src.salesforce.salesforce_api.CRM_USER_NAME', 'invalid_username') with pytest.raises(Exception): SalesforceApiClient() + + def test_raise_fetch_sf_count_auth_timeout(self, monkeypatch): + """ + Cases: + 認証タイムアウトが発生した場合、エラーが発生すること + Arranges: + 認証タイムアウト秒数を保持する環境変数に、0.0000000001を指定する + Expects: + コネクションタイムアウトエラーが発生すること + """ + monkeypatch.setattr('src.salesforce.salesforce_api.CRM_AUTH_TIMEOUT', 0.0000000001) + sf = SalesforceApiClient() + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-06T00:00:00.000Z + """ + with pytest.raises(ConnectTimeout): + sf.fetch_sf_count(soql) + + def test_raise_fetch_sf_count_read_timeout(self, monkeypatch): + """ + Cases: + 読み取りタイムアウトが発生した場合、エラーが発生すること + Arranges: + CRMの件数取得タイムアウト秒数を保持する環境変数に、0.0000000001を指定する + Expects: + 読み取りタイムアウトエラーが発生すること + """ + monkeypatch.setattr('src.salesforce.salesforce_api.CRM_GET_RECORD_COUNT_TIMEOUT', 0.0000000001) + sf = SalesforceApiClient() + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-06T00:00:00.000Z + """ + with pytest.raises(ReadTimeout): + sf.fetch_sf_count(soql) + + def test_raise_fetch_sf_count_invalid_soql(self, monkeypatch): + """ + Cases: + 不正なSOQLを指定した場合に、エラーが発生すること + Arranges: + 不正なSOQLを作成する + Expects: + エラーが発生すること + """ + sf = SalesforceApiClient() + soql = "SELECT" + with pytest.raises(Exception): + sf.fetch_sf_count(soql) + + def test_raise_fetch_sf_data_auth_timeout(self, monkeypatch): + """ + Cases: + 認証タイムアウトが発生した場合、エラーが発生すること + Arranges: + 認証タイムアウト秒数を保持する環境変数に、0.0000000001を指定する + Expects: + コネクションタイムアウトエラーが発生すること + """ + monkeypatch.setattr('src.salesforce.salesforce_api.CRM_AUTH_TIMEOUT', 0.0000000001) + sf = SalesforceApiClient() + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-06T00:00:00.000Z + """ + with pytest.raises(ConnectTimeout): + sf.fetch_sf_data(soql) + + def test_raise_fetch_sf_data_read_timeout(self, monkeypatch): + """ + Cases: + 読み取りタイムアウトが発生した場合、エラーが発生すること + Arranges: + CRMのデータ取得タイムアウト秒数を保持する環境変数に、0.0000000001を指定する + Expects: + 読み取りタイムアウトエラーが発生すること + """ + monkeypatch.setattr('src.salesforce.salesforce_api.CRM_FETCH_RECORD_TIMEOUT', 0.0000000001) + sf = SalesforceApiClient() + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-06T00:00:00.000Z + """ + with pytest.raises(ReadTimeout): + sf.fetch_sf_data(soql) + + def test_raise_fetch_sf_data_invalid_soql(self, monkeypatch): + """ + Cases: + 不正なSOQLを指定した場合に、エラーが発生すること + Arranges: + 不正なSOQLを作成する + Expects: + エラーが発生すること + """ + sf = SalesforceApiClient() + soql = "SELECT" + with pytest.raises(Exception): + sf.fetch_sf_count(soql) From 92238129fad0cae26cdb99a79582a7b095c5f62e Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 5 Aug 2022 21:32:04 +0900 Subject: [PATCH 167/275] =?UTF-8?q?feat:=20=E3=83=AD=E3=82=B0=E3=83=A1?= =?UTF-8?q?=E3=83=83=E3=82=BB=E3=83=BC=E3=82=B8=E7=94=9F=E6=88=90=E3=83=A6?= =?UTF-8?q?=E3=83=BC=E3=83=86=E3=82=A3=E3=83=AA=E3=83=86=E3=82=A3=E9=96=A2?= =?UTF-8?q?=E6=95=B0=E8=BF=BD=E5=8A=A0=EF=BC=88=E4=BB=96=E3=83=96=E3=83=A9?= =?UTF-8?q?=E3=83=B3=E3=83=81=E3=81=8B=E3=82=89Cherry-Pick=E3=81=97?= =?UTF-8?q?=E3=81=9F=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_utils/log_message.py | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 ecs/crm-datafetch/tests/test_utils/log_message.py diff --git a/ecs/crm-datafetch/tests/test_utils/log_message.py b/ecs/crm-datafetch/tests/test_utils/log_message.py new file mode 100644 index 00000000..25ce6e11 --- /dev/null +++ b/ecs/crm-datafetch/tests/test_utils/log_message.py @@ -0,0 +1,7 @@ +"""ログメッセージに関連するテストヘルパー""" + +import logging + + +def generate_log_message_tuple(logger_name='root', log_level=logging.INFO, log_message=''): + return (logger_name, log_level, log_message) From ede652b8af7b68624e2cc8ac7bfdd7f706bac378 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 16 Aug 2022 10:09:25 +0900 Subject: [PATCH 168/275] =?UTF-8?q?style:=20=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E5=90=8D=E3=81=AE=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/parser/{json_parse.py => json_parser.py} | 0 .../tests/parser/{test_json_parse.py => test_json_parser.py} | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename ecs/crm-datafetch/src/parser/{json_parse.py => json_parser.py} (100%) rename ecs/crm-datafetch/tests/parser/{test_json_parse.py => test_json_parser.py} (98%) diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parser.py similarity index 100% rename from ecs/crm-datafetch/src/parser/json_parse.py rename to ecs/crm-datafetch/src/parser/json_parser.py diff --git a/ecs/crm-datafetch/tests/parser/test_json_parse.py b/ecs/crm-datafetch/tests/parser/test_json_parser.py similarity index 98% rename from ecs/crm-datafetch/tests/parser/test_json_parse.py rename to ecs/crm-datafetch/tests/parser/test_json_parser.py index f670eb5e..c343752b 100644 --- a/ecs/crm-datafetch/tests/parser/test_json_parse.py +++ b/ecs/crm-datafetch/tests/parser/test_json_parser.py @@ -1,5 +1,5 @@ import pytest -from src.parser.json_parse import JsonParser +from src.parser.json_parser import JsonParser class TestJsonParser(): From 30641dafe7d0969f345daf599d29b3473f27a399 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Aug 2022 10:13:44 +0900 Subject: [PATCH 169/275] =?UTF-8?q?feat:=20=E3=82=AA=E3=83=96=E3=82=B8?= =?UTF-8?q?=E3=82=A7=E3=82=AF=E3=83=88=E6=83=85=E5=A0=B1=E5=BD=A2=E5=BC=8F?= =?UTF-8?q?=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF=E5=87=A6=E7=90=86=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_check_object_info_process.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 ecs/crm-datafetch/tests/test_check_object_info_process.py diff --git a/ecs/crm-datafetch/tests/test_check_object_info_process.py b/ecs/crm-datafetch/tests/test_check_object_info_process.py new file mode 100644 index 00000000..a5bc41ec --- /dev/null +++ b/ecs/crm-datafetch/tests/test_check_object_info_process.py @@ -0,0 +1,90 @@ +from unittest.mock import MagicMock, patch + +import pytest +from src.check_object_info_process import check_object_info_process +from src.config.objects import TargetObject +from src.error.exceptions import InvalidConfigException +from src.system_var.constants import CHK_JP_NAME +from src.util.execute_datetime import ExecuteDateTime + +from .test_utils.log_message import generate_log_message_tuple + + +class TestPrepareDataFetchProcess: + + def test_run_process_success(self, caplog): + """ + Cases: + オブジェクト情報形式チェック処理が正常終了し、期待通りの結果が返ること + Arranges: + - チェック対象のdictオブジェクトを宣言する + Expects: + - チェック後のオブジェクト情報コレクションクラスのインスタンスが返却される + - オブジェクト情報形式チェック処理の仕様に沿った正常系ログが出力されること(デバッグログは除く) + """ + # Arrange + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ] + } + + execute_datetime = ExecuteDateTime() + + # Act + actual_fetch_target_objects = check_object_info_process(target_objects_dict, execute_datetime) + + # Assert + + # 返り値の期待値チェック + assert isinstance(actual_fetch_target_objects, TargetObject), 'CRM取得オブジェクトクラスのインスタンスが返却される' + + # ログの確認 + assert generate_log_message_tuple(log_message='I-CHK-01 オブジェクト情報形式チェック処理を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-CHK-02 オブジェクト情報形式チェック処理を終了します') in caplog.record_tuples + + def test_call_depended_modules(self): + """ + Cases: + オブジェクト情報形式チェック処理内で依存しているモジュールが正しく呼ばれていること + Arranges: + - オブジェクト情報形式チェック処理の依存モジュールをモック化する + Expects: + - 依存しているモジュールが正しく呼ばれている + """ + + # Arrange + mock_target_object_init = MagicMock(return_value=None) + # Act + with patch('src.config.objects.TargetObject.__init__', mock_target_object_init): + check_object_info_process({}, {}) + + # Assert + assert mock_target_object_init.called is True + + def test_raise_check_object_info(self): + """ + Cases: + オブジェクト形式チェック処理でエラーが発生した場合、検査例外が発生すること + Arranges: + - オブジェクト形式チェック処理で例外が発生するようにする + Expects: + - 例外が発生する + - 形式チェックが失敗したエラーが返却される + """ + + # Arrange + mock_target_object_init = MagicMock(side_effect=Exception('形式チェックエラー')) + # Act + with patch('src.config.objects.TargetObject.__init__', mock_target_object_init): + with pytest.raises(InvalidConfigException) as e: + check_object_info_process({}, {}) + + # Assert + + assert mock_target_object_init.called is True + assert e.value.error_id == 'E-CHK-01' + assert e.value.func_name == CHK_JP_NAME + assert e.value.args[0] == f'オブジェクト情報形式チェック処理が失敗しました　エラー内容：[形式チェックエラー]' From 770148950ae4823af825bac6ca6bcc54394acc2b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Aug 2022 10:19:48 +0900 Subject: [PATCH 170/275] =?UTF-8?q?refactor:=20=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=82=AF=E3=83=A9=E3=82=B9=E5=90=8D=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_check_object_info_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/test_check_object_info_process.py b/ecs/crm-datafetch/tests/test_check_object_info_process.py index a5bc41ec..2e938504 100644 --- a/ecs/crm-datafetch/tests/test_check_object_info_process.py +++ b/ecs/crm-datafetch/tests/test_check_object_info_process.py @@ -10,7 +10,7 @@ from src.util.execute_datetime import ExecuteDateTime from .test_utils.log_message import generate_log_message_tuple -class TestPrepareDataFetchProcess: +class TestCheckObjectInfoProcess: def test_run_process_success(self, caplog): """ From 44b7256f3cef43e5ce79874d47cb811cf7176ec3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Aug 2022 11:08:05 +0900 Subject: [PATCH 171/275] =?UTF-8?q?feat:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=AF=BE=E5=BF=9C=20https://nds-tyo?= =?UTF-8?q?.backlog.com/git/NEWDWH2021/newsdwh2021/pullRequests/59#comment?= =?UTF-8?q?-1319332=20https://nds-tyo.backlog.com/git/NEWDWH2021/newsdwh20?= =?UTF-8?q?21/pullRequests/59#comment-1319342?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesfoce.py | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py index aac7f109..612c3679 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -41,6 +41,28 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual >= 1 + def test_fetch_sf_count_zero_record(self): + """ + Cases: + 取得範囲外の場合、Salesforceからオブジェクトの件数が取得できないこと + Arranges: + SalesforceのAccountオブジェクトに、レコードを作成する(手作業、コード上では行わない) + Expects: + 取得件数が0件になる + """ + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 1999-01-01T00:00:00.000Z AND + SystemModstamp <= 2000-01-01T00:00:00.000Z + """ + sut = SalesforceApiClient() + + actual = sut.fetch_sf_count(soql) + assert actual >= 0 + def test_fetch_sf_count_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): """ Cases: @@ -229,6 +251,28 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect + def test_fetch_sf_data_zero_record(self): + """ + Cases: + 取得範囲外の場合、Salesforceからオブジェクトが取得できないこと + Arranges: + SalesforceのAccountオブジェクトに、レコードを作成する(手作業、コード上では行わない) + Expects: + 取得件数が0件になる + """ + soql = """SELECT + COUNT(Id) + FROM + Account + WHERE + SystemModstamp > 1999-01-01T00:00:00.000Z AND + SystemModstamp <= 2000-01-01T00:00:00.000Z + """ + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) >= 0 + def test_fetch_sf_data_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): """ Cases: @@ -538,4 +582,4 @@ class TestSalesforceApiClient: sf = SalesforceApiClient() soql = "SELECT" with pytest.raises(Exception): - sf.fetch_sf_count(soql) + sf.fetch_sf_data(soql) From 63cdbd6c0331a43d9669ba366bb95e8a04760410 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 16 Aug 2022 12:09:37 +0900 Subject: [PATCH 172/275] =?UTF-8?q?fix:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E7=82=B9=E3=81=AE=E5=8F=8D=E6=98=A0?= =?UTF-8?q?=E3=80=81int=E5=9E=8B=E3=81=A8string=E5=9E=8B=E3=81=AE=E5=88=A4?= =?UTF-8?q?=E5=AE=9A=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/converter/convert_strategy.py | 25 ++++++-- .../tests/converter/test_convert_strategy.py | 62 ++++++++++--------- 2 files changed, 54 insertions(+), 33 deletions(-) diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index de8c124a..22fca8fc 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -15,7 +15,8 @@ class ConvertStrategyFactory: self.__float_convert_strategy = FloatConvertStrategy() self.__boolean_convert_strategy = BooleanConvertStrategy() self.__datetime_convert_strategy = DatetimeConvertStrategy() - self.__non_convert_strategy = NonConvertStrategy() + self.__int_convert_strategy = IntConvertStrategy() + self.__string_convert_strategy = StringConvertStrategy() def create(self, value): @@ -31,33 +32,49 @@ class ConvertStrategyFactory: elif type(value) == str and re.fullmatch(DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, value): convert_strategy = self.__datetime_convert_strategy + elif type(value) == int: + convert_strategy = self.__int_convert_strategy + else: - convert_strategy = self.__non_convert_strategy + convert_strategy = self.__string_convert_strategy return convert_strategy class NoneValueConvertStrategy: def convert_value(self, convert_value: None) -> str: + """Noneを''空文字に変換する処理""" return '' class BooleanConvertStrategy: def convert_value(self, convert_value: str) -> bool: + """booleanを数値に変換する処理""" return CSV_TRUE_VALUE if convert_value is True else CSV_FALSE_VALUE class DatetimeConvertStrategy: def convert_value(self, convert_value: str) -> str: + """UTCのdatetime文字列をJSTの日時文字列に変換する処理""" # データ登録処理がJSTとして登録するため、変換処理内で事前にJSTの日時文字列に変換する return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS) class FloatConvertStrategy: def convert_value(self, convert_value: str) -> int: + """float型をint型に変換する処理""" return int(convert_value) -class NonConvertStrategy: - def convert_value(self, convert_value): +class IntConvertStrategy: + def convert_value(self, convert_value: int): + """int型を変換せずに返す処理""" + # ConvertStrategyFactoryにて型チェックを行っているため値を変換せずに返す + return convert_value + + +class StringConvertStrategy: + def convert_value(self, convert_value: str): + """string型を変換せずに返す処理""" + # ConvertStrategyFactoryにて型チェックを行っているため値を変換せずに返す return convert_value diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py index 1018db5c..0e268fc4 100644 --- a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -2,8 +2,9 @@ from src.converter.convert_strategy import (BooleanConvertStrategy, ConvertStrategyFactory, DatetimeConvertStrategy, FloatConvertStrategy, - NonConvertStrategy, - NoneValueConvertStrategy) + IntConvertStrategy, + NoneValueConvertStrategy, + StringConvertStrategy) class TestConvertStrategyFactory: @@ -23,7 +24,7 @@ class TestConvertStrategyFactory: actual = sut.create(None) # Expects - assert type(actual).__name__ == "NoneValueConvertStrategy" + assert type(actual) == NoneValueConvertStrategy def test_create_float(self): """ @@ -40,7 +41,7 @@ class TestConvertStrategyFactory: actual = sut.create(1.2345678E7) # Expects - assert type(actual).__name__ == "FloatConvertStrategy" + assert type(actual) == FloatConvertStrategy def test_create_bool_true(self): """ @@ -57,7 +58,7 @@ class TestConvertStrategyFactory: actual = sut.create(True) # Expects - assert type(actual).__name__ == "BooleanConvertStrategy" + assert type(actual) == BooleanConvertStrategy def test_create_bool_false(self): """ @@ -74,7 +75,7 @@ class TestConvertStrategyFactory: actual = sut.create(False) # Expects - assert type(actual).__name__ == "BooleanConvertStrategy" + assert type(actual) == BooleanConvertStrategy def test_create_datetime(self): """ @@ -91,7 +92,7 @@ class TestConvertStrategyFactory: actual = sut.create('2022-06-13T10:15:32.000+0000') # Expects - assert type(actual).__name__ == "DatetimeConvertStrategy" + assert type(actual) == DatetimeConvertStrategy def test_create_other_str(self): """ @@ -108,7 +109,7 @@ class TestConvertStrategyFactory: actual = sut.create('test_string') # Expects - assert type(actual).__name__ == "NonConvertStrategy" + assert type(actual) == StringConvertStrategy def test_create_other_int(self): """ @@ -125,7 +126,7 @@ class TestConvertStrategyFactory: actual = sut.create(100) # Expects - assert type(actual).__name__ == "NonConvertStrategy" + assert type(actual) == IntConvertStrategy class TestNoneValueConvertStrategy: @@ -225,26 +226,9 @@ class TestFloatConvertStrategy: assert actual == 12345678 -class TestNonConvertStrategy: +class TestIntConvertStrategy: - def test_convert_value_str(self): - """ - Cases: - 引数に文字列を指定した場合、加工されず文字列が返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = NonConvertStrategy() - actual = sut.convert_value('テストデータ') - - # Expects - assert actual == 'テストデータ' - - def test_convert_value_int(self): + def test_convert_value(self): """ Cases: 引数に整数を指定した場合、加工されず整数が返ってくること @@ -255,8 +239,28 @@ class TestNonConvertStrategy: """ # Act - sut = NonConvertStrategy() + sut = IntConvertStrategy() actual = sut.convert_value(100) # Expects assert actual == 100 + + +class TestStringConvertStrategy: + + def test_convert_value(self): + """ + Cases: + 引数に文字列を指定した場合、加工されず文字列が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = StringConvertStrategy() + actual = sut.convert_value('テストデータ') + + # Expects + assert actual == 'テストデータ' From ee519c2205a35a6ecd8e04be6a3b1d82285ff149 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 16 Aug 2022 14:52:43 +0900 Subject: [PATCH 173/275] =?UTF-8?q?feat:=20logger=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/util/test_logger_out_of_class.py | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 ecs/crm-datafetch/tests/util/test_logger_out_of_class.py diff --git a/ecs/crm-datafetch/tests/util/test_logger_out_of_class.py b/ecs/crm-datafetch/tests/util/test_logger_out_of_class.py new file mode 100644 index 00000000..b5a54f01 --- /dev/null +++ b/ecs/crm-datafetch/tests/util/test_logger_out_of_class.py @@ -0,0 +1,145 @@ +import logging +from unittest.mock import MagicMock, patch + +import pytest +from src.util.logger import logger_instance as logger + + +class TestLogger: + + def test_logging_debug(self, caplog): + """ + Cases: + Debugレベルを設定したloggerについて、それ以下のログが出力されないこと + Arranges: + - 各レベルでのログを出力する + - ログレベルの設定 + Expects: + - 期待値通りのログが出力されること + """ + + caplog.set_level(logging.DEBUG) + + logger.debug('debugログ出力') + logger.info('infoログ出力') + logger.warning("warningログ出力") + logger.error("errorログ出力") + logger.critical("criticalログ出力") + + assert ("root", logging.DEBUG, "debugログ出力") in caplog.record_tuples + assert ("root", logging.INFO, "infoログ出力") in caplog.record_tuples + assert ("root", logging.WARNING, "warningログ出力") in caplog.record_tuples + assert ("root", logging.ERROR, "errorログ出力") in caplog.record_tuples + assert ("root", logging.CRITICAL, "criticalログ出力") in caplog.record_tuples + + def test_logging_info(self, caplog): + """ + Cases: + Infoレベルを設定したloggerについて、それ以下のログが出力されないこと + Arranges: + - 各レベルでのログを出力する + - ログレベルの設定 + Expects: + - 期待値通りのログが出力されること + """ + + caplog.set_level(logging.INFO) + + logger.debug('debugログ出力') + logger.info('infoログ出力') + logger.warning("warningログ出力") + logger.error("errorログ出力") + logger.critical("criticalログ出力") + + assert ("root", logging.DEBUG, "debugログ出力") not in caplog.record_tuples + assert ("root", logging.INFO, "infoログ出力") in caplog.record_tuples + assert ("root", logging.WARNING, "warningログ出力") in caplog.record_tuples + assert ("root", logging.ERROR, "errorログ出力") in caplog.record_tuples + assert ("root", logging.CRITICAL, "criticalログ出力") in caplog.record_tuples + + def test_logging_warning(self, caplog): + """ + Cases: + Warningレベルを設定したloggerについて、それ以下のログが出力されないこと + Arranges: + - 各レベルでのログを出力する + - ログレベルの設定 + Expects: + - 期待値通りのログが出力されること + """ + + caplog.set_level(logging.WARNING) + + logger.debug('debugログ出力') + logger.info('infoログ出力') + logger.warning("warningログ出力") + logger.error("errorログ出力") + logger.critical("criticalログ出力") + + assert ("root", logging.DEBUG, "debugログ出力") not in caplog.record_tuples + assert ("root", logging.INFO, "infoログ出力") not in caplog.record_tuples + assert ("root", logging.WARNING, "warningログ出力") in caplog.record_tuples + assert ("root", logging.ERROR, "errorログ出力") in caplog.record_tuples + assert ("root", logging.CRITICAL, "criticalログ出力") in caplog.record_tuples + + def test_logging_error(self, caplog): + """ + Cases: + Errorレベルを設定したloggerについて、それ以下のログが出力されないこと + Arranges: + - 各レベルでのログを出力する + - ログレベルの設定 + Expects: + - 期待値通りのログが出力されること + """ + + caplog.set_level(logging.ERROR) + + logger.debug('debugログ出力') + logger.info('infoログ出力') + logger.warning("warningログ出力") + logger.error("errorログ出力") + logger.critical("criticalログ出力") + + assert ("root", logging.DEBUG, "debugログ出力") not in caplog.record_tuples + assert ("root", logging.INFO, "infoログ出力") not in caplog.record_tuples + assert ("root", logging.WARNING, "warningログ出力") not in caplog.record_tuples + assert ("root", logging.ERROR, "errorログ出力") in caplog.record_tuples + assert ("root", logging.CRITICAL, "criticalログ出力") in caplog.record_tuples + + def test_logging_critical(self, caplog): + """ + Cases: + Criticalレベルを設定したloggerについて、それ以下のログが出力されないこと + Arranges: + - 各レベルでのログを出力する + - ログレベルの設定 + Expects: + - 期待値通りのログが出力されること + """ + + caplog.set_level(logging.CRITICAL) + + logger.debug('debugログ出力') + logger.info('infoログ出力') + logger.warning("warningログ出力") + logger.error("errorログ出力") + logger.critical("criticalログ出力") + + assert ("root", logging.DEBUG, "debugログ出力") not in caplog.record_tuples + assert ("root", logging.INFO, "infoログ出力") not in caplog.record_tuples + assert ("root", logging.WARNING, "warningログ出力") not in caplog.record_tuples + assert ("root", logging.ERROR, "errorログ出力") not in caplog.record_tuples + assert ("root", logging.CRITICAL, "criticalログ出力") in caplog.record_tuples + + def test_logging_getlogger_boto3(self): + assert logging.getLogger("boto3").getEffectiveLevel() == logging.WARNING + + def test_logging_getlogger_botocore(self): + assert logging.getLogger("botocore").getEffectiveLevel() == logging.WARNING + + def test_logging_getlogger_s3transfer(self): + assert logging.getLogger("s3transfer").getEffectiveLevel() == logging.WARNING + + def test_logging_getlogger_urllib3(self): + assert logging.getLogger("s3transfer").getEffectiveLevel() == logging.WARNING From 47407a5cef026873bb97ca60355aae89f3a3155b Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Fri, 5 Aug 2022 23:29:19 +0900 Subject: [PATCH 174/275] =?UTF-8?q?feat:=20objects=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/config/objects.py | 15 +- ecs/crm-datafetch/src/system_var/constants.py | 2 + ecs/crm-datafetch/src/util/dict_checker.py | 8 + .../tests/config/test_objects.py | 2506 +++++++++++++++++ 4 files changed, 2526 insertions(+), 5 deletions(-) create mode 100644 ecs/crm-datafetch/tests/config/test_objects.py diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 23d00f5c..2979e0f0 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -9,7 +9,9 @@ from src.system_var.constants import (COLUMNS_KEY, COLUMNS_TYPE, LAST_FETCH_DATETIME_FILE_NAME_KEY, LAST_FETCH_DATETIME_FILE_NAME_TYPE, LAST_FETCH_DATETIME_FROM_KEY, + LAST_FETCH_DATETIME_FROM_TYPE, LAST_FETCH_DATETIME_TO_KEY, + LAST_FETCH_DATETIME_TO_TYPE, OBJECT_NAME_KEY, OBJECT_NAME_TYPE, OBJECTS_KEY, OBJECTS_TYPE, UPLOAD_FILE_NAME_KEY, @@ -58,6 +60,7 @@ class TargetObject(): self.__dict_checker.assert_data_type(OBJECT_NAME_KEY, OBJECT_NAME_TYPE) self.__dict_checker.assert_key_exist(COLUMNS_KEY) self.__dict_checker.assert_data_type(COLUMNS_KEY, COLUMNS_TYPE) + self.__dict_checker.assert_list_empty(COLUMNS_KEY) return @@ -95,7 +98,7 @@ class TargetObject(): def is_update_last_fetch_datetime(self) -> bool: if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY): return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] - return False + return True @property def last_fetch_datetime_file_name(self) -> str: @@ -107,7 +110,7 @@ class TargetObject(): def upload_file_name(self) -> str: if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime.format_date()) - return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.format_date()}' + return f'CRM_{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.format_date()}' @property def datetime_column(self) -> str: @@ -122,9 +125,11 @@ class LastFetchDatetime(): self.__validate() def __validate(self) -> None: - if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FROM_KEY): - self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) + self.__dict_checker.assert_key_exist(LAST_FETCH_DATETIME_FROM_KEY) + self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_FROM_KEY, LAST_FETCH_DATETIME_FROM_TYPE) + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): + self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_TO_KEY, LAST_FETCH_DATETIME_TO_TYPE) self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) return @@ -136,4 +141,4 @@ class LastFetchDatetime(): def last_fetch_datetime_to(self) -> str: if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY] - return self.__execute_datetime + return str(self.__execute_datetime) diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index bf65010c..0214e5ac 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -93,4 +93,6 @@ DATETIME_COLUMN_KEY = 'datetime_column' DATETIME_COLUMN_TYPE = str DATETIME_COLUMN_DEFAULT_VALUE = 'SystemModstamp' LAST_FETCH_DATETIME_TO_KEY = 'last_fetch_datetime_to' +LAST_FETCH_DATETIME_TO_TYPE = str LAST_FETCH_DATETIME_FROM_KEY = 'last_fetch_datetime_from' +LAST_FETCH_DATETIME_FROM_TYPE = str diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py index 39ebf9f0..52c4f1ae 100644 --- a/ecs/crm-datafetch/src/util/dict_checker.py +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -9,6 +9,10 @@ class DictChecker: """辞書型バリュー空文字チェック""" return self.__object_dict[check_key] != '' and self.__object_dict[check_key] is not None + def is_list_empty(self, check_key): + """list型データ存在チェック""" + return len(self.__object_dict[check_key]) != 0 + def check_key_exist(self, check_key: str) -> bool: """辞書型キー存在チェック""" return check_key in self.__object_dict and self.is_empty(check_key) @@ -41,3 +45,7 @@ class DictChecker: raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') return + + def assert_list_empty(self, check_key: str): + if not self.is_list_empty(check_key): + raise Exception(f'「{check_key}」キーのリストの値は必須です') diff --git a/ecs/crm-datafetch/tests/config/test_objects.py b/ecs/crm-datafetch/tests/config/test_objects.py new file mode 100644 index 00000000..51c31981 --- /dev/null +++ b/ecs/crm-datafetch/tests/config/test_objects.py @@ -0,0 +1,2506 @@ +import pytest +from src.config.objects import (FetchTargetObjects, LastFetchDatetime, + TargetObject) +from src.parser.json_parse import JsonParser +from src.util.execute_datetime import ExecuteDateTime + + +class TestFetchTargetObjects(): + + def test_constructor(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キーがあるかまた、キーの型が正しいかをチェック + Arranges: + - オブジェクト情報文字列を準備する + - オブジェクト情報を辞書型にパースする + Expects: + - 例外が発生しないこと + """ + + # Arranges + fetch_objects = '''{ + "objects": [ + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true, + "datetime_column": "LastModifiedDate" + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + } + ] + }''' + + json_parser = JsonParser(fetch_objects) + fetch_objects_dict = json_parser.parse() + + # Act + FetchTargetObjects(fetch_objects_dict) + + # Expects + None + + def test_constructor_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、必要なキーがない場合、例外が発生すること + Arranges: + - オブジェクト情報を辞書型にパースする + Expects: + - 例外が発生し期待値と一致する + """ + + fetch_objects = '''{ + "test_objects": [ + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true, + "datetime_column": "LastModifiedDate" + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + } + ] + }''' + + # Arranges + json_parser = JsonParser(fetch_objects) + fetch_objects_dict = json_parser.parse() + + # Act + with pytest.raises(Exception) as e: + FetchTargetObjects(fetch_objects_dict) + + # Expects + assert str(e.value) == '「objects」キーは必須です' + + def test_constructor_no_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキーの値の型が想定と違う場合、例外が発生すること + Arranges: + - オブジェクト情報を辞書型にパースする + Expects: + - 例外が発生し期待値と一致する + """ + + fetch_objects = '''{ + "objects": "test_value" + }''' + + # Arranges + json_parser = JsonParser(fetch_objects) + fetch_objects_dict = json_parser.parse() + + # Act + with pytest.raises(Exception) as e: + FetchTargetObjects(fetch_objects_dict) + + # Expects + assert str(e.value) == '「objects」キーの値は「」でなければなりません' + + +class TestTargetObject(): + + def test_constructor(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、バリデーションチェックを行いチェックが通ることを確認する + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + # object_name + + def test_raise_constructor_object_name_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、必要なキー(object_name)がない場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「object_name」キーは必須です' + + def test_raise_constructor_object_name_no_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(object_name)の値が空文字の場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「object_name」キーは必須です' + + def test_raise_constructor_object_name_none_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(object_name)の値がNoneの場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": None, + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「object_name」キーは必須です' + + def test_raise_constructor_object_name_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(object_name)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": 1, + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「object_name」キーの値は「」でなければなりません' + + # columns + + def test_raise_constructor_columns_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、必要なキー(columns)がない場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「columns」キーは必須です' + + def test_raise_constructor_columns_no_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(columns)の値が空文字の場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": "", + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「columns」キーは必須です' + + def test_raise_constructor_columns_none_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(columns)の値がNoneの場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": None, + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「columns」キーは必須です' + + def test_raise_constructor_columns_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(columns)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": False, + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「columns」キーの値は「」でなければなりません' + + def test_raise_constructor_columns_no_value_list(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(columns)の値がリスト型で空の場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「columns」キーのリストの値は必須です' + + # is_skip + + def test_raise_constructor_is_skip_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(is_skip)がない場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_is_skip_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(is_skip)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": "", + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_is_skip_none_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(is_skip)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": None, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_raise_constructor_is_skip_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(is_skip)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": "False", + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「is_skip」キーの値は「」でなければなりません' + + # is_update_last_fetch_datetime + + def test_raise_constructor_is_update_last_fetch_datetime_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(is_update_last_fetch_datetime)がない場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_is_update_last_fetch_datetime_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(is_update_last_fetch_datetime)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": "", + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_is_update_last_fetch_datetime_none_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(is_update_last_fetch_datetime)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": None, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_raise_constructor_is_update_last_fetch_datetime_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(is_update_last_fetch_datetime)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": "False", + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「is_update_last_fetch_datetime」キーの値は「」でなければなりません' + + # last_fetch_datetime_file_name + + def test_raise_constructor_last_fetch_datetime_file_name_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(last_fetch_datetime_file_name)がない場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_last_fetch_datetime_file_name_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(last_fetch_datetime_file_name)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_last_fetch_datetime_file_name_none_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(last_fetch_datetime_file_name)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": None, + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_raise_constructor_last_fetch_datetime_file_name_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_file_name)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": 1, + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_file_name」キーの値は「」でなければなりません' + + # upload_file_name + + def test_raise_constructor_upload_file_name_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(upload_file_name)がない場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_upload_file_name_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(upload_file_name)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_upload_file_name_none_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(upload_file_name)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": None, + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_raise_constructor_upload_file_name_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(upload_file_name)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": 1, + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「upload_file_name」キーの値は「」でなければなりません' + + # datetime_column + + def test_raise_constructor_datetime_column_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(datetime_column)がない場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_datetime_column_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(datetime_column)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "" + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_constructor_datetime_column_none_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(datetime_column)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": None + } + + execute_datetime = ExecuteDateTime() + + # Act + TargetObject(object_info, execute_datetime) + + # Expects + None + + def test_raise_constructor_datetime_column_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(datetime_column)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": 1 + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + TargetObject(object_info, execute_datetime) + + # Expects + assert str(e.value) == '「datetime_column」キーの値は「」でなければなりません' + + # property + + def test_object_name(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.object_name + + # Expects + assert actual == 'AccountShare' + + def test_columns(self) -> list: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.columns + + # Expects + expected_value = [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ] + assert actual == expected_value + + def test_is_skip(self) -> bool: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.is_skip + + # Expects + assert actual is True + + def test_is_skip_default(self) -> bool: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": "", + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.is_skip + + # Expects + assert actual is False + + def test_is_update_last_fetch_datetime(self) -> bool: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.is_update_last_fetch_datetime + + # Expects + assert actual is False + + def test_is_update_last_fetch_datetime_default(self) -> bool: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": "", + "last_fetch_datetime_file_name": "AccountShare.json", + "upload_file_name": "CRM_AccountShare", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.is_update_last_fetch_datetime + + # Expects + assert actual is True + + def test_last_fetch_datetime_file_name(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare_Test.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_file_name + + # Expects + assert actual == 'AccountShare_Test.json' + + def test_last_fetch_datetime_file_name_default(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "", + "upload_file_name": "CRM_AccountShare", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_file_name + + # Expects + assert actual == 'AccountShare.json' + + def test_upload_file_name(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare_Test.json", + "upload_file_name": "CRM_AccountShare_Test_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.upload_file_name + + # Expects + assert actual == f'CRM_AccountShare_Test_{execute_datetime.format_date()}' + + def test_upload_file_name_default(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare_Test.json", + "upload_file_name": "", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.upload_file_name + + # Expects + assert actual == f'CRM_AccountShare_{execute_datetime.format_date()}' + + def test_datetime_column(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare_Test.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "LastModifiedDate" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.datetime_column + + # Expects + assert actual == 'LastModifiedDate' + + def test_datetime_column_default(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + - オブジェクト情報インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + object_info = { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": True, + "is_update_last_fetch_datetime": False, + "last_fetch_datetime_file_name": "AccountShare_Test.json", + "upload_file_name": "CRM_AccountShare_{execute_datetime}", + "datetime_column": "" + } + + execute_datetime = ExecuteDateTime() + + sut = TargetObject(object_info, execute_datetime) + + # Act + actual = sut.datetime_column + + # Expects + assert actual == 'SystemModstamp' + + +class TestLastFetchDatetime(): + def test_constructor(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、バリデーションチェックを行いチェックが通ることを確認する + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + None + + def test_raise_constructor_last_fetch_datetime_from_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、必要なキー(last_fetch_datetime_from)がない場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' + + def test_raise_constructor_last_fetch_datetime_from_no_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値が空文字の場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' + + def test_raise_constructor_last_fetch_datetime_from_none_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値がNoneの場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": None, + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' + + def test_raise_constructor_last_fetch_datetime_from_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": 1, + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーの値は「」でなければなりません' + + def test_raise_constructor_last_fetch_datetime_from_other_string(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値が正規表現と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "aaa", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str( + e.value) == '「last_fetch_datetime_from」キーの値の正規表現「[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z」チェックに失敗しました' + + def test_raise_constructor_last_fetch_datetime_to_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(last_fetch_datetime_to)がない場合、例外が発生しないこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + None + + def test_constructor_last_fetch_datetime_to_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + None + + def test_constructor_last_fetch_datetime_to_none__value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": None + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + None + + def test_raise_constructor_last_fetch_datetime_to_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": 1 + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_to」キーの値は「」でなければなりません' + + def test_raise_constructor_last_fetch_datetime_to_other_string(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値が正規表現と違う場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "aaa" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str( + e.value) == '「last_fetch_datetime_to」キーの値の正規表現「[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z」チェックに失敗しました' + + def test_last_fetch_datetime_from(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + - 前回取得日時インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_from + + # Expects + assert actual == '1900-01-01T00:00:00.000Z' + + def test_last_fetch_datetime_to(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + - 前回取得日時インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_to + + # Expects + assert actual == '2022-01-01T00:00:00.000Z' + + def test_last_fetch_datetime_to_default(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日次インスタンスを生成する + - 前回取得日時インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" + } + + execute_datetime = ExecuteDateTime() + + sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_to + + # Expects + assert actual == str(execute_datetime) From 25475138503541a6c89b3c8095cdb568574c132e Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 8 Aug 2022 13:24:03 +0900 Subject: [PATCH 175/275] =?UTF-8?q?fix:=20=E5=8D=98=E4=BD=93=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E6=8C=87=E6=91=98?= =?UTF-8?q?=E3=81=AB=E4=BC=B4=E3=81=86=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/config/objects.py | 32 +- ecs/crm-datafetch/src/system_var/constants.py | 2 + ecs/crm-datafetch/src/util/dict_checker.py | 12 +- .../test_objects_fetch_target_objects.py | 467 ++++++++++ .../test_objects_last_fetch_datetime.py | 380 ++++++++ ...jects.py => test_objects_target_object.py} | 822 ++---------------- 6 files changed, 926 insertions(+), 789 deletions(-) create mode 100644 ecs/crm-datafetch/tests/config/test_objects_fetch_target_objects.py create mode 100644 ecs/crm-datafetch/tests/config/test_objects_last_fetch_datetime.py rename ecs/crm-datafetch/tests/config/{test_objects.py => test_objects_target_object.py} (66%) diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 2979e0f0..70beb2eb 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,21 +1,13 @@ -from src.system_var.constants import (COLUMNS_KEY, COLUMNS_TYPE, - DATE_PATTERN_YYYYMMDDTHHMMSSTZ, - DATETIME_COLUMN_DEFAULT_VALUE, - DATETIME_COLUMN_KEY, - DATETIME_COLUMN_TYPE, IS_SKIP_KEY, - IS_SKIP_TYPE, - IS_UPDATE_LAST_FETCH_DATETIME_KEY, - IS_UPDATE_LAST_FETCH_DATETIME_TYPE, - LAST_FETCH_DATETIME_FILE_NAME_KEY, - LAST_FETCH_DATETIME_FILE_NAME_TYPE, - LAST_FETCH_DATETIME_FROM_KEY, - LAST_FETCH_DATETIME_FROM_TYPE, - LAST_FETCH_DATETIME_TO_KEY, - LAST_FETCH_DATETIME_TO_TYPE, - OBJECT_NAME_KEY, OBJECT_NAME_TYPE, - OBJECTS_KEY, OBJECTS_TYPE, - UPLOAD_FILE_NAME_KEY, - UPLOAD_FILE_NAME_TYPE) +from src.system_var.constants import ( + COLUMNS_KEY, COLUMNS_TYPE, DATE_PATTERN_YYYYMMDDTHHMMSSTZ, + DATETIME_COLUMN_DEFAULT_VALUE, DATETIME_COLUMN_KEY, DATETIME_COLUMN_TYPE, + DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ, IS_SKIP_KEY, IS_SKIP_TYPE, + IS_UPDATE_LAST_FETCH_DATETIME_KEY, IS_UPDATE_LAST_FETCH_DATETIME_TYPE, + LAST_FETCH_DATETIME_FILE_NAME_KEY, LAST_FETCH_DATETIME_FILE_NAME_TYPE, + LAST_FETCH_DATETIME_FROM_KEY, LAST_FETCH_DATETIME_FROM_TYPE, + LAST_FETCH_DATETIME_TO_KEY, LAST_FETCH_DATETIME_TO_TYPE, OBJECT_NAME_KEY, + OBJECT_NAME_TYPE, OBJECTS_KEY, OBJECTS_TYPE, UPLOAD_FILE_NAME_KEY, + UPLOAD_FILE_NAME_TYPE) from src.util.dict_checker import DictChecker from src.util.execute_datetime import ExecuteDateTime @@ -127,10 +119,10 @@ class LastFetchDatetime(): def __validate(self) -> None: self.__dict_checker.assert_key_exist(LAST_FETCH_DATETIME_FROM_KEY) self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_FROM_KEY, LAST_FETCH_DATETIME_FROM_TYPE) - self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ,DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_TO_KEY, LAST_FETCH_DATETIME_TO_TYPE) - self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ,DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ) return @property diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index 0214e5ac..9a8f6cd2 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -50,8 +50,10 @@ S3_CHAR_CODE = 'utf-8' # 正規表現チェック EXCLUDE_SYMBOL = ['#', '/'] DATE_PATTERN_YYYYMMDDTHHMMSSTZ = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' +DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ = 'YYYY-MM-DDTHH:MM:SS.000Z' DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.000\+0000' + # logger LOG_FORMAT = '[%(levelname)s]\t%(asctime)s\t%(message)s\n' LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py index 52c4f1ae..87e830e6 100644 --- a/ecs/crm-datafetch/src/util/dict_checker.py +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -7,15 +7,15 @@ class DictChecker: def is_empty(self, check_key): """辞書型バリュー空文字チェック""" - return self.__object_dict[check_key] != '' and self.__object_dict[check_key] is not None + return self.__object_dict[check_key] == '' or self.__object_dict[check_key] is None def is_list_empty(self, check_key): """list型データ存在チェック""" - return len(self.__object_dict[check_key]) != 0 + return len(self.__object_dict[check_key]) == 0 def check_key_exist(self, check_key: str) -> bool: """辞書型キー存在チェック""" - return check_key in self.__object_dict and self.is_empty(check_key) + return check_key in self.__object_dict and not self.is_empty(check_key) def check_data_type(self, check_key: str, check_type: type) -> bool: """辞書型バリュー型チェック""" @@ -39,13 +39,13 @@ class DictChecker: return - def assert_match_pattern(self, check_key: str, regex_str: str): + def assert_match_pattern(self, check_key: str, regex_str: str, expected_str: str): """正規表現検査""" if not self.check_match_pattern(regex_str, check_key): - raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') + raise Exception(f'「{check_key}」キーの値の正規表現チェックに失敗しました 「{expected_str}」形式である必要があります') return def assert_list_empty(self, check_key: str): - if not self.is_list_empty(check_key): + if self.is_list_empty(check_key): raise Exception(f'「{check_key}」キーのリストの値は必須です') diff --git a/ecs/crm-datafetch/tests/config/test_objects_fetch_target_objects.py b/ecs/crm-datafetch/tests/config/test_objects_fetch_target_objects.py new file mode 100644 index 00000000..3e7df076 --- /dev/null +++ b/ecs/crm-datafetch/tests/config/test_objects_fetch_target_objects.py @@ -0,0 +1,467 @@ +import pytest +from src.config.objects import FetchTargetObjects +from src.parser.json_parse import JsonParser + + +class TestFetchTargetObjects(): + + def test_constructor(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キーがあるかまた、キーの型が正しいかをチェック + Arranges: + - オブジェクト情報文字列を準備する + - オブジェクト情報を辞書型にパースする + Expects: + - 例外が発生しないこと + """ + + # Arranges + fetch_objects = '''{ + "objects": [ + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true, + "datetime_column": "LastModifiedDate" + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + } + ] + }''' + + json_parser = JsonParser(fetch_objects) + fetch_objects_dict = json_parser.parse() + + # Act + FetchTargetObjects(fetch_objects_dict) + + # Expects + pass + + def test_raise_constructor_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、必要なキーがない場合、例外が発生すること + Arranges: + - オブジェクト情報文字列を準備する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + fetch_objects_dict = { + "test_objects": [ + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "datetime_column": "LastModifiedDate" + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": False, + "is_update_last_fetch_datetime": True + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": False, + "is_update_last_fetch_datetime": True + } + ] + } + + # Act + with pytest.raises(Exception) as e: + FetchTargetObjects(fetch_objects_dict) + + # Expects + assert str(e.value) == '「objects」キーは必須です' + + def test_raise_constructor_no_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキーの値の型が想定と違う場合、例外が発生すること + Arranges: + - オブジェクト情報文字列を準備する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + fetch_objects_dict = { + "objects": "test_value" + } + + # Act + with pytest.raises(Exception) as e: + FetchTargetObjects(fetch_objects_dict) + + # Expects + assert str(e.value) == '「objects」キーの値は「」でなければなりません' + + def test_constructor_iterator(self) -> None: + """ + Cases: + インスタンス生成テスト + 登録されたオブジェクトリストをすべて取り出せること + Arranges: + - オブジェクト情報文字列を準備する + Expects: + - ループが最後まで回ること + """ + + fetch_objects_dict = { + "objects": [ + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": False, + "is_update_last_fetch_datetime": False, + "datetime_column": "LastModifiedDate" + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": False, + "is_update_last_fetch_datetime": True + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": False, + "is_update_last_fetch_datetime": True + } + ] + } + + # Act + sut = FetchTargetObjects(fetch_objects_dict) + for i, item in enumerate(sut, 1): + assert item is not None + + # Expects + assert i == 3 diff --git a/ecs/crm-datafetch/tests/config/test_objects_last_fetch_datetime.py b/ecs/crm-datafetch/tests/config/test_objects_last_fetch_datetime.py new file mode 100644 index 00000000..f3712372 --- /dev/null +++ b/ecs/crm-datafetch/tests/config/test_objects_last_fetch_datetime.py @@ -0,0 +1,380 @@ +import pytest +from src.config.objects import LastFetchDatetime +from src.util.execute_datetime import ExecuteDateTime + + +class TestLastFetchDatetime(): + def test_constructor(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、バリデーションチェックを行いチェックが通ることを確認する + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + pass + + def test_raise_constructor_last_fetch_datetime_from_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、必要なキー(last_fetch_datetime_from)がない場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' + + def test_raise_constructor_last_fetch_datetime_from_no_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値が空文字の場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' + + def test_raise_constructor_last_fetch_datetime_from_none_value(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値がNoneの場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": None, + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' + + def test_raise_constructor_last_fetch_datetime_from_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": 1, + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_from」キーの値は「」でなければなりません' + + def test_raise_constructor_last_fetch_datetime_from_other_string(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_from)の値が正規表現と違う場合、例外が発生すること + Arranges: + - 辞書型のオブジェクト情報を準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "aaa", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str( + e.value) == '「last_fetch_datetime_from」キーの値の正規表現チェックに失敗しました 「YYYY-MM-DDTHH:MM:SS.000Z」形式である必要があります' + + def test_raise_constructor_last_fetch_datetime_to_no_key(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータに対して、キー(last_fetch_datetime_to)がない場合、例外が発生しないこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + pass + + def test_constructor_last_fetch_datetime_to_no_value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型が空文字の場合、例外が発生しないこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + None + + def test_constructor_last_fetch_datetime_to_none__value(self) -> None: + """ + Cases: + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型がNoneの場合、例外が発生しないこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生しないこと + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": None + } + + execute_datetime = ExecuteDateTime() + + # Act + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + pass + + def test_raise_constructor_last_fetch_datetime_to_other_type(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型が想定と違う場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": 1 + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str(e.value) == '「last_fetch_datetime_to」キーの値は「」でなければなりません' + + def test_raise_constructor_last_fetch_datetime_to_other_string(self) -> None: + """ + Cases: + インスタンス生成テスト + 辞書型のデータの対象のキー(last_fetch_datetime_to)の値が正規表現と違う場合、例外が発生すること + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + Expects: + - 例外が発生し期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "aaa" + } + + execute_datetime = ExecuteDateTime() + + # Act + with pytest.raises(Exception) as e: + LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Expects + assert str( + e.value) == '「last_fetch_datetime_to」キーの値の正規表現チェックに失敗しました 「YYYY-MM-DDTHH:MM:SS.000Z」形式である必要があります' + + def test_last_fetch_datetime_from(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + - 前回取得日時インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_from + + # Expects + assert actual == '1900-01-01T00:00:00.000Z' + + def test_last_fetch_datetime_to(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + - 前回取得日時インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" + } + + execute_datetime = ExecuteDateTime() + + sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_to + + # Expects + assert actual == '2022-01-01T00:00:00.000Z' + + def test_last_fetch_datetime_to_default(self) -> str: + """ + Cases: + オブジェクト情報から対象の値を返すこと + Arranges: + - 辞書型の前回取得日時データを準備する + - 実行日時インスタンスを生成する + - 前回取得日時インスタンスを生成する + Expects: + - 戻り値が期待値と一致する + """ + + # Arranges + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" + } + + execute_datetime = ExecuteDateTime() + + sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + actual = sut.last_fetch_datetime_to + + # Expects + assert actual == str(execute_datetime) diff --git a/ecs/crm-datafetch/tests/config/test_objects.py b/ecs/crm-datafetch/tests/config/test_objects_target_object.py similarity index 66% rename from ecs/crm-datafetch/tests/config/test_objects.py rename to ecs/crm-datafetch/tests/config/test_objects_target_object.py index 51c31981..19b89db7 100644 --- a/ecs/crm-datafetch/tests/config/test_objects.py +++ b/ecs/crm-datafetch/tests/config/test_objects_target_object.py @@ -1,335 +1,8 @@ import pytest -from src.config.objects import (FetchTargetObjects, LastFetchDatetime, - TargetObject) -from src.parser.json_parse import JsonParser +from src.config.objects import TargetObject from src.util.execute_datetime import ExecuteDateTime -class TestFetchTargetObjects(): - - def test_constructor(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータに対して、キーがあるかまた、キーの型が正しいかをチェック - Arranges: - - オブジェクト情報文字列を準備する - - オブジェクト情報を辞書型にパースする - Expects: - - 例外が発生しないこと - """ - - # Arranges - fetch_objects = '''{ - "objects": [ - { - "object_name": "AccountShare", - "columns": [ - "Id", - "AccountId", - "UserOrGroupId", - "AccountAccessLevel", - "OpportunityAccessLevel", - "CaseAccessLevel", - "ContactAccessLevel", - "RowCause", - "LastModifiedDate", - "LastModifiedById", - "IsDeleted" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true, - "datetime_column": "LastModifiedDate" - }, - { - "object_name": "Contact", - "columns": [ - "Id", - "IsDeleted", - "MasterRecordId", - "AccountId", - "IsPersonAccount", - "LastName", - "FirstName", - "Salutation", - "Name", - "OtherStreet", - "OtherCity", - "OtherState", - "OtherPostalCode", - "OtherCountry", - "OtherLatitude", - "OtherLongitude", - "OtherGeocodeAccuracy", - "OtherAddress", - "MailingStreet", - "MailingCity", - "MailingState", - "MailingPostalCode", - "MailingCountry", - "MailingLatitude", - "MailingLongitude", - "MailingGeocodeAccuracy", - "MailingAddress", - "Phone", - "Fax", - "MobilePhone", - "HomePhone", - "OtherPhone", - "AssistantPhone", - "ReportsToId", - "Email", - "Title", - "Department", - "AssistantName", - "Birthdate", - "Description", - "OwnerId", - "HasOptedOutOfEmail", - "HasOptedOutOfFax", - "DoNotCall", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "LastCURequestDate", - "LastCUUpdateDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "EmailBouncedReason", - "EmailBouncedDate", - "IsEmailBounced", - "PhotoUrl", - "Jigsaw", - "JigsawContactId", - "IndividualId", - "Mobile_ID_vod__c", - "H1Insights__H1_NPI_Value_for_Testing__c", - "H1Insights__H1_Person_ID__c", - "H1Insights__H1_Request_Status__c", - "H1Insights__H1_URL__c", - "H1Insights__NPI_Number__c", - "H1Insights__NPI_Number_for_H1_Insights__c", - "MSJ_Marketing_Cloud_Integration__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Territory2", - "columns": [ - "Id", - "Name", - "Territory2TypeId", - "Territory2ModelId", - "ParentTerritory2Id", - "Description", - "ForecastUserId", - "AccountAccessLevel", - "OpportunityAccessLevel", - "CaseAccessLevel", - "ContactAccessLevel", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "DeveloperName", - "MSJ_Territory_Type__c", - "MSJ_Level__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - } - ] - }''' - - json_parser = JsonParser(fetch_objects) - fetch_objects_dict = json_parser.parse() - - # Act - FetchTargetObjects(fetch_objects_dict) - - # Expects - None - - def test_constructor_no_key(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータに対して、必要なキーがない場合、例外が発生すること - Arranges: - - オブジェクト情報を辞書型にパースする - Expects: - - 例外が発生し期待値と一致する - """ - - fetch_objects = '''{ - "test_objects": [ - { - "object_name": "AccountShare", - "columns": [ - "Id", - "AccountId", - "UserOrGroupId", - "AccountAccessLevel", - "OpportunityAccessLevel", - "CaseAccessLevel", - "ContactAccessLevel", - "RowCause", - "LastModifiedDate", - "LastModifiedById", - "IsDeleted" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true, - "datetime_column": "LastModifiedDate" - }, - { - "object_name": "Contact", - "columns": [ - "Id", - "IsDeleted", - "MasterRecordId", - "AccountId", - "IsPersonAccount", - "LastName", - "FirstName", - "Salutation", - "Name", - "OtherStreet", - "OtherCity", - "OtherState", - "OtherPostalCode", - "OtherCountry", - "OtherLatitude", - "OtherLongitude", - "OtherGeocodeAccuracy", - "OtherAddress", - "MailingStreet", - "MailingCity", - "MailingState", - "MailingPostalCode", - "MailingCountry", - "MailingLatitude", - "MailingLongitude", - "MailingGeocodeAccuracy", - "MailingAddress", - "Phone", - "Fax", - "MobilePhone", - "HomePhone", - "OtherPhone", - "AssistantPhone", - "ReportsToId", - "Email", - "Title", - "Department", - "AssistantName", - "Birthdate", - "Description", - "OwnerId", - "HasOptedOutOfEmail", - "HasOptedOutOfFax", - "DoNotCall", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "LastCURequestDate", - "LastCUUpdateDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "EmailBouncedReason", - "EmailBouncedDate", - "IsEmailBounced", - "PhotoUrl", - "Jigsaw", - "JigsawContactId", - "IndividualId", - "Mobile_ID_vod__c", - "H1Insights__H1_NPI_Value_for_Testing__c", - "H1Insights__H1_Person_ID__c", - "H1Insights__H1_Request_Status__c", - "H1Insights__H1_URL__c", - "H1Insights__NPI_Number__c", - "H1Insights__NPI_Number_for_H1_Insights__c", - "MSJ_Marketing_Cloud_Integration__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Territory2", - "columns": [ - "Id", - "Name", - "Territory2TypeId", - "Territory2ModelId", - "ParentTerritory2Id", - "Description", - "ForecastUserId", - "AccountAccessLevel", - "OpportunityAccessLevel", - "CaseAccessLevel", - "ContactAccessLevel", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "DeveloperName", - "MSJ_Territory_Type__c", - "MSJ_Level__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - } - ] - }''' - - # Arranges - json_parser = JsonParser(fetch_objects) - fetch_objects_dict = json_parser.parse() - - # Act - with pytest.raises(Exception) as e: - FetchTargetObjects(fetch_objects_dict) - - # Expects - assert str(e.value) == '「objects」キーは必須です' - - def test_constructor_no_type(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキーの値の型が想定と違う場合、例外が発生すること - Arranges: - - オブジェクト情報を辞書型にパースする - Expects: - - 例外が発生し期待値と一致する - """ - - fetch_objects = '''{ - "objects": "test_value" - }''' - - # Arranges - json_parser = JsonParser(fetch_objects) - fetch_objects_dict = json_parser.parse() - - # Act - with pytest.raises(Exception) as e: - FetchTargetObjects(fetch_objects_dict) - - # Expects - assert str(e.value) == '「objects」キーの値は「」でなければなりません' - - class TestTargetObject(): def test_constructor(self) -> None: @@ -339,7 +12,7 @@ class TestTargetObject(): 辞書型のデータに対して、バリデーションチェックを行いチェックが通ることを確認する Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -373,7 +46,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass # object_name @@ -384,7 +57,7 @@ class TestTargetObject(): 辞書型のデータに対して、必要なキー(object_name)がない場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -427,7 +100,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(object_name)の値が空文字の場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -471,7 +144,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(object_name)の値がNoneの場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -515,7 +188,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(object_name)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -561,7 +234,7 @@ class TestTargetObject(): 辞書型のデータに対して、必要なキー(columns)がない場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -592,7 +265,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(columns)の値が空文字の場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -624,7 +297,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(columns)の値がNoneの場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -656,7 +329,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(columns)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -688,7 +361,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(columns)の値がリスト型で空の場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -722,7 +395,7 @@ class TestTargetObject(): 辞書型のデータに対して、キー(is_skip)がない場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -755,7 +428,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_is_skip_no_value(self) -> None: """ @@ -763,7 +436,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(is_skip)の値の型が空文字の場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -797,7 +470,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_is_skip_none_value(self) -> None: """ @@ -805,7 +478,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(is_skip)の値の型がNoneの場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -839,7 +512,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_raise_constructor_is_skip_other_type(self) -> None: """ @@ -848,7 +521,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(is_skip)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -894,7 +567,7 @@ class TestTargetObject(): 辞書型のデータに対して、キー(is_update_last_fetch_datetime)がない場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -927,7 +600,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_is_update_last_fetch_datetime_no_value(self) -> None: """ @@ -935,7 +608,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(is_update_last_fetch_datetime)の値の型が空文字の場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -969,7 +642,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_is_update_last_fetch_datetime_none_value(self) -> None: """ @@ -977,7 +650,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(is_update_last_fetch_datetime)の値の型がNoneの場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1011,7 +684,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_raise_constructor_is_update_last_fetch_datetime_other_type(self) -> None: """ @@ -1020,7 +693,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(is_update_last_fetch_datetime)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -1066,7 +739,7 @@ class TestTargetObject(): 辞書型のデータに対して、キー(last_fetch_datetime_file_name)がない場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1099,7 +772,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_last_fetch_datetime_file_name_no_value(self) -> None: """ @@ -1107,7 +780,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(last_fetch_datetime_file_name)の値の型が空文字の場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1141,7 +814,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_last_fetch_datetime_file_name_none_value(self) -> None: """ @@ -1149,7 +822,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(last_fetch_datetime_file_name)の値の型がNoneの場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1183,7 +856,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_raise_constructor_last_fetch_datetime_file_name_other_type(self) -> None: """ @@ -1192,7 +865,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(last_fetch_datetime_file_name)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -1238,7 +911,7 @@ class TestTargetObject(): 辞書型のデータに対して、キー(upload_file_name)がない場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1271,7 +944,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_upload_file_name_no_value(self) -> None: """ @@ -1279,7 +952,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(upload_file_name)の値の型が空文字の場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1313,7 +986,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_upload_file_name_none_value(self) -> None: """ @@ -1321,7 +994,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(upload_file_name)の値の型がNoneの場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1355,7 +1028,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_raise_constructor_upload_file_name_other_type(self) -> None: """ @@ -1364,7 +1037,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(upload_file_name)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -1410,7 +1083,7 @@ class TestTargetObject(): 辞書型のデータに対して、キー(datetime_column)がない場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1443,7 +1116,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_datetime_column_no_value(self) -> None: """ @@ -1451,7 +1124,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(datetime_column)の値の型が空文字の場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1485,7 +1158,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_constructor_datetime_column_none_value(self) -> None: """ @@ -1493,7 +1166,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(datetime_column)の値の型がNoneの場合、例外が発生しないこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生しないこと """ @@ -1527,7 +1200,7 @@ class TestTargetObject(): TargetObject(object_info, execute_datetime) # Expects - None + pass def test_raise_constructor_datetime_column_other_type(self) -> None: """ @@ -1536,7 +1209,7 @@ class TestTargetObject(): 辞書型のデータの対象のキー(datetime_column)の値の型が想定と違う場合、例外が発生すること Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する Expects: - 例外が発生し期待値と一致する """ @@ -1581,7 +1254,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1626,7 +1299,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1684,7 +1357,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1729,7 +1402,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1774,7 +1447,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1819,7 +1492,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1864,7 +1537,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1909,7 +1582,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1954,7 +1627,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -1999,7 +1672,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -2044,7 +1717,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -2089,7 +1762,7 @@ class TestTargetObject(): オブジェクト情報から対象の値を返すこと Arranges: - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する + - 実行日時インスタンスを生成する - オブジェクト情報インスタンスを生成する Expects: - 戻り値が期待値と一致する @@ -2127,380 +1800,3 @@ class TestTargetObject(): # Expects assert actual == 'SystemModstamp' - - -class TestLastFetchDatetime(): - def test_constructor(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータに対して、バリデーションチェックを行いチェックが通ることを確認する - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生しないこと - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - None - - def test_raise_constructor_last_fetch_datetime_from_no_key(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータに対して、必要なキー(last_fetch_datetime_from)がない場合、例外が発生すること - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' - - def test_raise_constructor_last_fetch_datetime_from_no_value(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキー(last_fetch_datetime_from)の値が空文字の場合、例外が発生すること - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "", - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' - - def test_raise_constructor_last_fetch_datetime_from_none_value(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキー(last_fetch_datetime_from)の値がNoneの場合、例外が発生すること - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": None, - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str(e.value) == '「last_fetch_datetime_from」キーは必須です' - - def test_raise_constructor_last_fetch_datetime_from_other_type(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキー(last_fetch_datetime_from)の値の型が想定と違う場合、例外が発生すること - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": 1, - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str(e.value) == '「last_fetch_datetime_from」キーの値は「」でなければなりません' - - def test_raise_constructor_last_fetch_datetime_from_other_string(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキー(last_fetch_datetime_from)の値が正規表現と違う場合、例外が発生すること - Arranges: - - 辞書型のオブジェクト情報を準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "aaa", - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str( - e.value) == '「last_fetch_datetime_from」キーの値の正規表現「[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z」チェックに失敗しました' - - def test_raise_constructor_last_fetch_datetime_to_no_key(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータに対して、キー(last_fetch_datetime_to)がない場合、例外が発生しないこと - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生しないこと - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - # Act - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - None - - def test_constructor_last_fetch_datetime_to_no_value(self) -> None: - """ - Cases: - 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型が空文字の場合、例外が発生しないこと - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生しないこと - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "" - } - - execute_datetime = ExecuteDateTime() - - # Act - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - None - - def test_constructor_last_fetch_datetime_to_none__value(self) -> None: - """ - Cases: - 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型がNoneの場合、例外が発生しないこと - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生しないこと - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": None - } - - execute_datetime = ExecuteDateTime() - - # Act - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - None - - def test_raise_constructor_last_fetch_datetime_to_other_type(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキー(last_fetch_datetime_to)の値の型が想定と違う場合、例外が発生すること - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": 1 - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str(e.value) == '「last_fetch_datetime_to」キーの値は「」でなければなりません' - - def test_raise_constructor_last_fetch_datetime_to_other_string(self) -> None: - """ - Cases: - インスタンス生成テスト - 辞書型のデータの対象のキー(last_fetch_datetime_to)の値が正規表現と違う場合、例外が発生すること - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - Expects: - - 例外が発生し期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "aaa" - } - - execute_datetime = ExecuteDateTime() - - # Act - with pytest.raises(Exception) as e: - LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Expects - assert str( - e.value) == '「last_fetch_datetime_to」キーの値の正規表現「[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z」チェックに失敗しました' - - def test_last_fetch_datetime_from(self) -> str: - """ - Cases: - オブジェクト情報から対象の値を返すこと - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - - 前回取得日時インスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Act - actual = sut.last_fetch_datetime_from - - # Expects - assert actual == '1900-01-01T00:00:00.000Z' - - def test_last_fetch_datetime_to(self) -> str: - """ - Cases: - オブジェクト情報から対象の値を返すこと - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - - 前回取得日時インスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "2022-01-01T00:00:00.000Z" - } - - execute_datetime = ExecuteDateTime() - - sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Act - actual = sut.last_fetch_datetime_to - - # Expects - assert actual == '2022-01-01T00:00:00.000Z' - - def test_last_fetch_datetime_to_default(self) -> str: - """ - Cases: - オブジェクト情報から対象の値を返すこと - Arranges: - - 辞書型の前回取得日時データを準備する - - 実行日次インスタンスを生成する - - 前回取得日時インスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - last_fetch_datetime_dict = { - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "" - } - - execute_datetime = ExecuteDateTime() - - sut = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) - - # Act - actual = sut.last_fetch_datetime_to - - # Expects - assert actual == str(execute_datetime) From f220a8e1c0c364b52ac7da7acd616ba7fc3c90fb Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Aug 2022 15:24:04 +0900 Subject: [PATCH 176/275] =?UTF-8?q?fix:=20=E3=83=9E=E3=83=BC=E3=82=B8?= =?UTF-8?q?=E7=AB=B6=E5=90=88=E8=A7=A3=E6=B1=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/prepare_data_fetch_process.py | 2 +- .../src/set_datetime_period_process.py | 2 +- .../config/test_objects_fetch_target_objects.py | 2 +- ecs/crm-datafetch/tests/test_controller.py | 16 ++++++++-------- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index 03427265..f0ea9ab6 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -1,7 +1,7 @@ from src.aws.s3 import ConfigBucket from src.config.objects import FetchTargetObjects from src.error.exceptions import FileNotFoundException, InvalidConfigException -from src.parser.json_parse import JsonParser +from src.parser.json_parser import JsonParser from src.system_var.constants import PRE_JP_NAME from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, diff --git a/ecs/crm-datafetch/src/set_datetime_period_process.py b/ecs/crm-datafetch/src/set_datetime_period_process.py index bfebd5c2..062fac97 100644 --- a/ecs/crm-datafetch/src/set_datetime_period_process.py +++ b/ecs/crm-datafetch/src/set_datetime_period_process.py @@ -1,7 +1,7 @@ from src.aws.s3 import ConfigBucket from src.config.objects import LastFetchDatetime, TargetObject from src.error.exceptions import FileNotFoundException, InvalidConfigException -from src.parser.json_parse import JsonParser +from src.parser.json_parser import JsonParser from src.system_var.constants import DATE_JP_NAME from src.system_var.environments import (CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER) diff --git a/ecs/crm-datafetch/tests/config/test_objects_fetch_target_objects.py b/ecs/crm-datafetch/tests/config/test_objects_fetch_target_objects.py index 3e7df076..f626a6de 100644 --- a/ecs/crm-datafetch/tests/config/test_objects_fetch_target_objects.py +++ b/ecs/crm-datafetch/tests/config/test_objects_fetch_target_objects.py @@ -1,6 +1,6 @@ import pytest from src.config.objects import FetchTargetObjects -from src.parser.json_parse import JsonParser +from src.parser.json_parser import JsonParser class TestFetchTargetObjects(): diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index 8b9a464a..c142fcfd 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -19,17 +19,17 @@ COMMON_OBJECT_INFOS = { 'objects': [ { 'object_name': 'Account', - 'columns': [], + 'columns': ['Id'], 'upload_file_name': 'Account_YYYYMMDDHHMMSS' }, { 'object_name': 'Contact', - 'columns': [], + 'columns': ['Id'], 'upload_file_name': 'Contact_YYYYMMDDHHMMSS' }, { 'object_name': 'Call2_vod__c', - 'columns': [], + 'columns': ['Id'], 'upload_file_name': 'Call2_vod__c_YYYYMMDDHHMMSS' } ] @@ -254,7 +254,7 @@ class TestController: """ mock_check_object_info = { 'object_name': 'Account', - 'columns': [], + 'columns': ['id'], 'is_skip': True } mock_return_values = [TargetObject(mock_check_object_info, COMMON_EXECUTE_DATETIME)] @@ -307,17 +307,17 @@ class TestController: 'objects': [ { 'object_name': 'Account', - 'columns': [], + 'columns': ['id'], 'is_skip': False }, { 'object_name': 'Contact', - 'columns': [], + 'columns': ['id'], 'is_skip': True }, { 'object_name': 'Call2_vod__c', - 'columns': [], + 'columns': ['id'], 'is_skip': False } ] @@ -380,7 +380,7 @@ class TestController: 'objects': [ { 'object_name': 'Account', - 'columns': [], + 'columns': ['id'], 'is_skip': False } ] From 3b2241b9c6dbd1cb4da702b81606c1b062165309 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 16 Aug 2022 18:04:00 +0900 Subject: [PATCH 177/275] =?UTF-8?q?feat:=20=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E5=8F=96=E5=BE=97=E6=9C=9F=E9=96=93=E8=A8=AD=E5=AE=9A=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_set_datetime_period_process.py | 236 ++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 ecs/crm-datafetch/tests/test_set_datetime_period_process.py diff --git a/ecs/crm-datafetch/tests/test_set_datetime_period_process.py b/ecs/crm-datafetch/tests/test_set_datetime_period_process.py new file mode 100644 index 00000000..87a44a30 --- /dev/null +++ b/ecs/crm-datafetch/tests/test_set_datetime_period_process.py @@ -0,0 +1,236 @@ +import json +from unittest.mock import MagicMock, patch + +import pytest +from src.config.objects import LastFetchDatetime, TargetObject +from src.error.exceptions import FileNotFoundException, InvalidConfigException +from src.set_datetime_period_process import set_datetime_period_process +from src.util.execute_datetime import ExecuteDateTime + +from .test_utils.log_message import generate_log_message_tuple + + +class TestSetDatetimePeriodProcess: + + @pytest.fixture + def bucket_name(self): + return 'test-config-bucket' + + @pytest.fixture + def prepare_bucket(self, s3_client, bucket_name): + s3_client.create_bucket(Bucket=bucket_name) + yield + + def test_run_process_success(self, s3_client, prepare_bucket, bucket_name, monkeypatch, caplog): + """ + Cases: + データ取得期間設定処理が正常終了し、期待通りの結果が返ること + Arranges: + - チェック対象のdictオブジェクトを宣言する + Expects: + - チェック後のオブジェクト情報コレクションクラスのインスタンスが返却される + - データ取得期間設定処理の仕様に沿った正常系ログが出力されること(デバッグログは除く) + """ + # Arrange + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ] + } + + last_fetch_datetime_dict = { + 'last_fetch_datetime_from': '1999-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z', + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_objects_dict, execute_datetime) + s3_client.put_object(Bucket=bucket_name, Key='crm/last_fetch_datetime/Account.json', Body=json.dumps(last_fetch_datetime_dict)) + # 環境変数を編集 + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.set_datetime_period_process.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm/last_fetch_datetime') + # Act + actual_last_fetch_datetime = set_datetime_period_process(target_object, execute_datetime) + + # Assert + + # 返り値の期待値チェック + assert isinstance(actual_last_fetch_datetime, LastFetchDatetime), '最終取得日時オブジェクトクラスのインスタンスが返却される' + + # ログの確認 + assert generate_log_message_tuple(log_message='I-DATE-01 [Account] のデータ取得期間設定処理を開始します') in caplog.record_tuples + assert generate_log_message_tuple( + log_message='I-DATE-02 前回取得日時ファイルの取得開始します ファイルパス：[s3://test-config-bucket/crm/last_fetch_datetime/Account.json]') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-DATE-03 前回取得日時ファイルの取得成功しました') in caplog.record_tuples + assert generate_log_message_tuple( + log_message='I-DATE-06 取得範囲 From: [1999-01-01T00:00:00.000Z] To: [2100-12-31T23:59:59.000Z]') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-DATE-07 [Account] のデータ取得期間設定処理を終了します') in caplog.record_tuples + + def test_call_depended_modules(self, s3_client, prepare_bucket, bucket_name, monkeypatch, caplog): + """ + Cases: + データ取得期間設定処理内で依存しているモジュールが正しく呼ばれていること + Arranges: + - データ取得期間設定処理内の依存モジュールをモック化する + Expects: + - 依存しているモジュールが正しく呼ばれている + """ + + # Arrange + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ] + } + + last_fetch_datetime_dict = { + 'last_fetch_datetime_from': '1999-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z', + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_objects_dict, execute_datetime) + mock_config_bucket = MagicMock(return_value=last_fetch_datetime_dict) + mock_json_parser = MagicMock(return_value=last_fetch_datetime_dict) + # Act + with patch('src.aws.s3.ConfigBucket.get_last_fetch_datetime_file', mock_config_bucket), \ + patch('src.parser.json_parser.JsonParser.parse', mock_json_parser): + set_datetime_period_process(target_object, execute_datetime) + + # Assert + + assert mock_config_bucket.called is True + assert mock_json_parser.called is True + + def test_raise_get_last_fetch_datetime_file(self): + """ + Cases: + 最終取得日時ファイル取得処理でエラーが発生すること + Arranges: + - オブジェクト形式チェック処理で例外が発生するようにする + Expects: + - 例外が発生する + - 形式チェックが失敗したエラーが返却される + """ + + # Arrange + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ] + } + + last_fetch_datetime_dict = { + 'last_fetch_datetime_from': '1999-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z', + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_objects_dict, execute_datetime) + mock_config_bucket = MagicMock(side_effect=Exception('取得エラー')) + mock_json_parser = MagicMock(return_value=last_fetch_datetime_dict) + mock_last_fetch_datetime = MagicMock(return_value=None) + + # Act + with patch('src.aws.s3.ConfigBucket.get_last_fetch_datetime_file', mock_config_bucket), \ + patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ + patch('src.config.objects.LastFetchDatetime.__init__', mock_last_fetch_datetime): + with pytest.raises(FileNotFoundException) as e: + set_datetime_period_process(target_object, execute_datetime) + + # Assert + + assert mock_config_bucket.called is True + assert mock_json_parser.called is False + assert e.value.args[0] == f'前回取得日時ファイルが存在しません　ファイル名：[Account.json] エラー内容：[取得エラー]' + + def test_raise_json_parse(self): + """ + Cases: + Jsonパース処理でエラーが発生すること + Arranges: + - Jsonパース処理で例外が発生するようにする + Expects: + - 例外が発生する + - 形式チェックが失敗したエラーが返却される + """ + + # Arrange + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ] + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_objects_dict, execute_datetime) + mock_config_bucket = MagicMock(return_value='') + mock_json_parser = MagicMock(side_effect=Exception('変換エラー')) + mock_last_fetch_datetime = MagicMock(return_value=None) + + # Act + with patch('src.aws.s3.ConfigBucket.get_last_fetch_datetime_file', mock_config_bucket), \ + patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ + patch('src.config.objects.LastFetchDatetime.__init__', mock_last_fetch_datetime): + with pytest.raises(InvalidConfigException) as e: + set_datetime_period_process(target_object, execute_datetime) + + # Assert + + assert mock_config_bucket.called is True + assert mock_json_parser.called is True + assert e.value.args[0] == f'前回取得日時ファイルの形式チェック処理が失敗しました エラー内容：[変換エラー]' + + def test_raise_check_last_fetch_datetime(self): + """ + Cases: + Jsonパース処理でエラーが発生すること + Arranges: + - Jsonパース処理で例外が発生するようにする + Expects: + - 例外が発生する + - 形式チェックが失敗したエラーが返却される + """ + + # Arrange + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ] + } + + last_fetch_datetime_dict = { + 'last_fetch_datetime_from': '1999-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z', + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_objects_dict, execute_datetime) + mock_config_bucket = MagicMock(return_value='') + mock_json_parser = MagicMock(return_value=last_fetch_datetime_dict) + mock_last_fetch_datetime = MagicMock(side_effect=Exception('形式エラー')) + + # Act + with patch('src.aws.s3.ConfigBucket.get_last_fetch_datetime_file', mock_config_bucket), \ + patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ + patch('src.config.objects.LastFetchDatetime.__init__', mock_last_fetch_datetime): + with pytest.raises(InvalidConfigException) as e: + set_datetime_period_process(target_object, execute_datetime) + + # Assert + + assert mock_config_bucket.called is True + assert mock_json_parser.called is True + assert mock_last_fetch_datetime.called is True + assert e.value.args[0] == f'前回取得日時ファイルの形式チェック処理が失敗しました エラー内容：[形式エラー]' From d9c4268a024f14db618021f2e00048f417f08464 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 16 Aug 2022 18:17:40 +0900 Subject: [PATCH 178/275] =?UTF-8?q?feat:=20logger=E3=82=AF=E3=83=A9?= =?UTF-8?q?=E3=82=B9=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC?= =?UTF-8?q?=E3=83=89=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/util/test_logger_logger_class.py | 78 +++++++++++++++++++ .../tests/util/test_logger_out_of_class.py | 52 ++++++++++++- 2 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 ecs/crm-datafetch/tests/util/test_logger_logger_class.py diff --git a/ecs/crm-datafetch/tests/util/test_logger_logger_class.py b/ecs/crm-datafetch/tests/util/test_logger_logger_class.py new file mode 100644 index 00000000..172b7cf7 --- /dev/null +++ b/ecs/crm-datafetch/tests/util/test_logger_logger_class.py @@ -0,0 +1,78 @@ +import logging +from logging import handlers +from unittest.mock import MagicMock, patch + +from src.util.logger import Logger + + +class TestLogger: + + def test_constructor_loglevel_test(self, caplog, monkeypatch): + """ + Cases: + ログレベルに対象外の文字列を指定した場合、デフォルトであるInfoが指定されること + Arranges: + - LOG_LEVELの変数を想定外の文字列に置き換える + - loggerインスタンスの生成 + - ログ出力 + Expects: + - 期待値通りのログが出力されること + - ログレベルがInfoで設定されていること + """ + + # Arranges + monkeypatch.setattr('src.util.logger.LOG_LEVEL', "test") + + sut = Logger() + logger = sut.get_logger() + logger.info('infoログ出力') + + # Expects + assert ("root", logging.INFO, "infoログ出力") in caplog.record_tuples + assert logger.getEffectiveLevel() == logging.INFO + + def test_constructor_hashandlers(self, monkeypatch): + """ + Cases: + 生成したloggerインスタンスを利用し、期待値通りのログが出力されること + Arranges: + - `if not self.__logger.hasHandlers():`の分岐に入るようにモックを準備 + - loggerインスタンスの生成 + - ログ出力 + Expects: + - loggerインスタンスに1件以上のハンドラーが存在すること + - loggerインスタンスにStreamHandlerが含まれていること + """ + + # Arranges + def dummy_method(arg): + return False + + monkeypatch.setattr("logging.Logger.hasHandlers", dummy_method) + + sut = Logger() + logger = sut.get_logger() + logger.warning('warningログ出力') + + # Expects + assert len(logger.handlers) >= 1 + assert "StreamHandler" in str(logger.handlers) + + def test_get_logger(self, caplog): + """ + Cases: + 生成したloggerインスタンスを利用し、期待値通りのログが出力されること + Arranges: + - loggerインスタンスの生成 + - ログ出力 + Expects: + - 期待値通りのログが出力されること + """ + + # Arranges + sut = Logger() + logger = sut.get_logger() + logger.warning('warningログ出力') + + # Expects + assert ("root", logging.WARNING, "warningログ出力") in caplog.record_tuples diff --git a/ecs/crm-datafetch/tests/util/test_logger_out_of_class.py b/ecs/crm-datafetch/tests/util/test_logger_out_of_class.py index b5a54f01..170516ef 100644 --- a/ecs/crm-datafetch/tests/util/test_logger_out_of_class.py +++ b/ecs/crm-datafetch/tests/util/test_logger_out_of_class.py @@ -1,7 +1,5 @@ import logging -from unittest.mock import MagicMock, patch -import pytest from src.util.logger import logger_instance as logger @@ -18,6 +16,7 @@ class TestLogger: - 期待値通りのログが出力されること """ + # Arranges caplog.set_level(logging.DEBUG) logger.debug('debugログ出力') @@ -26,6 +25,7 @@ class TestLogger: logger.error("errorログ出力") logger.critical("criticalログ出力") + # Expects assert ("root", logging.DEBUG, "debugログ出力") in caplog.record_tuples assert ("root", logging.INFO, "infoログ出力") in caplog.record_tuples assert ("root", logging.WARNING, "warningログ出力") in caplog.record_tuples @@ -43,6 +43,7 @@ class TestLogger: - 期待値通りのログが出力されること """ + # Arranges caplog.set_level(logging.INFO) logger.debug('debugログ出力') @@ -51,6 +52,7 @@ class TestLogger: logger.error("errorログ出力") logger.critical("criticalログ出力") + # Expects assert ("root", logging.DEBUG, "debugログ出力") not in caplog.record_tuples assert ("root", logging.INFO, "infoログ出力") in caplog.record_tuples assert ("root", logging.WARNING, "warningログ出力") in caplog.record_tuples @@ -68,6 +70,7 @@ class TestLogger: - 期待値通りのログが出力されること """ + # Arranges caplog.set_level(logging.WARNING) logger.debug('debugログ出力') @@ -76,6 +79,7 @@ class TestLogger: logger.error("errorログ出力") logger.critical("criticalログ出力") + # Expects assert ("root", logging.DEBUG, "debugログ出力") not in caplog.record_tuples assert ("root", logging.INFO, "infoログ出力") not in caplog.record_tuples assert ("root", logging.WARNING, "warningログ出力") in caplog.record_tuples @@ -93,6 +97,7 @@ class TestLogger: - 期待値通りのログが出力されること """ + # Arranges caplog.set_level(logging.ERROR) logger.debug('debugログ出力') @@ -101,6 +106,7 @@ class TestLogger: logger.error("errorログ出力") logger.critical("criticalログ出力") + # Expects assert ("root", logging.DEBUG, "debugログ出力") not in caplog.record_tuples assert ("root", logging.INFO, "infoログ出力") not in caplog.record_tuples assert ("root", logging.WARNING, "warningログ出力") not in caplog.record_tuples @@ -118,6 +124,7 @@ class TestLogger: - 期待値通りのログが出力されること """ + # Arranges caplog.set_level(logging.CRITICAL) logger.debug('debugログ出力') @@ -126,6 +133,7 @@ class TestLogger: logger.error("errorログ出力") logger.critical("criticalログ出力") + # Expects assert ("root", logging.DEBUG, "debugログ出力") not in caplog.record_tuples assert ("root", logging.INFO, "infoログ出力") not in caplog.record_tuples assert ("root", logging.WARNING, "warningログ出力") not in caplog.record_tuples @@ -133,13 +141,53 @@ class TestLogger: assert ("root", logging.CRITICAL, "criticalログ出力") in caplog.record_tuples def test_logging_getlogger_boto3(self): + """ + Cases: + 個別設定したboto3モジュールのログレベル確認 + Arranges: + なし + Expects: + 設定されているログレベルが期待値と一致すること + """ + + # Expects assert logging.getLogger("boto3").getEffectiveLevel() == logging.WARNING def test_logging_getlogger_botocore(self): + """ + Cases: + 個別設定したbotocoreモジュールのログレベル確認 + Arranges: + なし + Expects: + 設定されているログレベルが期待値と一致すること + """ + + # Expects assert logging.getLogger("botocore").getEffectiveLevel() == logging.WARNING def test_logging_getlogger_s3transfer(self): + """ + Cases: + 個別設定したs3transferモジュールのログレベル確認 + Arranges: + なし + Expects: + 設定されているログレベルが期待値と一致すること + """ + + # Expects assert logging.getLogger("s3transfer").getEffectiveLevel() == logging.WARNING def test_logging_getlogger_urllib3(self): + """ + Cases: + 個別設定したurllib3モジュールのログレベル確認 + Arranges: + なし + Expects: + 設定されているログレベルが期待値と一致すること + """ + + # Expects assert logging.getLogger("s3transfer").getEffectiveLevel() == logging.WARNING From 00cb7602f602d9226ae52af4d97bf2c67037178a Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 16 Aug 2022 21:17:41 +0900 Subject: [PATCH 179/275] =?UTF-8?q?fix:=20parametrize=E3=82=92=E5=88=A9?= =?UTF-8?q?=E7=94=A8=E3=81=97=E3=81=9F=E3=83=86=E3=82=B9=E3=83=88=E3=83=91?= =?UTF-8?q?=E3=82=BF=E3=83=BC=E3=83=B3=E3=81=AE=E9=9B=86=E7=B4=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/util/test_dict_checker.py | 331 +++--------------- 1 file changed, 56 insertions(+), 275 deletions(-) diff --git a/ecs/crm-datafetch/tests/util/test_dict_checker.py b/ecs/crm-datafetch/tests/util/test_dict_checker.py index 6707b8be..508afa69 100644 --- a/ecs/crm-datafetch/tests/util/test_dict_checker.py +++ b/ecs/crm-datafetch/tests/util/test_dict_checker.py @@ -4,10 +4,17 @@ from src.util.dict_checker import DictChecker class TestDictChecker: - def test_is_empty_no_value(self): + @pytest.mark.parametrize('test_data,expect', [ + ('', True), + (None, True), + ('test_data', False), + ]) + def test_is_empty(self, test_data, expect): """ Cases: - 辞書型データの対象のキーの値が''(空文字)の場合、Trueが返る + 1. 辞書型データの対象のキーの値が''(空文字)の場合、Trueが返る + 2. 辞書型データの対象のキーの値がNoneの場合、Trueが返る + 3. 辞書型データの対象のキーの値が存在する場合、Falseが返る Arranges: - 辞書型のオブジェクト情報を準備する - 辞書型チェックインスタンスを生成する @@ -17,7 +24,7 @@ class TestDictChecker: # Arranges object_dict = { - "test_key": "" + "test_key": test_data } sut = DictChecker(object_dict) @@ -26,12 +33,23 @@ class TestDictChecker: actual = sut.is_empty("test_key") # Expects - assert actual is True + assert actual is expect - def test_is_empty_none(self): + @pytest.mark.parametrize('test_data,expect', [ + ([''], False), + ([None], False), + (['test_value'], False), + ([], True), + ('test_value', False) + ]) + def test_is_list_empty(self, test_data, expect): """ Cases: - 辞書型データの対象のキーの値がNoneの場合、Trueが返る + 1. 辞書型のデータの対象のキーのリストの値が''(空文字)の場合、Falseが返る + 2. 辞書型データの対象のキーのリストの値がNoneの場合、Falseが返る + 3. 辞書型データの対象のキーのリストの値が存在する場合、Falseが返る + 4. 辞書型データの対象のキーのリストの値が[](空)の場合、Trueが返る + 5. 辞書型データの対象のキーの値に文字列が存在する場合、Falseが返る Arranges: - 辞書型のオブジェクト情報を準備する - 辞書型チェックインスタンスを生成する @@ -41,57 +59,7 @@ class TestDictChecker: # Arranges object_dict = { - "test_key": None - } - - sut = DictChecker(object_dict) - - # Act - actual = sut.is_empty("test_key") - - # Expects - assert actual is True - - def test_is_empty_exist_value(self): - """ - Cases: - 辞書型データの対象のキーの値が存在する場合、Falseが返る - Arranges: - - 辞書型のオブジェクト情報を準備する - - 辞書型チェックインスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - object_dict = { - "test_key": "test_value" - } - - sut = DictChecker(object_dict) - - # Act - actual = sut.is_empty("test_key") - - # Expects - assert actual is False - - def test_is_list_empty_no_value(self): - """ - Cases: - 辞書型のデータの対象のキーのリストの値が''(空文字)の場合、Falseが返る - Arranges: - - 辞書型のオブジェクト情報を準備する - - 辞書型チェックインスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - object_dict = { - "test_key": [ - "" - ] + "test_key": test_data } sut = DictChecker(object_dict) @@ -100,12 +68,19 @@ class TestDictChecker: actual = sut.is_list_empty("test_key") # Expects - assert actual is False + assert actual is expect - def test_is_list_empty_none(self): + @pytest.mark.parametrize('test_data,expect', [ + ({"test_key": "test_value"}, True), + ({}, False), + ({"test_key": ""}, False), + ]) + def test_check_key_exist(self, test_data, expect) -> bool: """ Cases: - 辞書型データの対象のキーのリストの値がNoneの場合、Falseが返る + 1. 辞書型データの対象のキーが存在する、かつ値が存在する場合、Trueが返る + 2. 辞書型データの対象のキーが存在しない場合、Falseが返る + 3. 辞書型データの対象のキーの値が存在しない場合、Falseが返る Arranges: - 辞書型のオブジェクト情報を準備する - 辞書型チェックインスタンスを生成する @@ -114,109 +89,7 @@ class TestDictChecker: """ # Arranges - object_dict = { - "test_key": [ - None - ] - } - - sut = DictChecker(object_dict) - - # Act - actual = sut.is_list_empty("test_key") - - # Expects - assert actual is False - - def test_is_list_empty_blank(self): - """ - Cases: - 辞書型データの対象のキーのリストの値が[](空)の場合、Trueが返る - Arranges: - - 辞書型のオブジェクト情報を準備する - - 辞書型チェックインスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - object_dict = { - "test_key": [] - } - - sut = DictChecker(object_dict) - - # Act - actual = sut.is_list_empty("test_key") - - # Expects - assert actual is True - - def test_is_list_empty_exist_string(self): - """ - Cases: - 辞書型データの対象のキーの値に文字列が存在する場合、Falseが返る - Arranges: - - 辞書型のオブジェクト情報を準備する - - 辞書型チェックインスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - object_dict = { - "test_key": "test_value" - } - - sut = DictChecker(object_dict) - - # Act - actual = sut.is_list_empty("test_key") - - # Expects - assert actual is False - - def test_is_list_empty_exist_value(self): - """ - Cases: - 辞書型データの対象のキーのリストの値が存在する場合、Falseが返る - Arranges: - - 辞書型のオブジェクト情報を準備する - - 辞書型チェックインスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - object_dict = { - "test_key": [ - "test_value" - ] - } - - sut = DictChecker(object_dict) - - # Act - actual = sut.is_list_empty("test_key") - - # Expects - assert actual is False - - def test_check_key_exist(self) -> bool: - """ - Cases: - 辞書型データの対象のキーが存在する、かつ値が存在する場合、Trueが返る - Arranges: - - 辞書型のオブジェクト情報を準備する - - 辞書型チェックインスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - object_dict = { - "test_key": "test_value" - } + object_dict = test_data sut = DictChecker(object_dict) @@ -224,12 +97,17 @@ class TestDictChecker: actual = sut.check_key_exist("test_key") # Expects - assert actual is True + assert actual is expect - def test_check_key_exist_no_key(self) -> bool: + @pytest.mark.parametrize('test_data,test_type,expect', [ + ({"test_key": "test_value"}, str, True), + ({"test_key": 1}, str, False) + ]) + def test_check_data_type(self, test_data, test_type, expect) -> bool: """ Cases: - 辞書型データの対象のキーが存在しない場合、Falseが返る + 1. 辞書型データの対象のキーの値の型が指定した型と一致した場合、Trueが返る + 2. 辞書型データの対象のキーの値の型が指定した型と一致しない場合、Falseが返る Arranges: - 辞書型のオブジェクト情報を準備する - 辞書型チェックインスタンスを生成する @@ -238,93 +116,25 @@ class TestDictChecker: """ # Arranges - object_dict = { - } + object_dict = test_data sut = DictChecker(object_dict) # Act - actual = sut.check_key_exist("test_key") + actual = sut.check_data_type("test_key", test_type) # Expects - assert actual is False + assert actual is expect - def test_check_key_exist_no_value(self) -> bool: + @pytest.mark.parametrize('test_data,expect', [ + ({"test_key": "2022-08-01T10:00:00.000Z"}, True), + ({"test_key": "test_value"}, False) + ]) + def test_check_match_pattern(self, test_data, expect) -> bool: """ Cases: - 辞書型データの対象のキーの値が存在しない場合、Falseが返る - Arranges: - - 辞書型のオブジェクト情報を準備する - - 辞書型チェックインスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - object_dict = { - "test_key": "" - } - - sut = DictChecker(object_dict) - - # Act - actual = sut.check_key_exist("test_key") - - # Expects - assert actual is False - - def test_check_data_type(self) -> bool: - """ - Cases: - 辞書型データの対象のキーの値の型が指定した型と一致した場合、Trueが返る - Arranges: - - 辞書型のオブジェクト情報を準備する - - 辞書型チェックインスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - object_dict = { - "test_key": "test_value" - } - - sut = DictChecker(object_dict) - - # Act - actual = sut.check_data_type("test_key", str) - - # Expects - assert actual is True - - def test_check_data_type_other_type(self) -> bool: - """ - Cases: - 辞書型データの対象のキーの値の型が指定した型と一致しない場合、Falseが返る - Arranges: - - 辞書型のオブジェクト情報を準備する - - 辞書型チェックインスタンスを生成する - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - object_dict = { - "test_key": 1 - } - - sut = DictChecker(object_dict) - - # Act - actual = sut.check_data_type("test_key", str) - - # Expects - assert actual is False - - def test_check_match_pattern(self) -> bool: - """ - Cases: - 辞書型データの対象のキーの値の型が指定した正規表現と一致した場合、Trueが返る + 1. 辞書型データの対象のキーの値の型が指定した正規表現と一致した場合、Trueが返る + 2. 辞書型データの対象のキーの値の型が指定した正規表現と一致しない場合、Falseが返る Arranges: - 辞書型のオブジェクト情報を準備する - 辞書型チェックインスタンスを生成する @@ -334,9 +144,7 @@ class TestDictChecker: """ # Arranges - object_dict = { - "test_key": "2022-08-01T10:00:00.000Z" - } + object_dict = test_data sut = DictChecker(object_dict) @@ -346,34 +154,7 @@ class TestDictChecker: actual = sut.check_match_pattern(pattern, "test_key") # Expects - assert actual is True - - def test_check_match_pattern_not_match(self) -> bool: - """ - Cases: - 辞書型データの対象のキーの値の型が指定した正規表現と一致しない場合、Falseが返る - Arranges: - - 辞書型のオブジェクト情報を準備する - - 辞書型チェックインスタンスを生成する - - 正規表現パターンの生成 - Expects: - - 戻り値が期待値と一致する - """ - - # Arranges - object_dict = { - "test_key": "test_value" - } - - sut = DictChecker(object_dict) - - pattern = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' - - # Act - actual = sut.check_match_pattern(pattern, "test_key") - - # Expects - assert actual is False + assert actual is expect def test_assert_key_exist(self) -> None: """ From 2f230e2f15088fdda96fb3bba45e3b2a2f2a7a21 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 08:36:43 +0900 Subject: [PATCH 180/275] =?UTF-8?q?fix:=20=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E5=8F=96=E5=BE=97=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E3=81=AE?= =?UTF-8?q?SOQL=E6=96=87=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesfoce.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py index 612c3679..b815e772 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py @@ -261,7 +261,18 @@ class TestSalesforceApiClient: 取得件数が0件になる """ soql = """SELECT - COUNT(Id) + Id, + Name, + SystemModstamp, + LastModifiedDate, + CustomItem1__c, + CustomItem2__c, + CustomItem3__c, + CustomItem4__c, + CustomItem5__c, + CustomItem6__c, + CustomItem7__c, + CustomItem8__c FROM Account WHERE From cede4e0e122aa29faca2045f0625ba7d5b25a382 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Wed, 17 Aug 2022 10:23:31 +0900 Subject: [PATCH 181/275] =?UTF-8?q?feat:=20write=5Fto=5Fstring=E3=83=A1?= =?UTF-8?q?=E3=82=BD=E3=83=83=E3=83=89=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/converter/converter.py | 2 +- .../tests/converter/test_converter.py | 22 ++++++++++--------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index ace996ef..e1c01dcd 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -72,4 +72,4 @@ class CSVStringConverter: return csv_value except Exception as e: - raise Exception('CSVデータの取得に失敗しました', e) + raise Exception('CSVデータの出力に失敗しました', e) diff --git a/ecs/crm-datafetch/tests/converter/test_converter.py b/ecs/crm-datafetch/tests/converter/test_converter.py index 429a7510..0300381d 100644 --- a/ecs/crm-datafetch/tests/converter/test_converter.py +++ b/ecs/crm-datafetch/tests/converter/test_converter.py @@ -1,6 +1,5 @@ import textwrap from collections import OrderedDict -from unittest import mock import pytest from src.config.objects import TargetObject @@ -106,6 +105,7 @@ class TestCSVStringConverter: "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3234567","0"\r\n\ ''' + # expected_valueのインデントが半角スペースと認識されてしまうため、`textwrap.dedent`にて補正 assert actual == textwrap.dedent(expected_value) def test_raise_convert_extract_jsons(self) -> str: @@ -200,7 +200,7 @@ class TestCSVStringConverter: # Expects assert '必要なjsonのデータ成形に失敗しました' in str(e.value) - def test_raise_convert_convert_csv(self) -> str: + def test_raise_convert_convert_to_csv(self) -> str: """ Cases: 一部データのカラム（Id）の不足により、CSV変換で例外が発生すること @@ -292,16 +292,16 @@ class TestCSVStringConverter: # Expects assert 'CSV変換に失敗しました カラム名：[Id] 行番号: [1]' in str(e.value) - @mock.patch('src.converter.converter.CSVStringConverter.create') - def test_raise_convert_csv_string(self, execute_mock) -> str: + def test_raise_convert_write_csv_string(self, monkeypatch) -> str: """ Cases: - 一部データのカラム（Id）の不足により、CSV変換で例外が発生すること + csvデータ出力の、CSVデータ取得で例外が発生すること Arranges: - オブジェクト情報の作成 - データの作成 - 実行日時取得インスタンスの生成 - オブジェクト情報インスタンスの生成 + - csvデータ出力のエラーを発生させるためのモックを準備 Expects: 例外が期待値と一致すること """ @@ -377,15 +377,17 @@ class TestCSVStringConverter: execute_datetime = ExecuteDateTime() target_object = TargetObject(object_info, execute_datetime) + def dummy_method(arg): + raise Exception(e) + + # データ加工のみだと事前の処理によりエラーとなるため、csv出力モジュールをモック化する + monkeypatch.setattr("csv.writer", dummy_method) + # Act csv_string_converter = CSVStringConverter(target_object, data) - execute_mock.side_effect = Exception with pytest.raises(Exception) as e: csv_string_converter.convert() -# actual = csv_string_converter.convert() - # Expects - assert 'CSVデータの取得に失敗しました' in str(e.value) -# assert actual == 'CSVデータの取得に失敗しました' + assert 'CSVデータの出力に失敗しました' in str(e.value) From f4b03c0c4541fccbe906737998d0a16122b0381f Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 10:59:47 +0900 Subject: [PATCH 182/275] =?UTF-8?q?refactor:=20textwrap.dedent=E3=82=92?= =?UTF-8?q?=E4=BD=BF=E3=81=A3=E3=81=A6=E4=BD=99=E5=88=86=E3=81=AA=E5=8D=8A?= =?UTF-8?q?=E8=A7=92=E3=82=B9=E3=83=9A=E3=83=BC=E3=82=B9=E3=82=92=E6=B6=88?= =?UTF-8?q?=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/salesforce/soql_builder.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/ecs/crm-datafetch/src/salesforce/soql_builder.py b/ecs/crm-datafetch/src/salesforce/soql_builder.py index a84744a8..42d4ecea 100644 --- a/ecs/crm-datafetch/src/salesforce/soql_builder.py +++ b/ecs/crm-datafetch/src/salesforce/soql_builder.py @@ -1,12 +1,15 @@ -from src.config.objects import TargetObject, LastFetchDatetime +import textwrap + +from src.config.objects import LastFetchDatetime, TargetObject class SOQLBuilder: def __init__(self, target_object: TargetObject, last_fetch_datetime: LastFetchDatetime) -> None: - self.__SELECT_SOQL = """SELECT {column_or_expression} FROM {object_name} - WHERE {datetime_column} > {last_fetch_datetime_from} - AND {datetime_column} <= {last_fetch_datetime_to} - """ + self.__SELECT_SOQL = textwrap.dedent("""\ + SELECT {column_or_expression} FROM {object_name} + WHERE {datetime_column} > {last_fetch_datetime_from} + AND {datetime_column} <= {last_fetch_datetime_to} + """) self.__target_object = target_object self.__last_fetch_datetime = last_fetch_datetime From ff6b41b18d5295aff1abb6375e67d0c16cec54d0 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 11:59:43 +0900 Subject: [PATCH 183/275] =?UTF-8?q?feat:=20CRM=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E5=8F=96=E5=BE=97=E5=87=A6=E7=90=86=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E8=BF=BD=E5=8A=A0=E3=80=82=E3=83=AA=E3=83=88=E3=83=A9?= =?UTF-8?q?=E3=82=A4=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E3=81=8C=E9=80=94?= =?UTF-8?q?=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_fetch_crm_data_process.py | 308 ++++++++++++++++++ 1 file changed, 308 insertions(+) create mode 100644 ecs/crm-datafetch/tests/test_fetch_crm_data_process.py diff --git a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py new file mode 100644 index 00000000..6eca8099 --- /dev/null +++ b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py @@ -0,0 +1,308 @@ +import logging +from collections import OrderedDict +from unittest.mock import patch + +import pytest +from requests.exceptions import ConnectTimeout, ReadTimeout +from src.config.objects import LastFetchDatetime, TargetObject +from src.error.exceptions import SalesforceAPIException +from src.fetch_crm_data_process import fetch_crm_data_process +from src.system_var.constants import FETCH_JP_NAME +from src.util.execute_datetime import ExecuteDateTime + +from .test_utils.log_message import generate_log_message_tuple + +common_target_object_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'AccountNumber', + 'LastModifiedDate', + 'LastModifiedById', + 'SystemModstamp', + 'IsDeleted' + ] +} + +common_last_fetch_datetime_dict = { + 'last_fetch_datetime_from': '2000-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2010-01-01T00:00:00.000Z', +} + +common_execute_datetime = ExecuteDateTime() +common_target_object = TargetObject(common_target_object_dict, common_execute_datetime) +common_last_fetch_datetime = LastFetchDatetime(common_last_fetch_datetime_dict, common_execute_datetime) + +common_expect = [ + OrderedDict([ + ('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/TEST001')])), + ('Id', 'TEST001'), + ('AccountNumber', 'test001'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), + ('IsDeleted', False) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/TEST002')])), + ('Id', 'TEST002'), + ('AccountNumber', 'test002'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), + ('IsDeleted', False) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/TEST002')])), + ('Id', 'TEST002'), + ('AccountNumber', 'test002'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), + ('IsDeleted', False) + ]), +] + + +class TestFetchCrmDataProcess: + + def test_run_process_success(self, monkeypatch, caplog): + """ + Cases: + CRMデータ取得処理が正常終了し、期待通りの結果が返ること + Arranges: + - SalesforceApiClientクラスをモック化し、固定の値を返すようにする + Expects: + - CRMから取得した取得オブジェクトのリストが返却される + - CRMデータ取得処理の仕様に沿った正常系ログが出力されること(デバッグログは除く) + """ + # Arrange + + # モック化 + with patch('src.fetch_crm_data_process.SalesforceApiClient') as mock: + instance = mock.return_value + instance.fetch_sf_count.return_value = len(common_expect) + instance.fetch_sf_data.return_value = common_expect + # Act + actual_crm_data_response = fetch_crm_data_process(common_target_object, common_last_fetch_datetime) + + # Assert + + # 返り値の期待値チェック + assert isinstance(actual_crm_data_response, list) + assert isinstance(actual_crm_data_response[0], OrderedDict) + assert actual_crm_data_response == common_expect + + # ログの確認 + assert generate_log_message_tuple(log_message='I-FETCH-01 [Account] のCRMからのデータ取得処理を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-FETCH-02 [Account] の件数取得を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-FETCH-03 [Account] の件数：[3]') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-FETCH-04 [Account] のレコード取得を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-FETCH-05 [Account] のレコード取得が成功しました') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') in caplog.record_tuples + + def test_call_depended_modules(self): + """ + Cases: + CRMデータ取得処理内で依存しているモジュールが正しく呼ばれていること + Arranges: + - CRMデータ取得処理の依存モジュールをモック化する + Expects: + - 依存しているモジュールが正しく呼ばれている + """ + + # Arrange + with patch('src.fetch_crm_data_process.CounterObject', ) as mock_counter, \ + patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ + patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: + # モック化 + mock_counter_inst = mock_counter.return_value + mock_counter_inst.describe.return_value = 1 + mock_counter_inst.increment.return_value = 1 + mock_builder_inst = mock_soql_builder.return_value + mock_builder_inst.create_count_soql.return_value = '' + mock_builder_inst.create_fetch_soql.return_value = '' + mock_client_inst = mock_api_client.return_value + mock_client_inst.fetch_sf_count.return_value = 1 + mock_client_inst.fetch_sf_data.return_value = common_expect + + # Act + fetch_crm_data_process(common_target_object, common_last_fetch_datetime) + + # Assert + + assert mock_counter.call_count == 2, 'リトライカウント用クラスのインスタンスが生成されていること' + assert mock_counter_inst.describe.called is False, 'リトライ用のカウントが取得されていないこと' + assert mock_counter_inst.increment.called is False, 'リトライ用のカウントが足されていないこと' + assert mock_soql_builder.call_count == 1, 'SOQL生成クラスのインスタンスが生成されていること' + assert mock_builder_inst.create_count_soql.called is True, '件数取得SOQLが生成されていること' + assert mock_builder_inst.create_fetch_soql.called is True, 'データ取得SOQLが生成されていること' + assert mock_api_client.call_count == 2, 'Salesforce APIクライアントのインスタンスが生成されていること' + assert mock_client_inst.fetch_sf_count.called is True, '件数の取得が呼ばれていること' + assert mock_client_inst.fetch_sf_data.called is True, 'レコードの取得が呼ばれていること' + + def test_raise_create_count_soql(self, monkeypatch, caplog): + """ + Cases: + 件数取得用SOQLが生成できない場合、エラーが発生すること + Arranges: + - 件数取得用SOQL生成処理で例外が発生するようにする + Expects: + - 例外が発生する + - データ件数取得に失敗した胸のエラーが出力される + """ + + # Arrange + with patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ + patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: + # モック化 + mock_builder_inst = mock_soql_builder.return_value + mock_builder_inst.create_count_soql.side_effect = Exception('生成エラー') + mock_builder_inst.create_fetch_soql.return_value = '' + mock_client_inst = mock_api_client.return_value + mock_client_inst.fetch_sf_count.return_value = 1 + mock_client_inst.fetch_sf_data.return_value = common_expect + + # Act + with pytest.raises(SalesforceAPIException) as e: + fetch_crm_data_process(common_target_object, common_last_fetch_datetime) + + # Assert + + assert e.value.error_id == 'E-FETCH-01' + assert e.value.func_name == FETCH_JP_NAME + assert e.value.args[0] == f'[Account] の件数取得に失敗しました エラー内容：[生成エラー]' + + def test_raise_create_fetch_soql(self, monkeypatch, caplog): + """ + Cases: + データ取得用SOQLが生成できない場合、エラーが発生すること + Arranges: + - データ取得用SOQL生成処理で例外が発生するようにする + Expects: + - 例外が発生する + - データ取得に失敗した胸のエラーが出力される + """ + + # Arrange + with patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ + patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: + # モック化 + mock_builder_inst = mock_soql_builder.return_value + mock_builder_inst.create_count_soql.return_value = '' + mock_builder_inst.create_fetch_soql.side_effect = Exception('生成エラー') + mock_client_inst = mock_api_client.return_value + mock_client_inst.fetch_sf_count.return_value = 1 + mock_client_inst.fetch_sf_data.return_value = common_expect + + # Act + with pytest.raises(SalesforceAPIException) as e: + fetch_crm_data_process(common_target_object, common_last_fetch_datetime) + + # Assert + + assert e.value.error_id == 'E-FETCH-02' + assert e.value.func_name == FETCH_JP_NAME + assert e.value.args[0] == f'[Account] のレコード取得に失敗しました エラー内容：[生成エラー]' + + def test_raise_fetch_sf_count_connection_timeout_with_retry_success(self, monkeypatch, caplog): + """ + Cases: + データ件数取得処理で接続タイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること + Arranges: + - データ件数取得処理の最大リトライ試行回数を3に設定する + - データ件数取得処理の初回に接続タイムアウト例外が発生するようにする + Expects: + - 正常終了する + - データ件数取得に失敗した胸のエラーが出力されない + """ + + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_AUTH_TIMEOUT', 1) + # Arrange + with patch('src.fetch_crm_data_process.CounterObject', ) as mock_counter, \ + patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ + patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: + # モック化 + mock_counter_inst = mock_counter.return_value + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_builder_inst = mock_soql_builder.return_value + mock_builder_inst.create_count_soql.return_value = '' + mock_builder_inst.create_fetch_soql.return_value = '' + mock_client_inst = mock_api_client.return_value + mock_client_inst.fetch_sf_count.side_effect = [ConnectTimeout('接続タイムアウト'), len(common_expect)] + mock_client_inst.fetch_sf_data.return_value = common_expect + # Act + fetch_crm_data_process(common_target_object, common_last_fetch_datetime) + + # Assert + + assert mock_counter_inst.describe.call_count == 1 + assert mock_counter_inst.increment.call_count == 1 + + retry_log_message = 'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[1] エラー内容：[接続タイムアウト]' + assert generate_log_message_tuple( + log_level=logging.WARNING, + log_message=retry_log_message) in caplog.record_tuples + called_log_counts = len([log for log in caplog.messages if log == retry_log_message]) + assert called_log_counts == 1 + assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') in caplog.record_tuples + + def test_raise_fetch_sf_count_connection_timeout_with_retry_fail(self, monkeypatch, caplog): + """ + Cases: + データ件数取得処理で接続タイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること + Arranges: + - データ件数取得処理の最大リトライ試行回数を3に設定する + - データ件数取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする + Expects: + - 正常終了する + - データ件数取得に失敗した胸のエラーが出力されない + """ + + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_AUTH_TIMEOUT', 1) + # Arrange + with patch('src.fetch_crm_data_process.CounterObject', ) as mock_counter, \ + patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ + patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: + # モック化 + mock_counter_inst = mock_counter.return_value + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_builder_inst = mock_soql_builder.return_value + mock_builder_inst.create_count_soql.return_value = '' + mock_builder_inst.create_fetch_soql.return_value = '' + mock_client_inst = mock_api_client.return_value + mock_client_inst.fetch_sf_count.side_effect = [ConnectTimeout('接続タイムアウト'), ConnectTimeout('接続タイムアウト'), ConnectTimeout('接続タイムアウト')] + mock_client_inst.fetch_sf_data.return_value = common_expect + # Act + with pytest.raises(SalesforceAPIException) as e: + fetch_crm_data_process(common_target_object, common_last_fetch_datetime) + + # Assert + + # 取得は3回行われる + assert mock_counter_inst.describe.call_count == 3 + # 足し込みは2回のみ + assert mock_counter_inst.increment.call_count == 2 + + retry_log_message = 'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[1] エラー内容：[接続タイムアウト]' + assert generate_log_message_tuple( + log_level=logging.WARNING, + log_message=retry_log_message) in caplog.record_tuples + called_log_counts = len([log for log in caplog.messages if log == retry_log_message]) + assert called_log_counts == 2 + assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples + + assert e.value.error_id == 'E-FETCH-01' + assert e.value.func_name == FETCH_JP_NAME + # リトライ例外のオブジェクトIDが違うため、in句で比較 + assert f'[Account] の件数取得に失敗しました エラー内容：[RetryError' in e.value.args[0] From 9504b8ce50a9c7392667de556f33dddbc06f9ea0 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 13:05:29 +0900 Subject: [PATCH 184/275] =?UTF-8?q?feat:=20=E4=BB=B6=E6=95=B0=E5=8F=96?= =?UTF-8?q?=E5=BE=97=E6=99=82=E3=81=AE=E6=8E=A5=E7=B6=9A=E3=82=BF=E3=82=A4?= =?UTF-8?q?=E3=83=A0=E3=82=A2=E3=82=A6=E3=83=88=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_fetch_crm_data_process.py | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py index 6eca8099..3ce7b43f 100644 --- a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py @@ -306,3 +306,105 @@ class TestFetchCrmDataProcess: assert e.value.func_name == FETCH_JP_NAME # リトライ例外のオブジェクトIDが違うため、in句で比較 assert f'[Account] の件数取得に失敗しました エラー内容：[RetryError' in e.value.args[0] + + def test_raise_fetch_sf_count_read_timeout_with_retry_success(self, monkeypatch, caplog): + """ + Cases: + データ件数取得処理で読み取り例外が発生した場合、リトライした結果復旧し、正常終了すること + Arranges: + - データ件数取得処理の最大リトライ試行回数を3に設定する + - データ件数取得処理の初回に読み取り例外が発生するようにする + Expects: + - 正常終了する + - データ件数取得に失敗した胸のエラーが出力されない + """ + + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_TIMEOUT', 1) + # Arrange + with patch('src.fetch_crm_data_process.CounterObject', ) as mock_counter, \ + patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ + patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: + # モック化 + mock_counter_inst = mock_counter.return_value + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_builder_inst = mock_soql_builder.return_value + mock_builder_inst.create_count_soql.return_value = '' + mock_builder_inst.create_fetch_soql.return_value = '' + mock_client_inst = mock_api_client.return_value + mock_client_inst.fetch_sf_count.side_effect = [ReadTimeout('読み取りタイムアウト'), len(common_expect)] + mock_client_inst.fetch_sf_data.return_value = common_expect + # Act + fetch_crm_data_process(common_target_object, common_last_fetch_datetime) + + # Assert + + assert mock_counter_inst.describe.call_count == 1 + assert mock_counter_inst.increment.call_count == 1 + + retry_log_message = 'W-FETCH-02 [Account] の件数取得処理がタイムアウトしたため、リトライします：[1] エラー内容：[読み取りタイムアウト]' + assert generate_log_message_tuple( + log_level=logging.WARNING, + log_message=retry_log_message) in caplog.record_tuples + called_log_counts = len([log for log in caplog.messages if log == retry_log_message]) + assert called_log_counts == 1 + assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') in caplog.record_tuples + + def test_raise_fetch_sf_count_read_timeout_with_retry_fail(self, monkeypatch, caplog): + """ + Cases: + データ件数取得処理で読み取り例外が発生した場合、リトライした結果復旧せず、異常終了すること + Arranges: + - データ件数取得処理の最大リトライ試行回数を3に設定する + - データ件数取得処理の1回目、2回目、3回目で読み取り例外が発生するようにする + Expects: + - 正常終了する + - データ件数取得に失敗した胸のエラーが出力されない + """ + + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_TIMEOUT', 1) + # Arrange + with patch('src.fetch_crm_data_process.CounterObject', ) as mock_counter, \ + patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ + patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: + # モック化 + mock_counter_inst = mock_counter.return_value + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_builder_inst = mock_soql_builder.return_value + mock_builder_inst.create_count_soql.return_value = '' + mock_builder_inst.create_fetch_soql.return_value = '' + mock_client_inst = mock_api_client.return_value + mock_client_inst.fetch_sf_count.side_effect = [ReadTimeout('読み取りタイムアウト'), ReadTimeout('読み取りタイムアウト'), ReadTimeout('読み取りタイムアウト')] + mock_client_inst.fetch_sf_data.return_value = common_expect + # Act + with pytest.raises(SalesforceAPIException) as e: + fetch_crm_data_process(common_target_object, common_last_fetch_datetime) + + # Assert + + # 取得は3回行われる + assert mock_counter_inst.describe.call_count == 3 + # 足し込みは2回のみ + assert mock_counter_inst.increment.call_count == 2 + + retry_log_message = 'W-FETCH-02 [Account] の件数取得処理がタイムアウトしたため、リトライします：[1] エラー内容：[読み取りタイムアウト]' + assert generate_log_message_tuple( + log_level=logging.WARNING, + log_message=retry_log_message) in caplog.record_tuples + called_log_counts = len([log for log in caplog.messages if log == retry_log_message]) + assert called_log_counts == 2 + assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples + + assert e.value.error_id == 'E-FETCH-01' + assert e.value.func_name == FETCH_JP_NAME + # リトライ例外のオブジェクトIDが違うため、in句で比較 + assert f'[Account] の件数取得に失敗しました エラー内容：[RetryError' in e.value.args[0] From d0066a719b6be0834726c03fe329fb38bf299836 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 13:11:43 +0900 Subject: [PATCH 185/275] =?UTF-8?q?fix:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=AF=BE=E5=BF=9C=20https://nds-tyo?= =?UTF-8?q?.backlog.com/git/NEWDWH2021/newsdwh2021/pullRequests/66#comment?= =?UTF-8?q?-1321955?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_check_object_info_process.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ecs/crm-datafetch/tests/test_check_object_info_process.py b/ecs/crm-datafetch/tests/test_check_object_info_process.py index 2e938504..46125bc4 100644 --- a/ecs/crm-datafetch/tests/test_check_object_info_process.py +++ b/ecs/crm-datafetch/tests/test_check_object_info_process.py @@ -40,6 +40,12 @@ class TestCheckObjectInfoProcess: # 返り値の期待値チェック assert isinstance(actual_fetch_target_objects, TargetObject), 'CRM取得オブジェクトクラスのインスタンスが返却される' + assert actual_fetch_target_objects.object_name == 'Account' + assert actual_fetch_target_objects.columns == ['Id', 'Name'] + assert actual_fetch_target_objects.is_skip is False + assert actual_fetch_target_objects.is_update_last_fetch_datetime is True + assert actual_fetch_target_objects.datetime_column == 'SystemModstamp' + assert actual_fetch_target_objects.upload_file_name == f'CRM_Account_{execute_datetime.format_date()}' # ログの確認 assert generate_log_message_tuple(log_message='I-CHK-01 オブジェクト情報形式チェック処理を開始します') in caplog.record_tuples From cbffd5cd2c3dc3152ca97e39ae828c5158030f68 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 13:28:59 +0900 Subject: [PATCH 186/275] =?UTF-8?q?refactor:=20=E3=83=91=E3=83=A9=E3=83=A1?= =?UTF-8?q?=E3=82=BF=E5=8C=96=E3=83=86=E3=82=B9=E3=83=88=E3=81=AB=E3=81=97?= =?UTF-8?q?=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_fetch_crm_data_process.py | 148 ++++-------------- 1 file changed, 30 insertions(+), 118 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py index 3ce7b43f..3145ba32 100644 --- a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py @@ -205,12 +205,20 @@ class TestFetchCrmDataProcess: assert e.value.func_name == FETCH_JP_NAME assert e.value.args[0] == f'[Account] のレコード取得に失敗しました エラー内容：[生成エラー]' - def test_raise_fetch_sf_count_connection_timeout_with_retry_success(self, monkeypatch, caplog): + @pytest.mark.parametrize('timeout_env_name, exception, expect_message', [ + ('CRM_AUTH_TIMEOUT', ConnectTimeout('接続タイムアウト'), 'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[1] エラー内容：[接続タイムアウト]'), + ('CRM_GET_RECORD_COUNT_TIMEOUT', ReadTimeout('読み取りタイムアウト'), 'W-FETCH-02 [Account] の件数取得処理がタイムアウトしたため、リトライします：[1] エラー内容：[読み取りタイムアウト]'), + ('CRM_AUTH_TIMEOUT', Exception('予期せぬ例外'), 'W-FETCH-03 [Account] の件数取得に失敗したため、リトライします エラー内容：[予期せぬ例外]'), + ], ids=['connection_timeout', 'read_timeout', 'unexpected_exception']) + def test_raise_fetch_sf_count_with_retry_success(self, monkeypatch, caplog, timeout_env_name, exception, expect_message): """ Cases: - データ件数取得処理で接続タイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること + 1. データ件数取得処理で接続タイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること + 2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること + 3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること Arranges: - データ件数取得処理の最大リトライ試行回数を3に設定する + - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - データ件数取得処理の初回に接続タイムアウト例外が発生するようにする Expects: - 正常終了する @@ -221,7 +229,7 @@ class TestFetchCrmDataProcess: monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) - monkeypatch.setattr('src.fetch_crm_data_process.CRM_AUTH_TIMEOUT', 1) + monkeypatch.setattr(f'src.fetch_crm_data_process.{timeout_env_name}', 1) # Arrange with patch('src.fetch_crm_data_process.CounterObject', ) as mock_counter, \ patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ @@ -234,7 +242,7 @@ class TestFetchCrmDataProcess: mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' mock_client_inst = mock_api_client.return_value - mock_client_inst.fetch_sf_count.side_effect = [ConnectTimeout('接続タイムアウト'), len(common_expect)] + mock_client_inst.fetch_sf_count.side_effect = [exception, len(common_expect)] mock_client_inst.fetch_sf_data.return_value = common_expect # Act fetch_crm_data_process(common_target_object, common_last_fetch_datetime) @@ -244,31 +252,38 @@ class TestFetchCrmDataProcess: assert mock_counter_inst.describe.call_count == 1 assert mock_counter_inst.increment.call_count == 1 - retry_log_message = 'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[1] エラー内容：[接続タイムアウト]' assert generate_log_message_tuple( log_level=logging.WARNING, - log_message=retry_log_message) in caplog.record_tuples - called_log_counts = len([log for log in caplog.messages if log == retry_log_message]) + log_message=expect_message) in caplog.record_tuples + called_log_counts = len([log for log in caplog.messages if log == expect_message]) assert called_log_counts == 1 assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') in caplog.record_tuples - def test_raise_fetch_sf_count_connection_timeout_with_retry_fail(self, monkeypatch, caplog): + @pytest.mark.parametrize('timeout_env_name, exception, expect_message', [ + ('CRM_AUTH_TIMEOUT', ConnectTimeout('接続タイムアウト'), 'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[1] エラー内容：[接続タイムアウト]'), + ('CRM_GET_RECORD_COUNT_TIMEOUT', ReadTimeout('読み取りタイムアウト'), 'W-FETCH-02 [Account] の件数取得処理がタイムアウトしたため、リトライします：[1] エラー内容：[読み取りタイムアウト]'), + ('CRM_AUTH_TIMEOUT', Exception('予期せぬ例外'), 'W-FETCH-03 [Account] の件数取得に失敗したため、リトライします エラー内容：[予期せぬ例外]'), + ], ids=['connection_timeout', 'read_timeout', 'unexpected_exception']) + def test_raise_fetch_sf_count_with_retry_fail(self, monkeypatch, caplog, timeout_env_name, exception, expect_message): """ Cases: - データ件数取得処理で接続タイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること + 1. データ件数取得処理で接続タイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること + 2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること + 3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること Arranges: - データ件数取得処理の最大リトライ試行回数を3に設定する + - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - データ件数取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする Expects: - - 正常終了する - - データ件数取得に失敗した胸のエラーが出力されない + - 異常終了する + - データ件数取得に失敗した旨のエラーが出力される """ monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) - monkeypatch.setattr('src.fetch_crm_data_process.CRM_AUTH_TIMEOUT', 1) + monkeypatch.setattr(f'src.fetch_crm_data_process.{timeout_env_name}', 1) # Arrange with patch('src.fetch_crm_data_process.CounterObject', ) as mock_counter, \ patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ @@ -281,7 +296,7 @@ class TestFetchCrmDataProcess: mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' mock_client_inst = mock_api_client.return_value - mock_client_inst.fetch_sf_count.side_effect = [ConnectTimeout('接続タイムアウト'), ConnectTimeout('接続タイムアウト'), ConnectTimeout('接続タイムアウト')] + mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception] mock_client_inst.fetch_sf_data.return_value = common_expect # Act with pytest.raises(SalesforceAPIException) as e: @@ -294,113 +309,10 @@ class TestFetchCrmDataProcess: # 足し込みは2回のみ assert mock_counter_inst.increment.call_count == 2 - retry_log_message = 'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[1] エラー内容：[接続タイムアウト]' assert generate_log_message_tuple( log_level=logging.WARNING, - log_message=retry_log_message) in caplog.record_tuples - called_log_counts = len([log for log in caplog.messages if log == retry_log_message]) - assert called_log_counts == 2 - assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples - - assert e.value.error_id == 'E-FETCH-01' - assert e.value.func_name == FETCH_JP_NAME - # リトライ例外のオブジェクトIDが違うため、in句で比較 - assert f'[Account] の件数取得に失敗しました エラー内容：[RetryError' in e.value.args[0] - - def test_raise_fetch_sf_count_read_timeout_with_retry_success(self, monkeypatch, caplog): - """ - Cases: - データ件数取得処理で読み取り例外が発生した場合、リトライした結果復旧し、正常終了すること - Arranges: - - データ件数取得処理の最大リトライ試行回数を3に設定する - - データ件数取得処理の初回に読み取り例外が発生するようにする - Expects: - - 正常終了する - - データ件数取得に失敗した胸のエラーが出力されない - """ - - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_TIMEOUT', 1) - # Arrange - with patch('src.fetch_crm_data_process.CounterObject', ) as mock_counter, \ - patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ - patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: - # モック化 - mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] - mock_builder_inst = mock_soql_builder.return_value - mock_builder_inst.create_count_soql.return_value = '' - mock_builder_inst.create_fetch_soql.return_value = '' - mock_client_inst = mock_api_client.return_value - mock_client_inst.fetch_sf_count.side_effect = [ReadTimeout('読み取りタイムアウト'), len(common_expect)] - mock_client_inst.fetch_sf_data.return_value = common_expect - # Act - fetch_crm_data_process(common_target_object, common_last_fetch_datetime) - - # Assert - - assert mock_counter_inst.describe.call_count == 1 - assert mock_counter_inst.increment.call_count == 1 - - retry_log_message = 'W-FETCH-02 [Account] の件数取得処理がタイムアウトしたため、リトライします：[1] エラー内容：[読み取りタイムアウト]' - assert generate_log_message_tuple( - log_level=logging.WARNING, - log_message=retry_log_message) in caplog.record_tuples - called_log_counts = len([log for log in caplog.messages if log == retry_log_message]) - assert called_log_counts == 1 - assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') in caplog.record_tuples - - def test_raise_fetch_sf_count_read_timeout_with_retry_fail(self, monkeypatch, caplog): - """ - Cases: - データ件数取得処理で読み取り例外が発生した場合、リトライした結果復旧せず、異常終了すること - Arranges: - - データ件数取得処理の最大リトライ試行回数を3に設定する - - データ件数取得処理の1回目、2回目、3回目で読み取り例外が発生するようにする - Expects: - - 正常終了する - - データ件数取得に失敗した胸のエラーが出力されない - """ - - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_TIMEOUT', 1) - # Arrange - with patch('src.fetch_crm_data_process.CounterObject', ) as mock_counter, \ - patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ - patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: - # モック化 - mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] - mock_builder_inst = mock_soql_builder.return_value - mock_builder_inst.create_count_soql.return_value = '' - mock_builder_inst.create_fetch_soql.return_value = '' - mock_client_inst = mock_api_client.return_value - mock_client_inst.fetch_sf_count.side_effect = [ReadTimeout('読み取りタイムアウト'), ReadTimeout('読み取りタイムアウト'), ReadTimeout('読み取りタイムアウト')] - mock_client_inst.fetch_sf_data.return_value = common_expect - # Act - with pytest.raises(SalesforceAPIException) as e: - fetch_crm_data_process(common_target_object, common_last_fetch_datetime) - - # Assert - - # 取得は3回行われる - assert mock_counter_inst.describe.call_count == 3 - # 足し込みは2回のみ - assert mock_counter_inst.increment.call_count == 2 - - retry_log_message = 'W-FETCH-02 [Account] の件数取得処理がタイムアウトしたため、リトライします：[1] エラー内容：[読み取りタイムアウト]' - assert generate_log_message_tuple( - log_level=logging.WARNING, - log_message=retry_log_message) in caplog.record_tuples - called_log_counts = len([log for log in caplog.messages if log == retry_log_message]) + log_message=expect_message) in caplog.record_tuples + called_log_counts = len([log for log in caplog.messages if log == expect_message]) assert called_log_counts == 2 assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples From b99c27d54c4382a392d31564faf29f446397f4df Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 14:10:36 +0900 Subject: [PATCH 187/275] =?UTF-8?q?feat:=20=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E5=8F=96=E5=BE=97=E3=81=AE=E3=83=AA=E3=83=88=E3=83=A9=E3=82=A4?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_fetch_crm_data_process.py | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py index 3145ba32..5d35ac58 100644 --- a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py @@ -320,3 +320,119 @@ class TestFetchCrmDataProcess: assert e.value.func_name == FETCH_JP_NAME # リトライ例外のオブジェクトIDが違うため、in句で比較 assert f'[Account] の件数取得に失敗しました エラー内容：[RetryError' in e.value.args[0] + + @pytest.mark.parametrize('timeout_env_name, exception, expect_message', [ + ('CRM_AUTH_TIMEOUT', ConnectTimeout('接続タイムアウト'), 'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[1] エラー内容：[接続タイムアウト]'), + ('CRM_FETCH_RECORD_TIMEOUT', ReadTimeout('読み取りタイムアウト'), 'W-FETCH-05 [Account] のレコード取得処理がタイムアウトしたため、リトライします：[1] エラー内容：[読み取りタイムアウト]'), + ('CRM_AUTH_TIMEOUT', Exception('予期せぬ例外'), 'W-FETCH-06 [Account] のレコード取得に失敗したため、リトライします エラー内容：[予期せぬ例外]'), + ], ids=['connection_timeout', 'read_timeout', 'unexpected_exception']) + def test_raise_fetch_sf_data_with_retry_success(self, monkeypatch, caplog, timeout_env_name, exception, expect_message): + """ + Cases: + 1. レコード取得処理で接続タイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること + 2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること + 3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること + Arranges: + - レコード取得処理の最大リトライ試行回数を3に設定する + - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する + - レコード取得処理の初回に接続タイムアウト例外が発生するようにする + Expects: + - 正常終了する + - データレコード取得に失敗した胸のエラーが出力されない + """ + + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) + monkeypatch.setattr(f'src.fetch_crm_data_process.{timeout_env_name}', 1) + # Arrange + with patch('src.fetch_crm_data_process.CounterObject', ) as mock_counter, \ + patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ + patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: + # モック化 + mock_counter_inst = mock_counter.return_value + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_builder_inst = mock_soql_builder.return_value + mock_builder_inst.create_count_soql.return_value = '' + mock_builder_inst.create_fetch_soql.return_value = '' + mock_client_inst = mock_api_client.return_value + mock_client_inst.fetch_sf_count.return_value = 1 + mock_client_inst.fetch_sf_data.side_effect = [exception, common_expect] + # Act + fetch_crm_data_process(common_target_object, common_last_fetch_datetime) + + # Assert + + assert mock_counter_inst.describe.call_count == 1 + assert mock_counter_inst.increment.call_count == 1 + + assert generate_log_message_tuple( + log_level=logging.WARNING, + log_message=expect_message) in caplog.record_tuples + called_log_counts = len([log for log in caplog.messages if log == expect_message]) + assert called_log_counts == 1 + assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') in caplog.record_tuples + + @pytest.mark.parametrize('timeout_env_name, exception, expect_message', [ + ('CRM_AUTH_TIMEOUT', ConnectTimeout('接続タイムアウト'), 'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[1] エラー内容：[接続タイムアウト]'), + ('CRM_FETCH_RECORD_TIMEOUT', ReadTimeout('読み取りタイムアウト'), 'W-FETCH-05 [Account] のレコード取得処理がタイムアウトしたため、リトライします：[1] エラー内容：[読み取りタイムアウト]'), + ('CRM_AUTH_TIMEOUT', Exception('予期せぬ例外'), 'W-FETCH-06 [Account] のレコード取得に失敗したため、リトライします エラー内容：[予期せぬ例外]'), + ], ids=['connection_timeout', 'read_timeout', 'unexpected_exception']) + def test_raise_fetch_sf_data_with_retry_fail(self, monkeypatch, caplog, timeout_env_name, exception, expect_message): + """ + Cases: + 1. レコード取得処理で接続タイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること + 2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること + 3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること + Arranges: + - レコード取得処理の最大リトライ試行回数を3に設定する + - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する + - レコード取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする + Expects: + - 異常終了する + - データレコード取得に失敗した旨のエラーが出力される + """ + + monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL', 1) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_INTERVAL', 1) + monkeypatch.setattr(f'src.fetch_crm_data_process.{timeout_env_name}', 1) + # Arrange + with patch('src.fetch_crm_data_process.CounterObject', ) as mock_counter, \ + patch('src.fetch_crm_data_process.SOQLBuilder') as mock_soql_builder, \ + patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: + # モック化 + mock_counter_inst = mock_counter.return_value + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_builder_inst = mock_soql_builder.return_value + mock_builder_inst.create_count_soql.return_value = '' + mock_builder_inst.create_fetch_soql.return_value = '' + mock_client_inst = mock_api_client.return_value + mock_client_inst.fetch_sf_count.return_value = 1 + mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception] + # Act + with pytest.raises(SalesforceAPIException) as e: + fetch_crm_data_process(common_target_object, common_last_fetch_datetime) + + # Assert + + # 取得は3回行われる + assert mock_counter_inst.describe.call_count == 3 + # 足し込みは2回のみ + assert mock_counter_inst.increment.call_count == 2 + + assert generate_log_message_tuple( + log_level=logging.WARNING, + log_message=expect_message) in caplog.record_tuples + called_log_counts = len([log for log in caplog.messages if log == expect_message]) + assert called_log_counts == 2 + assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples + + assert e.value.error_id == 'E-FETCH-02' + assert e.value.func_name == FETCH_JP_NAME + # リトライ例外のオブジェクトIDが違うため、in句で比較 + assert f'[Account] のレコード取得に失敗しました エラー内容：[RetryError' in e.value.args[0] From 05406d17029bba5c988cc213a5767beeac9dab56 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Wed, 17 Aug 2022 14:15:33 +0900 Subject: [PATCH 188/275] =?UTF-8?q?feat:=20=E5=8F=96=E5=BE=97=E5=87=A6?= =?UTF-8?q?=E7=90=86=E5=AE=9F=E6=96=BD=E7=B5=90=E6=9E=9C=E3=82=A2=E3=83=83?= =?UTF-8?q?=E3=83=97=E3=83=AD=E3=83=BC=E3=83=89=E5=87=A6=E7=90=86=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/upload_result_data_process.py | 2 +- .../tests/test_upload_result_data_process.py | 124 ++++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 ecs/crm-datafetch/tests/test_upload_result_data_process.py diff --git a/ecs/crm-datafetch/src/upload_result_data_process.py b/ecs/crm-datafetch/src/upload_result_data_process.py index 8248fb78..e0c00a79 100644 --- a/ecs/crm-datafetch/src/upload_result_data_process.py +++ b/ecs/crm-datafetch/src/upload_result_data_process.py @@ -11,7 +11,7 @@ def upload_result_data_process(process_result: dict, execute_datetime: ExecuteDa Args: process_result (dict): 取得処理実行結果辞書オブジェクト - execute_datetime (ExecuteDateTime): データ取得期間設定インスタンス + execute_datetime (ExecuteDateTime): 実行日時取得インスタンス Raises: FileUploadException: S3のファイルアップロード失敗 diff --git a/ecs/crm-datafetch/tests/test_upload_result_data_process.py b/ecs/crm-datafetch/tests/test_upload_result_data_process.py new file mode 100644 index 00000000..abf7fb8d --- /dev/null +++ b/ecs/crm-datafetch/tests/test_upload_result_data_process.py @@ -0,0 +1,124 @@ +from unittest.mock import MagicMock, patch + +import pytest +from src.error.exceptions import FileUploadException +from src.system_var.constants import END_JP_NAME +from src.upload_result_data_process import upload_result_data_process +from src.util.execute_datetime import ExecuteDateTime + +from .test_utils.log_message import generate_log_message_tuple + + +class TestUploadResultDataProcess: + + @pytest.fixture + def bucket_name(self): + return 'test-config-bucket' + + @pytest.fixture + def prepare_bucket(self, s3_client, bucket_name): + s3_client.create_bucket(Bucket=bucket_name) + yield + + def test_run_process_success(self, s3_client, prepare_bucket, bucket_name, monkeypatch, caplog): + """ + Cases: + 取得処理実施結果アップロード処理が正常終了し、期待通りの結果が返ること + Arranges: + - prepare_bucketフィクスチャで、取得処理実施結果ファイルを置くためのモックバケットを作る + - モックバケットを指すように環境変数を設定する + - 取得処理実施結果を準備する + - 実行日時取得インスタンスを生成する + Expects: + - PUTした取得処理実施結果ファイルが存在し内容が想定と一致する + - 取得処理実施結果アップロード処理の仕様に沿った正常系ログが出力されること(デバッグログは除く) + """ + + # Arranges + + # 環境変数を編集 + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', 'data_import') + + process_result = { + "Account": "success", + "Contact": "fail" + } + + execute_datetime = ExecuteDateTime() + + # Act + upload_result_data_process(process_result, execute_datetime) + + # Assert + # ファイル確認 + actual = s3_client.get_object(Bucket=bucket_name, Key=f'data_import/{execute_datetime.to_path()}/process_result.json') + assert actual['Body'].read().decode('utf-8') == '{"Account": "success", "Contact": "fail"}' + + # ログの確認 + assert generate_log_message_tuple(log_message='I-END-01 取得処理実施結果アップロード処理を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-END-03 取得処理実施結果アップロード処理を終了します') in caplog.record_tuples + + def test_call_depended_modules(self): + """ + Cases: + 取得処理実施結果アップロード処理内で依存しているモジュールが正しく呼ばれていること + Arranges: + - 取得処理実施結果アップロード処理の依存モジュールをモック化する + - 取得処理実施結果を準備する + - 実行日時取得インスタンスを生成する + Expects: + - 依存しているモジュールが正しく呼ばれている + """ + + # Arrange + mock_backup_bucket = MagicMock(return_value=None) + + process_result = { + "Account": "success", + "Contact": "fail" + } + + execute_datetime = ExecuteDateTime() + + # Act + with patch('src.aws.s3.BackupBucket.put_result_json', mock_backup_bucket): + upload_result_data_process(process_result, execute_datetime) + + # Assert + assert mock_backup_bucket.called is True + + def test_raise_put_result_json(self, monkeypatch): + """ + Cases: + 取得処理実施結果をアップロードできない場合、エラーが発生すること + Arranges: + - 取得処理実施結果アップロード処理で例外を発生させるモックを生成する + - 取得処理実施結果を準備する + - 実行日時取得インスタンスを生成する + Expects: + - 例外が発生する + - ファイルがアップロードできないエラーが返却される + """ + + # Arrange + mock_backup_bucket = MagicMock(side_effect=Exception('ファイルアップロードエラー')) + + process_result = { + "Account": "success", + "Contact": "fail" + } + + execute_datetime = ExecuteDateTime() + + # Act + with patch('src.aws.s3.BackupBucket.put_result_json', mock_backup_bucket): + with pytest.raises(FileUploadException) as e: + upload_result_data_process(process_result, execute_datetime) + + # Assert + + assert mock_backup_bucket.called is True + assert e.value.error_id == 'E-END-01' + assert e.value.func_name == END_JP_NAME + assert e.value.args[0] == f'取得処理実施結果のアップロードに失敗しました　ファイル名：[process_result.json] エラー内容：[ファイルアップロードエラー]' From b9ca2c3a43414b2829aa4b1da419ad52f715d3f9 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 16:08:26 +0900 Subject: [PATCH 189/275] =?UTF-8?q?fix:=20=E3=82=A2=E3=82=B5=E3=83=BC?= =?UTF-8?q?=E3=82=B7=E3=83=A7=E3=83=B3=E3=81=AE=E4=B8=8D=E8=B6=B3=E3=82=92?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_check_object_info_process.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ecs/crm-datafetch/tests/test_check_object_info_process.py b/ecs/crm-datafetch/tests/test_check_object_info_process.py index 46125bc4..ebc8a917 100644 --- a/ecs/crm-datafetch/tests/test_check_object_info_process.py +++ b/ecs/crm-datafetch/tests/test_check_object_info_process.py @@ -46,6 +46,7 @@ class TestCheckObjectInfoProcess: assert actual_fetch_target_objects.is_update_last_fetch_datetime is True assert actual_fetch_target_objects.datetime_column == 'SystemModstamp' assert actual_fetch_target_objects.upload_file_name == f'CRM_Account_{execute_datetime.format_date()}' + assert actual_fetch_target_objects.last_fetch_datetime_file_name == 'Account.json' # ログの確認 assert generate_log_message_tuple(log_message='I-CHK-01 オブジェクト情報形式チェック処理を開始します') in caplog.record_tuples From d0e64b40ac0d4cd44fda1e4d942cc4fdccdffca9 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 16:10:40 +0900 Subject: [PATCH 190/275] =?UTF-8?q?fix:=20=E3=82=A2=E3=82=B5=E3=83=BC?= =?UTF-8?q?=E3=82=B7=E3=83=A7=E3=83=B3=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_set_datetime_period_process.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ecs/crm-datafetch/tests/test_set_datetime_period_process.py b/ecs/crm-datafetch/tests/test_set_datetime_period_process.py index 87a44a30..6315bb7b 100644 --- a/ecs/crm-datafetch/tests/test_set_datetime_period_process.py +++ b/ecs/crm-datafetch/tests/test_set_datetime_period_process.py @@ -59,6 +59,8 @@ class TestSetDatetimePeriodProcess: # 返り値の期待値チェック assert isinstance(actual_last_fetch_datetime, LastFetchDatetime), '最終取得日時オブジェクトクラスのインスタンスが返却される' + assert actual_last_fetch_datetime.last_fetch_datetime_from == '1999-01-01T00:00:00.000Z' + assert actual_last_fetch_datetime.last_fetch_datetime_to == '2100-12-31T23:59:59.000Z' # ログの確認 assert generate_log_message_tuple(log_message='I-DATE-01 [Account] のデータ取得期間設定処理を開始します') in caplog.record_tuples From 2d0471bb506da3e1c02b25d89d0b8d65baf13424 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Wed, 17 Aug 2022 18:34:01 +0900 Subject: [PATCH 191/275] =?UTF-8?q?feat:=20=E5=89=8D=E5=9B=9E=E5=8F=96?= =?UTF-8?q?=E5=BE=97=E6=97=A5=E6=99=82=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=87=A6=E7=90=86=E3=81=AE=E3=83=86=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...test_upload_last_fetch_datetime_process.py | 218 ++++++++++++++++++ 1 file changed, 218 insertions(+) create mode 100644 ecs/crm-datafetch/tests/test_upload_last_fetch_datetime_process.py diff --git a/ecs/crm-datafetch/tests/test_upload_last_fetch_datetime_process.py b/ecs/crm-datafetch/tests/test_upload_last_fetch_datetime_process.py new file mode 100644 index 00000000..e04c7943 --- /dev/null +++ b/ecs/crm-datafetch/tests/test_upload_last_fetch_datetime_process.py @@ -0,0 +1,218 @@ +from unittest.mock import MagicMock, patch + +import pytest +from src.config.objects import LastFetchDatetime, TargetObject +from src.error.exceptions import FileUploadException +from src.system_var.constants import UPD_JP_NAME +from src.upload_last_fetch_datetime_process import \ + upload_last_fetch_datetime_process +from src.util.execute_datetime import ExecuteDateTime + +from .test_utils.log_message import generate_log_message_tuple + + +class TestUploadLastFetchDatetimeProcess: + + @pytest.fixture + def bucket_name(self): + return 'test-config-bucket' + + @pytest.fixture + def prepare_bucket(self, s3_client, bucket_name): + s3_client.create_bucket(Bucket=bucket_name) + yield + + def test_run_process_success(self, s3_client, prepare_bucket, bucket_name, monkeypatch, caplog): + """ + Cases: + 前回取得日時ファイル更新処理が正常終了し、期待通りの結果が返ること + Arranges: + - prepare_bucketフィクスチャで、前回取得日時ファイルを置くためのモックバケットを作る + - モックバケットを指すように環境変数を設定する + - オブジェクト情報を準備する + - 最終更新日時インスタンスを準備する + Expects: + - PUTした前回取得日時ファイルが存在し内容が想定と一致する + - 前回取得日時ファイル更新処理の仕様に沿った正常系ログが出力されること(デバッグログは除く) + """ + + # Arranges + + # 環境変数を編集 + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm/last_fetch_datetime') + + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ], + 'is_update_last_fetch_datetime': True + } + + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1999-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2100-12-31T23:59:59.000Z", + } + + execute_datetime = ExecuteDateTime() + + target_object = TargetObject(target_objects_dict, execute_datetime) + last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + upload_last_fetch_datetime_process(target_object, last_fetch_datetime) + + # Assert + # ファイル確認 + actual = s3_client.get_object(Bucket=bucket_name, Key=f'crm/last_fetch_datetime/Account.json') + assert actual['Body'].read().decode( + 'utf-8') == '{"last_fetch_datetime_from": "2100-12-31T23:59:59.000Z", "last_fetch_datetime_to": ""}' + + # ログの確認 + assert generate_log_message_tuple(log_message='I-UPD-01 [Account] の前回取得日時ファイルの更新処理を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-UPD-02 [Account] の前回取得日時ファイルの更新処理をスキップします') not in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-UPD-04 [Account] の前回取得日時ファイルの更新処理を終了します') in caplog.record_tuples + + def test_run_process_success_skip(self, s3_client, prepare_bucket, bucket_name, monkeypatch, caplog): + """ + Cases: + 前回取得日時ファイル更新処理が正常終了し、期待通りの結果が返ること + Arranges: + - prepare_bucketフィクスチャで、前回取得日時ファイルを置くためのモックバケットを作る + - モックバケットを指すように環境変数を設定する + - ファイルPUT処理のモックを準備する + - オブジェクト情報を準備する + - 最終更新日時インスタンスを準備する + Expects: + - ファイルPUT処理が実行されないこと + - 前回取得日時ファイル更新処理の仕様に沿った正常系ログが出力されること(デバッグログは除く) + """ + + # Arranges + + # 環境変数を編集 + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', 'crm/last_fetch_datetime') + + mock_config_bucket = MagicMock(return_value=None) + + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ], + 'is_update_last_fetch_datetime': False + } + + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1999-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2100-12-31T23:59:59.000Z", + } + + execute_datetime = ExecuteDateTime() + + target_object = TargetObject(target_objects_dict, execute_datetime) + last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + with patch('src.aws.s3.ConfigBucket.put_last_fetch_datetime_file', mock_config_bucket): + upload_last_fetch_datetime_process(target_object, last_fetch_datetime) + + # Assert + # 処理実行確認 + assert mock_config_bucket.called is False + + # ログの確認 + assert generate_log_message_tuple(log_message='I-UPD-01 [Account] の前回取得日時ファイルの更新処理を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-UPD-02 [Account] の前回取得日時ファイルの更新処理をスキップします') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-UPD-04 [Account] の前回取得日時ファイルの更新処理を終了します') not in caplog.record_tuples + + def test_call_depended_modules(self): + """ + Cases: + 前回取得日時ファイル更新処理内で依存しているモジュールが正しく呼ばれていること + Arranges: + - 前回取得日時ファイル更新処理の依存モジュールをモック化する + - オブジェクト情報を準備する + - 最終更新日時インスタンスを準備する + Expects: + - 依存しているモジュールが正しく呼ばれている + """ + + # Arrange + mock_config_bucket = MagicMock(return_value=None) + + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ], + 'is_update_last_fetch_datetime': True + } + + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1999-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2100-12-31T23:59:59.000Z", + } + + execute_datetime = ExecuteDateTime() + + target_object = TargetObject(target_objects_dict, execute_datetime) + last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + with patch('src.aws.s3.ConfigBucket.put_last_fetch_datetime_file', mock_config_bucket): + upload_last_fetch_datetime_process(target_object, last_fetch_datetime) + + # Assert + assert mock_config_bucket.called is True + + def test_raise_put_last_fetch_datetime_file(self, monkeypatch): + """ + Cases: + 前回取得日時ファイルをアップロードできない場合、エラーが発生すること + Arranges: + - 前回取得日時ファイル更新処理で例外を発生させるモックを生成する + - 取得処理実施結果を準備する + - 実行日時取得インスタンスを生成する + Expects: + - 例外が発生する + - ファイルがアップロードできないエラーが返却される + """ + + # Arrange + mock_config_bucket = MagicMock(side_effect=Exception('ファイルアップロードエラー')) + + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ], + 'is_update_last_fetch_datetime': True + } + + last_fetch_datetime_dict = { + "last_fetch_datetime_from": "1999-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "2100-12-31T23:59:59.000Z", + } + + execute_datetime = ExecuteDateTime() + + target_object = TargetObject(target_objects_dict, execute_datetime) + last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_dict, execute_datetime) + + # Act + with patch('src.aws.s3.ConfigBucket.put_last_fetch_datetime_file', mock_config_bucket): + with pytest.raises(FileUploadException) as e: + upload_last_fetch_datetime_process(target_object, last_fetch_datetime) + + # Assert + assert mock_config_bucket.called is True + assert e.value.error_id == 'E-UPD-01' + assert e.value.func_name == UPD_JP_NAME + assert e.value.args[0] == f'[Account] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[Account.json] エラー内容:[ファイルアップロードエラー]' From 9f7e9a8e398c9dd7b9524cd28025d2ac2efca47d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 20:32:54 +0900 Subject: [PATCH 192/275] =?UTF-8?q?style:=20=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E5=90=8D=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/{test_salesfoce.py => test_salesforce.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ecs/crm-datafetch/tests/salesforce/{test_salesfoce.py => test_salesforce.py} (100%) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesfoce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py similarity index 100% rename from ecs/crm-datafetch/tests/salesforce/test_salesfoce.py rename to ecs/crm-datafetch/tests/salesforce/test_salesforce.py From d60d1a65364903d8dd53078f4a9a3fd76d2baf61 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 20:32:54 +0900 Subject: [PATCH 193/275] =?UTF-8?q?feat:=20CRM=E9=9B=BB=E6=96=87=E3=83=87?= =?UTF-8?q?=E3=83=BC=E3=82=BF=E3=83=90=E3=83=83=E3=82=AF=E3=82=A2=E3=83=83?= =?UTF-8?q?=E3=83=97=E5=87=A6=E7=90=86=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0=E3=80=82=E3=83=90=E3=83=83=E3=82=AF?= =?UTF-8?q?=E3=82=A2=E3=83=83=E3=83=97=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E5=90=8D=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/backup_crm_data_process.py | 10 +- ecs/crm-datafetch/src/controller.py | 2 +- .../tests/test_backup_crm_data_process.py | 99 +++++++++++++++++++ 3 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 ecs/crm-datafetch/tests/test_backup_crm_data_process.py diff --git a/ecs/crm-datafetch/src/backup_crm_data_process.py b/ecs/crm-datafetch/src/backup_crm_data_process.py index 0375d927..a5739e07 100644 --- a/ecs/crm-datafetch/src/backup_crm_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_data_process.py @@ -1,15 +1,16 @@ from src.aws.s3 import BackupBucket +from src.config.objects import TargetObject from src.error.exceptions import FileUploadException from src.system_var.constants import RESBK_JP_NAME from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def backup_crm_data_process(object_name: str, sf_object_dict: dict, execute_datetime: ExecuteDateTime): +def backup_crm_data_process(target_object: TargetObject, sf_object_dict: dict, execute_datetime: ExecuteDateTime): """CRM電文データバックアップ処理 Args: - object_name (str): 取得対象オブジェクト名 + target_object (TargetObject): 取得対象オブジェクト名 sf_object_dict (dict): Salesforceオブジェクトデータ execute_datetime (ExecuteDateTime): 実行日次取得インスタンス @@ -17,12 +18,13 @@ def backup_crm_data_process(object_name: str, sf_object_dict: dict, execute_date FileUploadException: S3のファイルアップロード失敗 """ + object_name = target_object.object_name # ① CRM電文データバックアップ処理の開始ログを出力する logger.info(f'I-RESBK-01 [{object_name}] のCRM電文データバックアップ処理を開始します') try: # ② CRMバックアップ保管用バケットに、CRMから取得したJSONの電文データのバックアップを保管する - file_name = f'{execute_datetime.to_path()}/{object_name}.json' + file_name = f'{execute_datetime.to_path()}/{target_object.upload_file_name}.json' backup_bucket = BackupBucket() backup_bucket.put_response_json(file_name, sf_object_dict) @@ -31,7 +33,7 @@ def backup_crm_data_process(object_name: str, sf_object_dict: dict, execute_date except Exception as e: raise FileUploadException( - 'E-RESBK-01', RESBK_JP_NAME, f'[{object_name}] 電文データのバックアップに失敗しました ファイル名：[{object_name}.json] エラー内容:[{e}]') + 'E-RESBK-01', RESBK_JP_NAME, f'[{object_name}] 電文データのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.json] エラー内容:[{e}]') # ③ CRM電文データバックアップ処理の終了ログを出力する logger.info(f'I-RESBK-03 [{object_name}] のCRM電文データバックアップ処理を終了します') diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index f9c87960..97c1f10e 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -152,7 +152,7 @@ def _fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateT # 8. CRM電文データバックアップ処理を呼び出す logger.info( f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し') - backup_crm_data_process(target_object_name, crm_data_response, execute_datetime) + backup_crm_data_process(target_object, crm_data_response, execute_datetime) # 9. CSV変換処理を呼び出す logger.info( diff --git a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py new file mode 100644 index 00000000..e476dce3 --- /dev/null +++ b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py @@ -0,0 +1,99 @@ +import json +from collections import OrderedDict +from unittest.mock import MagicMock, patch + +import pytest +from src.backup_crm_data_process import backup_crm_data_process +from src.config.objects import TargetObject +from src.error.exceptions import FileNotFoundException, InvalidConfigException +from src.system_var.constants import RESBK_JP_NAME +from src.util.execute_datetime import ExecuteDateTime + +from .test_utils.log_message import generate_log_message_tuple + + +class TestBackupCrmDataProcess: + + @pytest.fixture + def bucket_name(self): + return 'test-backup-bucket' + + @pytest.fixture + def prepare_bucket(self, s3_client, bucket_name): + s3_client.create_bucket(Bucket=bucket_name) + yield + + def test_run_process_success(self, s3_client, prepare_bucket, bucket_name, monkeypatch, caplog): + """ + Cases: + CRM電文データバックアップ処理が正常終了し、期待通りの結果となること + Arranges: + - prepare_bucketフィクスチャで、CRM電文データをバックアップするためのモックバケットを作る + - 作成したモックバケットを指すように環境変数を設定する + Expects: + - CRM電文データバックアップファイルがバケットに配置される + - CRM電文データバックアップ処理の仕様に沿った正常系ログが出力されること(デバッグログは除く) + """ + # Arrange + response_json = [ + OrderedDict([ + ('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/TEST001')])), + ('Id', 'TEST001'), + ('AccountNumber', 'test001'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), + ('IsDeleted', False) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/TEST002')])), + ('Id', 'TEST002'), + ('AccountNumber', 'test002'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), + ('IsDeleted', False) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/TEST002')])), + ('Id', 'TEST002'), + ('AccountNumber', 'test002'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), + ('IsDeleted', False) + ]), + ] + + target_object_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'AccountNumber', + 'LastModifiedDate', + 'LastModifiedById', + 'SystemModstamp', + 'IsDeleted' + ] + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_object_dict, execute_datetime) + + # 環境変数を編集 + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', 'response_json') + + # Act + backup_crm_data_process(target_object, response_json, execute_datetime) + + # Assert + + # ファイル確認 + actual = s3_client.get_object( + Bucket=bucket_name, Key=f'response_json/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.json') + assert actual['Body'].read().decode('utf-8') == json.dumps(response_json) + + # ログの確認 + assert generate_log_message_tuple(log_message='I-RESBK-01 [Account] のCRM電文データバックアップ処理を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-RESBK-03 [Account] のCRM電文データバックアップ処理を終了します') in caplog.record_tuples From ef14595e25abfcaf6a0e5f4409c7ca56fc65d9a6 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 20:32:54 +0900 Subject: [PATCH 194/275] =?UTF-8?q?refactor:=20=E3=81=BB=E3=81=8B=E3=83=A2?= =?UTF-8?q?=E3=82=B8=E3=83=A5=E3=83=BC=E3=83=AB=E3=81=AE=E6=A9=9F=E8=83=BD?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E3=81=AB=E3=82=88=E3=82=8A=E3=80=81=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E5=AE=9F=E6=96=BD=E4=B8=8D=E5=8F=AF=E3=81=A8?= =?UTF-8?q?=E3=81=AA=E3=81=A3=E3=81=9F=E3=83=86=E3=82=B9=E3=83=88=E3=82=92?= =?UTF-8?q?=E3=82=B9=E3=82=AD=E3=83=83=E3=83=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/salesforce/test_soql_builder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py index da42d486..56ff81ea 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py +++ b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py @@ -1,4 +1,5 @@ +import pytest from src.config.objects import ExecuteDateTime, LastFetchDatetime, TargetObject from src.salesforce.soql_builder import SOQLBuilder @@ -119,7 +120,8 @@ class TestSOQLBuilder: assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') - def test_create_fetch_soql_no_columns(self): + @pytest.mark.skip('データ取得用のSOQLがカラムがない状態は、TargetObjectクラス側で制御されるため、テストを実施しない') + def test_raise_create_fetch_soql_cause_no_columns(self): """ Cases: データ取得用のSOQLがカラムがない状態で生成されること From 2ae615671a4c5f98c7e3f3ff6db05773f22c5c75 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 20:32:54 +0900 Subject: [PATCH 195/275] =?UTF-8?q?feat:=20=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E9=80=94=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_backup_crm_data_process.py | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py index e476dce3..8028ebf0 100644 --- a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py @@ -97,3 +97,127 @@ class TestBackupCrmDataProcess: # ログの確認 assert generate_log_message_tuple(log_message='I-RESBK-01 [Account] のCRM電文データバックアップ処理を開始します') in caplog.record_tuples assert generate_log_message_tuple(log_message='I-RESBK-03 [Account] のCRM電文データバックアップ処理を終了します') in caplog.record_tuples + + def test_call_depended_modules(self): + """ + Cases: + CRM電文データバックアップ処理内で依存しているモジュールが正しく呼ばれていること + Arranges: + - CRM電文データバックアップ処理の依存モジュールをモック化する + Expects: + - 依存しているモジュールが正しく呼ばれている + """ + + # Arrange + with patch('src.backup_crm_data_process.BackupBucket') as mock_backup_bucket, \ + patch('src.backup_crm_data_process.ExecuteDateTime') as mock_execute_datetime, \ + patch('src.backup_crm_data_process.TargetObject') as mock_target_object: + mock_backup_bucket_inst = mock_backup_bucket.return_value + mock_backup_bucket_inst.put_response_json.return_value = '' + mock_execute_inst = mock_execute_datetime.return_value + mock_execute_inst.to_path.return_value = '' + mock_target_inst = mock_target_object.return_value + mock_target_inst.upload_file_name.return_value = '' + # Act + backup_crm_data_process(mock_target_object, {}, mock_execute_datetime, ) + + # Assert + + assert mock_backup_bucket_inst.put_response_json.called is True + assert mock_execute_inst.to_path.called is True + assert mock_target_object.upload_file_name.called is True + + def test_raise_get_object_info(self, bucket_name, monkeypatch, caplog): + """ + Cases: + CRM_取得オブジェクト情報ファイルを取得できない場合、エラーが発生すること + Arranges: + - オブジェクト情報ファイル取得処理で例外が発生するようにする + Expects: + - 例外が発生する + - ファイルが読み込めないエラーが返却される + """ + + # Arrange + monkeypatch.setattr('src.backup_crm_data_process.OBJECT_INFO_FILENAME', 'crm_object_list_diff.json') + mock_config_bucket = MagicMock(side_effect=Exception('ファイル取得エラー')) + mock_json_parser = MagicMock(return_value={}) + mock_fetch_target_objects = MagicMock(return_value=None) + # Act + with patch('src.aws.s3.ConfigBucket.get_object_info_file', mock_config_bucket), \ + patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ + patch('src.config.objects.FetchTargetObjects.__init__', mock_fetch_target_objects): + with pytest.raises(FileNotFoundException) as e: + backup_crm_data_process() + + # Assert + + assert mock_config_bucket.called is True + assert mock_json_parser.called is False + assert mock_fetch_target_objects.called is False + assert e.value.error_id == 'E-RESBK-01' + assert e.value.func_name == RESBK_JP_NAME + assert e.value.args[0] == f'CRM_取得オブジェクト情報ファイルが存在しません ファイル名：[crm_object_list_diff.json] エラー内容：[ファイル取得エラー]' + + def test_raise_parse_json_str(self, bucket_name, monkeypatch, caplog): + """ + Cases: + CRM_取得オブジェクト情報ファイルをパースできない場合、エラーが発生すること + Arranges: + - オブジェクト情報ファイルパース処理で例外が発生するようにする + Expects: + - 例外が発生する + - パースが失敗した旨、エラーメッセージが表示される + """ + + # Arrange + mock_config_bucket = MagicMock(return_value='') + mock_json_parser = MagicMock(side_effect=Exception('JSONパースエラー')) + mock_fetch_target_objects = MagicMock(return_value=None) + # Act + with patch('src.aws.s3.ConfigBucket.get_object_info_file', mock_config_bucket), \ + patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ + patch('src.config.objects.FetchTargetObjects.__init__', mock_fetch_target_objects): + with pytest.raises(InvalidConfigException) as e: + backup_crm_data_process() + + # Assert + + assert mock_config_bucket.called is True + assert mock_json_parser.called is True + assert mock_fetch_target_objects.called is False + assert e.value.error_id == 'E-RESBK-02' + assert e.value.func_name == RESBK_JP_NAME + assert e.value.args[0] == f'CRM_取得オブジェクト情報ファイルのパースに失敗しました エラー内容：[JSONパースエラー]' + + def test_raise_check_objects_format(self, bucket_name, monkeypatch, caplog): + """ + Cases: + CRM_取得オブジェクト情報のキーobjectsの形式チェックが不正の場合、エラーが発生すること + Arranges: + - CRM_取得オブジェクト情報のキーobjectsの形式チェックで例外が発生するようにする + Expects: + - 例外が発生する + - 形式チェックが失敗した旨、エラーメッセージが表示される + """ + + # Arrange + monkeypatch.setattr('src.backup_crm_data_process.OBJECT_INFO_FILENAME', 'crm_object_list_diff.json') + mock_config_bucket = MagicMock(return_value='') + mock_json_parser = MagicMock(return_value={}) + mock_fetch_target_objects = MagicMock(side_effect=Exception('形式チェックエラー')) + # Act + with patch('src.aws.s3.ConfigBucket.get_object_info_file', mock_config_bucket), \ + patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ + patch('src.config.objects.FetchTargetObjects.__init__', mock_fetch_target_objects): + with pytest.raises(InvalidConfigException) as e: + backup_crm_data_process() + + # Assert + + assert mock_config_bucket.called is True + assert mock_json_parser.called is True + assert mock_fetch_target_objects.called is True + assert e.value.error_id == 'E-RESBK-03' + assert e.value.func_name == RESBK_JP_NAME + assert e.value.args[0] == 'CRM_取得オブジェクト情報ファイルの形式チェックに失敗しました ファイル名：[crm_object_list_diff.json] エラー内容：[形式チェックエラー]' From b9e77ed045eb1c235ccb530315d89ddb16281cb7 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 20:32:54 +0900 Subject: [PATCH 196/275] =?UTF-8?q?feat:=E6=AE=8B=E3=82=8A=E3=81=AE?= =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=82=E8=BF=BD=E5=8A=A0=E3=80=82?= =?UTF-8?q?=E8=A3=BD=E5=93=81=E3=82=B3=E3=83=BC=E3=83=89=E3=81=AE=E3=83=95?= =?UTF-8?q?=E3=82=A9=E3=83=BC=E3=83=9E=E3=83=83=E3=83=88=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/backup_crm_data_process.py | 5 +- .../tests/test_backup_crm_data_process.py | 156 +++++++----------- 2 files changed, 62 insertions(+), 99 deletions(-) diff --git a/ecs/crm-datafetch/src/backup_crm_data_process.py b/ecs/crm-datafetch/src/backup_crm_data_process.py index a5739e07..b37a8a50 100644 --- a/ecs/crm-datafetch/src/backup_crm_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_data_process.py @@ -33,7 +33,10 @@ def backup_crm_data_process(target_object: TargetObject, sf_object_dict: dict, e except Exception as e: raise FileUploadException( - 'E-RESBK-01', RESBK_JP_NAME, f'[{object_name}] 電文データのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.json] エラー内容:[{e}]') + 'E-RESBK-01', + RESBK_JP_NAME, + f'[{object_name}] 電文データのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.json] エラー内容:[{e}]' + ) # ③ CRM電文データバックアップ処理の終了ログを出力する logger.info(f'I-RESBK-03 [{object_name}] のCRM電文データバックアップ処理を終了します') diff --git a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py index 8028ebf0..9ad35d52 100644 --- a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py @@ -1,11 +1,11 @@ import json from collections import OrderedDict -from unittest.mock import MagicMock, patch +from unittest.mock import patch import pytest from src.backup_crm_data_process import backup_crm_data_process from src.config.objects import TargetObject -from src.error.exceptions import FileNotFoundException, InvalidConfigException +from src.error.exceptions import FileUploadException from src.system_var.constants import RESBK_JP_NAME from src.util.execute_datetime import ExecuteDateTime @@ -37,7 +37,8 @@ class TestBackupCrmDataProcess: # Arrange response_json = [ OrderedDict([ - ('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/TEST001')])), + ('attributes', OrderedDict([('type', 'Account'), + ('url', '/services/data/v1.0/sobjects/Account/TEST001')])), ('Id', 'TEST001'), ('AccountNumber', 'test001'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), @@ -46,7 +47,8 @@ class TestBackupCrmDataProcess: ('IsDeleted', False) ]), OrderedDict([ - ('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/TEST002')])), + ('attributes', OrderedDict([('type', 'Account'), + ('url', '/services/data/v1.0/sobjects/Account/TEST002')])), ('Id', 'TEST002'), ('AccountNumber', 'test002'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), @@ -55,7 +57,8 @@ class TestBackupCrmDataProcess: ('IsDeleted', False) ]), OrderedDict([ - ('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/TEST002')])), + ('attributes', OrderedDict([('type', 'Account'), + ('url', '/services/data/v1.0/sobjects/Account/TEST002')])), ('Id', 'TEST002'), ('AccountNumber', 'test002'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), @@ -91,12 +94,15 @@ class TestBackupCrmDataProcess: # ファイル確認 actual = s3_client.get_object( - Bucket=bucket_name, Key=f'response_json/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.json') + Bucket=bucket_name, + Key=f'response_json/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.json') assert actual['Body'].read().decode('utf-8') == json.dumps(response_json) # ログの確認 - assert generate_log_message_tuple(log_message='I-RESBK-01 [Account] のCRM電文データバックアップ処理を開始します') in caplog.record_tuples - assert generate_log_message_tuple(log_message='I-RESBK-03 [Account] のCRM電文データバックアップ処理を終了します') in caplog.record_tuples + assert generate_log_message_tuple( + log_message='I-RESBK-01 [Account] のCRM電文データバックアップ処理を開始します') in caplog.record_tuples + assert generate_log_message_tuple( + log_message='I-RESBK-03 [Account] のCRM電文データバックアップ処理を終了します') in caplog.record_tuples def test_call_depended_modules(self): """ @@ -109,115 +115,69 @@ class TestBackupCrmDataProcess: """ # Arrange - with patch('src.backup_crm_data_process.BackupBucket') as mock_backup_bucket, \ - patch('src.backup_crm_data_process.ExecuteDateTime') as mock_execute_datetime, \ - patch('src.backup_crm_data_process.TargetObject') as mock_target_object: + target_object_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'AccountNumber', + 'LastModifiedDate', + 'LastModifiedById', + 'SystemModstamp', + 'IsDeleted' + ] + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_object_dict, execute_datetime) + + with patch('src.backup_crm_data_process.BackupBucket') as mock_backup_bucket: mock_backup_bucket_inst = mock_backup_bucket.return_value mock_backup_bucket_inst.put_response_json.return_value = '' - mock_execute_inst = mock_execute_datetime.return_value - mock_execute_inst.to_path.return_value = '' - mock_target_inst = mock_target_object.return_value - mock_target_inst.upload_file_name.return_value = '' # Act - backup_crm_data_process(mock_target_object, {}, mock_execute_datetime, ) + backup_crm_data_process(target_object, {}, execute_datetime) # Assert assert mock_backup_bucket_inst.put_response_json.called is True - assert mock_execute_inst.to_path.called is True - assert mock_target_object.upload_file_name.called is True - def test_raise_get_object_info(self, bucket_name, monkeypatch, caplog): + def test_raise_put_response_json(self, bucket_name, monkeypatch, caplog): """ Cases: - CRM_取得オブジェクト情報ファイルを取得できない場合、エラーが発生すること + CRM電文データをS3に配置できない場合、エラーが発生すること Arranges: - オブジェクト情報ファイル取得処理で例外が発生するようにする Expects: - 例外が発生する - - ファイルが読み込めないエラーが返却される + - ファイル登録できないエラーが返却される """ # Arrange - monkeypatch.setattr('src.backup_crm_data_process.OBJECT_INFO_FILENAME', 'crm_object_list_diff.json') - mock_config_bucket = MagicMock(side_effect=Exception('ファイル取得エラー')) - mock_json_parser = MagicMock(return_value={}) - mock_fetch_target_objects = MagicMock(return_value=None) - # Act - with patch('src.aws.s3.ConfigBucket.get_object_info_file', mock_config_bucket), \ - patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ - patch('src.config.objects.FetchTargetObjects.__init__', mock_fetch_target_objects): - with pytest.raises(FileNotFoundException) as e: - backup_crm_data_process() + target_object_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'AccountNumber', + 'LastModifiedDate', + 'LastModifiedById', + 'SystemModstamp', + 'IsDeleted' + ] + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_object_dict, execute_datetime) + + with patch('src.backup_crm_data_process.BackupBucket') as mock_backup_bucket: + mock_backup_bucket_inst = mock_backup_bucket.return_value + mock_backup_bucket_inst.put_response_json.side_effect = Exception('登録エラー') + # Act + with pytest.raises(FileUploadException) as e: + backup_crm_data_process(target_object, {}, execute_datetime) # Assert - assert mock_config_bucket.called is True - assert mock_json_parser.called is False - assert mock_fetch_target_objects.called is False + assert mock_backup_bucket_inst.put_response_json.called is True assert e.value.error_id == 'E-RESBK-01' assert e.value.func_name == RESBK_JP_NAME - assert e.value.args[0] == f'CRM_取得オブジェクト情報ファイルが存在しません ファイル名：[crm_object_list_diff.json] エラー内容：[ファイル取得エラー]' - - def test_raise_parse_json_str(self, bucket_name, monkeypatch, caplog): - """ - Cases: - CRM_取得オブジェクト情報ファイルをパースできない場合、エラーが発生すること - Arranges: - - オブジェクト情報ファイルパース処理で例外が発生するようにする - Expects: - - 例外が発生する - - パースが失敗した旨、エラーメッセージが表示される - """ - - # Arrange - mock_config_bucket = MagicMock(return_value='') - mock_json_parser = MagicMock(side_effect=Exception('JSONパースエラー')) - mock_fetch_target_objects = MagicMock(return_value=None) - # Act - with patch('src.aws.s3.ConfigBucket.get_object_info_file', mock_config_bucket), \ - patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ - patch('src.config.objects.FetchTargetObjects.__init__', mock_fetch_target_objects): - with pytest.raises(InvalidConfigException) as e: - backup_crm_data_process() - - # Assert - - assert mock_config_bucket.called is True - assert mock_json_parser.called is True - assert mock_fetch_target_objects.called is False - assert e.value.error_id == 'E-RESBK-02' - assert e.value.func_name == RESBK_JP_NAME - assert e.value.args[0] == f'CRM_取得オブジェクト情報ファイルのパースに失敗しました エラー内容：[JSONパースエラー]' - - def test_raise_check_objects_format(self, bucket_name, monkeypatch, caplog): - """ - Cases: - CRM_取得オブジェクト情報のキーobjectsの形式チェックが不正の場合、エラーが発生すること - Arranges: - - CRM_取得オブジェクト情報のキーobjectsの形式チェックで例外が発生するようにする - Expects: - - 例外が発生する - - 形式チェックが失敗した旨、エラーメッセージが表示される - """ - - # Arrange - monkeypatch.setattr('src.backup_crm_data_process.OBJECT_INFO_FILENAME', 'crm_object_list_diff.json') - mock_config_bucket = MagicMock(return_value='') - mock_json_parser = MagicMock(return_value={}) - mock_fetch_target_objects = MagicMock(side_effect=Exception('形式チェックエラー')) - # Act - with patch('src.aws.s3.ConfigBucket.get_object_info_file', mock_config_bucket), \ - patch('src.parser.json_parser.JsonParser.parse', mock_json_parser), \ - patch('src.config.objects.FetchTargetObjects.__init__', mock_fetch_target_objects): - with pytest.raises(InvalidConfigException) as e: - backup_crm_data_process() - - # Assert - - assert mock_config_bucket.called is True - assert mock_json_parser.called is True - assert mock_fetch_target_objects.called is True - assert e.value.error_id == 'E-RESBK-03' - assert e.value.func_name == RESBK_JP_NAME - assert e.value.args[0] == 'CRM_取得オブジェクト情報ファイルの形式チェックに失敗しました ファイル名：[crm_object_list_diff.json] エラー内容：[形式チェックエラー]' + assert e.value.args[0] == \ + f'[Account] 電文データのバックアップに失敗しました ファイル名：[CRM_Account_{execute_datetime.format_date()}.json] エラー内容:[登録エラー]' From f0b02efd8e3399d182013812cbced763b442f506 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 20:32:54 +0900 Subject: [PATCH 197/275] =?UTF-8?q?refactor:=20=E3=81=BB=E3=81=8B=E3=83=A2?= =?UTF-8?q?=E3=82=B8=E3=83=A5=E3=83=BC=E3=83=AB=E3=81=AE=E6=A9=9F=E8=83=BD?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E3=81=AB=E3=82=88=E3=82=8A=E3=80=81=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E5=AE=9F=E6=96=BD=E4=B8=8D=E5=8F=AF=E3=81=A8?= =?UTF-8?q?=E3=81=AA=E3=81=A3=E3=81=9F=E3=83=86=E3=82=B9=E3=83=88=E3=82=92?= =?UTF-8?q?=E3=82=B9=E3=82=AD=E3=83=83=E3=83=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/salesforce/test_soql_builder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py index da42d486..56ff81ea 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py +++ b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py @@ -1,4 +1,5 @@ +import pytest from src.config.objects import ExecuteDateTime, LastFetchDatetime, TargetObject from src.salesforce.soql_builder import SOQLBuilder @@ -119,7 +120,8 @@ class TestSOQLBuilder: assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') - def test_create_fetch_soql_no_columns(self): + @pytest.mark.skip('データ取得用のSOQLがカラムがない状態は、TargetObjectクラス側で制御されるため、テストを実施しない') + def test_raise_create_fetch_soql_cause_no_columns(self): """ Cases: データ取得用のSOQLがカラムがない状態で生成されること From 2d2ad65d16c5feed17afc6230086b67b5df93942 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Aug 2022 11:14:21 +0900 Subject: [PATCH 198/275] =?UTF-8?q?frat:=20CSV=E5=A4=89=E6=8F=9B=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test_convert_crm_csv_data_process.py | 169 ++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py diff --git a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py new file mode 100644 index 00000000..ec4ab64a --- /dev/null +++ b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py @@ -0,0 +1,169 @@ +import textwrap +from collections import OrderedDict +from unittest.mock import patch + +import pytest +from src.config.objects import TargetObject +from src.convert_crm_csv_data_process import convert_crm_csv_data_process +from src.error.exceptions import DataConvertException +from src.system_var.constants import CONV_JP_NAME +from src.util.execute_datetime import ExecuteDateTime + +from .test_utils.log_message import generate_log_message_tuple + + +class TestConvertCrmCsvDataProcess: + + def test_run_process_success(self, caplog): + """ + Cases: + CSV変換処理が正常終了し、期待通りの結果が返ること + Arranges: + - チェック対象のdictオブジェクトを宣言する + Expects: + - CSV変換処理の結果文字列が返却されること + - CSV変換処理の仕様に沿った正常系ログが出力されること(デバッグログは除く) + """ + # Arrange + response_json = [ + OrderedDict([ + ('attributes', OrderedDict([('type', 'Account'), + ('url', '/services/data/v1.0/sobjects/Account/TEST001')])), + ('Id', 'TEST001'), + ('AccountNumber', 'test001'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), + ('IsDeleted', True) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'Account'), + ('url', '/services/data/v1.0/sobjects/Account/TEST002')])), + ('Id', 'TEST002'), + ('AccountNumber', 'test002'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), + ('IsDeleted', False) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'Account'), + ('url', '/services/data/v1.0/sobjects/Account/TEST003')])), + ('Id', 'TEST003'), + ('AccountNumber', 'test003'), + ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), + ('LastModifiedById', 1.234567E+6), + ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), + ('IsDeleted', False) + ]), + ] + + target_object_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'AccountNumber', + 'LastModifiedDate', + 'LastModifiedById', + 'SystemModstamp', + 'IsDeleted' + ] + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_object_dict, execute_datetime) + + # Act + actual_csv_string = convert_crm_csv_data_process(target_object, response_json) + + # Assert + + expect_csv_string = """\ + "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\ + "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\ + "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ + "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ + """ + # 返り値の期待値チェック + assert isinstance(actual_csv_string, str), 'CSV文字列が返却される' + assert actual_csv_string == textwrap.dedent(expect_csv_string) + + # ログの確認 + assert generate_log_message_tuple(log_message='I-CONV-01 [Account] のCSV変換処理を開始します') in caplog.record_tuples + assert generate_log_message_tuple(log_message='I-CONV-03 [Account] のCSV変換処理を終了します') in caplog.record_tuples + + def test_call_depended_modules(self): + """ + Cases: + CSV変換処理内で依存しているモジュールが正しく呼ばれていること + Arranges: + - CSV変換処理の依存モジュールをモック化する + Expects: + - 依存しているモジュールが正しく呼ばれている + """ + + # Arrange + target_object_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'AccountNumber', + 'LastModifiedDate', + 'LastModifiedById', + 'SystemModstamp', + 'IsDeleted' + ] + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_object_dict, execute_datetime) + # Act + with patch('src.convert_crm_csv_data_process.CSVStringConverter') as mock_converter: + inst = mock_converter.return_value + inst.convert.return_value = '' + convert_crm_csv_data_process(target_object, {}) + + # Assert + assert mock_converter.called is True + assert inst.convert.called is True + + def test_raise_convert(self): + """ + Cases: + オブジェクト形式チェック処理でエラーが発生した場合、検査例外が発生すること + Arranges: + - オブジェクト形式チェック処理で例外が発生するようにする + Expects: + - 例外が発生する + - 形式チェックが失敗したエラーが返却される + """ + + # Arrange + target_object_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'AccountNumber', + 'LastModifiedDate', + 'LastModifiedById', + 'SystemModstamp', + 'IsDeleted' + ] + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_object_dict, execute_datetime) + # Act + with patch('src.convert_crm_csv_data_process.CSVStringConverter') as mock_converter: + inst = mock_converter.return_value + inst.convert.side_effect = Exception('変換エラー') + with pytest.raises(DataConvertException) as e: + convert_crm_csv_data_process(target_object, {}) + + # Assert + assert mock_converter.called is True + assert inst.convert.called is True + + assert e.value.error_id == 'E-CONV-01' + assert e.value.func_name == CONV_JP_NAME + assert e.value.args[0] == f'[Account] CSV変換に失敗しました エラー内容:[変換エラー]' From 5f302f5dc2c5f152c4c9a7aafe1178dec88a5e8e Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Aug 2022 11:14:42 +0900 Subject: [PATCH 199/275] =?UTF-8?q?refactor:=20VSCode=E3=81=AE=E6=8E=A8?= =?UTF-8?q?=E5=A5=A8=E8=A8=AD=E5=AE=9A=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.vscode/recommend_settings.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/.vscode/recommend_settings.json b/ecs/crm-datafetch/.vscode/recommend_settings.json index 66a8f2b5..d4822731 100644 --- a/ecs/crm-datafetch/.vscode/recommend_settings.json +++ b/ecs/crm-datafetch/.vscode/recommend_settings.json @@ -12,5 +12,8 @@ "python.linting.flake8Enabled": true, "python.linting.flake8Args": ["--max-line-length=150", "--ignore=F541"], "python.formatting.provider": "autopep8", - "python.formatting.autopep8Args": ["--max-line-length", "150"] + "python.formatting.autopep8Args": ["--max-line-length", "150"], + "python.testing.pytestArgs": ["tests"], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true } From e02d7a0c59bf32843b6ea30d1641e36803e83740 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 17 Aug 2022 20:32:54 +0900 Subject: [PATCH 200/275] =?UTF-8?q?refactor:=20=E3=81=BB=E3=81=8B=E3=83=A2?= =?UTF-8?q?=E3=82=B8=E3=83=A5=E3=83=BC=E3=83=AB=E3=81=AE=E6=A9=9F=E8=83=BD?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E3=81=AB=E3=82=88=E3=82=8A=E3=80=81=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E5=AE=9F=E6=96=BD=E4=B8=8D=E5=8F=AF=E3=81=A8?= =?UTF-8?q?=E3=81=AA=E3=81=A3=E3=81=9F=E3=83=86=E3=82=B9=E3=83=88=E3=82=92?= =?UTF-8?q?=E3=82=B9=E3=82=AD=E3=83=83=E3=83=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/salesforce/test_soql_builder.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py index da42d486..56ff81ea 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py +++ b/ecs/crm-datafetch/tests/salesforce/test_soql_builder.py @@ -1,4 +1,5 @@ +import pytest from src.config.objects import ExecuteDateTime, LastFetchDatetime, TargetObject from src.salesforce.soql_builder import SOQLBuilder @@ -119,7 +120,8 @@ class TestSOQLBuilder: assert actual.replace('\n', '').replace(' ', '') == expect.replace('\n', '').replace(' ', '') - def test_create_fetch_soql_no_columns(self): + @pytest.mark.skip('データ取得用のSOQLがカラムがない状態は、TargetObjectクラス側で制御されるため、テストを実施しない') + def test_raise_create_fetch_soql_cause_no_columns(self): """ Cases: データ取得用のSOQLがカラムがない状態で生成されること From d157edf5a849d81e07f27a807da691a014e449ec Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Aug 2022 12:15:09 +0900 Subject: [PATCH 201/275] =?UTF-8?q?feat:=20CSV=E3=83=90=E3=83=83=E3=82=AF?= =?UTF-8?q?=E3=82=A2=E3=83=83=E3=83=97=E5=87=A6=E7=90=86=E3=81=AE=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_backup_crm_csv_data_process.py | 157 ++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py diff --git a/ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py new file mode 100644 index 00000000..dbb9230d --- /dev/null +++ b/ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py @@ -0,0 +1,157 @@ +import textwrap +from unittest.mock import patch + +import pytest +from src.backup_crm_csv_data_process import backup_crm_csv_data_process +from src.config.objects import TargetObject +from src.error.exceptions import FileUploadException +from src.system_var.constants import CSVBK_JP_NAME +from src.util.execute_datetime import ExecuteDateTime + +from .test_utils.log_message import generate_log_message_tuple + + +class TestBackupCrmCsvDataProcess: + + @pytest.fixture + def bucket_name(self): + return 'test-backup-bucket' + + @pytest.fixture + def prepare_bucket(self, s3_client, bucket_name): + s3_client.create_bucket(Bucket=bucket_name) + yield + + def test_run_process_success(self, s3_client, prepare_bucket, bucket_name, monkeypatch, caplog): + """ + Cases: + CSVバックアップ処理が正常終了し、期待通りの結果となること + Arranges: + - prepare_bucketフィクスチャで、CRM電文データをバックアップするためのモックバケットを作る + - 作成したモックバケットを指すように環境変数を設定する + Expects: + - CSVバックアップファイルがバケットに配置される + - CSVバックアップ処理の仕様に沿った正常系ログが出力されること(デバッグログは除く) + """ + # Arrange + csv_string = textwrap.dedent("""\ + "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted" + "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1" + "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0" + "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0" + """) + target_object_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'AccountNumber', + 'LastModifiedDate', + 'LastModifiedById', + 'SystemModstamp', + 'IsDeleted' + ] + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_object_dict, execute_datetime) + + # 環境変数を編集 + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', 'data_import') + + # Act + backup_crm_csv_data_process(target_object, execute_datetime, csv_string) + + # Assert + + # ファイル確認 + actual = s3_client.get_object( + Bucket=bucket_name, + Key=f'data_import/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.csv') + + assert actual['Body'].read().decode('utf-8') == csv_string + + # ログの確認 + assert generate_log_message_tuple( + log_message=f'I-CSVBK-01 [Account] のCSVデータのバックアップ処理を開始します ファイル名：[CRM_Account_{execute_datetime.format_date()}.csv]' + ) in caplog.record_tuples + assert generate_log_message_tuple( + log_message='I-CSVBK-03 [Account] のCSVデータのバックアップ処理を終了します') in caplog.record_tuples + + def test_call_depended_modules(self): + """ + Cases: + CSVバックアップ処理内で依存しているモジュールが正しく呼ばれていること + Arranges: + - CSVバックアップ処理の依存モジュールをモック化する + Expects: + - 依存しているモジュールが正しく呼ばれている + """ + + # Arrange + target_object_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'AccountNumber', + 'LastModifiedDate', + 'LastModifiedById', + 'SystemModstamp', + 'IsDeleted' + ] + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_object_dict, execute_datetime) + + with patch('src.backup_crm_csv_data_process.BackupBucket') as mock_backup_bucket: + mock_backup_bucket_inst = mock_backup_bucket.return_value + mock_backup_bucket_inst.put_csv.return_value = '' + # Act + backup_crm_csv_data_process(target_object, execute_datetime, '') + + # Assert + + assert mock_backup_bucket_inst.put_csv.called is True + + def test_raise_put_csv(self, bucket_name, monkeypatch, caplog): + """ + Cases: + CRM電文データをS3に配置できない場合、エラーが発生すること + Arranges: + - オブジェクト情報ファイル取得処理で例外が発生するようにする + Expects: + - 例外が発生する + - ファイル登録できないエラーが返却される + """ + + # Arrange + target_object_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'AccountNumber', + 'LastModifiedDate', + 'LastModifiedById', + 'SystemModstamp', + 'IsDeleted' + ] + } + + execute_datetime = ExecuteDateTime() + target_object = TargetObject(target_object_dict, execute_datetime) + + with patch('src.backup_crm_csv_data_process.BackupBucket') as mock_backup_bucket: + mock_backup_bucket_inst = mock_backup_bucket.return_value + mock_backup_bucket_inst.put_csv.side_effect = Exception('登録エラー') + # Act + with pytest.raises(FileUploadException) as e: + backup_crm_csv_data_process(target_object, execute_datetime, '') + + # Assert + + assert mock_backup_bucket_inst.put_csv.called is True + assert e.value.error_id == 'E-CSVBK-01' + assert e.value.func_name == CSVBK_JP_NAME + assert e.value.args[0] == \ + f'[Account] CSVデータのバックアップに失敗しました ファイル名：[CRM_Account_{execute_datetime.format_date()}.csv] エラー内容:[登録エラー]' From 4fd36f35d201108da1b214e875a403bfd636f14a Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 18 Aug 2022 14:34:49 +0900 Subject: [PATCH 202/275] =?UTF-8?q?feat:=20CSV=E3=82=A2=E3=83=83=E3=83=97?= =?UTF-8?q?=E3=83=AD=E3=83=BC=E3=83=89=E5=87=A6=E7=90=86=E3=81=AE=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/test_copy_crm_csv_data_process.py | 160 ++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 ecs/crm-datafetch/tests/test_copy_crm_csv_data_process.py diff --git a/ecs/crm-datafetch/tests/test_copy_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_copy_crm_csv_data_process.py new file mode 100644 index 00000000..fbec7c7e --- /dev/null +++ b/ecs/crm-datafetch/tests/test_copy_crm_csv_data_process.py @@ -0,0 +1,160 @@ +from unittest.mock import MagicMock, patch + +import pytest +from src.config.objects import TargetObject +from src.copy_crm_csv_data_process import copy_crm_csv_data_process +from src.error.exceptions import FileUploadException +from src.system_var.constants import UPLD_JP_NAME +from src.util.execute_datetime import ExecuteDateTime + +from .test_utils.log_message import generate_log_message_tuple + + +class TestCopyCrmCsvDataProcess: + + @pytest.fixture + def bucket_name(self): + return 'test-bucket' + + @pytest.fixture + def prepare_bucket(self, s3_client, bucket_name): + s3_client.create_bucket(Bucket=bucket_name) + yield + + def test_run_process_success(self, s3_client, prepare_bucket, bucket_name, monkeypatch, caplog): + """ + Cases: + CSVアップロード処理が正常終了し、期待通りの結果が返ること + Arranges: + - オブジェクト情報の準備 + - コピー元のバケットのモックを生成する + - コピー先のバケットの環境変数を設定する + - コピー元にファイルを配置する + Expects: + - コピーしたしたCSVファイルが存在し内容が想定と一致する + - CSVアップロード処理の仕様に沿った正常系ログが出力されること(デバッグログは除く) + """ + + # Arranges + + # オブジェクト情報の準備 + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ] + } + + execute_datetime = ExecuteDateTime() + + target_object = TargetObject(target_objects_dict, execute_datetime) + + # コピー元のバケットをモック化 + from_copy_bucket = 'for-copy-bucket' + s3_client.create_bucket(Bucket=from_copy_bucket) + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', from_copy_bucket) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', 'data_import') + + # コピー先のバケットをモック化 + monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', bucket_name) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', 'crm/target') + + # コピー元のモックバケットにデータを配置 + s3_client.put_object(Bucket=from_copy_bucket, + Key=f'data_import/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.csv', Body=b'test,test,test') + + # Act + copy_crm_csv_data_process(target_object, execute_datetime) + + # Assert + # ファイル確認 + actual = s3_client.get_object( + Bucket=bucket_name, Key=f'crm/target/CRM_Account_{execute_datetime.format_date()}.csv') + assert actual['Body'].read().decode('utf-8') == 'test,test,test' + + # ログの確認 + assert generate_log_message_tuple( + log_message=f'I-UPLD-01 [Account] のCSVデータアップロード処理を開始します ファイル名：[CRM_Account_{execute_datetime.format_date()}.csv]') in caplog.record_tuples + assert generate_log_message_tuple(log_message=f'I-UPLD-03 [Account] のCSVデータのアップロード処理を終了します') in caplog.record_tuples + + def test_call_depended_modules(self): + """ + Cases: + CSVアップロード処理内で依存しているモジュールが正しく呼ばれていること + Arranges: + - CSVアップロード処理の依存モジュールをモック化する + - オブジェクト情報の準備 + Expects: + - 依存しているモジュールが正しく呼ばれている + """ + + # Arrange + + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ] + } + + execute_datetime = ExecuteDateTime() + + target_object = TargetObject(target_objects_dict, execute_datetime) + + mock_data_bucket = MagicMock(return_value=None) + mock_backup_bucket = MagicMock(return_value=None) + mock_copy_bucket = MagicMock(return_value=None) + + # Act + with patch('src.aws.s3.DataBucket.__init__', mock_data_bucket), \ + patch('src.aws.s3.BackupBucket.__init__', mock_backup_bucket), \ + patch('src.aws.s3.DataBucket.put_csv_from', mock_copy_bucket): + copy_crm_csv_data_process(target_object, execute_datetime) + + # Assert + assert mock_data_bucket.called is True + assert mock_backup_bucket.called is True + assert mock_copy_bucket.called is True + + def test_raise_run_process_success(self, s3_client, prepare_bucket, bucket_name, monkeypatch, caplog): + """ + Cases: + CSVデータをコピーできない場合、エラーが発生すること + Arranges: + - オブジェクト情報の準備 + - コピー元のバケットのモックを生成する + - コピー先のバケットの環境変数を設定する + - コピー元にファイルを配置する + Expects: + - 例外が発生する + - ファイルがアップロードできないエラーが返却される + """ + + # Arranges + + # オブジェクト情報の準備 + target_objects_dict = { + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name' + ] + } + + execute_datetime = ExecuteDateTime() + + target_object = TargetObject(target_objects_dict, execute_datetime) + + mock_copy_bucket = MagicMock(side_effect=Exception('ファイルコピーエラー')) + + # Act + with patch('src.aws.s3.DataBucket.put_csv_from', mock_copy_bucket): + with pytest.raises(FileUploadException) as e: + copy_crm_csv_data_process(target_object, execute_datetime) + + # Assert + assert e.value.error_id == 'E-UPLD-01' + assert e.value.func_name == UPLD_JP_NAME + assert e.value.args[0] == f'[Account] CSVデータのアップロードに失敗しました ファイル名：[CRM_Account_{execute_datetime.format_date()}.csv] エラー内容:[ファイルコピーエラー]' From 2a2316918596f9e404c047a92cf9c9e146ba7215 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Aug 2022 16:46:10 +0900 Subject: [PATCH 203/275] =?UTF-8?q?style:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py index ec4ab64a..49d960c0 100644 --- a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py +++ b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py @@ -130,9 +130,9 @@ class TestConvertCrmCsvDataProcess: def test_raise_convert(self): """ Cases: - オブジェクト形式チェック処理でエラーが発生した場合、検査例外が発生すること + CSV変換処理でエラーが発生した場合、検査例外が発生すること Arranges: - - オブジェクト形式チェック処理で例外が発生するようにする + - CSV変換処理で例外が発生するようにする Expects: - 例外が発生する - 形式チェックが失敗したエラーが返却される From 4b29344c74aca7ed3f3573435de6c4a75bed427e Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Aug 2022 16:47:03 +0900 Subject: [PATCH 204/275] =?UTF-8?q?style:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_fetch_crm_data_process.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py index 5d35ac58..66f5577e 100644 --- a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py @@ -149,7 +149,7 @@ class TestFetchCrmDataProcess: - 件数取得用SOQL生成処理で例外が発生するようにする Expects: - 例外が発生する - - データ件数取得に失敗した胸のエラーが出力される + - データ件数取得に失敗した旨のエラーが出力される """ # Arrange @@ -181,7 +181,7 @@ class TestFetchCrmDataProcess: - データ取得用SOQL生成処理で例外が発生するようにする Expects: - 例外が発生する - - データ取得に失敗した胸のエラーが出力される + - データ取得に失敗した旨のエラーが出力される """ # Arrange @@ -222,7 +222,7 @@ class TestFetchCrmDataProcess: - データ件数取得処理の初回に接続タイムアウト例外が発生するようにする Expects: - 正常終了する - - データ件数取得に失敗した胸のエラーが出力されない + - データ件数取得に失敗した旨のエラーが出力されない """ monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) @@ -338,7 +338,7 @@ class TestFetchCrmDataProcess: - レコード取得処理の初回に接続タイムアウト例外が発生するようにする Expects: - 正常終了する - - データレコード取得に失敗した胸のエラーが出力されない + - データレコード取得に失敗した旨のエラーが出力されない """ monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) From 25b1d36cc8321831c1848c22d8af14c713b9aad3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Aug 2022 16:49:10 +0900 Subject: [PATCH 205/275] =?UTF-8?q?style:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py index dbb9230d..267e87da 100644 --- a/ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py +++ b/ecs/crm-datafetch/tests/test_backup_crm_csv_data_process.py @@ -117,9 +117,9 @@ class TestBackupCrmCsvDataProcess: def test_raise_put_csv(self, bucket_name, monkeypatch, caplog): """ Cases: - CRM電文データをS3に配置できない場合、エラーが発生すること + CSVデータをバックアップできない場合、エラーが発生すること Arranges: - - オブジェクト情報ファイル取得処理で例外が発生するようにする + - CSVデータをバックアップ処理で例外が発生するようにする Expects: - 例外が発生する - ファイル登録できないエラーが返却される From 3b74a8ea9f58cc4f3757a4cf288d270c0b151ba2 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Aug 2022 16:57:36 +0900 Subject: [PATCH 206/275] =?UTF-8?q?feat:=20=E4=B8=80=E6=B0=97=E9=80=9A?= =?UTF-8?q?=E8=B2=AB=E3=83=86=E3=82=B9=E3=83=88=E3=81=A0=E3=81=91=E3=82=92?= =?UTF-8?q?=E5=AE=9F=E8=A1=8C=E3=81=99=E3=82=8B=E3=81=9F=E3=82=81=E3=81=AE?= =?UTF-8?q?=E3=83=9E=E3=83=BC=E3=82=AB=E3=83=BC=E3=81=A8=E3=82=B3=E3=83=9E?= =?UTF-8?q?=E3=83=B3=E3=83=89=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Pipfile | 1 + ecs/crm-datafetch/tests/conftest.py | 20 ++++++++++++++++++++ ecs/crm-datafetch/tests/test_walkthrough.py | 7 +++++++ 3 files changed, 28 insertions(+) create mode 100644 ecs/crm-datafetch/tests/test_walkthrough.py diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 82796598..6912e1a0 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -7,6 +7,7 @@ name = "pypi" test = "pytest tests/" "test:cov" = "pytest --cov=src --cov-branch --cov-report=term-missing tests/" "test:report" = "pytest --cov=src --cov-branch --cov-report=term-missing --html=.report/test_result.html tests/" +"test:walk-through" = "pytest tests/ --walk-through" [packages] boto3 = "*" diff --git a/ecs/crm-datafetch/tests/conftest.py b/ecs/crm-datafetch/tests/conftest.py index 55ade88c..b86b29d9 100644 --- a/ecs/crm-datafetch/tests/conftest.py +++ b/ecs/crm-datafetch/tests/conftest.py @@ -57,3 +57,23 @@ def pytest_runtest_makereport(item, call): report.cases = docstring.get("Cases", '') # 「テスト内容」を`report`に追加 report.arranges = docstring.get("Arranges", '') # 「準備作業」を`report`に追加 report.expects = docstring.get("Expects", '') # 「期待結果」を`report`に追加 + + +def pytest_addoption(parser): + parser.addoption( + "--walk-through", action="store_true", default=False, help="run walk through tests" + ) + + +def pytest_configure(config): + config.addinivalue_line("markers", "slow: mark test as slow to run") + + +def pytest_collection_modifyitems(config, items): + skip_slow = pytest.mark.skip(reason="need --walk-through option to run") + if config.getoption("--walk-through"): + [item.add_marker(skip_slow) for item in items if "walk_through" not in item.keywords] + return + for item in items: + if "walk_through" in item.keywords: + item.add_marker(skip_slow) diff --git a/ecs/crm-datafetch/tests/test_walkthrough.py b/ecs/crm-datafetch/tests/test_walkthrough.py new file mode 100644 index 00000000..2a3b09f2 --- /dev/null +++ b/ecs/crm-datafetch/tests/test_walkthrough.py @@ -0,0 +1,7 @@ +import pytest + + +@pytest.mark.walk_through +def test_walk_through(): + assert 0 + pass From 97141076c27af5b105f94b81794a3746b4e4e1fb Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Aug 2022 16:57:37 +0900 Subject: [PATCH 207/275] =?UTF-8?q?feat:=20VSCode=E3=81=A7=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=81=99=E3=82=8B=E3=81=A8=E3=81=8D=E3=82=82?= =?UTF-8?q?=E5=8F=8E=E9=9B=86=E5=AF=BE=E8=B1=A1=E3=81=8B=E3=82=89=E9=99=A4?= =?UTF-8?q?=E3=81=84=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.vscode/recommend_settings.json | 2 +- .../tests/{test_walkthrough.py => test_walk_through.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename ecs/crm-datafetch/tests/{test_walkthrough.py => test_walk_through.py} (100%) diff --git a/ecs/crm-datafetch/.vscode/recommend_settings.json b/ecs/crm-datafetch/.vscode/recommend_settings.json index d4822731..4fd1fddf 100644 --- a/ecs/crm-datafetch/.vscode/recommend_settings.json +++ b/ecs/crm-datafetch/.vscode/recommend_settings.json @@ -13,7 +13,7 @@ "python.linting.flake8Args": ["--max-line-length=150", "--ignore=F541"], "python.formatting.provider": "autopep8", "python.formatting.autopep8Args": ["--max-line-length", "150"], - "python.testing.pytestArgs": ["tests"], + "python.testing.pytestArgs": ["tests", "--ignore=tests/test_walk_through.py"], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true } diff --git a/ecs/crm-datafetch/tests/test_walkthrough.py b/ecs/crm-datafetch/tests/test_walk_through.py similarity index 100% rename from ecs/crm-datafetch/tests/test_walkthrough.py rename to ecs/crm-datafetch/tests/test_walk_through.py From 6221113b1ab8d258e3ba841abb8f2e99bee8759d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Aug 2022 16:57:37 +0900 Subject: [PATCH 208/275] =?UTF-8?q?feat:=20=E3=81=A8=E3=82=8A=E3=81=82?= =?UTF-8?q?=E3=81=88=E3=81=9A=E6=B5=81=E3=82=8C=E3=81=8D=E3=82=8B=E3=81=A8?= =?UTF-8?q?=E3=81=93=E3=82=8D=E3=81=BE=E3=81=A7=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/salesforce/soql_builder.py | 8 +- .../src/system_var/environments.py | 22 ++--- ecs/crm-datafetch/tests/test_walk_through.py | 89 ++++++++++++++++++- 3 files changed, 105 insertions(+), 14 deletions(-) diff --git a/ecs/crm-datafetch/src/salesforce/soql_builder.py b/ecs/crm-datafetch/src/salesforce/soql_builder.py index 42d4ecea..120b5bf2 100644 --- a/ecs/crm-datafetch/src/salesforce/soql_builder.py +++ b/ecs/crm-datafetch/src/salesforce/soql_builder.py @@ -1,6 +1,7 @@ import textwrap from src.config.objects import LastFetchDatetime, TargetObject +from src.system_var.environments import FETCH_LIMIT_CLAUSE class SOQLBuilder: @@ -9,6 +10,7 @@ class SOQLBuilder: SELECT {column_or_expression} FROM {object_name} WHERE {datetime_column} > {last_fetch_datetime_from} AND {datetime_column} <= {last_fetch_datetime_to} + {limit_clause} """) self.__target_object = target_object self.__last_fetch_datetime = last_fetch_datetime @@ -19,7 +21,8 @@ class SOQLBuilder: object_name=self.__target_object.object_name, last_fetch_datetime_from=self.__last_fetch_datetime.last_fetch_datetime_from, last_fetch_datetime_to=self.__last_fetch_datetime.last_fetch_datetime_to, - datetime_column=self.__target_object.datetime_column + datetime_column=self.__target_object.datetime_column, + limit_clause='' ) return count_soql @@ -31,7 +34,8 @@ class SOQLBuilder: object_name=self.__target_object.object_name, last_fetch_datetime_from=self.__last_fetch_datetime.last_fetch_datetime_from, last_fetch_datetime_to=self.__last_fetch_datetime.last_fetch_datetime_to, - datetime_column=self.__target_object.datetime_column + datetime_column=self.__target_object.datetime_column, + limit_clause=FETCH_LIMIT_CLAUSE ) return fetch_soql diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index b290cc7b..9ab6911d 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -7,14 +7,14 @@ import src.system_var.constants as constants LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # CRMへの認証処理のタイムアウト秒数 CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)) -# CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3)) -# CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5)) -# CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5)) -# CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50)) +# # CRMへの認証処理の最大リトライ試行回数 +# CRM_AUTH_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3)) +# # CRMへの認証処理のリトライ時の初回待ち秒数 +# CRM_AUTH_RETRY_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5)) +# # CRMへの認証処理のリトライ時の最小待ち秒数 +# CRM_AUTH_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5)) +# # CRMへの認証処理のリトライ時の最大待ち秒数 +# CRM_AUTH_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50)) # CRMのレコード件数取得処理のタイムアウト秒数 CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)) # CRMのレコード件数取得処理の最大リトライ試行回数 @@ -65,9 +65,11 @@ PROCESS_RESULT_FILENAME = os.environ.get(constants.PROCESS_RESULT_FILENAME, 'pro LAST_FETCH_DATE_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_FOLDER, 'crm/last_fetch_datetime') # CRMから取得し、取込用に変換したデータを格納するフォルダ CRM_IMPORT_DATA_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_FOLDER, 'crm/target') -# CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_BACKUP_FOLDER, 'last_fetch_datetime') +# # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +# LAST_FETCH_DATE_BACKUP_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_BACKUP_FOLDER, 'last_fetch_datetime') # CRMから取得した生データのバックアップを格納するフォルダパス RESPONSE_JSON_BACKUP_FOLDER = os.environ.get(constants.RESPONSE_JSON_BACKUP_FOLDER, 'response_json') # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_BACKUP_FOLDER, 'data_import') +# 一気通貫テスト用、取得件数を絞るためのSOQL句 +FETCH_LIMIT_CLAUSE = os.environ.get('FETCH_LIMIT_CLAUSE', '') diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index 2a3b09f2..cb7206aa 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -1,7 +1,92 @@ +import json +import os +import os.path as path + import pytest +from src.controller import controller + +ROOT_DIR = path.abspath(path.dirname(__name__)) +DATA_BUCKET = 'mbj-newdwh2021-staging-data' +CONFIG_BUCKET = 'mbj-newdwh2021-staging-config' +BACKUP_BUCKET = 'mbj-newdwh2021-staging-backup-crm' + +TARGET_FOLDER = 'crm/target' +OBJECT_INFO_FOLDER = 'crm/object_info' +LAST_FETCH_DATETIME_INFO_FOLDER = 'crm/last_fetch_datetime' +BACKUP_DATA_IMPORT_FOLDER = 'data_import' +BACKUP_RESPONSE_JSON_FOLDER = 'response_json' + + +@pytest.fixture +def s3_test(s3_client): + s3_client.create_bucket(Bucket=DATA_BUCKET) + s3_client.create_bucket(Bucket=CONFIG_BUCKET) + s3_client.create_bucket(Bucket=BACKUP_BUCKET) + yield @pytest.mark.walk_through -def test_walk_through(): +def test_walk_through(s3_test, s3_client, monkeypatch, caplog): + # Arrange + # バケットにファイルをアップロードしていく + object_info_list = get_object_config_list('object_info') + for object_info in object_info_list: + json_file = read_json(object_info) + json_file = to_upload_json(json_file) + upload_json(json_file, s3_client, CONFIG_BUCKET, f'{OBJECT_INFO_FOLDER}/{path.basename(object_info)}') + last_fetch_datetime_list = get_object_config_list('last_fetch_datetime') + for last_fetch_datetime in last_fetch_datetime_list: + json_file = read_json(last_fetch_datetime) + upload_json(json_file, s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{path.basename(last_fetch_datetime)}') + + # 環境変数を設定(CRMの認証情報は別途設定しておくこと) + monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', TARGET_FOLDER) + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', CONFIG_BUCKET) + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', OBJECT_INFO_FOLDER) + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'crm_object_list_diff.json') + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', LAST_FETCH_DATETIME_INFO_FOLDER) + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', BACKUP_BUCKET) + monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', 'process_result') + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) + monkeypatch.setattr('src.salesforce.soql_builder.FETCH_LIMIT_CLAUSE', ' LIMIT 10') + # 件数取得はログ出力用なので、0件が返るようにする + monkeypatch.setattr('src.fetch_crm_data_process.fetch_record_count_retry', lambda x, y, z: 0) + + # Act + controller() + print(caplog.messages) assert 0 - pass + # Assertion + + +def get_object_config_list(folder_name: str): + local_s3_path = path.join(ROOT_DIR, '..', '..', 's3', 'config', 'crm', folder_name) + config_list = [os.path.join(local_s3_path, config) for config in os.listdir(local_s3_path) if config.endswith('.json')] + return config_list + + +def read_json(json_path): + with open(json_path, 'r', encoding='utf8') as f: + json_file = json.load(f) + + return json_file + + +def upload_json(json_file, s3_client, bucket, folder): + json_str = json.dumps(json_file) + s3_client.put_object(Bucket=bucket, Key=folder, Body=json_str) + + +def to_upload_json(json_file): + for object_info in json_file['objects']: + if object_info['object_name'] != 'User': + continue + columns: list = object_info['columns'] + columns.remove('LastPasswordChangeDate') + columns.remove('NumberOfFailedLogins') + columns.remove('UserPreferencesNativeEmailClient') + + return json_file From 7b10186100c0a36d3fbf6810124a6bcbdc04a467 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Aug 2022 17:43:50 +0900 Subject: [PATCH 209/275] =?UTF-8?q?fix:=20CSV=E3=82=A2=E3=83=83=E3=83=97?= =?UTF-8?q?=E3=83=AD=E3=83=BC=E3=83=89=E5=87=A6=E7=90=86=E3=81=AE=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_copy_crm_csv_data_process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/test_copy_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_copy_crm_csv_data_process.py index fbec7c7e..3bb9f0ad 100644 --- a/ecs/crm-datafetch/tests/test_copy_crm_csv_data_process.py +++ b/ecs/crm-datafetch/tests/test_copy_crm_csv_data_process.py @@ -54,7 +54,7 @@ class TestCopyCrmCsvDataProcess: from_copy_bucket = 'for-copy-bucket' s3_client.create_bucket(Bucket=from_copy_bucket) monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', from_copy_bucket) - monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', 'data_import') + monkeypatch.setattr('src.copy_crm_csv_data_process.CRM_IMPORT_DATA_BACKUP_FOLDER', 'data_import') # コピー先のバケットをモック化 monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', bucket_name) From c40f5791cf6da922ac267067b79f9d441e2f4846 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 18 Aug 2022 18:01:05 +0900 Subject: [PATCH 210/275] =?UTF-8?q?fix:=20=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E7=BD=AE=E3=81=8D=E6=8F=9B=E3=81=88=E3=83=9F=E3=82=B9?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index cb7206aa..b680078d 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -48,6 +48,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', LAST_FETCH_DATETIME_INFO_FOLDER) monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', BACKUP_BUCKET) monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', 'process_result') + monkeypatch.setattr('src.copy_crm_csv_data_process.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) From cfcd8ec8dbdde8d245be429f8901d6fc396bcb21 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 18 Aug 2022 20:53:29 +0900 Subject: [PATCH 211/275] =?UTF-8?q?refactor:=20flake8=E8=A6=8F=E7=B4=84?= =?UTF-8?q?=E6=BA=96=E6=8B=A0=E3=81=AE=E3=81=9F=E3=82=81=E3=81=AE=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=E3=80=81=E4=BB=95=E6=A7=98=E6=9B=B8=E3=81=AB=E5=90=88?= =?UTF-8?q?=E3=82=8F=E3=81=9B=E3=81=9F=E8=AA=AC=E6=98=8E=E3=81=AE=E6=8E=A1?= =?UTF-8?q?=E7=95=AA=E3=80=81=E8=AA=A4=E5=AD=97=E4=BF=AE=E6=AD=A3=E3=80=81?= =?UTF-8?q?=E3=82=AA=E3=83=96=E3=82=B8=E3=82=A7=E3=82=AF=E3=83=88=E6=83=85?= =?UTF-8?q?=E5=A0=B1=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/backup_crm_csv_data_process.py | 2 +- .../src/backup_crm_data_process.py | 2 +- .../src/check_object_info_process.py | 2 +- ecs/crm-datafetch/src/config/objects.py | 35 ++++++++++++------- ecs/crm-datafetch/src/controller.py | 34 +++++++++--------- .../src/copy_crm_csv_data_process.py | 2 +- .../src/prepare_data_fetch_process.py | 2 +- .../src/set_datetime_period_process.py | 2 +- ecs/crm-datafetch/tests/test_controller.py | 2 +- .../crm/object_info/crm_object_list_diff.json | 3 +- 10 files changed, 50 insertions(+), 36 deletions(-) diff --git a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py index ba7bb8ea..11ecb03b 100644 --- a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py @@ -11,7 +11,7 @@ def backup_crm_csv_data_process(target_object: TargetObject, execute_datetime: E Args: target_object (TargetObject): 取得対象オブジェクト情報インスタンス - execute_datetime (ExecuteDateTime): 実行日次取得インスタンス + execute_datetime (ExecuteDateTime): 実行日時取得インスタンス csv_string (str): csvデータ Raises: diff --git a/ecs/crm-datafetch/src/backup_crm_data_process.py b/ecs/crm-datafetch/src/backup_crm_data_process.py index b37a8a50..763a17c8 100644 --- a/ecs/crm-datafetch/src/backup_crm_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_data_process.py @@ -12,7 +12,7 @@ def backup_crm_data_process(target_object: TargetObject, sf_object_dict: dict, e Args: target_object (TargetObject): 取得対象オブジェクト名 sf_object_dict (dict): Salesforceオブジェクトデータ - execute_datetime (ExecuteDateTime): 実行日次取得インスタンス + execute_datetime (ExecuteDateTime): 実行日時取得インスタンス Raises: FileUploadException: S3のファイルアップロード失敗 diff --git a/ecs/crm-datafetch/src/check_object_info_process.py b/ecs/crm-datafetch/src/check_object_info_process.py index 84f0db40..f187b62d 100644 --- a/ecs/crm-datafetch/src/check_object_info_process.py +++ b/ecs/crm-datafetch/src/check_object_info_process.py @@ -10,7 +10,7 @@ def check_object_info_process(object_info: dict, execute_datetime: ExecuteDateTi Args: object_info (dict): 取得対象オブジェクト情報 - execute_datetime (ExecuteDateTime): 実行日次取得インスタンス + execute_datetime (ExecuteDateTime): 実行日時取得インスタンス Raises: InvalidConfigException: オブジェクト情報定義が不正だった場合 diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 70beb2eb..35494ff9 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,13 +1,22 @@ -from src.system_var.constants import ( - COLUMNS_KEY, COLUMNS_TYPE, DATE_PATTERN_YYYYMMDDTHHMMSSTZ, - DATETIME_COLUMN_DEFAULT_VALUE, DATETIME_COLUMN_KEY, DATETIME_COLUMN_TYPE, - DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ, IS_SKIP_KEY, IS_SKIP_TYPE, - IS_UPDATE_LAST_FETCH_DATETIME_KEY, IS_UPDATE_LAST_FETCH_DATETIME_TYPE, - LAST_FETCH_DATETIME_FILE_NAME_KEY, LAST_FETCH_DATETIME_FILE_NAME_TYPE, - LAST_FETCH_DATETIME_FROM_KEY, LAST_FETCH_DATETIME_FROM_TYPE, - LAST_FETCH_DATETIME_TO_KEY, LAST_FETCH_DATETIME_TO_TYPE, OBJECT_NAME_KEY, - OBJECT_NAME_TYPE, OBJECTS_KEY, OBJECTS_TYPE, UPLOAD_FILE_NAME_KEY, - UPLOAD_FILE_NAME_TYPE) +from src.system_var.constants import (COLUMNS_KEY, COLUMNS_TYPE, + DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ, + DATE_PATTERN_YYYYMMDDTHHMMSSTZ, + DATETIME_COLUMN_DEFAULT_VALUE, + DATETIME_COLUMN_KEY, + DATETIME_COLUMN_TYPE, IS_SKIP_KEY, + IS_SKIP_TYPE, + IS_UPDATE_LAST_FETCH_DATETIME_KEY, + IS_UPDATE_LAST_FETCH_DATETIME_TYPE, + LAST_FETCH_DATETIME_FILE_NAME_KEY, + LAST_FETCH_DATETIME_FILE_NAME_TYPE, + LAST_FETCH_DATETIME_FROM_KEY, + LAST_FETCH_DATETIME_FROM_TYPE, + LAST_FETCH_DATETIME_TO_KEY, + LAST_FETCH_DATETIME_TO_TYPE, + OBJECT_NAME_KEY, OBJECT_NAME_TYPE, + OBJECTS_KEY, OBJECTS_TYPE, + UPLOAD_FILE_NAME_KEY, + UPLOAD_FILE_NAME_TYPE) from src.util.dict_checker import DictChecker from src.util.execute_datetime import ExecuteDateTime @@ -119,10 +128,12 @@ class LastFetchDatetime(): def __validate(self) -> None: self.__dict_checker.assert_key_exist(LAST_FETCH_DATETIME_FROM_KEY) self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_FROM_KEY, LAST_FETCH_DATETIME_FROM_TYPE) - self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ,DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ) + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ, + DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_TO_KEY, LAST_FETCH_DATETIME_TO_TYPE) - self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ,DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ) + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ, + DATE_PATTERN_EXPECTED_YYYYMMDDTHHMMSSTZ) return @property diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index 97c1f10e..7cacd906 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -71,7 +71,7 @@ def _fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: Args: fetch_target_objects (FetchTargetObjects): CRMオブジェクト情報インスタンス - execute_datetime (ExecuteDateTime): 実行日次取得インスタンス + execute_datetime (ExecuteDateTime): 実行日時取得インスタンス process_result (dict): 取得処理実行結果辞書オブジェクト Returns: @@ -80,10 +80,12 @@ def _fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: for object_info in fetch_target_objects: try: + # 1．処理結果出力用のマップの初期化 process_result[object_info.get(OBJECT_NAME_KEY)] = 'fail' _fetch_crm_data_per_object(object_info, execute_datetime) + # 16．処理結果出力用のマップの更新 process_result[object_info.get(OBJECT_NAME_KEY)] = 'success' except MeDaCaCRMDataFetchException as e: @@ -105,35 +107,35 @@ def _fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateT Args: object_info (dict): 取得対象オブジェクト情報 - execute_datetime (ExecuteDateTime): 実行日次取得インスタンス + execute_datetime (ExecuteDateTime): 実行日時取得インスタンス """ - # 1. オブジェクト処理結果の初期化 + # 2. 対象オブジェクト情報の内容をログに出力する logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：[{object_info}]') - # 2. オブジェクト情報形式チェック処理を呼び出す + # 3. オブジェクト情報形式チェック処理を呼び出す logger.info('I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') target_object = check_object_info_process(object_info, execute_datetime) target_object_name = target_object.object_name - # 3. 処理対象のオブジェクト名をログ出力する + # 4. 処理対象のオブジェクト名をログ出力する logger.info( f'I-CTRL-06 [{target_object_name}]のデータ取得を開始します') - # 4. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する + # 5. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する if target_object.is_skip is True: logger.info( f'I-CTRL-07 [{target_object_name}]のデータ取得処理をスキップします') return - # 5. データ取得期間設定処理を呼び出す + # 6. データ取得期間設定処理を呼び出す logger.info( f'I-CTRL-08 [{target_object_name}]のデータ取得期間設定処理呼び出し') last_fetch_datetime = set_datetime_period_process(target_object, execute_datetime) - # 6. CRMデータ取得処理を呼び出す + # 7. CRMデータ取得処理を呼び出す logger.info( f'I-CTRL-09 [{target_object_name}]のデータ取得処理呼び出し') @@ -145,41 +147,41 @@ def _fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateT f'I-CTRL-22 [{target_object_name}]のレコード件数が0件のため、ファイルアップロードをスキップします') return - # 7. 出力ファイル名をログ出力する + # 8. 出力ファイル名をログ出力する logger.info( f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{target_object.upload_file_name}] となります') - # 8. CRM電文データバックアップ処理を呼び出す + # 9. CRM電文データバックアップ処理を呼び出す logger.info( f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し') backup_crm_data_process(target_object, crm_data_response, execute_datetime) - # 9. CSV変換処理を呼び出す + # 10. CSV変換処理を呼び出す logger.info( f'I-CTRL-12 [{target_object.object_name}] CSV変換処理呼び出し') csv_string = convert_crm_csv_data_process(target_object, crm_data_response) - # 10. CSVバックアップ処理を呼び出す + # 11. CSVバックアップ処理を呼び出す logger.info( f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し') backup_crm_csv_data_process(target_object, execute_datetime, csv_string) - # 11. CSVアップロード処理を呼び出す + # 12. CSVアップロード処理を呼び出す logger.info( f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し') copy_crm_csv_data_process(target_object, execute_datetime) - # 12. メモリ解放 + # 13. メモリ解放 del crm_data_response del csv_string gc.collect() - # 13. 前回取得日時ファイル更新処理を呼びだす + # 14. 前回取得日時ファイル更新処理を呼びだす logger.info( f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し') upload_last_fetch_datetime_process(target_object, last_fetch_datetime) - # 14. オブジェクトのアップロードが完了した旨をログに出力する + # 15. 1オブジェクトのアップロードが完了した旨をログに出力する logger.info(f'I-CTRL-16 [{target_object_name}] 処理正常終了') return diff --git a/ecs/crm-datafetch/src/copy_crm_csv_data_process.py b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py index ba2831ea..c8dd08c0 100644 --- a/ecs/crm-datafetch/src/copy_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py @@ -12,7 +12,7 @@ def copy_crm_csv_data_process(target_object: TargetObject, execute_datetime: Exe Args: target_object (TargetObject): 取得対象オブジェクト情報インスタンス - execute_datetime (ExecuteDateTime): 実行日次取得インスタンス + execute_datetime (ExecuteDateTime): 実行日時取得インスタンス Raises: FileUploadException: S3のファイルアップロード失敗 diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index f0ea9ab6..d546c7b8 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -19,7 +19,7 @@ def prepare_data_fetch_process(): Returns: fetch_target_objects : CRMオブジェクト情報インスタンス - execute_datetime : 実行日次取得インスタンス + execute_datetime : 実行日時取得インスタンス process_result : 取得処理実行結果辞書オブジェクト """ diff --git a/ecs/crm-datafetch/src/set_datetime_period_process.py b/ecs/crm-datafetch/src/set_datetime_period_process.py index 062fac97..144c0888 100644 --- a/ecs/crm-datafetch/src/set_datetime_period_process.py +++ b/ecs/crm-datafetch/src/set_datetime_period_process.py @@ -14,7 +14,7 @@ def set_datetime_period_process(target_object: TargetObject, execute_datetime: E Args: target_object (TargetObject): 取得対象オブジェクト情報インスタンス - execute_datetime (ExecuteDateTime): 実行日次取得インスタンス + execute_datetime (ExecuteDateTime): 実行日時取得インスタンス Raises: FileNotFoundException: S3上のファイルが存在しない場合 diff --git a/ecs/crm-datafetch/tests/test_controller.py b/ecs/crm-datafetch/tests/test_controller.py index c142fcfd..b01e7a31 100644 --- a/ecs/crm-datafetch/tests/test_controller.py +++ b/ecs/crm-datafetch/tests/test_controller.py @@ -162,7 +162,7 @@ class TestController: assert self.mock_convert_crm_csv_data_process.call_count == 3, 'CSV変換処理が複数回実行されること' assert self.mock_backup_crm_csv_data_process.call_count == 3, 'CSVバックアップ処理が複数回実行されること' assert self.mock_copy_crm_csv_data_process.call_count == 3, 'CSVアップロード処理が複数回実行されること' - assert self.mock_upload_last_fetch_datetime_process.call_count == 3, '前回取得日次ファイル更新処理が複数回実行されること' + assert self.mock_upload_last_fetch_datetime_process.call_count == 3, '前回取得日時ファイル更新処理が複数回実行されること' assert self.mock_upload_result_data_process.called is True, '取得処理実施結果アップロード処理が1回のみ実行されること' assert self.mock_upload_result_data_process.call_count == 1, '取得処理実施結果アップロード処理が1回のみ実行されること' diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index aa20d187..a5f8da67 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -239,7 +239,8 @@ "MSJ_Evaluation_Comment__c", "Position_vod__c", "Talk_Title_vod__c", - "MSJ_Attendee_Reaction__c" + "MSJ_Attendee_Reaction__c", + "MSJ_Registration__c" ], "is_skip": false, "is_update_last_fetch_datetime": true From f106fef8597b55d4d94c3ae5fe0a5f3715eb7139 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 10:23:59 +0900 Subject: [PATCH 212/275] =?UTF-8?q?feat:=20=E3=83=AD=E3=82=B0=E3=81=AE?= =?UTF-8?q?=E7=A2=BA=E8=AA=8D=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 165 +++++++++++++++++-- 1 file changed, 150 insertions(+), 15 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index b680078d..c1094087 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -1,9 +1,13 @@ import json import os import os.path as path +from datetime import datetime, timezone import pytest from src.controller import controller +from src.parser.json_parser import JsonParser +from src.system_var.constants import YYYYMMDDTHHMMSSTZ +from src.util.execute_datetime import ExecuteDateTime ROOT_DIR = path.abspath(path.dirname(__name__)) DATA_BUCKET = 'mbj-newdwh2021-staging-data' @@ -29,38 +33,148 @@ def s3_test(s3_client): def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # Arrange # バケットにファイルをアップロードしていく + object_info_files = [] object_info_list = get_object_config_list('object_info') for object_info in object_info_list: json_file = read_json(object_info) json_file = to_upload_json(json_file) upload_json(json_file, s3_client, CONFIG_BUCKET, f'{OBJECT_INFO_FOLDER}/{path.basename(object_info)}') + object_info_files.append(json_file) last_fetch_datetime_list = get_object_config_list('last_fetch_datetime') for last_fetch_datetime in last_fetch_datetime_list: json_file = read_json(last_fetch_datetime) upload_json(json_file, s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{path.basename(last_fetch_datetime)}') # 環境変数を設定(CRMの認証情報は別途設定しておくこと) - monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET) - monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', TARGET_FOLDER) - monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', CONFIG_BUCKET) - monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', OBJECT_INFO_FOLDER) + set_environment(monkeypatch) + # 差分取得用に環境変数を設定 monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'crm_object_list_diff.json') - monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', LAST_FETCH_DATETIME_INFO_FOLDER) - monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', BACKUP_BUCKET) - monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', 'process_result') - monkeypatch.setattr('src.copy_crm_csv_data_process.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) - monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) - monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) - monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) + # 一気通貫テスト用に件数の制限をかける monkeypatch.setattr('src.salesforce.soql_builder.FETCH_LIMIT_CLAUSE', ' LIMIT 10') # 件数取得はログ出力用なので、0件が返るようにする - monkeypatch.setattr('src.fetch_crm_data_process.fetch_record_count_retry', lambda x, y, z: 0) + monkeypatch.setattr('src.fetch_crm_data_process.fetch_record_count_retry', lambda x, y, z: 10) + # 実行日時を固定する + now = datetime.now(timezone.utc).strftime(YYYYMMDDTHHMMSSTZ) + format_now = datetime.strptime(now, YYYYMMDDTHHMMSSTZ).strftime('%Y%m%d%H%M%S') + + class MockExecuteDateTime(ExecuteDateTime): + def __init__(self): + super().__init__() + self._ExecuteDateTime__execute_datetime = now + + monkeypatch.setattr('src.prepare_data_fetch_process.ExecuteDateTime', MockExecuteDateTime) # Act controller() - print(caplog.messages) - assert 0 # Assertion + log_messages = caplog.messages + # ループ前のログ確認 + assert 'I-CTRL-01 CRMデータ取得処理を開始します' in log_messages + assert 'I-CTRL-02 データ取得準備処理呼び出し' in log_messages + assert_prepare_process_log(log_messages, now) + assert 'I-CTRL-03 取得対象オブジェクトのループ処理開始' in log_messages + # オブジェクト情報を取得する(diff) + object_info_list = object_info_files[0] + + for object_info in object_info_list['objects']: + target_object_name = object_info['object_name'] + upload_file_name = f'CRM_{target_object_name}_{format_now}' + assert 'I-CTRL-05 オブジェクト情報形式チェック処理呼び出し' in log_messages + assert_check_process_log(log_messages) + assert f'I-CTRL-06 [{target_object_name}]のデータ取得を開始します' in log_messages + assert f'I-CTRL-08 [{target_object_name}]のデータ取得期間設定処理呼び出し' in log_messages + assert_period_process_log(log_messages, target_object_name, target_object_name, now) + assert f'I-CTRL-09 [{target_object_name}]のデータ取得処理呼び出し' in log_messages + assert_fetch_process_log(log_messages, target_object_name) + assert f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{upload_file_name}] となります' in log_messages + assert f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し' in log_messages + assert_backup_response_process_log(log_messages, target_object_name) + assert f'I-CTRL-12 [{target_object_name}] CSV変換処理呼び出し' in log_messages + assert_convert_process_log(log_messages, target_object_name) + assert f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し' in log_messages + assert_backup_csv_process_log(log_messages, target_object_name, upload_file_name) + assert f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し' in log_messages + assert_upload_csv_process_log(log_messages, target_object_name, upload_file_name) + assert f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し' in log_messages + assert_upload_fetch_datetime_process_log(log_messages, target_object_name, False) + assert f'I-CTRL-16 [{target_object_name}] 処理正常終了' in log_messages + + # ループ終了後のログの確認 + process_result_json = {obj['object_name']: 'success' for obj in object_info_list['objects']} + assert f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result_json}]' in log_messages + assert f'I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始' in log_messages + assert f'I-CTRL-19 すべてのデータの取得に成功しました' in log_messages + assert f'I-CTRL-20 CRMデータ取得処理を終了します' in log_messages + + assert 0 + + +""" +以下、アサーション関数 +""" + + +def assert_prepare_process_log(log_messages, now): + assert 'I-PRE-01 データ取得準備処理を開始します' in log_messages + assert f'I-PRE-02 データ取得処理開始日時：{now}' in log_messages + assert 'I-PRE-09 データ取得準備処理を終了します' in log_messages + + +def assert_check_process_log(log_messages): + assert 'I-CHK-01 オブジェクト情報形式チェック処理を開始します' in log_messages + assert 'I-CHK-02 オブジェクト情報形式チェック処理を終了します' in log_messages + + +def assert_period_process_log(log_messages, target_object_name, datetime_file_name, now): + assert f'I-DATE-01 [{target_object_name}] のデータ取得期間設定処理を開始します' in log_messages + assert f'I-DATE-02 前回取得日時ファイルの取得開始します ファイルパス：[s3://{CONFIG_BUCKET}/{LAST_FETCH_DATETIME_INFO_FOLDER}/{datetime_file_name}.json]' in log_messages + assert 'I-DATE-03 前回取得日時ファイルの取得成功しました' in log_messages + assert f'I-DATE-06 取得範囲 From: [1900-01-01T00:00:00.000Z] To: [{now}]' in log_messages + assert f'I-DATE-07 [{target_object_name}] のデータ取得期間設定処理を終了します' in log_messages + + +def assert_fetch_process_log(log_messages, target_object_name): + assert f'I-FETCH-01 [{target_object_name}] のCRMからのデータ取得処理を開始します' in log_messages + assert f'I-FETCH-02 [{target_object_name}] の件数取得を開始します' in log_messages + assert f'I-FETCH-03 [{target_object_name}] の件数：[10]' in log_messages + assert f'I-FETCH-04 [{target_object_name}] のレコード取得を開始します' in log_messages + assert f'I-FETCH-05 [{target_object_name}] のレコード取得が成功しました' in log_messages + assert f'I-FETCH-06 [{target_object_name}] のCRMからのデータ取得処理を終了します' in log_messages + + +def assert_backup_response_process_log(log_messages, target_object_name): + assert f'I-RESBK-01 [{target_object_name}] のCRM電文データバックアップ処理を開始します' in log_messages + assert f'I-RESBK-03 [{target_object_name}] のCRM電文データバックアップ処理を終了します' in log_messages + + +def assert_convert_process_log(log_messages, target_object_name): + assert f'I-CONV-01 [{target_object_name}] のCSV変換処理を開始します' in log_messages + assert f'I-CONV-03 [{target_object_name}] のCSV変換処理を終了します' in log_messages + + +def assert_backup_csv_process_log(log_messages, target_object_name, upload_file_name): + assert f'I-CSVBK-01 [{target_object_name}] のCSVデータのバックアップ処理を開始します ファイル名：[{upload_file_name}.csv]' in log_messages + assert f'I-CSVBK-03 [{target_object_name}] のCSVデータのバックアップ処理を終了します' in log_messages + + +def assert_upload_csv_process_log(log_messages, target_object_name, upload_file_name): + assert f'I-UPLD-01 [{target_object_name}] のCSVデータアップロード処理を開始します ファイル名：[{upload_file_name}.csv]' in log_messages + assert f'I-UPLD-03 [{target_object_name}] のCSVデータのアップロード処理を終了します' in log_messages + + +def assert_upload_fetch_datetime_process_log(log_messages, target_object_name, is_all=False): + assert f'I-UPD-01 [{target_object_name}] の前回取得日時ファイルの更新処理を開始します' in log_messages + if is_all: + assert f'I-UPD-02 [{target_object_name}] の前回取得日時ファイルの更新処理をスキップします' in log_messages + assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' not in log_messages + else: + assert f'I-UPD-02 [{target_object_name}] の前回取得日時ファイルの更新処理をスキップします' not in log_messages + assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' in log_messages + + +""" +以下、取得準備関数 +""" def get_object_config_list(folder_name: str): @@ -71,7 +185,9 @@ def get_object_config_list(folder_name: str): def read_json(json_path): with open(json_path, 'r', encoding='utf8') as f: - json_file = json.load(f) + json_str = f.read() + JsonParser + json_file = JsonParser(json_str).parse() return json_file @@ -82,6 +198,9 @@ def upload_json(json_file, s3_client, bucket, folder): def to_upload_json(json_file): + """Userオブジェクトの取得できないプロパティを取り除く + TODO: Userオブジェクトの恒久対応が確定したらこのメソッドは消す + """ for object_info in json_file['objects']: if object_info['object_name'] != 'User': continue @@ -91,3 +210,19 @@ def to_upload_json(json_file): columns.remove('UserPreferencesNativeEmailClient') return json_file + + +def set_environment(monkeypatch): + # 環境変数を設定(CRMの認証情報は別途設定しておくこと) + monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', TARGET_FOLDER) + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', CONFIG_BUCKET) + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', OBJECT_INFO_FOLDER) + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', LAST_FETCH_DATETIME_INFO_FOLDER) + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', BACKUP_BUCKET) + monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', 'process_result') + monkeypatch.setattr('src.copy_crm_csv_data_process.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.set_datetime_period_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) From 5bbcb9d7765126bcd73b2e6a963b3cc8a14e9a3a Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 10:52:13 +0900 Subject: [PATCH 213/275] =?UTF-8?q?feat:=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E3=81=AE=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF=E3=81=A8?= =?UTF-8?q?=E5=85=A8=E4=BB=B6=E7=94=A8=E5=8F=96=E5=BE=97=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=A4=E3=83=AB=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 53 +++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index c1094087..ce264721 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -19,6 +19,7 @@ OBJECT_INFO_FOLDER = 'crm/object_info' LAST_FETCH_DATETIME_INFO_FOLDER = 'crm/last_fetch_datetime' BACKUP_DATA_IMPORT_FOLDER = 'data_import' BACKUP_RESPONSE_JSON_FOLDER = 'response_json' +PROCESS_RESULT_JSON_FILE_NAME = 'process_result.json' @pytest.fixture @@ -56,6 +57,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # 実行日時を固定する now = datetime.now(timezone.utc).strftime(YYYYMMDDTHHMMSSTZ) format_now = datetime.strptime(now, YYYYMMDDTHHMMSSTZ).strftime('%Y%m%d%H%M%S') + path_now = datetime.strptime(now, YYYYMMDDTHHMMSSTZ).strftime('%Y/%m/%d/%H%M%S') class MockExecuteDateTime(ExecuteDateTime): def __init__(self): @@ -89,24 +91,64 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): assert f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{upload_file_name}] となります' in log_messages assert f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し' in log_messages assert_backup_response_process_log(log_messages, target_object_name) + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_RESPONSE_JSON_FOLDER}/{path_now}/{upload_file_name}.json') assert f'I-CTRL-12 [{target_object_name}] CSV変換処理呼び出し' in log_messages assert_convert_process_log(log_messages, target_object_name) assert f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し' in log_messages assert_backup_csv_process_log(log_messages, target_object_name, upload_file_name) + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_DATA_IMPORT_FOLDER}/{path_now}/{upload_file_name}.csv') assert f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し' in log_messages assert_upload_csv_process_log(log_messages, target_object_name, upload_file_name) + assert_file_exist(s3_client, DATA_BUCKET, f'{TARGET_FOLDER}/{upload_file_name}.csv') assert f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し' in log_messages assert_upload_fetch_datetime_process_log(log_messages, target_object_name, False) + assert_file_exist(s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{target_object_name}.json') assert f'I-CTRL-16 [{target_object_name}] 処理正常終了' in log_messages # ループ終了後のログの確認 process_result_json = {obj['object_name']: 'success' for obj in object_info_list['objects']} assert f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result_json}]' in log_messages assert f'I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始' in log_messages + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_DATA_IMPORT_FOLDER}/{path_now}/{PROCESS_RESULT_JSON_FILE_NAME}') assert f'I-CTRL-19 すべてのデータの取得に成功しました' in log_messages assert f'I-CTRL-20 CRMデータ取得処理を終了します' in log_messages - assert 0 + # 全件要取得処理を実行する + + # 全件取得用に環境変数を設定 + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'crm_object_list_all.json') + controller() + # ログ再取得 + log_messages_all = caplog.messages + object_info_list_all = object_info_files[1] + # 開始ログなどはテスト済みなのでチェックを省く + for object_info in object_info_list_all['objects']: + target_object_name = object_info['object_name'] + upload_file_name = f'CRM_{target_object_name}_ALL_{format_now}' + datetime_filename = f'{target_object_name}_ALL' + assert 'I-CTRL-05 オブジェクト情報形式チェック処理呼び出し' in log_messages_all + assert_check_process_log(log_messages_all) + assert f'I-CTRL-06 [{target_object_name}]のデータ取得を開始します' in log_messages_all + assert f'I-CTRL-08 [{target_object_name}]のデータ取得期間設定処理呼び出し' in log_messages_all + assert_period_process_log(log_messages_all, target_object_name, datetime_filename, now) + assert f'I-CTRL-09 [{target_object_name}]のデータ取得処理呼び出し' in log_messages_all + assert_fetch_process_log(log_messages_all, target_object_name) + assert f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{upload_file_name}] となります' in log_messages_all + assert f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し' in log_messages_all + assert_backup_response_process_log(log_messages_all, target_object_name) + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_RESPONSE_JSON_FOLDER}/{path_now}/{upload_file_name}.json') + assert f'I-CTRL-12 [{target_object_name}] CSV変換処理呼び出し' in log_messages_all + assert_convert_process_log(log_messages_all, target_object_name) + assert f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し' in log_messages_all + assert_backup_csv_process_log(log_messages_all, target_object_name, upload_file_name) + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_DATA_IMPORT_FOLDER}/{path_now}/{upload_file_name}.csv') + assert f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し' in log_messages_all + assert_upload_csv_process_log(log_messages_all, target_object_name, upload_file_name) + assert_file_exist(s3_client, DATA_BUCKET, f'{TARGET_FOLDER}/{upload_file_name}.csv') + assert f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し' in log_messages_all + assert_upload_fetch_datetime_process_log(log_messages_all, target_object_name, True) + assert_file_exist(s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{target_object_name}.json') + assert f'I-CTRL-16 [{target_object_name}] 処理正常終了' in log_messages_all """ @@ -172,6 +214,13 @@ def assert_upload_fetch_datetime_process_log(log_messages, target_object_name, i assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' in log_messages +def assert_file_exist(s3_client, bucket_name, file_key): + try: + assert s3_client.head_object(Bucket=bucket_name, Key=file_key) is not None + except Exception: + raise Exception(f'ファイルが存在しません バケット名：{bucket_name}, ファイルパス：{file_key}') + + """ 以下、取得準備関数 """ @@ -220,7 +269,7 @@ def set_environment(monkeypatch): monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', OBJECT_INFO_FOLDER) monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', LAST_FETCH_DATETIME_INFO_FOLDER) monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', BACKUP_BUCKET) - monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', 'process_result') + monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', PROCESS_RESULT_JSON_FILE_NAME) monkeypatch.setattr('src.copy_crm_csv_data_process.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.set_datetime_period_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET) monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) From 0ff005c45097dcb5b4db767ca360a7af694deccf Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 10:56:38 +0900 Subject: [PATCH 214/275] =?UTF-8?q?docs:=20=E3=83=89=E3=82=AD=E3=83=A5?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index ce264721..c815f90f 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -32,6 +32,18 @@ def s3_test(s3_client): @pytest.mark.walk_through def test_walk_through(s3_test, s3_client, monkeypatch, caplog): + """ + Cases: + コントロール処理の頭から最後まで処理が流れきることを確認する + Arranges: + - 実行に必要なモックバケットを用意する + - 実行に必要なファイルをアップロードする + - 実行に必要な環境変数をmonkeypatchで設定する + - CRMデータカウント取得処理をモック化する（ログ出力にのみ使用しているため） + Expects: + - すべてのログが仕様どおり二出力されていること（デバッグログを除く） + - すべてのファイルが使用どおりの場所にアップロードされていること + """ # Arrange # バケットにファイルをアップロードしていく object_info_files = [] From 526cce1d423c37f2fede9ade97c876cfcf589e40 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 11:09:05 +0900 Subject: [PATCH 215/275] =?UTF-8?q?fix:=20=E5=85=A8=E4=BB=B6=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=81=AE=E3=83=AD=E3=82=B0=E6=A4=9C=E6=9F=BB?= =?UTF-8?q?=E3=81=A7=E5=A4=B1=E6=95=97=E3=81=97=E3=81=A6=E3=81=84=E3=81=9F?= =?UTF-8?q?=E3=81=AE=E3=81=A7=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index c815f90f..51071707 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -220,10 +220,8 @@ def assert_upload_fetch_datetime_process_log(log_messages, target_object_name, i assert f'I-UPD-01 [{target_object_name}] の前回取得日時ファイルの更新処理を開始します' in log_messages if is_all: assert f'I-UPD-02 [{target_object_name}] の前回取得日時ファイルの更新処理をスキップします' in log_messages - assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' not in log_messages - else: - assert f'I-UPD-02 [{target_object_name}] の前回取得日時ファイルの更新処理をスキップします' not in log_messages - assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' in log_messages + return + assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' in log_messages def assert_file_exist(s3_client, bucket_name, file_key): From a610b9f2d8bc1aeb064634c7625d1fc18e6cbe52 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 11:09:22 +0900 Subject: [PATCH 216/275] =?UTF-8?q?docs:=20README=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Pipfile | 4 ++-- ecs/crm-datafetch/README.md | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 6912e1a0..da57afc8 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -6,8 +6,8 @@ name = "pypi" [scripts] test = "pytest tests/" "test:cov" = "pytest --cov=src --cov-branch --cov-report=term-missing tests/" -"test:report" = "pytest --cov=src --cov-branch --cov-report=term-missing --html=.report/test_result.html tests/" -"test:walk-through" = "pytest tests/ --walk-through" +"test:report" = "pytest --cov=src --cov-branch --cov-report=term-missing --html=.report/unit_test/test_result.html tests/" +"test:walk-through" = "pytest tests/test_walk_through.py --walk-through --cov-report=term-missing --html=.report/walk_through/test_result.html" [packages] boto3 = "*" diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md index f2928706..4e0072cf 100644 --- a/ecs/crm-datafetch/README.md +++ b/ecs/crm-datafetch/README.md @@ -146,6 +146,19 @@ pipenv install --dev pipenv run test:cov ``` +#### 一気通貫テストを実行する場合の設定 + +- `.env`の以下に示す環境変数の値をメルク様提供のFullSandboxのものに書き換える + - CRM_AUTH_DOMAIN + - CRM_USER_NAME + - CRM_USER_PASSWORD + - CRM_USER_SECURITY_TOKEN +- 以下のコマンドを実行して単体テストを起動する + +```sh +pipenv run test:walk-through +``` + #### 拡張機能：Python Test Explorer UIを導入する場合 - VSCodeの拡張機能メニューから、「Python Test Explorer for Visual Studio Code」をインストール @@ -163,6 +176,9 @@ pipenv run test:cov - `pipenv run test:report` - pytestのテスト終了時にテスト結果をHTMLで出力する - `.report/test_result.html`が出力される +- `test:walk-through` + - 一気通貫テストが実行される + - 上記のテスト意外はスキップされる ## 単体テストの追加方法 From 3385495b6fa6fed9da2b8c93b448b7649069cf1c Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 11:59:15 +0900 Subject: [PATCH 217/275] =?UTF-8?q?feat:=20=E3=82=8F=E3=81=8B=E3=82=8A?= =?UTF-8?q?=E3=82=84=E3=81=99=E3=81=84=E3=82=88=E3=81=86=E3=81=AB=E3=83=AD?= =?UTF-8?q?=E3=82=B0=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/pytest.ini | 3 +++ ecs/crm-datafetch/tests/test_walk_through.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 ecs/crm-datafetch/pytest.ini diff --git a/ecs/crm-datafetch/pytest.ini b/ecs/crm-datafetch/pytest.ini new file mode 100644 index 00000000..5dbe2661 --- /dev/null +++ b/ecs/crm-datafetch/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +log_format = %(levelname)s %(asctime)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index 51071707..42da7fa4 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -1,4 +1,5 @@ import json +import logging import os import os.path as path from datetime import datetime, timezone @@ -79,7 +80,15 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): monkeypatch.setattr('src.prepare_data_fetch_process.ExecuteDateTime', MockExecuteDateTime) # Act + logger = logging.getLogger() + logger.setLevel(logging.INFO) + logger.info(f'##########################') + logger.info(f'# 差分データ取得処理:実行開始 #') + logger.info(f'##########################') controller() + logger.info(f'##########################') + logger.info(f'# 差分データ取得処理:実行終了 #') + logger.info(f'##########################') # Assertion log_messages = caplog.messages # ループ前のログ確認 @@ -129,7 +138,13 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # 全件取得用に環境変数を設定 monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'crm_object_list_all.json') + logger.info(f'##########################') + logger.info(f'# 全件データ取得処理:実行開始 #') + logger.info(f'##########################') controller() + logger.info(f'##########################') + logger.info(f'# 全件データ取得処理:実行終了 #') + logger.info(f'##########################') # ログ再取得 log_messages_all = caplog.messages object_info_list_all = object_info_files[1] From 40aca22a33c1f2af7a4ae53a29e6057f16cf645c Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:00:26 +0900 Subject: [PATCH 218/275] =?UTF-8?q?feat:=20=E4=B8=80=E6=B0=97=E9=80=9A?= =?UTF-8?q?=E8=B2=AB=E3=83=86=E3=82=B9=E3=83=88=E3=81=A0=E3=81=91=E3=82=92?= =?UTF-8?q?=E5=AE=9F=E8=A1=8C=E3=81=99=E3=82=8B=E3=81=9F=E3=82=81=E3=81=AE?= =?UTF-8?q?=E3=83=9E=E3=83=BC=E3=82=AB=E3=83=BC=E3=81=A8=E3=82=B3=E3=83=9E?= =?UTF-8?q?=E3=83=B3=E3=83=89=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Pipfile | 1 + ecs/crm-datafetch/tests/conftest.py | 20 ++++++++++++++++++++ ecs/crm-datafetch/tests/test_walkthrough.py | 7 +++++++ 3 files changed, 28 insertions(+) create mode 100644 ecs/crm-datafetch/tests/test_walkthrough.py diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 82796598..6912e1a0 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -7,6 +7,7 @@ name = "pypi" test = "pytest tests/" "test:cov" = "pytest --cov=src --cov-branch --cov-report=term-missing tests/" "test:report" = "pytest --cov=src --cov-branch --cov-report=term-missing --html=.report/test_result.html tests/" +"test:walk-through" = "pytest tests/ --walk-through" [packages] boto3 = "*" diff --git a/ecs/crm-datafetch/tests/conftest.py b/ecs/crm-datafetch/tests/conftest.py index 55ade88c..b86b29d9 100644 --- a/ecs/crm-datafetch/tests/conftest.py +++ b/ecs/crm-datafetch/tests/conftest.py @@ -57,3 +57,23 @@ def pytest_runtest_makereport(item, call): report.cases = docstring.get("Cases", '') # 「テスト内容」を`report`に追加 report.arranges = docstring.get("Arranges", '') # 「準備作業」を`report`に追加 report.expects = docstring.get("Expects", '') # 「期待結果」を`report`に追加 + + +def pytest_addoption(parser): + parser.addoption( + "--walk-through", action="store_true", default=False, help="run walk through tests" + ) + + +def pytest_configure(config): + config.addinivalue_line("markers", "slow: mark test as slow to run") + + +def pytest_collection_modifyitems(config, items): + skip_slow = pytest.mark.skip(reason="need --walk-through option to run") + if config.getoption("--walk-through"): + [item.add_marker(skip_slow) for item in items if "walk_through" not in item.keywords] + return + for item in items: + if "walk_through" in item.keywords: + item.add_marker(skip_slow) diff --git a/ecs/crm-datafetch/tests/test_walkthrough.py b/ecs/crm-datafetch/tests/test_walkthrough.py new file mode 100644 index 00000000..2a3b09f2 --- /dev/null +++ b/ecs/crm-datafetch/tests/test_walkthrough.py @@ -0,0 +1,7 @@ +import pytest + + +@pytest.mark.walk_through +def test_walk_through(): + assert 0 + pass From e7aea6c818a2afafc2d0fbe7656fef5ed7ca5d63 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:00:26 +0900 Subject: [PATCH 219/275] =?UTF-8?q?feat:=20VSCode=E3=81=A7=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=81=99=E3=82=8B=E3=81=A8=E3=81=8D=E3=82=82?= =?UTF-8?q?=E5=8F=8E=E9=9B=86=E5=AF=BE=E8=B1=A1=E3=81=8B=E3=82=89=E9=99=A4?= =?UTF-8?q?=E3=81=84=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.vscode/recommend_settings.json | 2 +- .../tests/{test_walkthrough.py => test_walk_through.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename ecs/crm-datafetch/tests/{test_walkthrough.py => test_walk_through.py} (100%) diff --git a/ecs/crm-datafetch/.vscode/recommend_settings.json b/ecs/crm-datafetch/.vscode/recommend_settings.json index d4822731..4fd1fddf 100644 --- a/ecs/crm-datafetch/.vscode/recommend_settings.json +++ b/ecs/crm-datafetch/.vscode/recommend_settings.json @@ -13,7 +13,7 @@ "python.linting.flake8Args": ["--max-line-length=150", "--ignore=F541"], "python.formatting.provider": "autopep8", "python.formatting.autopep8Args": ["--max-line-length", "150"], - "python.testing.pytestArgs": ["tests"], + "python.testing.pytestArgs": ["tests", "--ignore=tests/test_walk_through.py"], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true } diff --git a/ecs/crm-datafetch/tests/test_walkthrough.py b/ecs/crm-datafetch/tests/test_walk_through.py similarity index 100% rename from ecs/crm-datafetch/tests/test_walkthrough.py rename to ecs/crm-datafetch/tests/test_walk_through.py From 75274f60fa361f206d4b450468b30c0f53a3ade3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:00:26 +0900 Subject: [PATCH 220/275] =?UTF-8?q?feat:=20=E3=81=A8=E3=82=8A=E3=81=82?= =?UTF-8?q?=E3=81=88=E3=81=9A=E6=B5=81=E3=82=8C=E3=81=8D=E3=82=8B=E3=81=A8?= =?UTF-8?q?=E3=81=93=E3=82=8D=E3=81=BE=E3=81=A7=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/salesforce/soql_builder.py | 8 +- .../src/system_var/environments.py | 22 ++--- ecs/crm-datafetch/tests/test_walk_through.py | 89 ++++++++++++++++++- 3 files changed, 105 insertions(+), 14 deletions(-) diff --git a/ecs/crm-datafetch/src/salesforce/soql_builder.py b/ecs/crm-datafetch/src/salesforce/soql_builder.py index 42d4ecea..120b5bf2 100644 --- a/ecs/crm-datafetch/src/salesforce/soql_builder.py +++ b/ecs/crm-datafetch/src/salesforce/soql_builder.py @@ -1,6 +1,7 @@ import textwrap from src.config.objects import LastFetchDatetime, TargetObject +from src.system_var.environments import FETCH_LIMIT_CLAUSE class SOQLBuilder: @@ -9,6 +10,7 @@ class SOQLBuilder: SELECT {column_or_expression} FROM {object_name} WHERE {datetime_column} > {last_fetch_datetime_from} AND {datetime_column} <= {last_fetch_datetime_to} + {limit_clause} """) self.__target_object = target_object self.__last_fetch_datetime = last_fetch_datetime @@ -19,7 +21,8 @@ class SOQLBuilder: object_name=self.__target_object.object_name, last_fetch_datetime_from=self.__last_fetch_datetime.last_fetch_datetime_from, last_fetch_datetime_to=self.__last_fetch_datetime.last_fetch_datetime_to, - datetime_column=self.__target_object.datetime_column + datetime_column=self.__target_object.datetime_column, + limit_clause='' ) return count_soql @@ -31,7 +34,8 @@ class SOQLBuilder: object_name=self.__target_object.object_name, last_fetch_datetime_from=self.__last_fetch_datetime.last_fetch_datetime_from, last_fetch_datetime_to=self.__last_fetch_datetime.last_fetch_datetime_to, - datetime_column=self.__target_object.datetime_column + datetime_column=self.__target_object.datetime_column, + limit_clause=FETCH_LIMIT_CLAUSE ) return fetch_soql diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index b290cc7b..9ab6911d 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -7,14 +7,14 @@ import src.system_var.constants as constants LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # CRMへの認証処理のタイムアウト秒数 CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)) -# CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3)) -# CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5)) -# CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5)) -# CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50)) +# # CRMへの認証処理の最大リトライ試行回数 +# CRM_AUTH_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3)) +# # CRMへの認証処理のリトライ時の初回待ち秒数 +# CRM_AUTH_RETRY_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5)) +# # CRMへの認証処理のリトライ時の最小待ち秒数 +# CRM_AUTH_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5)) +# # CRMへの認証処理のリトライ時の最大待ち秒数 +# CRM_AUTH_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50)) # CRMのレコード件数取得処理のタイムアウト秒数 CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)) # CRMのレコード件数取得処理の最大リトライ試行回数 @@ -65,9 +65,11 @@ PROCESS_RESULT_FILENAME = os.environ.get(constants.PROCESS_RESULT_FILENAME, 'pro LAST_FETCH_DATE_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_FOLDER, 'crm/last_fetch_datetime') # CRMから取得し、取込用に変換したデータを格納するフォルダ CRM_IMPORT_DATA_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_FOLDER, 'crm/target') -# CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_BACKUP_FOLDER, 'last_fetch_datetime') +# # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +# LAST_FETCH_DATE_BACKUP_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_BACKUP_FOLDER, 'last_fetch_datetime') # CRMから取得した生データのバックアップを格納するフォルダパス RESPONSE_JSON_BACKUP_FOLDER = os.environ.get(constants.RESPONSE_JSON_BACKUP_FOLDER, 'response_json') # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_BACKUP_FOLDER, 'data_import') +# 一気通貫テスト用、取得件数を絞るためのSOQL句 +FETCH_LIMIT_CLAUSE = os.environ.get('FETCH_LIMIT_CLAUSE', '') diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index 2a3b09f2..cb7206aa 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -1,7 +1,92 @@ +import json +import os +import os.path as path + import pytest +from src.controller import controller + +ROOT_DIR = path.abspath(path.dirname(__name__)) +DATA_BUCKET = 'mbj-newdwh2021-staging-data' +CONFIG_BUCKET = 'mbj-newdwh2021-staging-config' +BACKUP_BUCKET = 'mbj-newdwh2021-staging-backup-crm' + +TARGET_FOLDER = 'crm/target' +OBJECT_INFO_FOLDER = 'crm/object_info' +LAST_FETCH_DATETIME_INFO_FOLDER = 'crm/last_fetch_datetime' +BACKUP_DATA_IMPORT_FOLDER = 'data_import' +BACKUP_RESPONSE_JSON_FOLDER = 'response_json' + + +@pytest.fixture +def s3_test(s3_client): + s3_client.create_bucket(Bucket=DATA_BUCKET) + s3_client.create_bucket(Bucket=CONFIG_BUCKET) + s3_client.create_bucket(Bucket=BACKUP_BUCKET) + yield @pytest.mark.walk_through -def test_walk_through(): +def test_walk_through(s3_test, s3_client, monkeypatch, caplog): + # Arrange + # バケットにファイルをアップロードしていく + object_info_list = get_object_config_list('object_info') + for object_info in object_info_list: + json_file = read_json(object_info) + json_file = to_upload_json(json_file) + upload_json(json_file, s3_client, CONFIG_BUCKET, f'{OBJECT_INFO_FOLDER}/{path.basename(object_info)}') + last_fetch_datetime_list = get_object_config_list('last_fetch_datetime') + for last_fetch_datetime in last_fetch_datetime_list: + json_file = read_json(last_fetch_datetime) + upload_json(json_file, s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{path.basename(last_fetch_datetime)}') + + # 環境変数を設定(CRMの認証情報は別途設定しておくこと) + monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', TARGET_FOLDER) + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', CONFIG_BUCKET) + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', OBJECT_INFO_FOLDER) + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'crm_object_list_diff.json') + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', LAST_FETCH_DATETIME_INFO_FOLDER) + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', BACKUP_BUCKET) + monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', 'process_result') + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) + monkeypatch.setattr('src.salesforce.soql_builder.FETCH_LIMIT_CLAUSE', ' LIMIT 10') + # 件数取得はログ出力用なので、0件が返るようにする + monkeypatch.setattr('src.fetch_crm_data_process.fetch_record_count_retry', lambda x, y, z: 0) + + # Act + controller() + print(caplog.messages) assert 0 - pass + # Assertion + + +def get_object_config_list(folder_name: str): + local_s3_path = path.join(ROOT_DIR, '..', '..', 's3', 'config', 'crm', folder_name) + config_list = [os.path.join(local_s3_path, config) for config in os.listdir(local_s3_path) if config.endswith('.json')] + return config_list + + +def read_json(json_path): + with open(json_path, 'r', encoding='utf8') as f: + json_file = json.load(f) + + return json_file + + +def upload_json(json_file, s3_client, bucket, folder): + json_str = json.dumps(json_file) + s3_client.put_object(Bucket=bucket, Key=folder, Body=json_str) + + +def to_upload_json(json_file): + for object_info in json_file['objects']: + if object_info['object_name'] != 'User': + continue + columns: list = object_info['columns'] + columns.remove('LastPasswordChangeDate') + columns.remove('NumberOfFailedLogins') + columns.remove('UserPreferencesNativeEmailClient') + + return json_file From fa5fbab0691567fbaaac21bbf4fdb5cda5f44f12 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:00:26 +0900 Subject: [PATCH 221/275] =?UTF-8?q?fix:=20=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E7=BD=AE=E3=81=8D=E6=8F=9B=E3=81=88=E3=83=9F=E3=82=B9?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 1 + 1 file changed, 1 insertion(+) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index cb7206aa..b680078d 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -48,6 +48,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', LAST_FETCH_DATETIME_INFO_FOLDER) monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', BACKUP_BUCKET) monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', 'process_result') + monkeypatch.setattr('src.copy_crm_csv_data_process.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) From e1ff423b400f1a3c9e7702e334c65be13ae36027 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:00:26 +0900 Subject: [PATCH 222/275] =?UTF-8?q?feat:=20=E3=83=AD=E3=82=B0=E3=81=AE?= =?UTF-8?q?=E7=A2=BA=E8=AA=8D=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 165 +++++++++++++++++-- 1 file changed, 150 insertions(+), 15 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index b680078d..c1094087 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -1,9 +1,13 @@ import json import os import os.path as path +from datetime import datetime, timezone import pytest from src.controller import controller +from src.parser.json_parser import JsonParser +from src.system_var.constants import YYYYMMDDTHHMMSSTZ +from src.util.execute_datetime import ExecuteDateTime ROOT_DIR = path.abspath(path.dirname(__name__)) DATA_BUCKET = 'mbj-newdwh2021-staging-data' @@ -29,38 +33,148 @@ def s3_test(s3_client): def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # Arrange # バケットにファイルをアップロードしていく + object_info_files = [] object_info_list = get_object_config_list('object_info') for object_info in object_info_list: json_file = read_json(object_info) json_file = to_upload_json(json_file) upload_json(json_file, s3_client, CONFIG_BUCKET, f'{OBJECT_INFO_FOLDER}/{path.basename(object_info)}') + object_info_files.append(json_file) last_fetch_datetime_list = get_object_config_list('last_fetch_datetime') for last_fetch_datetime in last_fetch_datetime_list: json_file = read_json(last_fetch_datetime) upload_json(json_file, s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{path.basename(last_fetch_datetime)}') # 環境変数を設定(CRMの認証情報は別途設定しておくこと) - monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET) - monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', TARGET_FOLDER) - monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', CONFIG_BUCKET) - monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', OBJECT_INFO_FOLDER) + set_environment(monkeypatch) + # 差分取得用に環境変数を設定 monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'crm_object_list_diff.json') - monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', LAST_FETCH_DATETIME_INFO_FOLDER) - monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', BACKUP_BUCKET) - monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', 'process_result') - monkeypatch.setattr('src.copy_crm_csv_data_process.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) - monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) - monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) - monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) + # 一気通貫テスト用に件数の制限をかける monkeypatch.setattr('src.salesforce.soql_builder.FETCH_LIMIT_CLAUSE', ' LIMIT 10') # 件数取得はログ出力用なので、0件が返るようにする - monkeypatch.setattr('src.fetch_crm_data_process.fetch_record_count_retry', lambda x, y, z: 0) + monkeypatch.setattr('src.fetch_crm_data_process.fetch_record_count_retry', lambda x, y, z: 10) + # 実行日時を固定する + now = datetime.now(timezone.utc).strftime(YYYYMMDDTHHMMSSTZ) + format_now = datetime.strptime(now, YYYYMMDDTHHMMSSTZ).strftime('%Y%m%d%H%M%S') + + class MockExecuteDateTime(ExecuteDateTime): + def __init__(self): + super().__init__() + self._ExecuteDateTime__execute_datetime = now + + monkeypatch.setattr('src.prepare_data_fetch_process.ExecuteDateTime', MockExecuteDateTime) # Act controller() - print(caplog.messages) - assert 0 # Assertion + log_messages = caplog.messages + # ループ前のログ確認 + assert 'I-CTRL-01 CRMデータ取得処理を開始します' in log_messages + assert 'I-CTRL-02 データ取得準備処理呼び出し' in log_messages + assert_prepare_process_log(log_messages, now) + assert 'I-CTRL-03 取得対象オブジェクトのループ処理開始' in log_messages + # オブジェクト情報を取得する(diff) + object_info_list = object_info_files[0] + + for object_info in object_info_list['objects']: + target_object_name = object_info['object_name'] + upload_file_name = f'CRM_{target_object_name}_{format_now}' + assert 'I-CTRL-05 オブジェクト情報形式チェック処理呼び出し' in log_messages + assert_check_process_log(log_messages) + assert f'I-CTRL-06 [{target_object_name}]のデータ取得を開始します' in log_messages + assert f'I-CTRL-08 [{target_object_name}]のデータ取得期間設定処理呼び出し' in log_messages + assert_period_process_log(log_messages, target_object_name, target_object_name, now) + assert f'I-CTRL-09 [{target_object_name}]のデータ取得処理呼び出し' in log_messages + assert_fetch_process_log(log_messages, target_object_name) + assert f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{upload_file_name}] となります' in log_messages + assert f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し' in log_messages + assert_backup_response_process_log(log_messages, target_object_name) + assert f'I-CTRL-12 [{target_object_name}] CSV変換処理呼び出し' in log_messages + assert_convert_process_log(log_messages, target_object_name) + assert f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し' in log_messages + assert_backup_csv_process_log(log_messages, target_object_name, upload_file_name) + assert f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し' in log_messages + assert_upload_csv_process_log(log_messages, target_object_name, upload_file_name) + assert f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し' in log_messages + assert_upload_fetch_datetime_process_log(log_messages, target_object_name, False) + assert f'I-CTRL-16 [{target_object_name}] 処理正常終了' in log_messages + + # ループ終了後のログの確認 + process_result_json = {obj['object_name']: 'success' for obj in object_info_list['objects']} + assert f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result_json}]' in log_messages + assert f'I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始' in log_messages + assert f'I-CTRL-19 すべてのデータの取得に成功しました' in log_messages + assert f'I-CTRL-20 CRMデータ取得処理を終了します' in log_messages + + assert 0 + + +""" +以下、アサーション関数 +""" + + +def assert_prepare_process_log(log_messages, now): + assert 'I-PRE-01 データ取得準備処理を開始します' in log_messages + assert f'I-PRE-02 データ取得処理開始日時：{now}' in log_messages + assert 'I-PRE-09 データ取得準備処理を終了します' in log_messages + + +def assert_check_process_log(log_messages): + assert 'I-CHK-01 オブジェクト情報形式チェック処理を開始します' in log_messages + assert 'I-CHK-02 オブジェクト情報形式チェック処理を終了します' in log_messages + + +def assert_period_process_log(log_messages, target_object_name, datetime_file_name, now): + assert f'I-DATE-01 [{target_object_name}] のデータ取得期間設定処理を開始します' in log_messages + assert f'I-DATE-02 前回取得日時ファイルの取得開始します ファイルパス：[s3://{CONFIG_BUCKET}/{LAST_FETCH_DATETIME_INFO_FOLDER}/{datetime_file_name}.json]' in log_messages + assert 'I-DATE-03 前回取得日時ファイルの取得成功しました' in log_messages + assert f'I-DATE-06 取得範囲 From: [1900-01-01T00:00:00.000Z] To: [{now}]' in log_messages + assert f'I-DATE-07 [{target_object_name}] のデータ取得期間設定処理を終了します' in log_messages + + +def assert_fetch_process_log(log_messages, target_object_name): + assert f'I-FETCH-01 [{target_object_name}] のCRMからのデータ取得処理を開始します' in log_messages + assert f'I-FETCH-02 [{target_object_name}] の件数取得を開始します' in log_messages + assert f'I-FETCH-03 [{target_object_name}] の件数：[10]' in log_messages + assert f'I-FETCH-04 [{target_object_name}] のレコード取得を開始します' in log_messages + assert f'I-FETCH-05 [{target_object_name}] のレコード取得が成功しました' in log_messages + assert f'I-FETCH-06 [{target_object_name}] のCRMからのデータ取得処理を終了します' in log_messages + + +def assert_backup_response_process_log(log_messages, target_object_name): + assert f'I-RESBK-01 [{target_object_name}] のCRM電文データバックアップ処理を開始します' in log_messages + assert f'I-RESBK-03 [{target_object_name}] のCRM電文データバックアップ処理を終了します' in log_messages + + +def assert_convert_process_log(log_messages, target_object_name): + assert f'I-CONV-01 [{target_object_name}] のCSV変換処理を開始します' in log_messages + assert f'I-CONV-03 [{target_object_name}] のCSV変換処理を終了します' in log_messages + + +def assert_backup_csv_process_log(log_messages, target_object_name, upload_file_name): + assert f'I-CSVBK-01 [{target_object_name}] のCSVデータのバックアップ処理を開始します ファイル名：[{upload_file_name}.csv]' in log_messages + assert f'I-CSVBK-03 [{target_object_name}] のCSVデータのバックアップ処理を終了します' in log_messages + + +def assert_upload_csv_process_log(log_messages, target_object_name, upload_file_name): + assert f'I-UPLD-01 [{target_object_name}] のCSVデータアップロード処理を開始します ファイル名：[{upload_file_name}.csv]' in log_messages + assert f'I-UPLD-03 [{target_object_name}] のCSVデータのアップロード処理を終了します' in log_messages + + +def assert_upload_fetch_datetime_process_log(log_messages, target_object_name, is_all=False): + assert f'I-UPD-01 [{target_object_name}] の前回取得日時ファイルの更新処理を開始します' in log_messages + if is_all: + assert f'I-UPD-02 [{target_object_name}] の前回取得日時ファイルの更新処理をスキップします' in log_messages + assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' not in log_messages + else: + assert f'I-UPD-02 [{target_object_name}] の前回取得日時ファイルの更新処理をスキップします' not in log_messages + assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' in log_messages + + +""" +以下、取得準備関数 +""" def get_object_config_list(folder_name: str): @@ -71,7 +185,9 @@ def get_object_config_list(folder_name: str): def read_json(json_path): with open(json_path, 'r', encoding='utf8') as f: - json_file = json.load(f) + json_str = f.read() + JsonParser + json_file = JsonParser(json_str).parse() return json_file @@ -82,6 +198,9 @@ def upload_json(json_file, s3_client, bucket, folder): def to_upload_json(json_file): + """Userオブジェクトの取得できないプロパティを取り除く + TODO: Userオブジェクトの恒久対応が確定したらこのメソッドは消す + """ for object_info in json_file['objects']: if object_info['object_name'] != 'User': continue @@ -91,3 +210,19 @@ def to_upload_json(json_file): columns.remove('UserPreferencesNativeEmailClient') return json_file + + +def set_environment(monkeypatch): + # 環境変数を設定(CRMの認証情報は別途設定しておくこと) + monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_FOLDER', TARGET_FOLDER) + monkeypatch.setattr('src.aws.s3.CRM_CONFIG_BUCKET', CONFIG_BUCKET) + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', OBJECT_INFO_FOLDER) + monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', LAST_FETCH_DATETIME_INFO_FOLDER) + monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', BACKUP_BUCKET) + monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', 'process_result') + monkeypatch.setattr('src.copy_crm_csv_data_process.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.set_datetime_period_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET) + monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) + monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) From 50753dde5097f075ca6c6f64170aa648b471e5b6 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:00:26 +0900 Subject: [PATCH 223/275] =?UTF-8?q?feat:=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E3=81=AE=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF=E3=81=A8?= =?UTF-8?q?=E5=85=A8=E4=BB=B6=E7=94=A8=E5=8F=96=E5=BE=97=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=A4=E3=83=AB=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 53 +++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index c1094087..ce264721 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -19,6 +19,7 @@ OBJECT_INFO_FOLDER = 'crm/object_info' LAST_FETCH_DATETIME_INFO_FOLDER = 'crm/last_fetch_datetime' BACKUP_DATA_IMPORT_FOLDER = 'data_import' BACKUP_RESPONSE_JSON_FOLDER = 'response_json' +PROCESS_RESULT_JSON_FILE_NAME = 'process_result.json' @pytest.fixture @@ -56,6 +57,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # 実行日時を固定する now = datetime.now(timezone.utc).strftime(YYYYMMDDTHHMMSSTZ) format_now = datetime.strptime(now, YYYYMMDDTHHMMSSTZ).strftime('%Y%m%d%H%M%S') + path_now = datetime.strptime(now, YYYYMMDDTHHMMSSTZ).strftime('%Y/%m/%d/%H%M%S') class MockExecuteDateTime(ExecuteDateTime): def __init__(self): @@ -89,24 +91,64 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): assert f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{upload_file_name}] となります' in log_messages assert f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し' in log_messages assert_backup_response_process_log(log_messages, target_object_name) + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_RESPONSE_JSON_FOLDER}/{path_now}/{upload_file_name}.json') assert f'I-CTRL-12 [{target_object_name}] CSV変換処理呼び出し' in log_messages assert_convert_process_log(log_messages, target_object_name) assert f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し' in log_messages assert_backup_csv_process_log(log_messages, target_object_name, upload_file_name) + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_DATA_IMPORT_FOLDER}/{path_now}/{upload_file_name}.csv') assert f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し' in log_messages assert_upload_csv_process_log(log_messages, target_object_name, upload_file_name) + assert_file_exist(s3_client, DATA_BUCKET, f'{TARGET_FOLDER}/{upload_file_name}.csv') assert f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し' in log_messages assert_upload_fetch_datetime_process_log(log_messages, target_object_name, False) + assert_file_exist(s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{target_object_name}.json') assert f'I-CTRL-16 [{target_object_name}] 処理正常終了' in log_messages # ループ終了後のログの確認 process_result_json = {obj['object_name']: 'success' for obj in object_info_list['objects']} assert f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result_json}]' in log_messages assert f'I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始' in log_messages + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_DATA_IMPORT_FOLDER}/{path_now}/{PROCESS_RESULT_JSON_FILE_NAME}') assert f'I-CTRL-19 すべてのデータの取得に成功しました' in log_messages assert f'I-CTRL-20 CRMデータ取得処理を終了します' in log_messages - assert 0 + # 全件要取得処理を実行する + + # 全件取得用に環境変数を設定 + monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'crm_object_list_all.json') + controller() + # ログ再取得 + log_messages_all = caplog.messages + object_info_list_all = object_info_files[1] + # 開始ログなどはテスト済みなのでチェックを省く + for object_info in object_info_list_all['objects']: + target_object_name = object_info['object_name'] + upload_file_name = f'CRM_{target_object_name}_ALL_{format_now}' + datetime_filename = f'{target_object_name}_ALL' + assert 'I-CTRL-05 オブジェクト情報形式チェック処理呼び出し' in log_messages_all + assert_check_process_log(log_messages_all) + assert f'I-CTRL-06 [{target_object_name}]のデータ取得を開始します' in log_messages_all + assert f'I-CTRL-08 [{target_object_name}]のデータ取得期間設定処理呼び出し' in log_messages_all + assert_period_process_log(log_messages_all, target_object_name, datetime_filename, now) + assert f'I-CTRL-09 [{target_object_name}]のデータ取得処理呼び出し' in log_messages_all + assert_fetch_process_log(log_messages_all, target_object_name) + assert f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{upload_file_name}] となります' in log_messages_all + assert f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し' in log_messages_all + assert_backup_response_process_log(log_messages_all, target_object_name) + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_RESPONSE_JSON_FOLDER}/{path_now}/{upload_file_name}.json') + assert f'I-CTRL-12 [{target_object_name}] CSV変換処理呼び出し' in log_messages_all + assert_convert_process_log(log_messages_all, target_object_name) + assert f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し' in log_messages_all + assert_backup_csv_process_log(log_messages_all, target_object_name, upload_file_name) + assert_file_exist(s3_client, BACKUP_BUCKET, f'{BACKUP_DATA_IMPORT_FOLDER}/{path_now}/{upload_file_name}.csv') + assert f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し' in log_messages_all + assert_upload_csv_process_log(log_messages_all, target_object_name, upload_file_name) + assert_file_exist(s3_client, DATA_BUCKET, f'{TARGET_FOLDER}/{upload_file_name}.csv') + assert f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し' in log_messages_all + assert_upload_fetch_datetime_process_log(log_messages_all, target_object_name, True) + assert_file_exist(s3_client, CONFIG_BUCKET, f'{LAST_FETCH_DATETIME_INFO_FOLDER}/{target_object_name}.json') + assert f'I-CTRL-16 [{target_object_name}] 処理正常終了' in log_messages_all """ @@ -172,6 +214,13 @@ def assert_upload_fetch_datetime_process_log(log_messages, target_object_name, i assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' in log_messages +def assert_file_exist(s3_client, bucket_name, file_key): + try: + assert s3_client.head_object(Bucket=bucket_name, Key=file_key) is not None + except Exception: + raise Exception(f'ファイルが存在しません バケット名：{bucket_name}, ファイルパス：{file_key}') + + """ 以下、取得準備関数 """ @@ -220,7 +269,7 @@ def set_environment(monkeypatch): monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FOLDER', OBJECT_INFO_FOLDER) monkeypatch.setattr('src.aws.s3.LAST_FETCH_DATE_FOLDER', LAST_FETCH_DATETIME_INFO_FOLDER) monkeypatch.setattr('src.aws.s3.CRM_BACKUP_BUCKET', BACKUP_BUCKET) - monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', 'process_result') + monkeypatch.setattr('src.upload_result_data_process.PROCESS_RESULT_FILENAME', PROCESS_RESULT_JSON_FILE_NAME) monkeypatch.setattr('src.copy_crm_csv_data_process.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.set_datetime_period_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET) monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) From 9fd4392f0e02aba038b4905c90452ddc2be339ab Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:00:26 +0900 Subject: [PATCH 224/275] =?UTF-8?q?docs:=20=E3=83=89=E3=82=AD=E3=83=A5?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index ce264721..c815f90f 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -32,6 +32,18 @@ def s3_test(s3_client): @pytest.mark.walk_through def test_walk_through(s3_test, s3_client, monkeypatch, caplog): + """ + Cases: + コントロール処理の頭から最後まで処理が流れきることを確認する + Arranges: + - 実行に必要なモックバケットを用意する + - 実行に必要なファイルをアップロードする + - 実行に必要な環境変数をmonkeypatchで設定する + - CRMデータカウント取得処理をモック化する（ログ出力にのみ使用しているため） + Expects: + - すべてのログが仕様どおり二出力されていること（デバッグログを除く） + - すべてのファイルが使用どおりの場所にアップロードされていること + """ # Arrange # バケットにファイルをアップロードしていく object_info_files = [] From 11a3191727d907d0eacbf61311c8db6c8339c1db Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:00:26 +0900 Subject: [PATCH 225/275] =?UTF-8?q?fix:=20=E5=85=A8=E4=BB=B6=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=81=AE=E3=83=AD=E3=82=B0=E6=A4=9C=E6=9F=BB?= =?UTF-8?q?=E3=81=A7=E5=A4=B1=E6=95=97=E3=81=97=E3=81=A6=E3=81=84=E3=81=9F?= =?UTF-8?q?=E3=81=AE=E3=81=A7=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index c815f90f..51071707 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -220,10 +220,8 @@ def assert_upload_fetch_datetime_process_log(log_messages, target_object_name, i assert f'I-UPD-01 [{target_object_name}] の前回取得日時ファイルの更新処理を開始します' in log_messages if is_all: assert f'I-UPD-02 [{target_object_name}] の前回取得日時ファイルの更新処理をスキップします' in log_messages - assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' not in log_messages - else: - assert f'I-UPD-02 [{target_object_name}] の前回取得日時ファイルの更新処理をスキップします' not in log_messages - assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' in log_messages + return + assert f'I-UPD-04 [{target_object_name}] の前回取得日時ファイルの更新処理を終了します' in log_messages def assert_file_exist(s3_client, bucket_name, file_key): From bdf8274f8ecf72d7e2efff3411506b93045e0b9d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:00:26 +0900 Subject: [PATCH 226/275] =?UTF-8?q?docs:=20README=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Pipfile | 4 ++-- ecs/crm-datafetch/README.md | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 6912e1a0..da57afc8 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -6,8 +6,8 @@ name = "pypi" [scripts] test = "pytest tests/" "test:cov" = "pytest --cov=src --cov-branch --cov-report=term-missing tests/" -"test:report" = "pytest --cov=src --cov-branch --cov-report=term-missing --html=.report/test_result.html tests/" -"test:walk-through" = "pytest tests/ --walk-through" +"test:report" = "pytest --cov=src --cov-branch --cov-report=term-missing --html=.report/unit_test/test_result.html tests/" +"test:walk-through" = "pytest tests/test_walk_through.py --walk-through --cov-report=term-missing --html=.report/walk_through/test_result.html" [packages] boto3 = "*" diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md index f2928706..4e0072cf 100644 --- a/ecs/crm-datafetch/README.md +++ b/ecs/crm-datafetch/README.md @@ -146,6 +146,19 @@ pipenv install --dev pipenv run test:cov ``` +#### 一気通貫テストを実行する場合の設定 + +- `.env`の以下に示す環境変数の値をメルク様提供のFullSandboxのものに書き換える + - CRM_AUTH_DOMAIN + - CRM_USER_NAME + - CRM_USER_PASSWORD + - CRM_USER_SECURITY_TOKEN +- 以下のコマンドを実行して単体テストを起動する + +```sh +pipenv run test:walk-through +``` + #### 拡張機能：Python Test Explorer UIを導入する場合 - VSCodeの拡張機能メニューから、「Python Test Explorer for Visual Studio Code」をインストール @@ -163,6 +176,9 @@ pipenv run test:cov - `pipenv run test:report` - pytestのテスト終了時にテスト結果をHTMLで出力する - `.report/test_result.html`が出力される +- `test:walk-through` + - 一気通貫テストが実行される + - 上記のテスト意外はスキップされる ## 単体テストの追加方法 From 63eeb96b0fbfed0fb3f5c098858674373343f19d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:00:26 +0900 Subject: [PATCH 227/275] =?UTF-8?q?feat:=20=E3=82=8F=E3=81=8B=E3=82=8A?= =?UTF-8?q?=E3=82=84=E3=81=99=E3=81=84=E3=82=88=E3=81=86=E3=81=AB=E3=83=AD?= =?UTF-8?q?=E3=82=B0=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/pytest.ini | 3 +++ ecs/crm-datafetch/tests/test_walk_through.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 ecs/crm-datafetch/pytest.ini diff --git a/ecs/crm-datafetch/pytest.ini b/ecs/crm-datafetch/pytest.ini new file mode 100644 index 00000000..5dbe2661 --- /dev/null +++ b/ecs/crm-datafetch/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +log_format = %(levelname)s %(asctime)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index 51071707..42da7fa4 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -1,4 +1,5 @@ import json +import logging import os import os.path as path from datetime import datetime, timezone @@ -79,7 +80,15 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): monkeypatch.setattr('src.prepare_data_fetch_process.ExecuteDateTime', MockExecuteDateTime) # Act + logger = logging.getLogger() + logger.setLevel(logging.INFO) + logger.info(f'##########################') + logger.info(f'# 差分データ取得処理:実行開始 #') + logger.info(f'##########################') controller() + logger.info(f'##########################') + logger.info(f'# 差分データ取得処理:実行終了 #') + logger.info(f'##########################') # Assertion log_messages = caplog.messages # ループ前のログ確認 @@ -129,7 +138,13 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # 全件取得用に環境変数を設定 monkeypatch.setattr('src.aws.s3.OBJECT_INFO_FILENAME', 'crm_object_list_all.json') + logger.info(f'##########################') + logger.info(f'# 全件データ取得処理:実行開始 #') + logger.info(f'##########################') controller() + logger.info(f'##########################') + logger.info(f'# 全件データ取得処理:実行終了 #') + logger.info(f'##########################') # ログ再取得 log_messages_all = caplog.messages object_info_list_all = object_info_files[1] From dcf342470ea2f4e895f5cd8b78f47fc17b99139d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:48:00 +0900 Subject: [PATCH 228/275] =?UTF-8?q?feat:=20docker=20build=E6=99=82?= =?UTF-8?q?=E3=81=AB=E3=83=93=E3=83=AB=E3=83=89=E3=82=B3=E3=83=B3=E3=83=86?= =?UTF-8?q?=E3=82=AD=E3=82=B9=E3=83=88=E4=B9=97=E3=81=9B=E3=81=AA=E3=81=84?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=92.dockeringore?= =?UTF-8?q?=E3=81=AB=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.dockerignore | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 ecs/crm-datafetch/.dockerignore diff --git a/ecs/crm-datafetch/.dockerignore b/ecs/crm-datafetch/.dockerignore new file mode 100644 index 00000000..01840c61 --- /dev/null +++ b/ecs/crm-datafetch/.dockerignore @@ -0,0 +1,11 @@ +tests/* +.coverage +.env +.env.example +.report/* +.vscode/* +.pytest_cache/* +*/__pychache__/* +Dockerfile +pytest.ini +README.md From a8868d14b995de1f6b179826d9f95ebb8e568389 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:50:38 +0900 Subject: [PATCH 229/275] =?UTF-8?q?style:=20=E6=8E=A8=E5=A5=A8=E8=A8=AD?= =?UTF-8?q?=E5=AE=9A=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.vscode/recommend_settings.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/.vscode/recommend_settings.json b/ecs/crm-datafetch/.vscode/recommend_settings.json index 4fd1fddf..873f9075 100644 --- a/ecs/crm-datafetch/.vscode/recommend_settings.json +++ b/ecs/crm-datafetch/.vscode/recommend_settings.json @@ -13,7 +13,10 @@ "python.linting.flake8Args": ["--max-line-length=150", "--ignore=F541"], "python.formatting.provider": "autopep8", "python.formatting.autopep8Args": ["--max-line-length", "150"], - "python.testing.pytestArgs": ["tests", "--ignore=tests/test_walk_through.py"], + "python.testing.pytestArgs": [ + "tests" + // "--walk-through" // 一気通貫テストを行いたい場合はコメントを外す + ], "python.testing.unittestEnabled": false, "python.testing.pytestEnabled": true } From b99bc8e6e4921da5fd5fbee6043c9531bc526c3c Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:52:44 +0900 Subject: [PATCH 230/275] =?UTF-8?q?docs:=20README.md=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md index 4e0072cf..75d82e79 100644 --- a/ecs/crm-datafetch/README.md +++ b/ecs/crm-datafetch/README.md @@ -175,10 +175,11 @@ pipenv run test:walk-through - 標準出力とカバレッジファイル(`.coverage`)に出力される - `pipenv run test:report` - pytestのテスト終了時にテスト結果をHTMLで出力する - - `.report/test_result.html`が出力される + - `.report/unit_test/test_result.html`が出力される - `test:walk-through` - 一気通貫テストが実行される - 上記のテスト意外はスキップされる + - `.report/walk_through/test_result.html`に結果のHTMLが出力される ## 単体テストの追加方法 From 5ba69420fe7077c0c8a08ede39040c7765214cc0 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 12:55:36 +0900 Subject: [PATCH 231/275] =?UTF-8?q?fix:=20=E4=BD=BF=E7=94=A8=E3=81=97?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=AA=E3=81=84=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E3=82=92=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/system_var/environments.py | 10 ---------- s3/config/crm/task_settings/task_settings.env | 4 ---- 2 files changed, 14 deletions(-) diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index 9ab6911d..297cfb2b 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -7,14 +7,6 @@ import src.system_var.constants as constants LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # CRMへの認証処理のタイムアウト秒数 CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)) -# # CRMへの認証処理の最大リトライ試行回数 -# CRM_AUTH_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3)) -# # CRMへの認証処理のリトライ時の初回待ち秒数 -# CRM_AUTH_RETRY_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5)) -# # CRMへの認証処理のリトライ時の最小待ち秒数 -# CRM_AUTH_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5)) -# # CRMへの認証処理のリトライ時の最大待ち秒数 -# CRM_AUTH_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50)) # CRMのレコード件数取得処理のタイムアウト秒数 CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)) # CRMのレコード件数取得処理の最大リトライ試行回数 @@ -65,8 +57,6 @@ PROCESS_RESULT_FILENAME = os.environ.get(constants.PROCESS_RESULT_FILENAME, 'pro LAST_FETCH_DATE_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_FOLDER, 'crm/last_fetch_datetime') # CRMから取得し、取込用に変換したデータを格納するフォルダ CRM_IMPORT_DATA_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_FOLDER, 'crm/target') -# # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -# LAST_FETCH_DATE_BACKUP_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_BACKUP_FOLDER, 'last_fetch_datetime') # CRMから取得した生データのバックアップを格納するフォルダパス RESPONSE_JSON_BACKUP_FOLDER = os.environ.get(constants.RESPONSE_JSON_BACKUP_FOLDER, 'response_json') # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env index 1998012c..580bf564 100644 --- a/s3/config/crm/task_settings/task_settings.env +++ b/s3/config/crm/task_settings/task_settings.env @@ -1,9 +1,5 @@ LOG_LEVEL=INFO CRM_AUTH_TIMEOUT=100 -CRM_AUTH_MAX_RETRY_ATTEMPT=3 -CRM_AUTH_RETRY_INTERVAL=5 -CRM_AUTH_RETRY_MIN_INTERVAL=5 -CRM_AUTH_RETRY_MAX_INTERVAL=50 CRM_GET_RECORD_COUNT_TIMEOUT=300 CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=3 CRM_GET_RECORD_COUNT_RETRY_INTERVAL=5 From 08f87d2abfd3360941f70b577a1b2fe381878e92 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 16:02:26 +0900 Subject: [PATCH 232/275] =?UTF-8?q?feat:=20=E4=BD=8F=E6=89=80=E5=9E=8B?= =?UTF-8?q?=E3=81=8C=E8=A4=87=E5=90=88=E3=82=AA=E3=83=96=E3=82=B8=E3=82=A7?= =?UTF-8?q?=E3=82=AF=E3=83=88=E9=A0=85=E7=9B=AE=E3=81=AB=E3=81=AA=E3=81=A3?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=9F=E3=81=9F=E3=82=81=E3=80=81=E5=80=8B?= =?UTF-8?q?=E5=88=A5=E3=81=AB=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesforce.py | 67 +++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index b815e772..2b02a5cb 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -9,6 +9,8 @@ Accountオブジェクトの下記SFIDのレコードはいじらないように 変更してしまった場合は各SOQLの取得日付とデータを修正してください """ +from typing import OrderedDict + import pytest from requests.exceptions import ConnectTimeout, ReadTimeout from src.config.objects import LastFetchDatetime, TargetObject @@ -160,7 +162,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得件数が16になる + 取得件数が17になる """ execute_datetime = ExecuteDateTime() @@ -190,7 +192,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_count(soql) - assert actual == 16 + assert actual == 17 def test_fetch_sf_data_one_record(self): """ @@ -422,6 +424,63 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect + def test_fetch_sf_data_by_soql_builder_address_item_check(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること + - SystemModStampのFromが2000年1月1日以降のレコードが取得できること + - SystemModStampのToが2100年12月31日未満のレコードが取得できること + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromに2000年1月1日を指定する + - LastFetchDatetimeのToに2100年12月31日を指定する + Expects: + 取得できたオブジェクト件数が1件になる + 住所項目(BillingAddress)が想定通りの値になっていること + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2022-08-19T05:53:29.000Z', + 'last_fetch_datetime_to': '2022-08-19T05:53:30.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'BillingAddress', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_fetch_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) == 1 + expect_address = OrderedDict([ + ("city", '〇〇区'), + ("country", "日本"), + ("geocodeAccuracy", None), + ("latitude", None), + ("longitude", None), + ("postalCode", '999-9999'), + ("state", '東京都'), + ("street", '△△-✗✗'), + ]) + + assert actual[0]['BillingAddress'] == expect_address + def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self): """ Cases: @@ -433,7 +492,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得できたオブジェクト件数が16件になる + 取得できたオブジェクト件数が17件になる """ execute_datetime = ExecuteDateTime() @@ -463,7 +522,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_data(soql) - assert len(actual) == 16 + assert len(actual) == 17 # 内容の確認は別のケースで行っているため省略 def test_raise_create_instance_cause_auth_failed(self, monkeypatch): From 2e24ce85f2079b6abeadf38b16721961148df3b0 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 16:11:33 +0900 Subject: [PATCH 233/275] =?UTF-8?q?feat:=20dict=E5=9E=8B=E3=81=AE=E5=A4=89?= =?UTF-8?q?=E6=8F=9B=E5=99=A8=E7=94=9F=E6=88=90=E5=87=A6=E7=90=86=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/converter/convert_strategy.py | 10 +++++ .../tests/converter/test_convert_strategy.py | 45 +++++++++++++++++-- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 22fca8fc..3e092315 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -1,4 +1,5 @@ import re +from collections import OrderedDict from datetime import datetime from dateutil.tz import gettz @@ -17,6 +18,7 @@ class ConvertStrategyFactory: self.__datetime_convert_strategy = DatetimeConvertStrategy() self.__int_convert_strategy = IntConvertStrategy() self.__string_convert_strategy = StringConvertStrategy() + self.__dict_convert_strategy = DictConvertStrategy() def create(self, value): @@ -35,6 +37,8 @@ class ConvertStrategyFactory: elif type(value) == int: convert_strategy = self.__int_convert_strategy + elif type(value) == dict or type(value) == OrderedDict: + convert_strategy = self.__dict_convert_strategy else: convert_strategy = self.__string_convert_strategy @@ -78,3 +82,9 @@ class StringConvertStrategy: """string型を変換せずに返す処理""" # ConvertStrategyFactoryにて型チェックを行っているため値を変換せずに返す return convert_value + + +class DictConvertStrategy: + def convert_value(self, convert_value: str): + """dict型の項目を文字列に変換して返す処理""" + return convert_value diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py index 0e268fc4..77ecfea2 100644 --- a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -1,6 +1,9 @@ +from typing import OrderedDict + from src.converter.convert_strategy import (BooleanConvertStrategy, ConvertStrategyFactory, DatetimeConvertStrategy, + DictConvertStrategy, FloatConvertStrategy, IntConvertStrategy, NoneValueConvertStrategy, @@ -94,10 +97,10 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == DatetimeConvertStrategy - def test_create_other_str(self): + def test_create_str(self): """ Cases: - 引数にSalesforce日付型以外の文字列を指定した場合、NonConvertStrategyインスタンスが返ってくること + 引数にSalesforce日付型以外の文字列を指定した場合、StringConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -111,10 +114,10 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == StringConvertStrategy - def test_create_other_int(self): + def test_create_int(self): """ Cases: - 引数に整数を指定した場合、NonConvertStrategyインスタンスが返ってくること + 引数に整数を指定した場合、IntConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -128,6 +131,40 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == IntConvertStrategy + def test_create_dict(self): + """ + Cases: + 引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create({'key': 'value'}) + + # Expects + assert type(actual) == DictConvertStrategy + + def test_create_ordered_dict_dict(self): + """ + Cases: + 引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(OrderedDict([('key', 'value')])) + + # Expects + assert type(actual) == DictConvertStrategy + class TestNoneValueConvertStrategy: From 5e857aa9c6cab1a37502def748ba3fb4835923c7 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 16:33:49 +0900 Subject: [PATCH 234/275] =?UTF-8?q?feat:=20=E8=BE=9E=E6=9B=B8=E5=9E=8B?= =?UTF-8?q?=E9=A0=85=E7=9B=AE=E3=82=92=E6=96=87=E5=AD=97=E5=88=97=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=8F=9B=E3=81=99=E3=82=8B=E5=87=A6=E7=90=86=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/converter/convert_strategy.py | 5 +- .../tests/converter/test_convert_strategy.py | 58 ++++++++++++++++++- .../tests/salesforce/test_salesforce.py | 1 + 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 3e092315..5f3daf9d 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -1,3 +1,4 @@ +import json import re from collections import OrderedDict from datetime import datetime @@ -85,6 +86,6 @@ class StringConvertStrategy: class DictConvertStrategy: - def convert_value(self, convert_value: str): + def convert_value(self, convert_value: dict): """dict型の項目を文字列に変換して返す処理""" - return convert_value + return json.dumps(convert_value, ensure_ascii=False) diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py index 77ecfea2..20d5f655 100644 --- a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -1,4 +1,4 @@ -from typing import OrderedDict +from collections import OrderedDict from src.converter.convert_strategy import (BooleanConvertStrategy, ConvertStrategyFactory, @@ -301,3 +301,59 @@ class TestStringConvertStrategy: # Expects assert actual == 'テストデータ' + + +class TestDictConvertStrategy: + + def test_convert_value_dict(self): + """ + Cases: + 引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DictConvertStrategy() + actual = sut.convert_value({'テストデータキー': 'テストデータバリュー'}) + + # Expects + assert actual == '{"テストデータキー": "テストデータバリュー"}' + + def test_convert_value_dict_in_line_break(self): + """ + Cases: + 引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること(バリューに改行を含む) + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DictConvertStrategy() + actual = sut.convert_value({'テストデータキー': 'テスト\nデータ\nバリュー'}) + + # Expects + assert actual == '{"テストデータキー": "テスト\\nデータ\\nバリュー"}' + + def test_convert_value_ordered_dict(self): + """ + Cases: + 引数に整列された辞書型のデータを指定した場合、JSONの文字列が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DictConvertStrategy() + actual = sut.convert_value(OrderedDict( + [('テストデータキー', 'テストデータバリュー')] + )) + + # Expects + assert actual == '{"テストデータキー": "テストデータバリュー"}' diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index 2b02a5cb..9547b97d 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -507,6 +507,7 @@ class TestSalesforceApiClient: 'Name', 'SystemModstamp', 'LastModifiedDate', + 'BillingAddress', 'CustomItem1__c', 'CustomItem2__c', 'CustomItem3__c', From dfaa405d724073ba559ff4306e37a87247616420 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 19 Aug 2022 17:17:38 +0900 Subject: [PATCH 235/275] =?UTF-8?q?feat:=20=E5=A4=89=E6=8F=9B=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=81=AE=E5=8D=98=E4=BD=93=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=81=A7=E3=80=81JSON=E5=9E=8B=E3=81=AE=E3=83=87=E3=83=BC?= =?UTF-8?q?=E3=82=BF=E3=81=AE=E5=A4=89=E6=8F=9B=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test_convert_crm_csv_data_process.py | 47 +++++++++++++++---- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py index 49d960c0..9615d234 100644 --- a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py +++ b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py @@ -34,7 +34,17 @@ class TestConvertCrmCsvDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', True) + ('IsDeleted', True), + ('PersonMailingAddress', OrderedDict([ + ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 + ('PersonMailingCity', 'New york city'), + ('PersonMailingState', 'Ohaio'), + ('PersonMailingPostalCode', '999-9999'), + ('PersonMailingCountry', 'US'), + ('PersonMailingLatitude', 50.1234567), + ('PersonMailingLongitude', 103.1234567), + ('PersonMailingGeocodeAccuracy', 'Address'), + ])), ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -44,7 +54,17 @@ class TestConvertCrmCsvDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('PersonMailingAddress', OrderedDict([ + ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 + ('PersonMailingCity', 'New york city'), + ('PersonMailingState', 'Ohaio'), + ('PersonMailingPostalCode', '999-9999'), + ('PersonMailingCountry', 'US'), + ('PersonMailingLatitude', 50.1234567), + ('PersonMailingLongitude', 103.1234567), + ('PersonMailingGeocodeAccuracy', 'Address'), + ])), ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -54,7 +74,17 @@ class TestConvertCrmCsvDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('PersonMailingAddress', OrderedDict([ + ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 + ('PersonMailingCity', 'New york city'), + ('PersonMailingState', 'Ohaio'), + ('PersonMailingPostalCode', '999-9999'), + ('PersonMailingCountry', 'US'), + ('PersonMailingLatitude', 50.1234567), + ('PersonMailingLongitude', 103.1234567), + ('PersonMailingGeocodeAccuracy', 'Address'), + ])), ]), ] @@ -66,7 +96,8 @@ class TestConvertCrmCsvDataProcess: 'LastModifiedDate', 'LastModifiedById', 'SystemModstamp', - 'IsDeleted' + 'IsDeleted', + 'PersonMailingAddress' ] } @@ -79,10 +110,10 @@ class TestConvertCrmCsvDataProcess: # Assert expect_csv_string = """\ - "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\ - "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\ - "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ - "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ + "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted","PersonMailingAddress"\r\n\ + "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ + "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ + "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ """ # 返り値の期待値チェック assert isinstance(actual_csv_string, str), 'CSV文字列が返却される' From 4d179f6be5aef52e113b24c963670643d13bdc55 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 22 Aug 2022 11:35:00 +0900 Subject: [PATCH 236/275] =?UTF-8?q?fix:=20=E6=8C=87=E6=95=B0=E8=A1=A8?= =?UTF-8?q?=E8=A8=98=E3=81=AE=E5=A4=89=E6=8F=9B=E3=82=92=E8=A1=8C=E3=81=A3?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=9F=E3=81=93=E3=81=A8=E3=81=AB=E3=82=88?= =?UTF-8?q?=E3=82=8A=E3=80=81=E4=BB=96=E3=81=AE=E5=B0=91=E6=95=B0=E9=A0=85?= =?UTF-8?q?=E7=9B=AE=E3=81=8C=E6=95=B4=E6=95=B0=E3=81=AB=E3=81=AA=E3=81=A3?= =?UTF-8?q?=E3=81=A6=E3=81=97=E3=81=BE=E3=81=A3=E3=81=A6=E3=81=84=E3=81=9F?= =?UTF-8?q?=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/converter/convert_strategy.py | 10 ----- .../tests/converter/test_convert_strategy.py | 44 +++++++++---------- .../tests/converter/test_converter.py | 10 ++--- .../test_convert_crm_csv_data_process.py | 10 ++--- 4 files changed, 31 insertions(+), 43 deletions(-) diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 22fca8fc..ab1e0626 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -12,7 +12,6 @@ from src.system_var.environments import CONVERT_TZ class ConvertStrategyFactory: def __init__(self) -> None: self.__none_value_convert_strategy = NoneValueConvertStrategy() - self.__float_convert_strategy = FloatConvertStrategy() self.__boolean_convert_strategy = BooleanConvertStrategy() self.__datetime_convert_strategy = DatetimeConvertStrategy() self.__int_convert_strategy = IntConvertStrategy() @@ -23,9 +22,6 @@ class ConvertStrategyFactory: if value is None: convert_strategy = self.__none_value_convert_strategy - elif type(value) == float: - convert_strategy = self.__float_convert_strategy - elif type(value) == bool: convert_strategy = self.__boolean_convert_strategy @@ -60,12 +56,6 @@ class DatetimeConvertStrategy: return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS) -class FloatConvertStrategy: - def convert_value(self, convert_value: str) -> int: - """float型をint型に変換する処理""" - return int(convert_value) - - class IntConvertStrategy: def convert_value(self, convert_value: int): """int型を変換せずに返す処理""" diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py index 0e268fc4..4b4552a8 100644 --- a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -1,7 +1,6 @@ from src.converter.convert_strategy import (BooleanConvertStrategy, ConvertStrategyFactory, DatetimeConvertStrategy, - FloatConvertStrategy, IntConvertStrategy, NoneValueConvertStrategy, StringConvertStrategy) @@ -29,7 +28,7 @@ class TestConvertStrategyFactory: def test_create_float(self): """ Cases: - 引数に指数表記を指定した場合、FloatConvertStrategyインスタンスが返ってくること + 引数に指数表記を指定した場合、StringConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -41,7 +40,26 @@ class TestConvertStrategyFactory: actual = sut.create(1.2345678E7) # Expects - assert type(actual) == FloatConvertStrategy + # 変換しない + assert type(actual) == StringConvertStrategy + + def test_create_float_scale(self): + """ + Cases: + 引数に少数を指定した場合、StringConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(1.2345678E7) + + # Expects + # 変換しない + assert type(actual) == StringConvertStrategy def test_create_bool_true(self): """ @@ -206,26 +224,6 @@ class TestDatetimeConvertStrategy: assert actual == "2022-06-14 05:15:32" -class TestFloatConvertStrategy: - - def test_convert_value(self) -> int: - """ - Cases: - 引数に指数表記を指定した場合、整数で返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = FloatConvertStrategy() - actual = sut.convert_value(1.2345678E7) - - # Expects - assert actual == 12345678 - - class TestIntConvertStrategy: def test_convert_value(self): diff --git a/ecs/crm-datafetch/tests/converter/test_converter.py b/ecs/crm-datafetch/tests/converter/test_converter.py index 0300381d..da80982b 100644 --- a/ecs/crm-datafetch/tests/converter/test_converter.py +++ b/ecs/crm-datafetch/tests/converter/test_converter.py @@ -71,7 +71,7 @@ class TestCSVStringConverter: ('ContactAccessLevel', 8), ('RowCause', 'テストのため2'), ('LastModifiedDate', '2022-06-02T16:30:30.000+0000'), - ('LastModifiedById', 2.234567E+6), + ('LastModifiedById', 2.23E+0), ('IsDeleted', True) ]), OrderedDict([ @@ -85,7 +85,7 @@ class TestCSVStringConverter: ('ContactAccessLevel', 12), ('RowCause', 'テストのため3'), ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), - ('LastModifiedById', 3.234567E+6), + ('LastModifiedById', 3.234567), ('IsDeleted', False) ]) ] @@ -100,9 +100,9 @@ class TestCSVStringConverter: # Expects expected_value = '''\ "Id","AccountId","UserOrGroupId","AccountAccessLevel","OpportunityAccessLevel","CaseAccessLevel","ContactAccessLevel","RowCause","LastModifiedDate","LastModifiedById","IsDeleted"\r\n\ - "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567","0"\r\n\ - "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2234567","1"\r\n\ - "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3234567","0"\r\n\ + "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567.0","0"\r\n\ + "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2.23","1"\r\n\ + "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3.234567","0"\r\n\ ''' # expected_valueのインデントが半角スペースと認識されてしまうため、`textwrap.dedent`にて補正 diff --git a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py index 49d960c0..6d1b63f1 100644 --- a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py +++ b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py @@ -42,7 +42,7 @@ class TestConvertCrmCsvDataProcess: ('Id', 'TEST002'), ('AccountNumber', 'test002'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), - ('LastModifiedById', 1.234567E+6), + ('LastModifiedById', 1.23E+0), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), ('IsDeleted', False) ]), @@ -52,7 +52,7 @@ class TestConvertCrmCsvDataProcess: ('Id', 'TEST003'), ('AccountNumber', 'test003'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), - ('LastModifiedById', 1.234567E+6), + ('LastModifiedById', 1.234567), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), ('IsDeleted', False) ]), @@ -80,9 +80,9 @@ class TestConvertCrmCsvDataProcess: expect_csv_string = """\ "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\ - "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\ - "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ - "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ + "TEST001","test001","2022-06-01 09:00:00","1234567.0","2022-06-01 09:00:00","1"\r\n\ + "TEST002","test002","2022-06-01 09:00:00","1.23","2022-06-01 09:00:00","0"\r\n\ + "TEST003","test003","2022-06-01 09:00:00","1.234567","2022-06-01 09:00:00","0"\r\n\ """ # 返り値の期待値チェック assert isinstance(actual_csv_string, str), 'CSV文字列が返却される' From f6713f2c094e4fb9232fd81404fbc93ee949c0e9 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 22 Aug 2022 11:36:54 +0900 Subject: [PATCH 237/275] =?UTF-8?q?fix:=20salesforce=E3=81=AE=E3=83=87?= =?UTF-8?q?=E3=83=BC=E3=82=BF=E5=A4=89=E6=9B=B4=E3=81=AB=E4=BC=B4=E3=81=86?= =?UTF-8?q?=E4=BB=B6=E6=95=B0=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/salesforce/test_salesforce.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index b815e772..4a3667f5 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -160,7 +160,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得件数が16になる + 取得件数が17になる """ execute_datetime = ExecuteDateTime() @@ -190,7 +190,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_count(soql) - assert actual == 16 + assert actual == 17 def test_fetch_sf_data_one_record(self): """ @@ -433,7 +433,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得できたオブジェクト件数が16件になる + 取得できたオブジェクト件数が17件になる """ execute_datetime = ExecuteDateTime() @@ -463,7 +463,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_data(soql) - assert len(actual) == 16 + assert len(actual) == 17 # 内容の確認は別のケースで行っているため省略 def test_raise_create_instance_cause_auth_failed(self, monkeypatch): From 7ca19a998e075feceb59c609b727c0608fd20fcc Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 22 Aug 2022 13:47:13 +0900 Subject: [PATCH 238/275] =?UTF-8?q?feat:=20=E3=83=87=E3=83=90=E3=83=83?= =?UTF-8?q?=E3=82=B0=E3=83=AD=E3=82=B0=E3=82=92=E5=87=BA=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index 42da7fa4..bc2d7249 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -67,6 +67,8 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): monkeypatch.setattr('src.salesforce.soql_builder.FETCH_LIMIT_CLAUSE', ' LIMIT 10') # 件数取得はログ出力用なので、0件が返るようにする monkeypatch.setattr('src.fetch_crm_data_process.fetch_record_count_retry', lambda x, y, z: 10) + # ログレベルをDEBUGに + monkeypatch.setattr('src.util.logger.LOG_LEVEL', 'DEBUG') # 実行日時を固定する now = datetime.now(timezone.utc).strftime(YYYYMMDDTHHMMSSTZ) format_now = datetime.strptime(now, YYYYMMDDTHHMMSSTZ).strftime('%Y%m%d%H%M%S') From 7fd9937b10c96201aa68abb4a986051e681c7e31 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 22 Aug 2022 16:22:19 +0900 Subject: [PATCH 239/275] =?UTF-8?q?feat:=20AWS=E4=B8=8A=E3=81=AES3?= =?UTF-8?q?=E3=82=92=E5=88=A9=E7=94=A8=E3=81=99=E3=82=8B=E6=A9=9F=E8=83=BD?= =?UTF-8?q?=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 46 +++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index bc2d7249..e8c3b6e4 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -4,6 +4,7 @@ import os import os.path as path from datetime import datetime, timezone +import boto3 import pytest from src.controller import controller from src.parser.json_parser import JsonParser @@ -11,9 +12,38 @@ from src.system_var.constants import YYYYMMDDTHHMMSSTZ from src.util.execute_datetime import ExecuteDateTime ROOT_DIR = path.abspath(path.dirname(__name__)) -DATA_BUCKET = 'mbj-newdwh2021-staging-data' -CONFIG_BUCKET = 'mbj-newdwh2021-staging-config' -BACKUP_BUCKET = 'mbj-newdwh2021-staging-backup-crm' + + +# ↓↓↓モックテスト用(バケットをモック化する際、コメントアウトを外す。AWS利用テスト用はコメントアウトする)↓↓↓ +# DATA_BUCKET = 'mbj-newdwh2021-staging-data' +# CONFIG_BUCKET = 'mbj-newdwh2021-staging-config' +# BACKUP_BUCKET = 'mbj-newdwh2021-staging-backup-crm' +# +# @pytest.fixture +# def s3_test(s3_client): +# s3_client.create_bucket(Bucket=DATA_BUCKET) +# s3_client.create_bucket(Bucket=CONFIG_BUCKET) +# s3_client.create_bucket(Bucket=BACKUP_BUCKET) +# yield +# ↑↑↑モックテスト用↑↑↑ + +# ↓↓↓AWS利用テスト用(AWS上のバケットを利用する際、コメントアウトを外す。モックテスト用はコメントアウトする)↓↓↓ +DATA_BUCKET = 'test-mbj-newdwh2021-test-data' +CONFIG_BUCKET = 'test-mbj-newdwh2021-test-config' +BACKUP_BUCKET = 'test-mbj-newdwh2021-test-backup-crm' + + +@pytest.fixture +def s3_test(s3_client): + yield + + +@pytest.fixture +def s3_client(): + s3_client = boto3.client("s3") + yield s3_client +# ↑↑↑AWS利用テスト用↑↑↑ + TARGET_FOLDER = 'crm/target' OBJECT_INFO_FOLDER = 'crm/object_info' @@ -23,14 +53,6 @@ BACKUP_RESPONSE_JSON_FOLDER = 'response_json' PROCESS_RESULT_JSON_FILE_NAME = 'process_result.json' -@pytest.fixture -def s3_test(s3_client): - s3_client.create_bucket(Bucket=DATA_BUCKET) - s3_client.create_bucket(Bucket=CONFIG_BUCKET) - s3_client.create_bucket(Bucket=BACKUP_BUCKET) - yield - - @pytest.mark.walk_through def test_walk_through(s3_test, s3_client, monkeypatch, caplog): """ @@ -256,7 +278,7 @@ def assert_file_exist(s3_client, bucket_name, file_key): def get_object_config_list(folder_name: str): local_s3_path = path.join(ROOT_DIR, '..', '..', 's3', 'config', 'crm', folder_name) config_list = [os.path.join(local_s3_path, config) for config in os.listdir(local_s3_path) if config.endswith('.json')] - return config_list + return sorted(config_list, reverse=True) def read_json(json_path): From 76d8b37be9f5d420f0874c174f973474112eef7b Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 22 Aug 2022 16:27:25 +0900 Subject: [PATCH 240/275] =?UTF-8?q?docs:=20=E6=BC=A2=E5=AD=97=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md index 75d82e79..5da3ae51 100644 --- a/ecs/crm-datafetch/README.md +++ b/ecs/crm-datafetch/README.md @@ -178,7 +178,7 @@ pipenv run test:walk-through - `.report/unit_test/test_result.html`が出力される - `test:walk-through` - 一気通貫テストが実行される - - 上記のテスト意外はスキップされる + - 上記のテスト以外はスキップされる - `.report/walk_through/test_result.html`に結果のHTMLが出力される ## 単体テストの追加方法 From f052732e20e9df31c0dfe2f2ff80d068291f306b Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 23 Aug 2022 11:46:31 +0900 Subject: [PATCH 241/275] =?UTF-8?q?feat:=20=E4=BD=8F=E6=89=80=E9=A0=85?= =?UTF-8?q?=E7=9B=AE=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesforce.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index 9547b97d..c06ce546 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -428,12 +428,10 @@ class TestSalesforceApiClient: """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること - - SystemModStampのFromが2000年1月1日以降のレコードが取得できること - - SystemModStampのToが2100年12月31日未満のレコードが取得できること + - できること Arranges: - - SalesforceのAccountオブジェクトに、レコードを作成する - - LastFetchDatetimeのFromに2000年1月1日を指定する - - LastFetchDatetimeのToに2100年12月31日を指定する + - SalesforceのAccountオブジェクトに、住所項目を持つレコードを作成する + - 住所項目を持つレコードだけが取れるよう日付を設定する Expects: 取得できたオブジェクト件数が1件になる 住所項目(BillingAddress)が想定通りの値になっていること @@ -441,8 +439,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-19T05:53:29.000Z', - 'last_fetch_datetime_to': '2022-08-19T05:53:30.000Z', + 'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z', + 'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -451,15 +449,15 @@ class TestSalesforceApiClient: 'Name', 'SystemModstamp', 'LastModifiedDate', + 'BillingStreet', + 'BillingCity', + 'BillingState', + 'BillingPostalCode', + 'BillingCountry', + 'BillingLatitude', + 'BillingLongitude', + 'BillingGeocodeAccuracy', 'BillingAddress', - 'CustomItem1__c', - 'CustomItem2__c', - 'CustomItem3__c', - 'CustomItem4__c', - 'CustomItem5__c', - 'CustomItem6__c', - 'CustomItem7__c', - 'CustomItem8__c' ] }, execute_datetime) soql_builder = SOQLBuilder(target_object, last_fetch_datetime) @@ -480,6 +478,14 @@ class TestSalesforceApiClient: ]) assert actual[0]['BillingAddress'] == expect_address + assert actual[0]['BillingCity'] == '〇〇区' + assert actual[0]['BillingCountry'] == '日本' + assert actual[0]['BillingGeocodeAccuracy'] is None + assert actual[0]['BillingLatitude'] is None + assert actual[0]['BillingLongitude'] is None + assert actual[0]['BillingPostalCode'] == '999-9999' + assert actual[0]['BillingState'] == '東京都' + assert actual[0]['BillingStreet'] == '△△-✗✗' def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self): """ From 4ffb55563ba27b64154c20e1e15146c3fa078d85 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 23 Aug 2022 11:48:13 +0900 Subject: [PATCH 242/275] =?UTF-8?q?fix:=20JSON=E5=86=85=E3=81=AE=E6=97=A5?= =?UTF-8?q?=E6=9C=AC=E8=AA=9E=E3=81=8CHex=E3=82=B3=E3=83=BC=E3=83=89?= =?UTF-8?q?=E5=80=A4=E3=81=A7=E5=87=BA=E5=8A=9B=E3=81=95=E3=82=8C=E3=81=A6?= =?UTF-8?q?=E3=81=84=E3=82=8B=E5=95=8F=E9=A1=8C=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/aws/s3.py | 4 ++-- .../tests/test_backup_crm_data_process.py | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index d5bd03f0..7b63861f 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -87,7 +87,7 @@ class BackupBucket: def put_response_json(self, file_path: str, data: dict) -> None: object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_path}' - self.__s3_resource.put_object(object_key, json.dumps(data)) + self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False)) return def put_csv(self, file_path: str, data: str) -> None: @@ -97,5 +97,5 @@ class BackupBucket: def put_result_json(self, file_path: str, data: dict) -> None: object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}' - self.__s3_resource.put_object(object_key, json.dumps(data)) + self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False)) return diff --git a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py index 9ad35d52..4e48c419 100644 --- a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py @@ -44,7 +44,8 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('Name', 'テスト取引先1') ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -54,7 +55,8 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('Name', 'テスト取引先2') ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -64,7 +66,8 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('Name', 'テスト取引先3') ]), ] @@ -96,7 +99,7 @@ class TestBackupCrmDataProcess: actual = s3_client.get_object( Bucket=bucket_name, Key=f'response_json/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.json') - assert actual['Body'].read().decode('utf-8') == json.dumps(response_json) + assert actual['Body'].read().decode('utf-8') == json.dumps(response_json, ensure_ascii=False) # ログの確認 assert generate_log_message_tuple( From d77fafa2de7c5e9f640e957dbe5058ac28b90b00 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 23 Aug 2022 11:51:26 +0900 Subject: [PATCH 243/275] =?UTF-8?q?feat:=20=E4=BD=8F=E6=89=80=E5=9E=8B?= =?UTF-8?q?=E3=81=8C=E8=A4=87=E5=90=88=E3=82=AA=E3=83=96=E3=82=B8=E3=82=A7?= =?UTF-8?q?=E3=82=AF=E3=83=88=E9=A0=85=E7=9B=AE=E3=81=AB=E3=81=AA=E3=81=A3?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=9F=E3=81=9F=E3=82=81=E3=80=81=E5=80=8B?= =?UTF-8?q?=E5=88=A5=E3=81=AB=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesforce.py | 67 +++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index b815e772..2b02a5cb 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -9,6 +9,8 @@ Accountオブジェクトの下記SFIDのレコードはいじらないように 変更してしまった場合は各SOQLの取得日付とデータを修正してください """ +from typing import OrderedDict + import pytest from requests.exceptions import ConnectTimeout, ReadTimeout from src.config.objects import LastFetchDatetime, TargetObject @@ -160,7 +162,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得件数が16になる + 取得件数が17になる """ execute_datetime = ExecuteDateTime() @@ -190,7 +192,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_count(soql) - assert actual == 16 + assert actual == 17 def test_fetch_sf_data_one_record(self): """ @@ -422,6 +424,63 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect + def test_fetch_sf_data_by_soql_builder_address_item_check(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること + - SystemModStampのFromが2000年1月1日以降のレコードが取得できること + - SystemModStampのToが2100年12月31日未満のレコードが取得できること + Arranges: + - SalesforceのAccountオブジェクトに、レコードを作成する + - LastFetchDatetimeのFromに2000年1月1日を指定する + - LastFetchDatetimeのToに2100年12月31日を指定する + Expects: + 取得できたオブジェクト件数が1件になる + 住所項目(BillingAddress)が想定通りの値になっていること + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2022-08-19T05:53:29.000Z', + 'last_fetch_datetime_to': '2022-08-19T05:53:30.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'BillingAddress', + 'CustomItem1__c', + 'CustomItem2__c', + 'CustomItem3__c', + 'CustomItem4__c', + 'CustomItem5__c', + 'CustomItem6__c', + 'CustomItem7__c', + 'CustomItem8__c' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_fetch_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) == 1 + expect_address = OrderedDict([ + ("city", '〇〇区'), + ("country", "日本"), + ("geocodeAccuracy", None), + ("latitude", None), + ("longitude", None), + ("postalCode", '999-9999'), + ("state", '東京都'), + ("street", '△△-✗✗'), + ]) + + assert actual[0]['BillingAddress'] == expect_address + def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self): """ Cases: @@ -433,7 +492,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得できたオブジェクト件数が16件になる + 取得できたオブジェクト件数が17件になる """ execute_datetime = ExecuteDateTime() @@ -463,7 +522,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_data(soql) - assert len(actual) == 16 + assert len(actual) == 17 # 内容の確認は別のケースで行っているため省略 def test_raise_create_instance_cause_auth_failed(self, monkeypatch): From 35730fc45f3d8670674b2e41dd8d1771ecc3a8c3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 23 Aug 2022 11:51:26 +0900 Subject: [PATCH 244/275] =?UTF-8?q?feat:=20dict=E5=9E=8B=E3=81=AE=E5=A4=89?= =?UTF-8?q?=E6=8F=9B=E5=99=A8=E7=94=9F=E6=88=90=E5=87=A6=E7=90=86=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/converter/convert_strategy.py | 10 +++++ .../tests/converter/test_convert_strategy.py | 45 +++++++++++++++++-- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 22fca8fc..3e092315 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -1,4 +1,5 @@ import re +from collections import OrderedDict from datetime import datetime from dateutil.tz import gettz @@ -17,6 +18,7 @@ class ConvertStrategyFactory: self.__datetime_convert_strategy = DatetimeConvertStrategy() self.__int_convert_strategy = IntConvertStrategy() self.__string_convert_strategy = StringConvertStrategy() + self.__dict_convert_strategy = DictConvertStrategy() def create(self, value): @@ -35,6 +37,8 @@ class ConvertStrategyFactory: elif type(value) == int: convert_strategy = self.__int_convert_strategy + elif type(value) == dict or type(value) == OrderedDict: + convert_strategy = self.__dict_convert_strategy else: convert_strategy = self.__string_convert_strategy @@ -78,3 +82,9 @@ class StringConvertStrategy: """string型を変換せずに返す処理""" # ConvertStrategyFactoryにて型チェックを行っているため値を変換せずに返す return convert_value + + +class DictConvertStrategy: + def convert_value(self, convert_value: str): + """dict型の項目を文字列に変換して返す処理""" + return convert_value diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py index 0e268fc4..77ecfea2 100644 --- a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -1,6 +1,9 @@ +from typing import OrderedDict + from src.converter.convert_strategy import (BooleanConvertStrategy, ConvertStrategyFactory, DatetimeConvertStrategy, + DictConvertStrategy, FloatConvertStrategy, IntConvertStrategy, NoneValueConvertStrategy, @@ -94,10 +97,10 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == DatetimeConvertStrategy - def test_create_other_str(self): + def test_create_str(self): """ Cases: - 引数にSalesforce日付型以外の文字列を指定した場合、NonConvertStrategyインスタンスが返ってくること + 引数にSalesforce日付型以外の文字列を指定した場合、StringConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -111,10 +114,10 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == StringConvertStrategy - def test_create_other_int(self): + def test_create_int(self): """ Cases: - 引数に整数を指定した場合、NonConvertStrategyインスタンスが返ってくること + 引数に整数を指定した場合、IntConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -128,6 +131,40 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == IntConvertStrategy + def test_create_dict(self): + """ + Cases: + 引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create({'key': 'value'}) + + # Expects + assert type(actual) == DictConvertStrategy + + def test_create_ordered_dict_dict(self): + """ + Cases: + 引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(OrderedDict([('key', 'value')])) + + # Expects + assert type(actual) == DictConvertStrategy + class TestNoneValueConvertStrategy: From 977c5bf7ef35bb30a2f005bd9ae4d701de72fe9e Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 23 Aug 2022 11:51:26 +0900 Subject: [PATCH 245/275] =?UTF-8?q?feat:=20=E8=BE=9E=E6=9B=B8=E5=9E=8B?= =?UTF-8?q?=E9=A0=85=E7=9B=AE=E3=82=92=E6=96=87=E5=AD=97=E5=88=97=E3=81=AB?= =?UTF-8?q?=E5=A4=89=E6=8F=9B=E3=81=99=E3=82=8B=E5=87=A6=E7=90=86=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/converter/convert_strategy.py | 5 +- .../tests/converter/test_convert_strategy.py | 58 ++++++++++++++++++- .../tests/salesforce/test_salesforce.py | 1 + 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 3e092315..5f3daf9d 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -1,3 +1,4 @@ +import json import re from collections import OrderedDict from datetime import datetime @@ -85,6 +86,6 @@ class StringConvertStrategy: class DictConvertStrategy: - def convert_value(self, convert_value: str): + def convert_value(self, convert_value: dict): """dict型の項目を文字列に変換して返す処理""" - return convert_value + return json.dumps(convert_value, ensure_ascii=False) diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py index 77ecfea2..20d5f655 100644 --- a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -1,4 +1,4 @@ -from typing import OrderedDict +from collections import OrderedDict from src.converter.convert_strategy import (BooleanConvertStrategy, ConvertStrategyFactory, @@ -301,3 +301,59 @@ class TestStringConvertStrategy: # Expects assert actual == 'テストデータ' + + +class TestDictConvertStrategy: + + def test_convert_value_dict(self): + """ + Cases: + 引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DictConvertStrategy() + actual = sut.convert_value({'テストデータキー': 'テストデータバリュー'}) + + # Expects + assert actual == '{"テストデータキー": "テストデータバリュー"}' + + def test_convert_value_dict_in_line_break(self): + """ + Cases: + 引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること(バリューに改行を含む) + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DictConvertStrategy() + actual = sut.convert_value({'テストデータキー': 'テスト\nデータ\nバリュー'}) + + # Expects + assert actual == '{"テストデータキー": "テスト\\nデータ\\nバリュー"}' + + def test_convert_value_ordered_dict(self): + """ + Cases: + 引数に整列された辞書型のデータを指定した場合、JSONの文字列が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DictConvertStrategy() + actual = sut.convert_value(OrderedDict( + [('テストデータキー', 'テストデータバリュー')] + )) + + # Expects + assert actual == '{"テストデータキー": "テストデータバリュー"}' diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index 2b02a5cb..9547b97d 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -507,6 +507,7 @@ class TestSalesforceApiClient: 'Name', 'SystemModstamp', 'LastModifiedDate', + 'BillingAddress', 'CustomItem1__c', 'CustomItem2__c', 'CustomItem3__c', From 8353a535137a11afe32217a82abdf7a15c025120 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 23 Aug 2022 11:51:26 +0900 Subject: [PATCH 246/275] =?UTF-8?q?feat:=20=E5=A4=89=E6=8F=9B=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=81=AE=E5=8D=98=E4=BD=93=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=81=A7=E3=80=81JSON=E5=9E=8B=E3=81=AE=E3=83=87=E3=83=BC?= =?UTF-8?q?=E3=82=BF=E3=81=AE=E5=A4=89=E6=8F=9B=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test_convert_crm_csv_data_process.py | 47 +++++++++++++++---- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py index 49d960c0..9615d234 100644 --- a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py +++ b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py @@ -34,7 +34,17 @@ class TestConvertCrmCsvDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', True) + ('IsDeleted', True), + ('PersonMailingAddress', OrderedDict([ + ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 + ('PersonMailingCity', 'New york city'), + ('PersonMailingState', 'Ohaio'), + ('PersonMailingPostalCode', '999-9999'), + ('PersonMailingCountry', 'US'), + ('PersonMailingLatitude', 50.1234567), + ('PersonMailingLongitude', 103.1234567), + ('PersonMailingGeocodeAccuracy', 'Address'), + ])), ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -44,7 +54,17 @@ class TestConvertCrmCsvDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('PersonMailingAddress', OrderedDict([ + ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 + ('PersonMailingCity', 'New york city'), + ('PersonMailingState', 'Ohaio'), + ('PersonMailingPostalCode', '999-9999'), + ('PersonMailingCountry', 'US'), + ('PersonMailingLatitude', 50.1234567), + ('PersonMailingLongitude', 103.1234567), + ('PersonMailingGeocodeAccuracy', 'Address'), + ])), ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -54,7 +74,17 @@ class TestConvertCrmCsvDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('PersonMailingAddress', OrderedDict([ + ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 + ('PersonMailingCity', 'New york city'), + ('PersonMailingState', 'Ohaio'), + ('PersonMailingPostalCode', '999-9999'), + ('PersonMailingCountry', 'US'), + ('PersonMailingLatitude', 50.1234567), + ('PersonMailingLongitude', 103.1234567), + ('PersonMailingGeocodeAccuracy', 'Address'), + ])), ]), ] @@ -66,7 +96,8 @@ class TestConvertCrmCsvDataProcess: 'LastModifiedDate', 'LastModifiedById', 'SystemModstamp', - 'IsDeleted' + 'IsDeleted', + 'PersonMailingAddress' ] } @@ -79,10 +110,10 @@ class TestConvertCrmCsvDataProcess: # Assert expect_csv_string = """\ - "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\ - "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\ - "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ - "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ + "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted","PersonMailingAddress"\r\n\ + "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ + "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ + "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ """ # 返り値の期待値チェック assert isinstance(actual_csv_string, str), 'CSV文字列が返却される' From 7a194b8585107421feaa8f07c2023859c2162213 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 23 Aug 2022 11:51:26 +0900 Subject: [PATCH 247/275] =?UTF-8?q?feat:=20=E4=BD=8F=E6=89=80=E9=A0=85?= =?UTF-8?q?=E7=9B=AE=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesforce.py | 36 +++++++++++-------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index 9547b97d..c06ce546 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -428,12 +428,10 @@ class TestSalesforceApiClient: """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること - - SystemModStampのFromが2000年1月1日以降のレコードが取得できること - - SystemModStampのToが2100年12月31日未満のレコードが取得できること + - できること Arranges: - - SalesforceのAccountオブジェクトに、レコードを作成する - - LastFetchDatetimeのFromに2000年1月1日を指定する - - LastFetchDatetimeのToに2100年12月31日を指定する + - SalesforceのAccountオブジェクトに、住所項目を持つレコードを作成する + - 住所項目を持つレコードだけが取れるよう日付を設定する Expects: 取得できたオブジェクト件数が1件になる 住所項目(BillingAddress)が想定通りの値になっていること @@ -441,8 +439,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-19T05:53:29.000Z', - 'last_fetch_datetime_to': '2022-08-19T05:53:30.000Z', + 'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z', + 'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -451,15 +449,15 @@ class TestSalesforceApiClient: 'Name', 'SystemModstamp', 'LastModifiedDate', + 'BillingStreet', + 'BillingCity', + 'BillingState', + 'BillingPostalCode', + 'BillingCountry', + 'BillingLatitude', + 'BillingLongitude', + 'BillingGeocodeAccuracy', 'BillingAddress', - 'CustomItem1__c', - 'CustomItem2__c', - 'CustomItem3__c', - 'CustomItem4__c', - 'CustomItem5__c', - 'CustomItem6__c', - 'CustomItem7__c', - 'CustomItem8__c' ] }, execute_datetime) soql_builder = SOQLBuilder(target_object, last_fetch_datetime) @@ -480,6 +478,14 @@ class TestSalesforceApiClient: ]) assert actual[0]['BillingAddress'] == expect_address + assert actual[0]['BillingCity'] == '〇〇区' + assert actual[0]['BillingCountry'] == '日本' + assert actual[0]['BillingGeocodeAccuracy'] is None + assert actual[0]['BillingLatitude'] is None + assert actual[0]['BillingLongitude'] is None + assert actual[0]['BillingPostalCode'] == '999-9999' + assert actual[0]['BillingState'] == '東京都' + assert actual[0]['BillingStreet'] == '△△-✗✗' def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self): """ From 570d229c8367e7d399ef1c8557a564a56bcd1e14 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 23 Aug 2022 11:51:27 +0900 Subject: [PATCH 248/275] =?UTF-8?q?fix:=20JSON=E5=86=85=E3=81=AE=E6=97=A5?= =?UTF-8?q?=E6=9C=AC=E8=AA=9E=E3=81=8CHex=E3=82=B3=E3=83=BC=E3=83=89?= =?UTF-8?q?=E5=80=A4=E3=81=A7=E5=87=BA=E5=8A=9B=E3=81=95=E3=82=8C=E3=81=A6?= =?UTF-8?q?=E3=81=84=E3=82=8B=E5=95=8F=E9=A1=8C=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/aws/s3.py | 4 ++-- .../tests/test_backup_crm_data_process.py | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index d5bd03f0..7b63861f 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -87,7 +87,7 @@ class BackupBucket: def put_response_json(self, file_path: str, data: dict) -> None: object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_path}' - self.__s3_resource.put_object(object_key, json.dumps(data)) + self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False)) return def put_csv(self, file_path: str, data: str) -> None: @@ -97,5 +97,5 @@ class BackupBucket: def put_result_json(self, file_path: str, data: dict) -> None: object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}' - self.__s3_resource.put_object(object_key, json.dumps(data)) + self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False)) return diff --git a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py index 9ad35d52..4e48c419 100644 --- a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py @@ -44,7 +44,8 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('Name', 'テスト取引先1') ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -54,7 +55,8 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('Name', 'テスト取引先2') ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -64,7 +66,8 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('Name', 'テスト取引先3') ]), ] @@ -96,7 +99,7 @@ class TestBackupCrmDataProcess: actual = s3_client.get_object( Bucket=bucket_name, Key=f'response_json/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.json') - assert actual['Body'].read().decode('utf-8') == json.dumps(response_json) + assert actual['Body'].read().decode('utf-8') == json.dumps(response_json, ensure_ascii=False) # ログの確認 assert generate_log_message_tuple( From 062ac4daa8e3f9add3b85bbd12ecccee9eb46c1c Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 23 Aug 2022 11:54:43 +0900 Subject: [PATCH 249/275] =?UTF-8?q?fix:=20=E6=8C=87=E6=95=B0=E8=A1=A8?= =?UTF-8?q?=E8=A8=98=E3=81=AE=E5=A4=89=E6=8F=9B=E3=82=92=E8=A1=8C=E3=81=A3?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=9F=E3=81=93=E3=81=A8=E3=81=AB=E3=82=88?= =?UTF-8?q?=E3=82=8A=E3=80=81=E4=BB=96=E3=81=AE=E5=B0=91=E6=95=B0=E9=A0=85?= =?UTF-8?q?=E7=9B=AE=E3=81=8C=E6=95=B4=E6=95=B0=E3=81=AB=E3=81=AA=E3=81=A3?= =?UTF-8?q?=E3=81=A6=E3=81=97=E3=81=BE=E3=81=A3=E3=81=A6=E3=81=84=E3=81=9F?= =?UTF-8?q?=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/converter/convert_strategy.py | 10 ----- .../tests/converter/test_convert_strategy.py | 44 +++++++++---------- .../tests/converter/test_converter.py | 10 ++--- .../test_convert_crm_csv_data_process.py | 10 ++--- 4 files changed, 31 insertions(+), 43 deletions(-) diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 22fca8fc..ab1e0626 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -12,7 +12,6 @@ from src.system_var.environments import CONVERT_TZ class ConvertStrategyFactory: def __init__(self) -> None: self.__none_value_convert_strategy = NoneValueConvertStrategy() - self.__float_convert_strategy = FloatConvertStrategy() self.__boolean_convert_strategy = BooleanConvertStrategy() self.__datetime_convert_strategy = DatetimeConvertStrategy() self.__int_convert_strategy = IntConvertStrategy() @@ -23,9 +22,6 @@ class ConvertStrategyFactory: if value is None: convert_strategy = self.__none_value_convert_strategy - elif type(value) == float: - convert_strategy = self.__float_convert_strategy - elif type(value) == bool: convert_strategy = self.__boolean_convert_strategy @@ -60,12 +56,6 @@ class DatetimeConvertStrategy: return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS) -class FloatConvertStrategy: - def convert_value(self, convert_value: str) -> int: - """float型をint型に変換する処理""" - return int(convert_value) - - class IntConvertStrategy: def convert_value(self, convert_value: int): """int型を変換せずに返す処理""" diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py index 0e268fc4..4b4552a8 100644 --- a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -1,7 +1,6 @@ from src.converter.convert_strategy import (BooleanConvertStrategy, ConvertStrategyFactory, DatetimeConvertStrategy, - FloatConvertStrategy, IntConvertStrategy, NoneValueConvertStrategy, StringConvertStrategy) @@ -29,7 +28,7 @@ class TestConvertStrategyFactory: def test_create_float(self): """ Cases: - 引数に指数表記を指定した場合、FloatConvertStrategyインスタンスが返ってくること + 引数に指数表記を指定した場合、StringConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -41,7 +40,26 @@ class TestConvertStrategyFactory: actual = sut.create(1.2345678E7) # Expects - assert type(actual) == FloatConvertStrategy + # 変換しない + assert type(actual) == StringConvertStrategy + + def test_create_float_scale(self): + """ + Cases: + 引数に少数を指定した場合、StringConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(1.2345678E7) + + # Expects + # 変換しない + assert type(actual) == StringConvertStrategy def test_create_bool_true(self): """ @@ -206,26 +224,6 @@ class TestDatetimeConvertStrategy: assert actual == "2022-06-14 05:15:32" -class TestFloatConvertStrategy: - - def test_convert_value(self) -> int: - """ - Cases: - 引数に指数表記を指定した場合、整数で返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = FloatConvertStrategy() - actual = sut.convert_value(1.2345678E7) - - # Expects - assert actual == 12345678 - - class TestIntConvertStrategy: def test_convert_value(self): diff --git a/ecs/crm-datafetch/tests/converter/test_converter.py b/ecs/crm-datafetch/tests/converter/test_converter.py index 0300381d..da80982b 100644 --- a/ecs/crm-datafetch/tests/converter/test_converter.py +++ b/ecs/crm-datafetch/tests/converter/test_converter.py @@ -71,7 +71,7 @@ class TestCSVStringConverter: ('ContactAccessLevel', 8), ('RowCause', 'テストのため2'), ('LastModifiedDate', '2022-06-02T16:30:30.000+0000'), - ('LastModifiedById', 2.234567E+6), + ('LastModifiedById', 2.23E+0), ('IsDeleted', True) ]), OrderedDict([ @@ -85,7 +85,7 @@ class TestCSVStringConverter: ('ContactAccessLevel', 12), ('RowCause', 'テストのため3'), ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), - ('LastModifiedById', 3.234567E+6), + ('LastModifiedById', 3.234567), ('IsDeleted', False) ]) ] @@ -100,9 +100,9 @@ class TestCSVStringConverter: # Expects expected_value = '''\ "Id","AccountId","UserOrGroupId","AccountAccessLevel","OpportunityAccessLevel","CaseAccessLevel","ContactAccessLevel","RowCause","LastModifiedDate","LastModifiedById","IsDeleted"\r\n\ - "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567","0"\r\n\ - "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2234567","1"\r\n\ - "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3234567","0"\r\n\ + "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567.0","0"\r\n\ + "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2.23","1"\r\n\ + "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3.234567","0"\r\n\ ''' # expected_valueのインデントが半角スペースと認識されてしまうため、`textwrap.dedent`にて補正 diff --git a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py index 49d960c0..6d1b63f1 100644 --- a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py +++ b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py @@ -42,7 +42,7 @@ class TestConvertCrmCsvDataProcess: ('Id', 'TEST002'), ('AccountNumber', 'test002'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), - ('LastModifiedById', 1.234567E+6), + ('LastModifiedById', 1.23E+0), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), ('IsDeleted', False) ]), @@ -52,7 +52,7 @@ class TestConvertCrmCsvDataProcess: ('Id', 'TEST003'), ('AccountNumber', 'test003'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), - ('LastModifiedById', 1.234567E+6), + ('LastModifiedById', 1.234567), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), ('IsDeleted', False) ]), @@ -80,9 +80,9 @@ class TestConvertCrmCsvDataProcess: expect_csv_string = """\ "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\ - "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\ - "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ - "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ + "TEST001","test001","2022-06-01 09:00:00","1234567.0","2022-06-01 09:00:00","1"\r\n\ + "TEST002","test002","2022-06-01 09:00:00","1.23","2022-06-01 09:00:00","0"\r\n\ + "TEST003","test003","2022-06-01 09:00:00","1.234567","2022-06-01 09:00:00","0"\r\n\ """ # 返り値の期待値チェック assert isinstance(actual_csv_string, str), 'CSV文字列が返却される' From 5841af2bbed8731147ba293182a871ba7cf47ae4 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 23 Aug 2022 11:54:43 +0900 Subject: [PATCH 250/275] =?UTF-8?q?fix:=20salesforce=E3=81=AE=E3=83=87?= =?UTF-8?q?=E3=83=BC=E3=82=BF=E5=A4=89=E6=9B=B4=E3=81=AB=E4=BC=B4=E3=81=86?= =?UTF-8?q?=E4=BB=B6=E6=95=B0=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/salesforce/test_salesforce.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index b815e772..4a3667f5 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -160,7 +160,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得件数が16になる + 取得件数が17になる """ execute_datetime = ExecuteDateTime() @@ -190,7 +190,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_count(soql) - assert actual == 16 + assert actual == 17 def test_fetch_sf_data_one_record(self): """ @@ -433,7 +433,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得できたオブジェクト件数が16件になる + 取得できたオブジェクト件数が17件になる """ execute_datetime = ExecuteDateTime() @@ -463,7 +463,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_data(soql) - assert len(actual) == 16 + assert len(actual) == 17 # 内容の確認は別のケースで行っているため省略 def test_raise_create_instance_cause_auth_failed(self, monkeypatch): From 5c5d7cc0a68185e32d6b99f6c9fa7f5a521558a9 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 23 Aug 2022 13:06:21 +0900 Subject: [PATCH 251/275] =?UTF-8?q?fix:=20=E3=83=AC=E3=82=B3=E3=83=BC?= =?UTF-8?q?=E3=83=89=E3=82=92=E7=B7=A8=E9=9B=86=E3=81=97=E3=81=9F=E3=81=93?= =?UTF-8?q?=E3=81=A8=E3=81=AB=E3=82=88=E3=82=8A=E3=83=86=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=81=AE=E6=9C=9F=E5=BE=85=E5=80=A4=E3=81=8C=E3=81=9A=E3=82=8C?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=9F=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/salesforce/test_salesforce.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index c06ce546..64530071 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -35,8 +35,8 @@ class TestSalesforceApiClient: FROM Account WHERE - SystemModstamp > 2022-08-04T00:00:00.000Z AND - SystemModstamp <= 2022-08-06T00:00:00.000Z + SystemModstamp > 2022-08-23T01:56:39.000Z AND + SystemModstamp <= 2022-08-24T00:00:00.000Z """ sut = SalesforceApiClient() @@ -65,7 +65,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual >= 0 - def test_fetch_sf_count_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): + def test_fetch_sf_count_by_soql_builder_system_modstamp_to_ge(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること @@ -81,8 +81,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', - 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', + 'last_fetch_datetime_from': '2022-08-23T02:38:59.000Z', + 'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -108,7 +108,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual == 1 - def test_fetch_sf_count_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): + def test_fetch_sf_count_by_soql_builder_system_modstamp_to_lt(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること @@ -124,8 +124,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', - 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', + 'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z', + 'last_fetch_datetime_to': '2022-08-23T02:39:01.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -219,8 +219,8 @@ class TestSalesforceApiClient: FROM Account WHERE - SystemModstamp > 2022-08-04T00:00:00.000Z AND - SystemModstamp <= 2022-08-06T00:00:00.000Z + SystemModstamp > 2022-08-23T01:56:39.000Z AND + SystemModstamp <= 2022-08-24T00:00:00.000Z """ sut = SalesforceApiClient() @@ -286,7 +286,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_data(soql) assert len(actual) >= 0 - def test_fetch_sf_data_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): + def test_fetch_sf_data_by_soql_builder_system_modstamp_to_ge(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること @@ -302,8 +302,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', - 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', + 'last_fetch_datetime_from': '2022-08-23T03:48:39.000Z', + 'last_fetch_datetime_to': '2022-08-23T03:48:40.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -355,7 +355,7 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect - def test_fetch_sf_data_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): + def test_fetch_sf_data_by_soql_builder_system_modstamp_to_gt(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること @@ -363,16 +363,16 @@ class TestSalesforceApiClient: - SystemModStampのToが指定日付未満のレコードは取得できないこと Arranges: - SalesforceのAccountオブジェクトに、レコードを作成する - - LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定） - - LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定） + - LastFetchDatetimeのFromがSystemModstampより小さくなるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstampより大きくなるように指定する（UTC指定） Expects: 取得できたオブジェクト1件が期待値どおりであること """ execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', - 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', + 'last_fetch_datetime_from': '2022-08-23T03:42:24.000Z', + 'last_fetch_datetime_to': '2022-08-23T03:42:26.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', From 58318e22a649a7fac51007e301bcfdb92d15778f Mon Sep 17 00:00:00 2001 From: "shibata.r" Date: Tue, 23 Aug 2022 14:19:25 +0900 Subject: [PATCH 252/275] =?UTF-8?q?style:MSJ=5FRegistration=5F=5Fc?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt index d5d6c0dc..813f86ee 100644 --- a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt +++ b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt @@ -4,9 +4,9 @@ utf-8 " CRLF 1 -31 -Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c -id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c +32 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c,MSJ_Registration__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c,msj_registration__c src02.crm_event_attendee_vod__c org02.crm_event_attendee_vod__c From 4816cadf2aa3a891507c0e00b3be36f2a659a6a7 Mon Sep 17 00:00:00 2001 From: "shibata.r" Date: Tue, 23 Aug 2022 16:33:36 +0900 Subject: [PATCH 253/275] =?UTF-8?q?style:=E9=A0=85=E7=9B=AE=E3=82=92?= =?UTF-8?q?=E6=89=80=E5=BE=97=E3=81=97=E3=81=AA=E3=81=84=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../crm/object_info/crm_object_list_diff.json | 6015 ++++++++--------- s3/data/crm/settings/CRM_User.txt | 26 +- 2 files changed, 3019 insertions(+), 3022 deletions(-) diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index a5f8da67..ea846d62 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -1,3010 +1,3007 @@ -{ - "objects": [ - { - "object_name": "Clm_Presentation_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Mobile_ID_vod__c", - "Presentation_Id_vod__c", - "Product_vod__c", - "Default_Presentation_vod__c", - "Training_vod__c", - "ParentId_vod__c", - "Hidden_vod__c", - "Type_vod__c", - "Approved_vod__c", - "Copied_From_vod__c", - "Copy_Date_vod__c", - "Survey_vod__c", - "Original_Record_ID_vod__c", - "Directory_vod__c", - "End_Date_vod__c", - "Start_Date_vod__c", - "Status_vod__c", - "VExternal_Id_vod__c", - "Vault_DNS_vod__c", - "Vault_Doc_Id_vod__c", - "Vault_External_Id_vod__c", - "Vault_GUID_vod__c", - "Vault_Last_Modified_Date_Time_vod__c", - "Version_vod__c", - "Enable_Survey_Overlay_vod__c", - "Description_vod__c", - "Keywords_vod__c", - "Content_Channel_vod__c", - "original_material_approved_in_veritas__c", - "keywords__c", - "trade_team__c", - "ewizard_link__c", - "business_function__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Clm_Presentation_Slide_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Clm_Presentation_vod__c", - "Key_Message_vod__c", - "Display_Order_vod__c", - "Sub_Presentation_vod__c", - "Mobile_ID_vod__c", - "External_ID_vod__c", - "VExternal_Id_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Medical_Insight_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_vod__c", - "Clinical_Trial_vod__c", - "Date_vod__c", - "Description_vod__c", - "Entity_Reference_Id_vod__c", - "Interaction_vod__c", - "Medical_Event_vod__c", - "Mobile_ID_vod__c", - "Other_Source_vod__c", - "Override_Lock_vod__c", - "Publication_vod__c", - "Status_vod__c", - "Summary_vod__c", - "Unlock_vod__c", - "Commercial_Medical__c", - "MSJ_Level_1A__c", - "MSJ_Level_1B__c", - "MSJ_Level_2A__c", - "MSJ_Level_2B__c", - "MSJ_Level_3A__c", - "MSJ_Level_3B__c", - "MSJ_Level_4A__c", - "MSJ_Level_4B__c", - "MSJ_SubStatus__c", - "MSJ_Type_A__c", - "MSJ_Type_B__c", - "MSJ_Description_Backup__c", - "MSJ_Country__c", - "MSJ_Received_at_Boomi__c", - "MSJ_Level_1A_Value__c", - "MSJ_Level_1B_Value__c", - "MSJ_Level_2A_Value__c", - "MSJ_Level_2B_Value__c", - "MSJ_Level_3A_Value__c", - "MSJ_Level_3B_Value__c", - "MSJ_Level_4A_Value__c", - "MSJ_Level_4B_Value__c", - "MSJ_Hospital_ID__c", - "MSJ_Hospital_Name__c", - "MSJ_Hospital__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "MSJ_MR_Weekly_Report__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "MSJ_Account_Name__c", - "MSJ_Activity_Results_Summary__c", - "MSJ_MUID__c", - "MSJ_Next_Week_Action_What__c", - "MSJ_Next_Week_Action_When__c", - "MSJ_Next_Week_Action_Where__c", - "MSJ_Next_Week_Action_Who__c", - "MSJ_Report_Week__c", - "MSJ_Target_Patient_Count__c", - "Mobile_ID_vod__c", - "MSJ_Activity_Results_Summary_HN__c", - "MSJ_Next_Week_Action_Where_HN__c", - "MSJ_Next_Week_Action_Who_HN__c", - "MSJ_Next_Week_Action_What_HN__c", - "MSJ_Next_Week_Action_When_HN__c", - "MSJ_Target_Patient_Count_HN__c", - "MSJ_Activity_Results_Summary_MCC__c", - "MSJ_Next_Week_Action_Where_MCC__c", - "MSJ_Next_Week_Action_Who_MCC__c", - "MSJ_Next_Week_Action_What_MCC__c", - "MSJ_Next_Week_Action_When_MCC__c", - "MSJ_Target_Patient_Count_MCC__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Account_Territory_Loader_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_vod__c", - "External_ID_vod__c", - "Territory_vod__c", - "Mobile_ID_vod__c", - "Territory_To_Add_vod__c", - "Territory_to_Drop_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Event_Attendee_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Attendee_vod__c", - "User_vod__c", - "Medical_Event_vod__c", - "Attendee_Type_vod__c", - "Status_vod__c", - "Contact_vod__c", - "Attendee_Name_vod__c", - "Account_vod__c", - "Start_Date_vod__c", - "Signature_vod__c", - "Signature_Datetime_vod__c", - "MSJ_Copy_Account_Type__c", - "MSJ_Evaluation__c", - "MSJ_Hospital__c", - "MSJ_Role__c", - "Mobile_ID_vod__c", - "MSJ_Evaluation_Comment__c", - "Position_vod__c", - "Talk_Title_vod__c", - "MSJ_Attendee_Reaction__c", - "MSJ_Registration__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "ObjectTerritory2Association", - "columns": [ - "Id", - "ObjectId", - "Territory2Id", - "AssociationCause", - "SobjectType", - "IsDeleted", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Key_Message_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Description_vod__c", - "Product_vod__c", - "Product_Strategy_vod__c", - "Display_Order_vod__c", - "Active_vod__c", - "Category_vod__c", - "Vehicle_vod__c", - "CLM_ID_vod__c", - "Custom_Reaction_vod__c", - "Slide_Version_vod__c", - "Language_vod__c", - "Media_File_CRC_vod__c", - "Media_File_Name_vod__c", - "Media_File_Size_vod__c", - "Segment_vod__c", - "Disable_Actions_vod__c", - "VExternal_Id_vod__c", - "CDN_Path_vod__c", - "Status_vod__c", - "Vault_DNS_vod__c", - "Vault_Doc_Id_vod__c", - "Vault_External_Id_vod__c", - "Vault_GUID_vod__c", - "Vault_Last_Modified_Date_Time_vod__c", - "Is_Shared_Resource_vod__c", - "iOS_Viewer_vod__c", - "iOS_Resolution_vod__c", - "approved_for_distribution_date__c", - "approved_for_use_date__c", - "ewizard_link__c", - "expiration_date__c", - "keywords__c", - "trade_team__c", - "business_function__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Group", - "columns": [ - "Id", - "Name", - "DeveloperName", - "RelatedId", - "Type", - "Email", - "OwnerId", - "DoesSendEmailToMembers", - "DoesIncludeBosses", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Medical_Event_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Primary_Product__c", - "Description_vod__c", - "Start_Date_vod__c", - "Location__c", - "End_Date_vod__c", - "Secondary_Product__c", - "Website__c", - "Active_vod__c", - "Event_Type__c", - "MSJ_Area__c", - "MSJ_Business_Unit__c", - "MSJ_Comment__c", - "MSJ_Company__c", - "MSJ_Expense_App_No__c", - "MSJ_Form__c", - "MSJ_HQ_Area__c", - "MSJ_Location__c", - "MSJ_MR__c", - "MSJ_Number_of_Attendee__c", - "MSJ_Product__c", - "MSJ_Site__c", - "MSJ_Type__c", - "MSJ_Number_of_Attendee_Auto_Calc__c", - "MSJ_Number_of_Attendee_Invited__c", - "Account_vod__c", - "MSJ_MUID__c", - "Country_Name_vod__c", - "MSJ_CE_SIPAGL_Updater__c", - "MSJ_CE_SIPAGL_1A__c", - "MSJ_CE_SIPAGL_1B__c", - "MSJ_CE_SIPAGL_2__c", - "MSJ_CE_SIPAGL_3__c", - "MSJ_CE_SIPAGL_4__c", - "MSJ_CE_SIPAGL_5__c", - "MSJ_CE_SIPAGL_Comment__c", - "MSJ_CE_SIPAGL_1A_date__c", - "MSJ_CE_SIPAGL_6__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "MSJ_Medical_Event_Evaluation__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "MSJ_Medical_Event__c", - "MSJ_Evaluation_Comment__c", - "MSJ_Evaluation__c", - "Mobile_ID_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Coaching_Report_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Mobile_ID_vod__c", - "Manager_vod__c", - "Employee_vod__c", - "Review_Date__c", - "Review_Period__c", - "Status__c", - "Comments__c", - "Strategic_Planning__c", - "Customer_Focus__c", - "Knowledge_Expertise__c", - "Business_Account_Planning__c", - "Call_Productivity__c", - "Overall_Rating__c", - "MSJ_A01__c", - "MSJ_A02__c", - "MSJ_A03__c", - "MSJ_AM_Memo__c", - "MSJ_Aid_Total__c", - "MSJ_C0_GC__c", - "MSJ_C1_GC__c", - "MSJ_C2_GC__c", - "MSJ_Countermeasure__c", - "MSJ_Deadline__c", - "MSJ_Double_Visit_Time__c", - "MSJ_Hospital__c", - "MSJ_K01_FE__c", - "MSJ_K01_ONC__c", - "MSJ_K02_FE__c", - "MSJ_K02_ONC__c", - "MSJ_K03_FE__c", - "MSJ_K03_ONC__c", - "MSJ_K04_FE__c", - "MSJ_K04_ONC__c", - "MSJ_K05_FE__c", - "MSJ_K05_ONC__c", - "MSJ_K06_FE__c", - "MSJ_K06_ONC__c", - "MSJ_K0_GC__c", - "MSJ_K1_GC__c", - "MSJ_K2_GC__c", - "MSJ_Knowledge_Total__c", - "MSJ_L0_GC__c", - "MSJ_L1_GC__c", - "MSJ_L2_GC__c", - "MSJ_MR_GC__c", - "MSJ_MR_Problems__c", - "MSJ_N0_GC__c", - "MSJ_N1_GC__c", - "MSJ_N2_GC__c", - "MSJ_Num_of_DTL__c", - "MSJ_P01__c", - "MSJ_P02__c", - "MSJ_P03__c", - "MSJ_P04__c", - "MSJ_P05__c", - "MSJ_P0_GC__c", - "MSJ_P1_GC__c", - "MSJ_P2_GC__c", - "MSJ_PlanningTotal__c", - "MSJ_R0_GC__c", - "MSJ_R1_GC__c", - "MSJ_R2_GC__c", - "MSJ_S01__c", - "MSJ_S02__c", - "MSJ_S03__c", - "MSJ_S04__c", - "MSJ_S05__c", - "MSJ_S06__c", - "MSJ_S07__c", - "MSJ_S08__c", - "MSJ_S09__c", - "MSJ_S10__c", - "MSJ_S11__c", - "MSJ_S12__c", - "MSJ_Skill_Total__c", - "MSJ_After_Call_01__c", - "MSJ_After_Call_02__c", - "MSJ_After_Call_03__c", - "MSJ_After_Call_04__c", - "MSJ_Closing__c", - "MSJ_Comment_by_MR__c", - "MSJ_Confirmed_by_MR__c", - "MSJ_Createdby__c", - "MSJ_FT_AM_Name__c", - "MSJ_Interview_Preparation__c", - "MSJ_Interview_Reflection__c", - "MSJ_Notify_To_MR__c", - "MSJ_Opening__c", - "MSJ_Others_01_Result__c", - "MSJ_Others_01__c", - "MSJ_Others_02_Result__c", - "MSJ_Others_02__c", - "MSJ_Patient_Thinking__c", - "MSJ_Probing__c", - "MSJ_Supporting__c", - "MSJ_Patient_Thinking_for_FE__c", - "MSJ_After_Call_05__c", - "MSJ_After_Call_06__c", - "MSJ_After_Call_07__c", - "MSJ_After_Call_08__c", - "MSJ_Createdby_FE__c", - "MSJ_Createdby_ONC__c", - "MSJ_Development_Level__c", - "MSJ_Interview_Prep_01__c", - "MSJ_Interview_Prep_02__c", - "MSJ_Leadership_Style__c", - "MSJ_Overcome_01__c", - "MSJ_Overcome_02__c", - "MSJ_Overcome_03__c", - "MSJ_Overcome_04__c", - "MSJ_Review_01__c", - "MSJ_Review_02__c", - "MSJ_SK_01__c", - "MSJ_SK_02__c", - "MSJ_SK_03__c", - "MSJ_SK_04__c", - "MSJ_SK_05__c", - "MSJ_SK_06__c", - "MSJ_SK_07__c", - "MSJ_SK_08__c", - "MSJ_SK_09__c", - "MSJ_SK_10__c", - "MSJ_Specific_Action__c", - "MSJ_Training_Point__c", - "MSJ_Efforts_of_Year__c", - "MSJ_Efforts_of_Month__c", - "MSJ_Skill_Task__c", - "MSJ_Action_of_This_Month__c", - "MSJ_Achievement_of_This_Month__c", - "MSJ_Comment_from_AM__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Call2_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Call_Comments_vod__c", - "Sample_Card_vod__c", - "Add_Detail_vod__c", - "Property_vod__c", - "Account_vod__c", - "zvod_Product_Discussion_vod__c", - "Status_vod__c", - "Parent_Address_vod__c", - "Account_Plan_vod__c", - "zvod_SaveNew_vod__c", - "Next_Call_Notes_vod__c", - "Pre_Call_Notes_vod__c", - "Mobile_ID_vod__c", - "zvod_Account_Credentials_vod_c_vod__c", - "zvod_Account_Preferred_Name_vod_c_vod__c", - "zvod_Account_Sample_Status_vod_c_vod__c", - "zvod_Attendees_vod__c", - "zvod_Key_Messages_vod__c", - "zvod_Detailing_vod__c", - "zvod_Expenses_vod__c", - "zvod_Followup_vod__c", - "zvod_Samples_vod__c", - "zvod_Save_vod__c", - "zvod_Submit_vod__c", - "zvod_Delete_vod__c", - "Activity_Type__c", - "Significant_Event__c", - "Location_vod__c", - "Subject_vod__c", - "Unlock_vod__c", - "Call_Datetime_vod__c", - "Disbursed_To_vod__c", - "Disclaimer_vod__c", - "Request_Receipt_vod__c", - "Signature_Date_vod__c", - "Signature_vod__c", - "Territory_vod__c", - "Submitted_By_Mobile_vod__c", - "Call_Type_vod__c", - "Add_Key_Message_vod__c", - "Address_vod__c", - "Attendees_vod__c", - "Attendee_Type_vod__c", - "Call_Date_vod__c", - "Detailed_Products_vod__c", - "No_Disbursement_vod__c", - "Parent_Call_vod__c", - "User_vod__c", - "Contact_vod__c", - "zvod_Entity_vod__c", - "Medical_Event_vod__c", - "Mobile_Created_Datetime_vod__c", - "Mobile_Last_Modified_Datetime_vod__c", - "License_vod__c", - "Is_Parent_Call_vod__c", - "Entity_Display_Name_vod__c", - "Override_Lock_vod__c", - "Last_Device_vod__c", - "Ship_Address_Line_1_vod__c", - "Ship_Address_Line_2_vod__c", - "Ship_City_vod__c", - "Ship_Country_vod__c", - "Ship_License_Expiration_Date_vod__c", - "Ship_License_Status_vod__c", - "Ship_License_vod__c", - "Ship_State_vod__c", - "Ship_To_Address_vod__c", - "Ship_Zip_vod__c", - "Ship_To_Address_Text_vod__c", - "CLM_vod__c", - "zvod_CLMDetails_vod__c", - "Is_Sampled_Call_vod__c", - "zvod_Surveys_vod__c", - "Presentations_vod__c", - "Entity_Reference_Id_vod__c", - "Error_Reference_Call_vod__c", - "Duration_vod__c", - "Color_vod__c", - "Allowed_Products_vod__c", - "zvod_Attachments_vod__c", - "Sample_Card_Reason_vod__c", - "ASSMCA_vod__c", - "Address_Line_1_vod__c", - "Address_Line_2_vod__c", - "City_vod__c", - "DEA_Address_Line_1_vod__c", - "DEA_Address_Line_2_vod__c", - "DEA_Address_vod__c", - "DEA_City_vod__c", - "DEA_Expiration_Date_vod__c", - "DEA_State_vod__c", - "DEA_Zip_4_vod__c", - "DEA_Zip_vod__c", - "DEA_vod__c", - "Ship_Zip_4_vod__c", - "State_vod__c", - "Zip_4_vod__c", - "Zip_vod__c", - "Sample_Send_Card_vod__c", - "zvod_Address_vod_c_DEA_Status_vod_c_vod__c", - "Signature_Page_Image_vod__c", - "Credentials_vod__c", - "Salutation_vod__c", - "zvod_Account_Call_Reminder_vod_c_vod__c", - "MSJ_Meeting_Duration__c", - "MSJ_Double_Visit_AM__c", - "zvod_Business_Account_vod__c", - "Product_Priority_1_vod__c", - "Product_Priority_2_vod__c", - "Product_Priority_3_vod__c", - "Product_Priority_4_vod__c", - "Product_Priority_5_vod__c", - "zvod_More_Actions_vod__c", - "zvod_Call_Conflict_Status_vod__c", - "Signature_Timestamp_vod__c", - "Expense_Amount_vod__c", - "Total_Expense_Attendees_Count_vod__c", - "Attendee_list_vod__c", - "Expense_Post_Status_vod__c", - "Attendee_Post_Status_vod__c", - "Expense_System_External_ID_vod__c", - "Incurred_Expense_vod__c", - "Assigner_vod__c", - "Assignment_Datetime_vod__c", - "zvod_Call_Objective_vod__c", - "Signature_Location_Longitude_vod__c", - "Signature_Location_Latitude_vod__c", - "Location_Services_Status_vod__c", - "MSJ_Double_Visit_Other__c", - "MSJ_Comment__c", - "MSJ_For_Reporting__c", - "MSJ_Number_of_Attendees__c", - "MSJ_Main_Dept__c", - "Planned_Type_vjh__c", - "Cobrowse_URL_Participant_vod__c", - "MSJ_Activity_Method_Text__c", - "MSJ_Activity_Method__c", - "MSJ_Classification__c", - "MSJ_Double_Visit_MSL__c", - "MSJ_MSL_Comment_for_MR__c", - "MSJ_APD__c", - "Medical_Inquiry_vod__c", - "MSJ_Call_Type_MSJ__c", - "MSJ_Prescription_Request__c", - "MSJ_Patient_Follow__c", - "Child_Account_Id_vod__c", - "Child_Account_vod__c", - "Location_Id_vod__c", - "Location_Name_vod__c", - "MSJ_Comments_about_technology__c", - "Remote_Meeting_vod__c", - "Veeva_Remote_Meeting_Id_vod__c", - "MSJ_Activity_Type_Report__c", - "MSJ_Activity_Type__c", - "MSJ_Activity__c", - "MSJ_Comments__c", - "MSJ_Therapy__c", - "MSJ_Time_Hrs__c", - "EMDS_CO_Reference__c", - "EMDS_Call_Sub_Type__c", - "EMDS_Call_Type__c", - "EMDS_Call_Unsuccessful__c", - "EMDS_Congress_Type__c", - "EMDS_Date_of_Service__c", - "EMDS_Fertility_DisInterest__c", - "EMDS_Fertility_Interest__c", - "EMDS_Installed_Equipment__c", - "EMDS_Pipeline_Stage_Value__c", - "EMDS_Pipeline_Stage__c", - "EMDS_Pipeline__c", - "EMDS_Reason_for_Call__c", - "EMDS_Training_Completed__c", - "MSJ_BrainStorming__c", - "MSJ_SIPAGL_1A__c", - "MSJ_SIPAGL_1B__c", - "MSJ_SIPAGL_2__c", - "MSJ_SIPAGL_3__c", - "MSJ_SIPAGL_4A__c", - "MSJ_SIPAGL_5A__c", - "MSJ_SIPAGL_comment__c", - "MSJ_SIPAGL_4B__c", - "MSJ_SIPAGL_5B__c", - "Location_Text_vod__c", - "Call_Channel_vod__c", - "MSJ_Scientific_Interaction__c", - "MSJ_Activity_Email_Reply__c", - "MSJ_Interaction_Duration__c", - "MSJ_SIPAGL_1A_date__c", - "MSJ_CoPromotion__c", - "Call_Channel_Formula_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Call2_Detail_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Is_Parent_Call_vod__c", - "Call2_vod__c", - "Product_vod__c", - "Detail_Priority_vod__c", - "Mobile_ID_vod__c", - "Override_Lock_vod__c", - "Type_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Call2_Key_Message_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Account_vod__c", - "Call2_vod__c", - "Reaction_vod__c", - "Product_vod__c", - "Key_Message_vod__c", - "Mobile_ID_vod__c", - "Contact_vod__c", - "Call_Date_vod__c", - "User_vod__c", - "Category_vod__c", - "Vehicle_vod__c", - "Is_Parent_Call_vod__c", - "Override_Lock_vod__c", - "CLM_ID_vod__c", - "Slide_Version_vod__c", - "Duration_vod__c", - "Presentation_ID_vod__c", - "Start_Time_vod__c", - "Attendee_Type_vod__c", - "Entity_Reference_Id_vod__c", - "Segment_vod__c", - "Display_Order_vod__c", - "Clm_Presentation_Name_vod__c", - "Clm_Presentation_Version_vod__c", - "Clm_Presentation_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Call_Clickstream_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Answer_vod__c", - "Call_vod__c", - "Key_Message_vod__c", - "Mobile_ID_vod__c", - "Popup_Opened_vod__c", - "Possible_Answers_vod__c", - "Presentation_ID_vod__c", - "Product_vod__c", - "Range_Value_vod__c", - "Rollover_Entered_vod__c", - "Selected_Items_vod__c", - "CLM_ID_vod__c", - "Question_vod__c", - "Survey_Type_vod__c", - "Text_Entered_vod__c", - "Toggle_Button_On_vod__c", - "Track_Element_Description_vod__c", - "Track_Element_Id_vod__c", - "Track_Element_Type_vod__c", - "Usage_Duration_vod__c", - "Usage_Start_Time_vod__c", - "AuxillaryId_vod__c", - "ParentId_vod__c", - "Revision_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Call2_Discussion_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Account_vod__c", - "Call2_vod__c", - "Activity__c", - "Comments__c", - "Contact_vod__c", - "Call_Date_vod__c", - "Product_Strategy_vod__c", - "Product_Tactic_vod__c", - "Restricted_Comments__c", - "Product_vod__c", - "Presentation__c", - "Discussion_Topics__c", - "Slides__c", - "User_vod__c", - "Indication__c", - "Mobile_ID_vod__c", - "Medical_Event_vod__c", - "Is_Parent_Call_vod__c", - "Override_Lock_vod__c", - "zvod_Product_Map_vod__c", - "Attendee_Type_vod__c", - "Entity_Reference_Id_vod__c", - "Account_Tactic_vod__c", - "MSJ_Material_Type__c", - "MSJ_Discussion_Contents__c", - "MSJ_IST_Minutes__c", - "MSJ_Off_Label_Minutes__c", - "MSJ_Discussion_Objectives__c", - "MSJ_Insight__c", - "EMDS_Materials__c", - "EMDS_Topic__c", - "MSJ_Visit_Purpose__c", - "MSJ_Insight_Count__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Time_Off_Territory_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Reason_vod__c", - "Territory_vod__c", - "Date_vod__c", - "Status_vod__c", - "Time_vod__c", - "Hours_vod__c", - "Mobile_ID_vod__c", - "Hours_off_vod__c", - "Start_Time_vod__c", - "MSJ_Day__c", - "MSJ_Comment__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Dynamic_Attribute_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Account_vod__c", - "Active_vod__c", - "Dynamic_Attribute_Configuration_vod__c", - "Dynamic_Attribute_Description_vod__c", - "Dynamic_Attribute_Help_Text_vod__c", - "Dynamic_Attribute_Label_vod__c", - "Dynamic_Attribute_Name_vod__c", - "Dynamic_Attribute_Record_Type_vod__c", - "Dynamic_Attribute_Value_Checkbox_vod__c", - "Dynamic_Attribute_Value_Date_Time_vod__c", - "Dynamic_Attribute_Value_Date_vod__c", - "Dynamic_Attribute_Value_Number_vod__c", - "Dynamic_Attribute_Value_Text_Area_vod__c", - "Dynamic_Attribute_Value_Text_vod__c", - "Mobile_ID_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Dynamic_Attribute_Configuration_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Applies_To_vod__c", - "Attribute_Label_vod__c", - "Attribute_Name_vod__c", - "Available_Values_vod__c", - "Description_vod__c", - "Detail_Group_vod__c", - "Display_Order_vod__c", - "External_ID_vod__c", - "Help_Text_vod__c", - "Product_vod__c", - "Read_Only_vod__c", - "Section_Name_vod__c", - "Sharing_Group_vod__c", - "Status_vod__c", - "Track_Changes_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Territory2", - "columns": [ - "Id", - "Name", - "Territory2TypeId", - "Territory2ModelId", - "ParentTerritory2Id", - "Description", - "ForecastUserId", - "AccountAccessLevel", - "OpportunityAccessLevel", - "CaseAccessLevel", - "ContactAccessLevel", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "DeveloperName", - "MSJ_Territory_Type__c", - "MSJ_Level__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Profile", - "columns": [ - "Id", - "Name", - "PermissionsEmailSingle", - "PermissionsEmailMass", - "PermissionsEditTask", - "PermissionsEditEvent", - "PermissionsExportReport", - "PermissionsImportPersonal", - "PermissionsDataExport", - "PermissionsManageUsers", - "PermissionsEditPublicFilters", - "PermissionsEditPublicTemplates", - "PermissionsModifyAllData", - "PermissionsManageCases", - "PermissionsManageSolutions", - "PermissionsCustomizeApplication", - "PermissionsEditReadonlyFields", - "PermissionsRunReports", - "PermissionsViewSetup", - "PermissionsTransferAnyEntity", - "PermissionsNewReportBuilder", - "PermissionsManageSelfService", - "PermissionsManageCssUsers", - "PermissionsActivateContract", - "PermissionsApproveContract", - "PermissionsImportLeads", - "PermissionsManageLeads", - "PermissionsTransferAnyLead", - "PermissionsViewAllData", - "PermissionsEditPublicDocuments", - "PermissionsViewEncryptedData", - "PermissionsEditBrandTemplates", - "PermissionsEditHtmlTemplates", - "PermissionsManageTranslation", - "PermissionsDeleteActivatedContract", - "PermissionsSendSitRequests", - "PermissionsApiUserOnly", - "PermissionsManageRemoteAccess", - "PermissionsCanUseNewDashboardBuilder", - "PermissionsManageCategories", - "PermissionsConvertLeads", - "PermissionsTestInstanceCreate", - "PermissionsPasswordNeverExpires", - "PermissionsUseTeamReassignWizards", - "PermissionsInstallMultiforce", - "PermissionsPublishMultiforce", - "PermissionsEditOppLineItemUnitPrice", - "PermissionsManageTerritories", - "PermissionsCreateMultiforce", - "PermissionsBulkApiHardDelete", - "PermissionsInboundMigrationToolsUser", - "PermissionsSolutionImport", - "PermissionsManageCallCenters", - "PermissionsManageSynonyms", - "PermissionsOutboundMigrationToolsUser", - "PermissionsViewContent", - "PermissionsManageEmailClientConfig", - "PermissionsEnableNotifications", - "PermissionsManageDataIntegrations", - "PermissionsDistributeFromPersWksp", - "PermissionsViewDataCategories", - "PermissionsManageDataCategories", - "PermissionsAuthorApex", - "PermissionsManageMobile", - "PermissionsApiEnabled", - "PermissionsManageCustomReportTypes", - "PermissionsEditCaseComments", - "PermissionsTransferAnyCase", - "PermissionsContentAdministrator", - "PermissionsCreateWorkspaces", - "PermissionsManageContentPermissions", - "PermissionsManageContentProperties", - "PermissionsManageContentTypes", - "PermissionsScheduleJob", - "PermissionsManageExchangeConfig", - "PermissionsManageAnalyticSnapshots", - "PermissionsScheduleReports", - "PermissionsManageBusinessHourHolidays", - "PermissionsManageDynamicDashboards", - "PermissionsManageInteraction", - "PermissionsViewMyTeamsDashboards", - "PermissionsResetPasswords", - "PermissionsFlowUFLRequired", - "PermissionsActivitiesAccess", - "PermissionsEmailTemplateManagement", - "PermissionsEmailAdministration", - "PermissionsChatterFileLink", - "PermissionsForceTwoFactor", - "PermissionsViewEventLogFiles", - "PermissionsManageNetworks", - "PermissionsManageAuthProviders", - "PermissionsRunFlow", - "PermissionsCreateCustomizeDashboards", - "PermissionsCreateDashboardFolders", - "PermissionsViewPublicDashboards", - "PermissionsManageDashbdsInPubFolders", - "PermissionsCreateCustomizeReports", - "PermissionsCreateReportFolders", - "PermissionsViewPublicReports", - "PermissionsManageReportsInPubFolders", - "PermissionsEditMyDashboards", - "PermissionsEditMyReports", - "PermissionsViewAllUsers", - "PermissionsConnectOrgToEnvironmentHub", - "PermissionsCreateCustomizeFilters", - "PermissionsContentHubUser", - "PermissionsGovernNetworks", - "PermissionsSalesConsole", - "PermissionsTwoFactorApi", - "PermissionsDeleteTopics", - "PermissionsEditTopics", - "PermissionsCreateTopics", - "PermissionsAssignTopics", - "PermissionsIdentityEnabled", - "PermissionsIdentityConnect", - "PermissionsContentWorkspaces", - "PermissionsCustomMobileAppsAccess", - "PermissionsViewHelpLink", - "PermissionsManageProfilesPermissionsets", - "PermissionsAssignPermissionSets", - "PermissionsManageRoles", - "PermissionsManageIpAddresses", - "PermissionsManageSharing", - "PermissionsManageInternalUsers", - "PermissionsManagePasswordPolicies", - "PermissionsManageLoginAccessPolicies", - "PermissionsManageCustomPermissions", - "PermissionsStdAutomaticActivityCapture", - "PermissionsManageTwoFactor", - "PermissionsDebugApex", - "PermissionsLightningExperienceUser", - "PermissionsConfigCustomRecs", - "PermissionsSubmitMacrosAllowed", - "PermissionsBulkMacrosAllowed", - "PermissionsManageSessionPermissionSets", - "PermissionsCreateAuditFields", - "PermissionsUpdateWithInactiveOwner", - "PermissionsManageSandboxes", - "PermissionsAutomaticActivityCapture", - "PermissionsImportCustomObjects", - "PermissionsDelegatedTwoFactor", - "PermissionsSelectFilesFromSalesforce", - "PermissionsModerateNetworkUsers", - "PermissionsMergeTopics", - "PermissionsSubscribeToLightningReports", - "PermissionsManagePvtRptsAndDashbds", - "PermissionsAllowLightningLogin", - "PermissionsCampaignInfluence2", - "PermissionsViewDataAssessment", - "PermissionsCanApproveFeedPost", - "PermissionsAllowViewEditConvertedLeads", - "PermissionsShowCompanyNameAsUserBadge", - "PermissionsAccessCMC", - "PermissionsViewHealthCheck", - "PermissionsManageHealthCheck", - "PermissionsPackaging2", - "PermissionsManageCertificates", - "PermissionsCreateReportInLightning", - "PermissionsPreventClassicExperience", - "PermissionsListEmailSend", - "PermissionsChangeDashboardColors", - "PermissionsManageRecommendationStrategies", - "PermissionsManagePropositions", - "PermissionsSubscribeReportRolesGrps", - "PermissionsSubscribeDashboardRolesGrps", - "PermissionsUseWebLink", - "PermissionsHasUnlimitedNBAExecutions", - "PermissionsViewOnlyEmbeddedAppUser", - "PermissionsViewAllActivities", - "PermissionsSubscribeReportToOtherUsers", - "PermissionsLightningConsoleAllowedForUser", - "PermissionsSubscribeReportsRunAsUser", - "PermissionsSubscribeToLightningDashboards", - "PermissionsSubscribeDashboardToOtherUsers", - "PermissionsCreateLtngTempInPub", - "PermissionsTransactionalEmailSend", - "PermissionsViewPrivateStaticResources", - "PermissionsCreateLtngTempFolder", - "PermissionsApexRestServices", - "PermissionsEnableCommunityAppLauncher", - "PermissionsGiveRecognitionBadge", - "PermissionsUseMySearch", - "PermissionsLtngPromoReserved01UserPerm", - "PermissionsManageSubscriptions", - "PermissionsManageSurveys", - "PermissionsUseAssistantDialog", - "PermissionsUseQuerySuggestions", - "PermissionsViewRoles", - "PermissionsLMOutboundMessagingUserPerm", - "PermissionsModifyDataClassification", - "PermissionsPrivacyDataAccess", - "PermissionsQueryAllFiles", - "PermissionsModifyMetadata", - "PermissionsManageCMS", - "PermissionsSandboxTestingInCommunityApp", - "PermissionsCanEditPrompts", - "PermissionsViewUserPII", - "PermissionsManageHubConnections", - "PermissionsB2BMarketingAnalyticsUser", - "PermissionsTraceXdsQueries", - "PermissionsViewAllCustomSettings", - "PermissionsViewAllForeignKeyNames", - "PermissionsHeadlessCMSAccess", - "PermissionsLMEndMessagingSessionUserPerm", - "PermissionsConsentApiUpdate", - "PermissionsAccessContentBuilder", - "PermissionsAccountSwitcherUser", - "PermissionsManageC360AConnections", - "PermissionsManageReleaseUpdates", - "PermissionsViewAllProfiles", - "PermissionsSkipIdentityConfirmation", - "PermissionsSendCustomNotifications", - "PermissionsPackaging2Delete", - "PermissionsFSCComprehensiveUserAccess", - "PermissionsManageTrustMeasures", - "PermissionsViewTrustMeasures", - "PermissionsIsotopeCToCUser", - "PermissionsIsotopeAccess", - "PermissionsIsotopeLEX", - "PermissionsQuipMetricsAccess", - "PermissionsQuipUserEngagementMetrics", - "PermissionsManageExternalConnections", - "PermissionsAIViewInsightObjects", - "PermissionsAICreateInsightObjects", - "PermissionsNativeWebviewScrolling", - "PermissionsViewDeveloperName", - "Type", - "UserLicenseId", - "UserType", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "Description", - "LastViewedDate", - "LastReferencedDate" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "My_Setup_Products_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Product_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Multichannel_Activity_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_External_ID_Map_vod__c", - "Account_vod__c", - "Call_vod__c", - "City_vod__c", - "Client_Name_vod__c", - "Client_OS_vod__c", - "Client_Type_vod__c", - "Country_vod__c", - "Debug_vod__c", - "Device_vod__c", - "IP_Address_vod__c", - "Multichannel_Activity_vod__c", - "Referring_Site_vod__c", - "Region_vod__c", - "Sent_Email_vod__c", - "Session_Id_vod__c", - "Site_vod__c", - "Start_DateTime_vod__c", - "Total_Duration_vod__c", - "URL_vod__c", - "User_Agent_vod__c", - "VExternal_Id_vod__c", - "Viewport_Height_vod__c", - "Viewport_Width_vod__c", - "Color_vod__c", - "Icon_vod__c", - "MCD_Primary_Key_vod__c", - "Record_Type_Name_vod__c", - "MSJ_Date_Opened__c", - "MSJ_Sent_Date__c", - "MSJ_Email_Subject__c", - "MSJ_Opens__c", - "MSJ_Email_Status__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Multichannel_Activity_Line_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Multichannel_Activity_vod__c", - "Call_vod__c", - "Custom_vod__c", - "DateTime_vod__c", - "Debug_vod__c", - "Detail_Group_VExternal_Id_vod__c", - "Detail_Group_vod__c", - "Duration_vod__c", - "Event_Subtype_vod__c", - "Event_Type_vod__c", - "Key_Message_VExternal_Id_vod__c", - "Key_Message_vod__c", - "Multichannel_Content_Asset_Id_vod__c", - "Multichannel_Content_Asset_Version_vod__c", - "Multichannel_Content_Asset_vod__c", - "Multichannel_Content_vod__c", - "Product_VExternal_Id_vod__c", - "Product_vod__c", - "Sent_Email_vod__c", - "VExternal_Id_vod__c", - "Video_Last_Viewed_Time_vod__c", - "Video_Length_vod__c", - "Video_Total_Time_Spent_vod__c", - "View_Order_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Multichannel_Consent_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_vod__c", - "Capture_Datetime_vod__c", - "Channel_Value_vod__c", - "Detail_Group_vod__c", - "External_ID_vod__c", - "Last_Device_vod__c", - "Mobile_ID_vod__c", - "Opt_Expiration_Date_vod__c", - "Opt_Type_vod__c", - "Optout_Event_Type_vod__c", - "Product_vod__c", - "Signature_Datetime_vod__c", - "Signature_ID_vod__c", - "Signature_vod__c", - "Sample_Consent_Template_Data_vod__c", - "Sample_Consent_Template_vod__c", - "Consent_Line_vod__c", - "Consent_Type_vod__c", - "Default_Consent_Text_vod__c", - "Disclaimer_Text_vod__c", - "Sub_Channel_Key_vod__c", - "Consent_Confirm_Datetime_vod__c", - "Related_Transaction_Id_vod__c", - "Sent_Email_vod__c", - "Content_Type_vod__c", - "Receipt_Email_vod__c", - "Receipt_Sent_Email_Transaction_Id_vod__c", - "Receipt_Sent_Email_vod__c", - "Captured_By_vod__c", - "Opt_Out_Disclaimer_Text_vod__c", - "Channel_Source_vod__c", - "Union_Id_vod__c", - "User_Last_Notified_vod__c", - "Sub_Channel_Display_Name__c", - "MSJ_Consent_Source__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Medical_Inquiry_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_vod__c", - "Address_Line_1_vod__c", - "Address_Line_2_vod__c", - "City_vod__c", - "Delivery_Method_vod__c", - "Email_vod__c", - "Fax_Number_vod__c", - "Inquiry_Text__c", - "Lock_vod__c", - "Mobile_ID_vod__c", - "Phone_Number_vod__c", - "Product__c", - "Rush_Delivery__c", - "Signature_Date_vod__c", - "Signature_vod__c", - "State_vod__c", - "Status_vod__c", - "Zip_vod__c", - "zvod_Delivery_Method_vod__c", - "zvod_Disclaimer_vod__c", - "Submitted_By_Mobile_vod__c", - "Disclaimer_vod__c", - "Entity_Reference_Id_vod__c", - "Call2_vod__c", - "Country_vod__c", - "Override_Lock_vod__c", - "MSJ_Department__c", - "MSJ_Doctor_Name__c", - "MSJ_Hospital_Name__c", - "MSJ_Indication__c", - "MSJ_Inquiry_Assignment__c", - "MSJ_Inquiry_Date__c", - "MSJ_Inquiry_Input_Manager__c", - "MSJ_Inquiry_Input_User__c", - "MSJ_MSL_Manager__c", - "MSJ_Notice_to_MR__c", - "MSJ_Person_in_charge_1__c", - "MSJ_Person_in_charge_2__c", - "MSJ_Product_for_MEC__c", - "MSJ_Product_for_MR__c", - "MSJ_Reply_Date__c", - "MSJ_Reply_User__c", - "MSJ_Reply__c", - "MSJ_Title__c", - "MSJ_AE_Infomation__c", - "MSJ_FAQ_Number_Report__c", - "MSJ_Return_Call_Report__c", - "MSJ_Inquiry_Origin_Report__c", - "MSJ_AE_Report__c", - "MSJ_Background__c", - "MSJ_MSL_Support__c", - "MSJ_Material_Requirement__c", - "MSJ_Hospital_Name_Disp__c", - "MSJ_Hospital__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Email_Activity_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "Sent_Email_vod__c", - "Activity_DateTime_vod__c", - "City_vod__c", - "Click_URL_vod__c", - "Client_Name_vod__c", - "Client_OS_vod__c", - "Client_Type_vod__c", - "Country_vod__c", - "Device_Type_vod__c", - "Event_Msg_vod__c", - "Event_type_vod__c", - "IP_Address_vod__c", - "Region_vod__c", - "User_Agent_vod__c", - "Vault_Doc_ID_vod__c", - "Vault_Doc_Name_vod__c", - "Vault_Document_Major_Version_vod__c", - "Vault_Document_Minor_Version_vod__c", - "Vault_Document_Number_vod__c", - "Vault_Document_Title_vod__c", - "Vault_Instance_ID_vod__c", - "Preference_Modification_vod__c", - "Approved_Document_vod__c", - "Link_Name_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "User", - "columns": [ - "Id", - "Username", - "LastName", - "FirstName", - "Name", - "CompanyName", - "Division", - "Department", - "Title", - "Street", - "City", - "State", - "PostalCode", - "Country", - "Latitude", - "Longitude", - "GeocodeAccuracy", - "Address", - "Email", - "EmailPreferencesAutoBcc", - "EmailPreferencesAutoBccStayInTouch", - "EmailPreferencesStayInTouchReminder", - "SenderEmail", - "SenderName", - "Signature", - "StayInTouchSubject", - "StayInTouchSignature", - "StayInTouchNote", - "Phone", - "Fax", - "MobilePhone", - "Alias", - "CommunityNickname", - "BadgeText", - "IsActive", - "TimeZoneSidKey", - "UserRoleId", - "LocaleSidKey", - "ReceivesInfoEmails", - "ReceivesAdminInfoEmails", - "EmailEncodingKey", - "ProfileId", - "UserType", - "LanguageLocaleKey", - "EmployeeNumber", - "DelegatedApproverId", - "ManagerId", - "LastLoginDate", - "LastPasswordChangeDate", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "NumberOfFailedLogins", - "OfflineTrialExpirationDate", - "OfflinePdaTrialExpirationDate", - "UserPermissionsMarketingUser", - "UserPermissionsOfflineUser", - "UserPermissionsWirelessUser", - "UserPermissionsAvantgoUser", - "UserPermissionsCallCenterAutoLogin", - "UserPermissionsSFContentUser", - "UserPermissionsInteractionUser", - "UserPermissionsSupportUser", - "UserPermissionsChatterAnswersUser", - "ForecastEnabled", - "UserPreferencesActivityRemindersPopup", - "UserPreferencesEventRemindersCheckboxDefault", - "UserPreferencesTaskRemindersCheckboxDefault", - "UserPreferencesReminderSoundOff", - "UserPreferencesDisableAllFeedsEmail", - "UserPreferencesApexPagesDeveloperMode", - "UserPreferencesReceiveNoNotificationsAsApprover", - "UserPreferencesReceiveNotificationsAsDelegatedApprover", - "UserPreferencesHideCSNGetChatterMobileTask", - "UserPreferencesHideCSNDesktopTask", - "UserPreferencesHideChatterOnboardingSplash", - "UserPreferencesHideSecondChatterOnboardingSplash", - "UserPreferencesShowTitleToExternalUsers", - "UserPreferencesShowManagerToExternalUsers", - "UserPreferencesShowEmailToExternalUsers", - "UserPreferencesShowWorkPhoneToExternalUsers", - "UserPreferencesShowMobilePhoneToExternalUsers", - "UserPreferencesShowFaxToExternalUsers", - "UserPreferencesShowStreetAddressToExternalUsers", - "UserPreferencesShowCityToExternalUsers", - "UserPreferencesShowStateToExternalUsers", - "UserPreferencesShowPostalCodeToExternalUsers", - "UserPreferencesShowCountryToExternalUsers", - "UserPreferencesShowProfilePicToGuestUsers", - "UserPreferencesShowTitleToGuestUsers", - "UserPreferencesShowCityToGuestUsers", - "UserPreferencesShowStateToGuestUsers", - "UserPreferencesShowPostalCodeToGuestUsers", - "UserPreferencesShowCountryToGuestUsers", - "UserPreferencesHideInvoicesRedirectConfirmation", - "UserPreferencesHideStatementsRedirectConfirmation", - "UserPreferencesPathAssistantCollapsed", - "UserPreferencesCacheDiagnostics", - "UserPreferencesShowEmailToGuestUsers", - "UserPreferencesShowManagerToGuestUsers", - "UserPreferencesShowWorkPhoneToGuestUsers", - "UserPreferencesShowMobilePhoneToGuestUsers", - "UserPreferencesShowFaxToGuestUsers", - "UserPreferencesShowStreetAddressToGuestUsers", - "UserPreferencesLightningExperiencePreferred", - "UserPreferencesPreviewLightning", - "UserPreferencesHideEndUserOnboardingAssistantModal", - "UserPreferencesHideLightningMigrationModal", - "UserPreferencesHideSfxWelcomeMat", - "UserPreferencesHideBiggerPhotoCallout", - "UserPreferencesGlobalNavBarWTShown", - "UserPreferencesGlobalNavGridMenuWTShown", - "UserPreferencesCreateLEXAppsWTShown", - "UserPreferencesFavoritesWTShown", - "UserPreferencesRecordHomeSectionCollapseWTShown", - "UserPreferencesRecordHomeReservedWTShown", - "UserPreferencesFavoritesShowTopFavorites", - "UserPreferencesExcludeMailAppAttachments", - "UserPreferencesSuppressTaskSFXReminders", - "UserPreferencesSuppressEventSFXReminders", - "UserPreferencesPreviewCustomTheme", - "UserPreferencesHasCelebrationBadge", - "UserPreferencesUserDebugModePref", - "UserPreferencesSRHOverrideActivities", - "UserPreferencesNewLightningReportRunPageEnabled", - "UserPreferencesReverseOpenActivitiesView", - "UserPreferencesNativeEmailClient", - "UserPreferencesHideBrowseProductRedirectConfirmation", - "UserPreferencesHideOnlineSalesAppWelcomeMat", - "ContactId", - "AccountId", - "CallCenterId", - "Extension", - "FederationIdentifier", - "AboutMe", - "FullPhotoUrl", - "SmallPhotoUrl", - "IsExtIndicatorVisible", - "OutOfOfficeMessage", - "MediumPhotoUrl", - "DigestFrequency", - "DefaultGroupNotificationFrequency", - "LastViewedDate", - "LastReferencedDate", - "BannerPhotoUrl", - "SmallBannerPhotoUrl", - "MediumBannerPhotoUrl", - "IsProfilePhotoActive", - "IndividualId", - "Last_Mobile_Connect_vod__c", - "Last_Tablet_Connect_vod__c", - "Last_Mobile_Connect_Version_vod__c", - "Last_Tablet_Connect_Version_vod__c", - "Last_Mobile_Sync_vod__c", - "Last_Tablet_Sync_vod__c", - "RaiseLoggingLevel_vod__c", - "SendDetailedLog_vod__c", - "Last_Blackberry_Connect_vod__c", - "Last_Blackberry_Connect_Version_vod__c", - "Last_Blackberry_Sync_vod__c", - "Force_Full_Refresh_vod__c", - "Override_SystemModstamp_Timestamp_vod__c", - "Facetime_Email_vod__c", - "Facetime_Phone_vod__c", - "Product_Expertise_vod__c", - "Available_vod__c", - "Available_Last_Update_vod__c", - "Last_iPad_Connect_Version_vod__c", - "Last_iPad_Connect_vod__c", - "Last_iPad_Sync_vod__c", - "Inventory_Order_Allocation_Group_vod__c", - "Concur_User_Id_vod__c", - "Last_iPad_iOS_Version_vod__c", - "Approved_Email_Admin_vod__c", - "Last_WinModern_Connect_Version_vod__c", - "Last_WinModern_Connect_vod__c", - "Last_WinModern_Sync_vod__c", - "Primary_Territory_vod__c", - "Analytics_Admin_vod__c", - "Content_Admin_vod__c", - "Last_WinModern_Windows_Version_vod__c", - "Upload_VTrans_vod__c", - "Sync_Frequency_vjh__c", - "Clear_Client_Sync_Errors_vod__c", - "Remote_Meeting_Host_Id_vod__c", - "Remote_Meeting_Host_Token_vod__c", - "Last_iPhone_Connect_Version_vod__c", - "Last_iPhone_Connect_vod__c", - "Last_iPhone_Sync_vod__c", - "Last_iPhone_iOS_Version_vod__c", - "Remote_Meeting_Start_From_CRM_Online_vod__c", - "Country_Code_vod__c", - "User_Type_vod__c", - "Engage_Group_Provisioning_Status_vod__c", - "Engage_Group_Request_vod__c", - "Engage_Group_vod__c", - "Last_CRMDesktop_Mac_Sync_vod__c", - "Last_CRMDesktop_Mac_Version_vod__c", - "Last_CRMDesktop_Windows_Sync_vod__c", - "Last_CRMDesktop_Windows_Version_vod__c", - "MSJ_Test_User__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "UserTerritory2Association", - "columns": [ - "Id", - "UserId", - "Territory2Id", - "IsActive", - "RoleInTerritory2", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Remote_Meeting_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Meeting_Id_vod__c", - "Meeting_Name_vod__c", - "Mobile_ID_vod__c", - "Scheduled_DateTime_vod__c", - "Scheduled_vod__c", - "Attendance_Report_Process_Status_vod__c", - "Latest_Meeting_Start_Datetime_vod__c", - "Meeting_Password_vod__c", - "Meeting_Outcome_Status_vod__c", - "Allow_for_Joining_via_Zoom_vod__c", - "Zoom_Join_Token_vod__c", - "VExternal_Id_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "RecordType", - "columns": [ - "Id", - "Name", - "DeveloperName", - "NamespacePrefix", - "Description", - "BusinessProcessId", - "SobjectType", - "IsActive", - "IsPersonType", - "CreatedById", - "CreatedDate", - "LastModifiedById", - "LastModifiedDate", - "SystemModstamp" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "UserRole", - "columns": [ - "Id", - "Name", - "ParentRoleId", - "RollupDescription", - "OpportunityAccessForAccountOwner", - "CaseAccessForAccountOwner", - "ContactAccessForAccountOwner", - "ForecastUserId", - "MayForecastManagerShare", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "DeveloperName", - "PortalAccountId", - "PortalType", - "PortalAccountOwnerId" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Account", - "columns": [ - "Id", - "IsDeleted", - "MasterRecordId", - "Name", - "LastName", - "FirstName", - "Salutation", - "RecordTypeId", - "Phone", - "Fax", - "Website", - "PhotoUrl", - "NumberOfEmployees", - "Ownership", - "OwnerId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "IsExcludedFromRealign", - "PersonContactId", - "IsPersonAccount", - "PersonMailingStreet", - "PersonMailingCity", - "PersonMailingState", - "PersonMailingPostalCode", - "PersonMailingCountry", - "PersonMailingLatitude", - "PersonMailingLongitude", - "PersonMailingGeocodeAccuracy", - "PersonMailingAddress", - "PersonOtherStreet", - "PersonOtherCity", - "PersonOtherState", - "PersonOtherPostalCode", - "PersonOtherCountry", - "PersonOtherLatitude", - "PersonOtherLongitude", - "PersonOtherGeocodeAccuracy", - "PersonOtherAddress", - "PersonMobilePhone", - "PersonHomePhone", - "PersonOtherPhone", - "PersonAssistantPhone", - "PersonEmail", - "PersonTitle", - "PersonDepartment", - "PersonAssistantName", - "PersonBirthdate", - "PersonHasOptedOutOfEmail", - "PersonHasOptedOutOfFax", - "PersonDoNotCall", - "PersonLastCURequestDate", - "PersonLastCUUpdateDate", - "PersonEmailBouncedReason", - "PersonEmailBouncedDate", - "PersonIndividualId", - "Jigsaw", - "JigsawCompanyId", - "AccountSource", - "SicDesc", - "External_ID_vod__c", - "Credentials_vod__c", - "Territory_vod__c", - "Exclude_from_Zip_to_Terr_Processing_vod__c", - "Group_Specialty_1_vod__c", - "Group_Specialty_2_vod__c", - "Specialty_1_vod__c", - "Specialty_2_vod__c", - "Formatted_Name_vod__c", - "Territory_Test_vod__c", - "Mobile_ID_vod__c", - "Gender_vod__c", - "ID_vod__c", - "Do_Not_Sync_Sales_Data_vod__c", - "ID2_vod__c", - "Preferred_Name_vod__c", - "Sample_Default_vod__c", - "Segmentations_vod__c", - "Restricted_Products_vod__c", - "Payer_Id_vod__c", - "Alternate_Name_vod__c", - "Do_Not_Call_vod__c", - "MSJ_Beds__c", - "Spend_Amount__c", - "PDRP_Opt_Out_vod__c", - "Spend_Status_Value_vod__c", - "PDRP_Opt_Out_Date_vod__c", - "Spend_Status_vod__c", - "Enable_Restricted_Products_vod__c", - "Call_Reminder_vod__c", - "Account_Group_vod__c", - "Primary_Parent_vod__c", - "Color_vod__c", - "Middle_vod__c", - "Suffix_vod__c", - "MSJ_Type__c", - "No_Orders_vod__c", - "MSJ_BU_ONC__c", - "MSJ_BU_FE__c", - "Account_Search_FirstLast_vod__c", - "Account_Search_LastFirst_vod__c", - "MSJ_Operation__c", - "Practice_at_Hospital_vod__c", - "Practice_Near_Hospital_vod__c", - "Do_Not_Create_Child_Account_vod__c", - "Total_MDs_DOs__c", - "AHA__c", - "Order_Type_vod__c", - "NPI_vod__c", - "ME__c", - "Speaker__c", - "Investigator_vod__c", - "Default_Order_Type_vod__c", - "Tax_Status__c", - "Model__c", - "Offerings__c", - "Departments__c", - "Account_Type__c", - "MSJ_ONC_Tier__c", - "Account_Search_Business_vod__c", - "Business_Professional_Person_vod__c", - "Hospital_Type_vod__c", - "Account_Class_vod__c", - "Furigana_vod__c", - "MSJ_JISART__c", - "Total_Revenue_000__c", - "Net_Income_Loss_000__c", - "PMPM_Income_Loss_000__c", - "Commercial_Premiums_PMPM__c", - "Medical_Loss_Ratio__c", - "Medical_Expenses_PMPM__c", - "Commercial_Patient_Days_1000__c", - "HMO_Market_Shr__c", - "HMO__c", - "HMO_POS__c", - "PPO__c", - "PPO_POS__c", - "Medicare__c", - "Medicaid__c", - "MSJ_HP_Name_E__c", - "MSJ_Department__c", - "MSJ_Date_Of_Birth__c", - "MSJ_FE_GF_Potential__c", - "MSJ_FE_SZ_Potential__c", - "MSJ_EB_CRC_Ladder__c", - "MSJ_EB_CRC_Segment__c", - "MSJ_EB_HN_Segment__c", - "Business_Description__c", - "Regional_Strategy__c", - "Contracts_Process__c", - "MSJ_GF_segment__c", - "MSJ_DCF_DR_Code__c", - "MSJ_SZ_Segment__c", - "MSJ_Remark__c", - "MSJ_Title__c", - "MSJ_Role__c", - "MSJ_Kana__c", - "MSJ_Specialism__c", - "MSJ_Graduated_from__c", - "MSJ_Year_Graduation__c", - "Target__c", - "KOL_vod__c", - "MSJ_EPPV_Code__c", - "MSJ_DCF_HP_Code__c", - "Total_Lives__c", - "Total_Physicians_Enrolled__c", - "MSJ_Delete__c", - "MSJ_KOL_LOL__c", - "MSJ_ONC_Status__c", - "Account_Identifier_vod__c", - "Approved_Email_Opt_Type_vod__c", - "Language_vod__c", - "MSJ_KRAS_Routine_Date__c", - "MSJ_KRAS_Routine__c", - "MSJ_DRP_Target__c", - "MSJ_Fertility_Evaluation_Score__c", - "MSJ_Fertility_Tracking_Last_Modify_Date__c", - "Total_Pharmacists__c", - "MSJ_Number_of_Gonadotropin__c", - "MSJ_Number_of_IUI_cycle__c", - "MSJ_Number_of_OI_monthly_cycle__c", - "MSJ_OI_Protocol_learning_level__c", - "MSJ_H_N_Tier__c", - "MSJ_XLK_Segment__c", - "MSJ_XLK_Tier__c", - "Career_Status_vod__c", - "Photo_vod__c", - "MSJ_EB_H_N_LA_Segment__c", - "MSJ_EB_H_N_RM_Segment__c", - "MSJ_FE_CE_Potential__c", - "MSJ_FE_1C_potential__c", - "MSJ_FE_OV_potential__c", - "MSJ_FE_Tech_potential__c", - "MSJ_CE_segment__c", - "MSJ_1C_segment__c", - "MSJ_OV_segment__c", - "MSJ_Tech_segment__c", - "MSJ_Target_Call_Num__c", - "MSJ_DR_Change_Log__c", - "MSJ_Global_scientific_exposure__c", - "MSJ_H_index__c", - "MSJ_Num_of_Article_3Y__c", - "MSJ_Num_of_Article__c", - "MSJ_Num_of_Article_as_1st_Author_3Y__c", - "MSJ_Num_of_article_growth_rate_3Y__c", - "MSJ_Num_of_cited_3Y__c", - "MSJ_Num_of_impact_factor_3Y__c", - "MSJ_impact_factor_as_1st_Author_3Y__c", - "EMDS_Has_Pipeline_Opportunity__c", - "EMDS_Pipeline_Count__c", - "MSJ_BVC_Segment__c", - "MSJ_BVC_Tier__c", - "MSJ_BVC_AcctOpen__c", - "MSJ_BVC_MCC_Patients__c", - "MSJ_ONC_HP_Segment__c", - "MSJ_AE_Department__c", - "MSJ_AE_Facility__c", - "MSJ_AE_Name__c", - "MSJ_AE_Title__c", - "MSJ_Email__c", - "MSJ_FE_GF2_Potential__c", - "MSJ_FE_Location_potential__c", - "MSJ_GF2_segment__c", - "MSJ_OPTIN_target__c", - "MSJ_Merck_Specialty1__c", - "MSJ_Merck_Specialty2__c", - "MSJ_Marketing_Cloud_Integration__c", - "MSJ_Marketing_Cloud1__c", - "MSJ_Marketing_Cloud2__c", - "MSJ_Marketing_Cloud3__c", - "MSJ_Marketing_Cloud4__c", - "MSJ_Medical_Department__c", - "MSJ_Marketing_Cloud0__c", - "Mobile_ID_vod__pc", - "H1Insights__H1_NPI_Value_for_Testing__pc", - "H1Insights__H1_Person_ID__pc", - "H1Insights__H1_Request_Status__pc", - "H1Insights__H1_URL__pc", - "H1Insights__NPI_Number__pc", - "H1Insights__NPI_Number_for_H1_Insights__pc", - "MSJ_Marketing_Cloud_Integration__pc" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "AccountShare", - "columns": [ - "Id", - "AccountId", - "UserOrGroupId", - "AccountAccessLevel", - "OpportunityAccessLevel", - "CaseAccessLevel", - "ContactAccessLevel", - "RowCause", - "LastModifiedDate", - "LastModifiedById", - "IsDeleted" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true, - "datetime_column": "LastModifiedDate" - }, - { - "object_name": "Contact", - "columns": [ - "Id", - "IsDeleted", - "MasterRecordId", - "AccountId", - "IsPersonAccount", - "LastName", - "FirstName", - "Salutation", - "Name", - "OtherStreet", - "OtherCity", - "OtherState", - "OtherPostalCode", - "OtherCountry", - "OtherLatitude", - "OtherLongitude", - "OtherGeocodeAccuracy", - "OtherAddress", - "MailingStreet", - "MailingCity", - "MailingState", - "MailingPostalCode", - "MailingCountry", - "MailingLatitude", - "MailingLongitude", - "MailingGeocodeAccuracy", - "MailingAddress", - "Phone", - "Fax", - "MobilePhone", - "HomePhone", - "OtherPhone", - "AssistantPhone", - "ReportsToId", - "Email", - "Title", - "Department", - "AssistantName", - "Birthdate", - "Description", - "OwnerId", - "HasOptedOutOfEmail", - "HasOptedOutOfFax", - "DoNotCall", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "LastCURequestDate", - "LastCUUpdateDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "EmailBouncedReason", - "EmailBouncedDate", - "IsEmailBounced", - "PhotoUrl", - "Jigsaw", - "JigsawContactId", - "IndividualId", - "Mobile_ID_vod__c", - "H1Insights__H1_NPI_Value_for_Testing__c", - "H1Insights__H1_Person_ID__c", - "H1Insights__H1_Request_Status__c", - "H1Insights__H1_URL__c", - "H1Insights__NPI_Number__c", - "H1Insights__NPI_Number_for_H1_Insights__c", - "MSJ_Marketing_Cloud_Integration__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Consent_Type_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Consent_Header_vod__c", - "Channel_Label_vod__c", - "Channel_Source_vod__c", - "Consent_Expires_In_vod__c", - "Default_Consent_Type_vod__c", - "Disclaimer_Text_vod__c", - "Display_Order_vod__c", - "Product_Preference_vod__c", - "zvod_Consent_Default_Consent_Text_vod__c", - "zvod_Consent_Line_vod__c", - "zvod_Signature_Capture_vod__c", - "Double_Opt_In_vod__c", - "zvod_Consent_Activity_Tracking_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Consent_Header_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Consent_Header_Help_Text_vod__c", - "Country_vod__c", - "Inactive_Datetime_vod__c", - "Language_vod__c", - "Status_vod__c", - "Signature_Required_On_Opt_Out_vod__c", - "Request_Receipt_vod__c", - "Subscription_Option_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Consent_Line_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Consent_Type_vod__c", - "Detail_Group_Display_Name_vod__c", - "Detail_Group_vod__c", - "Display_Order_vod__c", - "End_Date_vod__c", - "Group_By_vod__c", - "Product_Display_Name_vod__c", - "Product_vod__c", - "Start_Date_vod__c", - "Sub_Channel_Description_vod__c", - "Sub_Channel_Display_Name_vod__c", - "Sub_Channel_Key_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "MSJ_Inquiry_Assignment__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "MSJ_Medical_Inquiry__c", - "MSJ_Close__c", - "MSJ_Doctor_Name__c", - "MSJ_Hospital_Name__c", - "MSJ_Indication__c", - "MSJ_Inquiry_Text__c", - "MSJ_MEC_User__c", - "MSJ_MSL_Manager__c", - "MSJ_MSL_User__c", - "MSJ_Notice_to_MR__c", - "MSJ_Product_for_MEC__c", - "MSJ_Product_for_MR__c", - "MSJ_Reply_Date__c", - "MSJ_Reply__c", - "MSJ_AE_Infomation__c", - "MSJ_Cancel__c", - "MSJ_FAQ_number_c__c", - "MSJ_Return_Call__c", - "MSJ_Inquiry_Origin__c", - "First_Response__c", - "Inquiry_Created_Date__c", - "Inquiry_Type_1__c", - "Inquiry_Type_2__c", - "MSJ_First_User__c", - "MSJ_MEC_Comment__c", - "MSJ_Send_Email__c", - "MSJ_Temp_Aggregated_Info__c", - "MSJ_AE_Report__c", - "MSJ_Background__c", - "MSJ_Inquiry_Date__c", - "MSJ_MSL_Support__c", - "MSJ_Handover_Comment__c", - "MSJ_Handover_Email__c", - "MSJ_Material_Requirement__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Approved_Document_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Detail_Group_vod__c", - "Document_Description_vod__c", - "Document_Host_URL_vod__c", - "Document_ID_vod__c", - "Document_Last_Mod_DateTime_vod__c", - "Email_Allows_Documents_vod__c", - "Email_Domain_vod__c", - "Email_Fragment_HTML_vod__c", - "Email_From_Address_vod__c", - "Email_From_Name_vod__c", - "Email_HTML_1_vod__c", - "Email_HTML_2_vod__c", - "Email_ReplyTo_Address_vod__c", - "Email_ReplyTo_Name_vod__c", - "Email_Subject_vod__c", - "Email_Template_Fragment_Document_ID_vod__c", - "Email_Template_Fragment_HTML_vod__c", - "ISI_Document_ID_vod__c", - "Language_vod__c", - "Other_Document_ID_List_vod__c", - "PI_Document_ID_vod__c", - "Piece_Document_ID_vod__c", - "Product_vod__c", - "Status_vod__c", - "Territory_vod__c", - "Vault_Instance_ID_vod__c", - "Allow_Any_Product_Fragment_vod__c", - "Allowed_Document_IDs_vod__c", - "Engage_Document_Id_vod__c", - "Vault_Document_ID_vod__c", - "Key_Message_vod__c", - "Events_Management_Subtype_vod__c", - "Survey_vod__c", - "Content_Type_vod__c", - "Bcc_vod__c", - "Audience_vod__c", - "WeChat_Template_ID_vod__c", - "Check_Consent_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Child_Account_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Parent_Account_vod__c", - "Child_Account_vod__c", - "External_ID_vod__c", - "Mobile_ID_vod__c", - "Primary_vod__c", - "Copy_Address_vod__c", - "Child_Name_vod__c", - "Parent_Name_vod__c", - "Parent_Child_Name_vod__c", - "Account_Code__c", - "Child_Department__c", - "Child_Role__c", - "Child_Title__c", - "Child_Remark__c", - "MSJ_1C_segment__c", - "MSJ_BU_FE__c", - "MSJ_BU_ONC__c", - "MSJ_BVC_Segment__c", - "MSJ_CE_segment__c", - "MSJ_Child_Account_Link__c", - "MSJ_DCF_DR_Code__c", - "MSJ_DCF_HP_Code__c", - "MSJ_DR_Change_Log__c", - "MSJ_Delete__c", - "MSJ_Department__c", - "MSJ_EB_CRC_Segment__c", - "MSJ_EB_HN_Segment__c", - "MSJ_EB_H_N_LA_Segment__c", - "MSJ_EB_H_N_RM_Segment__c", - "MSJ_External_ID__c", - "MSJ_Fax__c", - "MSJ_GF2_segment__c", - "MSJ_GF_segment__c", - "MSJ_KOL_LOL__c", - "MSJ_KOL__c", - "MSJ_ONC_HP_Segment__c", - "MSJ_OPTIN_target__c", - "MSJ_OV_segment__c", - "MSJ_Parent_Child_Name__c", - "MSJ_Phone__c", - "MSJ_Remark__c", - "MSJ_Target_Call_Num__c", - "MSJ_Tech_segment__c", - "MSJ_Title__c", - "MSJ_XLK_Segment__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "MSJ_Hospital_Medical_Regimen__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "MSJ_Account_Name__c", - "MSJ_Delete_Date__c", - "MSJ_Delete_Flag__c", - "MSJ_Indication__c", - "MSJ_Line__c", - "MSJ_Medical_Regimen__c", - "Mobile_ID_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "MSJ_Medical_Regimen__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "MSJ_Delete_Date__c", - "MSJ_Delete_Flag__c", - "MSJ_Indication__c", - "MSJ_Remark__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "MSJ_Patient__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "MSJ_Account_Name__c", - "MSJ_CRC_Group__c", - "MSJ_Casus_or_Transfer_Point__c", - "MSJ_Entry_Date__c", - "MSJ_IST_Name__c", - "MSJ_Indication__c", - "MSJ_Line__c", - "MSJ_MR_Comments__c", - "MSJ_MUID__c", - "MSJ_Medical_Regimen__c", - "MSJ_Month__c", - "MSJ_Report_Comments__c", - "MSJ_Start_Date_Of_Administration__c", - "MSJ_Year__c", - "Mobile_ID_vod__c", - "MSJ_CRC_RAS_KRAS__c", - "MSJ_End_Date_Of_Administration__c", - "MSJ_End_Date_of_Stop_Administration__c", - "MSJ_HN_Hospitalized_Type__c", - "MSJ_Start_Date_of_Stop_Administration__c", - "MSJ_Patient_Status__c", - "MSJ_Patient_TA__c", - "MSJ_Child_Account_Name__c", - "MSJ_Child_Account__c", - "MSJ_Parent_Account_Name__c", - "MSJ_Parent_Child_Name__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Product_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Consumer_site__c", - "Product_info__c", - "Therapeutic_Class_vod__c", - "Parent_Product_vod__c", - "Therapeutic_Area_vod__c", - "Product_Type_vod__c", - "Require_Key_Message_vod__c", - "Cost_vod__c", - "External_ID_vod__c", - "Manufacturer_vod__c", - "Company_Product_vod__c", - "Controlled_Substance_vod__c", - "Description_vod__c", - "Sample_Quantity_Picklist_vod__c", - "Display_Order_vod__c", - "No_Metrics_vod__c", - "Distributor_vod__c", - "Sample_Quantity_Bound_vod__c", - "Sample_U_M_vod__c", - "No_Details_vod__c", - "Quantity_Per_Case_vod__c", - "Schedule_vod__c", - "Restricted_vod__c", - "Pricing_Rule_Quantity_Bound_vod__c", - "No_Promo_Items_vod__c", - "User_Aligned_vod__c", - "Restricted_States_vod__c", - "Sort_Code_vod__c", - "No_Cycle_Plans_vod__c", - "Inventory_Order_UOM_vod__c", - "Inventory_Quantity_Per_Case_vod__c", - "VExternal_Id_vod__c", - "Country__c", - "MSJ_Product_Classification__c", - "MSJ_Indication__c", - "MSJ_Therapeutic_Area__c", - "MSJ_Global_Brand__c", - "MSJ_Global_Business_Unit__c", - "MSJ_Molecules__c", - "MSJ_SBU__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Product_Group_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Description_vod__c", - "Product_vod__c", - "Product_Catalog_vod__c", - "Start_Date_vod__c", - "End_Date_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Product_Metrics_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_vod__c", - "Awareness__c", - "Selling_Stage__c", - "Formulary_Status__c", - "Movement__c", - "Products_vod__c", - "Segment__c", - "X12_mo_trx_chg__c", - "Speaker_Skills__c", - "Investigator_Readiness__c", - "Engagements__c", - "Mobile_ID_vod__c", - "External_ID_vod__c", - "MSJ_Patient__c", - "Detail_Group_vod__c", - "MSJ_EB_1st_Line_Liver_Meta__c", - "MSJ_EB_1st_Line_Multi_Meta__c", - "MSJ_EB_2nd_Line_Mono__c", - "MSJ_EB_2nd_Line_Combination__c", - "MSJ_EB_3rd_Line_Mono__c", - "MSJ_EB_3rd_Line_Combination__c", - "EMDS_Ability__c", - "EMDS_Brand_Loyalty__c", - "EMDS_Decision_Maker__c", - "EMDS_Early_Tech_Adopter__c", - "EMDS_Influence__c", - "EMDS_Main_Driver__c", - "EMDS_Priority__c", - "EMDS_Willingness__c", - "MSJ_KTL_Type__c", - "MSJ_KTL_Tier__c", - "MSJ_Publications__c", - "MSJ_Clinical_Trials__c", - "MSJ_Speaker_for_Medical_Events__c", - "MSJ_Advisor_to_Medical_Affairs__c", - "MSJ_Guidelines_Treatment_Standards__c", - "MSJ_Therapeutic_Area_Expertise__c", - "MSJ_MAP_GAP__c", - "MSJ_Associations__c", - "MSJ_Tier_Score__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Survey_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Assignment_Type_vod__c", - "Channels_vod__c", - "End_Date_vod__c", - "Expired_vod__c", - "External_ID_vod__c", - "Language_vod__c", - "Lock_vod__c", - "Open_vod__c", - "Product_vod__c", - "Region_vod__c", - "Segment_vod__c", - "Start_Date_vod__c", - "Status_vod__c", - "Territory_vod__c", - "zvod_Questions_vod__c", - "zvod_Segments_vod__c", - "zvod_Targets_vod__c", - "Max_Score_vod__c", - "Min_Score_vod__c", - "Autotarget_vod__c", - "Territories_vod__c", - "Target_Type_vod__c", - "MSJ_External_ID__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Survey_Target_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_Display_Name_vod__c", - "Account_vod__c", - "Channels_vod__c", - "End_Date_vod__c", - "Entity_Reference_Id_vod__c", - "External_ID_vod__c", - "Language_vod__c", - "Lock_vod__c", - "Mobile_ID_vod__c", - "No_Autoassign_vod__c", - "Not_Completed_vod__c", - "Region_vod__c", - "Segment_vod__c", - "Start_Date_vod__c", - "Status_vod__c", - "Survey_vod__c", - "Territory_vod__c", - "zvod_Address_vod__c", - "zvod_Specialty_vod__c", - "Score_vod__c", - "User_vod__c", - "Child_Account_vod__c", - "Location_Entity_Reference_Id_vod__c", - "Location_vod__c", - "Target_Type_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Survey_Question_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Survey_vod__c", - "Answer_Choice_vod__c", - "External_ID_vod__c", - "Max_Score_vod__c", - "Min_Score_vod__c", - "Order_vod__c", - "Question_vod__c", - "Required_vod__c", - "Text_vod__c", - "Condition_vod__c", - "Source_ID_vod__c", - "MSJ_External_ID__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Question_Response_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Survey_Target_vod__c", - "Answer_Choice_vod__c", - "Date_vod__c", - "Datetime_vod__c", - "External_ID_vod__c", - "Mobile_ID_vod__c", - "Number_vod__c", - "Order_vod__c", - "Question_Text_vod__c", - "Required_vod__c", - "Response_Hash_vod__c", - "Response_vod__c", - "Score_vod__c", - "Survey_Question_vod__c", - "Text_vod__c", - "Type_vod__c", - "Condition_vod__c", - "Inactive_Condition_vod__c", - "Source_ID_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Sent_Fragment_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "Sent_Email_vod__c", - "Account_vod__c", - "Email_Template_vod__c", - "Sent_Fragment_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Sent_Email_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "Account_Email_vod__c", - "Account_vod__c", - "Approved_Email_Template_vod__c", - "Capture_Datetime_vod__c", - "Detail_Group_vod__c", - "Email_Config_Values_vod__c", - "Email_Content2_vod__c", - "Email_Content_vod__c", - "Email_Fragments_vod__c", - "Email_Sent_Date_vod__c", - "Failure_Msg_vod__c", - "Last_Activity_Date_vod__c", - "Last_Device_vod__c", - "MC_Capture_Datetime_vod__c", - "Mobile_ID_vod__c", - "Opened_vod__c", - "Product_Display_vod__c", - "Product_vod__c", - "Sender_Email_vod__c", - "Status_vod__c", - "Valid_Consent_Exists_vod__c", - "Approved_Document_Views_vod__c", - "Click_Count_vod__c", - "Last_Click_Date_vod__c", - "Last_Open_Date_vod__c", - "Open_Count_vod__c", - "Receipt_Entity_Type_vod__c", - "Receipt_Record_Id_vod__c", - "Territory_vod__c", - "Call2_vod__c", - "Medical_Inquiry_vod__c", - "Parent_Email_vod__c", - "Related_Transaction_ID_vod__c", - "Case_vod__c", - "Key_Message_vod__c", - "Suggestion_vod__c", - "EM_Attendee_vod__c", - "EM_Event_Speaker_vod__c", - "EM_Event_Team_Member_vod__c", - "Event_Attendee_vod__c", - "Event_vod__c", - "Medical_Event_vod__c", - "Scheduled_Send_Datetime_vod__c", - "User_vod__c", - "Content_Type_vod__c", - "Bcc_vod__c", - "Event_Attendee_Mobile_Id_vod__c", - "Event_Mobile_Id_vod__c", - "Activity_Tracking_Mode_vod__c", - "Email_Source_vod__c", - "Subject_vod__c", - "User_Input_Text_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - } - ] +{ + "objects": [ + { + "object_name": "Clm_Presentation_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Mobile_ID_vod__c", + "Presentation_Id_vod__c", + "Product_vod__c", + "Default_Presentation_vod__c", + "Training_vod__c", + "ParentId_vod__c", + "Hidden_vod__c", + "Type_vod__c", + "Approved_vod__c", + "Copied_From_vod__c", + "Copy_Date_vod__c", + "Survey_vod__c", + "Original_Record_ID_vod__c", + "Directory_vod__c", + "End_Date_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "VExternal_Id_vod__c", + "Vault_DNS_vod__c", + "Vault_Doc_Id_vod__c", + "Vault_External_Id_vod__c", + "Vault_GUID_vod__c", + "Vault_Last_Modified_Date_Time_vod__c", + "Version_vod__c", + "Enable_Survey_Overlay_vod__c", + "Description_vod__c", + "Keywords_vod__c", + "Content_Channel_vod__c", + "original_material_approved_in_veritas__c", + "keywords__c", + "trade_team__c", + "ewizard_link__c", + "business_function__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Clm_Presentation_Slide_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Clm_Presentation_vod__c", + "Key_Message_vod__c", + "Display_Order_vod__c", + "Sub_Presentation_vod__c", + "Mobile_ID_vod__c", + "External_ID_vod__c", + "VExternal_Id_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Insight_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Clinical_Trial_vod__c", + "Date_vod__c", + "Description_vod__c", + "Entity_Reference_Id_vod__c", + "Interaction_vod__c", + "Medical_Event_vod__c", + "Mobile_ID_vod__c", + "Other_Source_vod__c", + "Override_Lock_vod__c", + "Publication_vod__c", + "Status_vod__c", + "Summary_vod__c", + "Unlock_vod__c", + "Commercial_Medical__c", + "MSJ_Level_1A__c", + "MSJ_Level_1B__c", + "MSJ_Level_2A__c", + "MSJ_Level_2B__c", + "MSJ_Level_3A__c", + "MSJ_Level_3B__c", + "MSJ_Level_4A__c", + "MSJ_Level_4B__c", + "MSJ_SubStatus__c", + "MSJ_Type_A__c", + "MSJ_Type_B__c", + "MSJ_Description_Backup__c", + "MSJ_Country__c", + "MSJ_Received_at_Boomi__c", + "MSJ_Level_1A_Value__c", + "MSJ_Level_1B_Value__c", + "MSJ_Level_2A_Value__c", + "MSJ_Level_2B_Value__c", + "MSJ_Level_3A_Value__c", + "MSJ_Level_3B_Value__c", + "MSJ_Level_4A_Value__c", + "MSJ_Level_4B_Value__c", + "MSJ_Hospital_ID__c", + "MSJ_Hospital_Name__c", + "MSJ_Hospital__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_MR_Weekly_Report__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Account_Name__c", + "MSJ_Activity_Results_Summary__c", + "MSJ_MUID__c", + "MSJ_Next_Week_Action_What__c", + "MSJ_Next_Week_Action_When__c", + "MSJ_Next_Week_Action_Where__c", + "MSJ_Next_Week_Action_Who__c", + "MSJ_Report_Week__c", + "MSJ_Target_Patient_Count__c", + "Mobile_ID_vod__c", + "MSJ_Activity_Results_Summary_HN__c", + "MSJ_Next_Week_Action_Where_HN__c", + "MSJ_Next_Week_Action_Who_HN__c", + "MSJ_Next_Week_Action_What_HN__c", + "MSJ_Next_Week_Action_When_HN__c", + "MSJ_Target_Patient_Count_HN__c", + "MSJ_Activity_Results_Summary_MCC__c", + "MSJ_Next_Week_Action_Where_MCC__c", + "MSJ_Next_Week_Action_Who_MCC__c", + "MSJ_Next_Week_Action_What_MCC__c", + "MSJ_Next_Week_Action_When_MCC__c", + "MSJ_Target_Patient_Count_MCC__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Account_Territory_Loader_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "External_ID_vod__c", + "Territory_vod__c", + "Mobile_ID_vod__c", + "Territory_To_Add_vod__c", + "Territory_to_Drop_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Event_Attendee_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Attendee_vod__c", + "User_vod__c", + "Medical_Event_vod__c", + "Attendee_Type_vod__c", + "Status_vod__c", + "Contact_vod__c", + "Attendee_Name_vod__c", + "Account_vod__c", + "Start_Date_vod__c", + "Signature_vod__c", + "Signature_Datetime_vod__c", + "MSJ_Copy_Account_Type__c", + "MSJ_Evaluation__c", + "MSJ_Hospital__c", + "MSJ_Role__c", + "Mobile_ID_vod__c", + "MSJ_Evaluation_Comment__c", + "Position_vod__c", + "Talk_Title_vod__c", + "MSJ_Attendee_Reaction__c", + "MSJ_Registration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "ObjectTerritory2Association", + "columns": [ + "Id", + "ObjectId", + "Territory2Id", + "AssociationCause", + "SobjectType", + "IsDeleted", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Key_Message_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Description_vod__c", + "Product_vod__c", + "Product_Strategy_vod__c", + "Display_Order_vod__c", + "Active_vod__c", + "Category_vod__c", + "Vehicle_vod__c", + "CLM_ID_vod__c", + "Custom_Reaction_vod__c", + "Slide_Version_vod__c", + "Language_vod__c", + "Media_File_CRC_vod__c", + "Media_File_Name_vod__c", + "Media_File_Size_vod__c", + "Segment_vod__c", + "Disable_Actions_vod__c", + "VExternal_Id_vod__c", + "CDN_Path_vod__c", + "Status_vod__c", + "Vault_DNS_vod__c", + "Vault_Doc_Id_vod__c", + "Vault_External_Id_vod__c", + "Vault_GUID_vod__c", + "Vault_Last_Modified_Date_Time_vod__c", + "Is_Shared_Resource_vod__c", + "iOS_Viewer_vod__c", + "iOS_Resolution_vod__c", + "approved_for_distribution_date__c", + "approved_for_use_date__c", + "ewizard_link__c", + "expiration_date__c", + "keywords__c", + "trade_team__c", + "business_function__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Group", + "columns": [ + "Id", + "Name", + "DeveloperName", + "RelatedId", + "Type", + "Email", + "OwnerId", + "DoesSendEmailToMembers", + "DoesIncludeBosses", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Event_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Primary_Product__c", + "Description_vod__c", + "Start_Date_vod__c", + "Location__c", + "End_Date_vod__c", + "Secondary_Product__c", + "Website__c", + "Active_vod__c", + "Event_Type__c", + "MSJ_Area__c", + "MSJ_Business_Unit__c", + "MSJ_Comment__c", + "MSJ_Company__c", + "MSJ_Expense_App_No__c", + "MSJ_Form__c", + "MSJ_HQ_Area__c", + "MSJ_Location__c", + "MSJ_MR__c", + "MSJ_Number_of_Attendee__c", + "MSJ_Product__c", + "MSJ_Site__c", + "MSJ_Type__c", + "MSJ_Number_of_Attendee_Auto_Calc__c", + "MSJ_Number_of_Attendee_Invited__c", + "Account_vod__c", + "MSJ_MUID__c", + "Country_Name_vod__c", + "MSJ_CE_SIPAGL_Updater__c", + "MSJ_CE_SIPAGL_1A__c", + "MSJ_CE_SIPAGL_1B__c", + "MSJ_CE_SIPAGL_2__c", + "MSJ_CE_SIPAGL_3__c", + "MSJ_CE_SIPAGL_4__c", + "MSJ_CE_SIPAGL_5__c", + "MSJ_CE_SIPAGL_Comment__c", + "MSJ_CE_SIPAGL_1A_date__c", + "MSJ_CE_SIPAGL_6__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Medical_Event_Evaluation__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Medical_Event__c", + "MSJ_Evaluation_Comment__c", + "MSJ_Evaluation__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Coaching_Report_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Mobile_ID_vod__c", + "Manager_vod__c", + "Employee_vod__c", + "Review_Date__c", + "Review_Period__c", + "Status__c", + "Comments__c", + "Strategic_Planning__c", + "Customer_Focus__c", + "Knowledge_Expertise__c", + "Business_Account_Planning__c", + "Call_Productivity__c", + "Overall_Rating__c", + "MSJ_A01__c", + "MSJ_A02__c", + "MSJ_A03__c", + "MSJ_AM_Memo__c", + "MSJ_Aid_Total__c", + "MSJ_C0_GC__c", + "MSJ_C1_GC__c", + "MSJ_C2_GC__c", + "MSJ_Countermeasure__c", + "MSJ_Deadline__c", + "MSJ_Double_Visit_Time__c", + "MSJ_Hospital__c", + "MSJ_K01_FE__c", + "MSJ_K01_ONC__c", + "MSJ_K02_FE__c", + "MSJ_K02_ONC__c", + "MSJ_K03_FE__c", + "MSJ_K03_ONC__c", + "MSJ_K04_FE__c", + "MSJ_K04_ONC__c", + "MSJ_K05_FE__c", + "MSJ_K05_ONC__c", + "MSJ_K06_FE__c", + "MSJ_K06_ONC__c", + "MSJ_K0_GC__c", + "MSJ_K1_GC__c", + "MSJ_K2_GC__c", + "MSJ_Knowledge_Total__c", + "MSJ_L0_GC__c", + "MSJ_L1_GC__c", + "MSJ_L2_GC__c", + "MSJ_MR_GC__c", + "MSJ_MR_Problems__c", + "MSJ_N0_GC__c", + "MSJ_N1_GC__c", + "MSJ_N2_GC__c", + "MSJ_Num_of_DTL__c", + "MSJ_P01__c", + "MSJ_P02__c", + "MSJ_P03__c", + "MSJ_P04__c", + "MSJ_P05__c", + "MSJ_P0_GC__c", + "MSJ_P1_GC__c", + "MSJ_P2_GC__c", + "MSJ_PlanningTotal__c", + "MSJ_R0_GC__c", + "MSJ_R1_GC__c", + "MSJ_R2_GC__c", + "MSJ_S01__c", + "MSJ_S02__c", + "MSJ_S03__c", + "MSJ_S04__c", + "MSJ_S05__c", + "MSJ_S06__c", + "MSJ_S07__c", + "MSJ_S08__c", + "MSJ_S09__c", + "MSJ_S10__c", + "MSJ_S11__c", + "MSJ_S12__c", + "MSJ_Skill_Total__c", + "MSJ_After_Call_01__c", + "MSJ_After_Call_02__c", + "MSJ_After_Call_03__c", + "MSJ_After_Call_04__c", + "MSJ_Closing__c", + "MSJ_Comment_by_MR__c", + "MSJ_Confirmed_by_MR__c", + "MSJ_Createdby__c", + "MSJ_FT_AM_Name__c", + "MSJ_Interview_Preparation__c", + "MSJ_Interview_Reflection__c", + "MSJ_Notify_To_MR__c", + "MSJ_Opening__c", + "MSJ_Others_01_Result__c", + "MSJ_Others_01__c", + "MSJ_Others_02_Result__c", + "MSJ_Others_02__c", + "MSJ_Patient_Thinking__c", + "MSJ_Probing__c", + "MSJ_Supporting__c", + "MSJ_Patient_Thinking_for_FE__c", + "MSJ_After_Call_05__c", + "MSJ_After_Call_06__c", + "MSJ_After_Call_07__c", + "MSJ_After_Call_08__c", + "MSJ_Createdby_FE__c", + "MSJ_Createdby_ONC__c", + "MSJ_Development_Level__c", + "MSJ_Interview_Prep_01__c", + "MSJ_Interview_Prep_02__c", + "MSJ_Leadership_Style__c", + "MSJ_Overcome_01__c", + "MSJ_Overcome_02__c", + "MSJ_Overcome_03__c", + "MSJ_Overcome_04__c", + "MSJ_Review_01__c", + "MSJ_Review_02__c", + "MSJ_SK_01__c", + "MSJ_SK_02__c", + "MSJ_SK_03__c", + "MSJ_SK_04__c", + "MSJ_SK_05__c", + "MSJ_SK_06__c", + "MSJ_SK_07__c", + "MSJ_SK_08__c", + "MSJ_SK_09__c", + "MSJ_SK_10__c", + "MSJ_Specific_Action__c", + "MSJ_Training_Point__c", + "MSJ_Efforts_of_Year__c", + "MSJ_Efforts_of_Month__c", + "MSJ_Skill_Task__c", + "MSJ_Action_of_This_Month__c", + "MSJ_Achievement_of_This_Month__c", + "MSJ_Comment_from_AM__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Call_Comments_vod__c", + "Sample_Card_vod__c", + "Add_Detail_vod__c", + "Property_vod__c", + "Account_vod__c", + "zvod_Product_Discussion_vod__c", + "Status_vod__c", + "Parent_Address_vod__c", + "Account_Plan_vod__c", + "zvod_SaveNew_vod__c", + "Next_Call_Notes_vod__c", + "Pre_Call_Notes_vod__c", + "Mobile_ID_vod__c", + "zvod_Account_Credentials_vod_c_vod__c", + "zvod_Account_Preferred_Name_vod_c_vod__c", + "zvod_Account_Sample_Status_vod_c_vod__c", + "zvod_Attendees_vod__c", + "zvod_Key_Messages_vod__c", + "zvod_Detailing_vod__c", + "zvod_Expenses_vod__c", + "zvod_Followup_vod__c", + "zvod_Samples_vod__c", + "zvod_Save_vod__c", + "zvod_Submit_vod__c", + "zvod_Delete_vod__c", + "Activity_Type__c", + "Significant_Event__c", + "Location_vod__c", + "Subject_vod__c", + "Unlock_vod__c", + "Call_Datetime_vod__c", + "Disbursed_To_vod__c", + "Disclaimer_vod__c", + "Request_Receipt_vod__c", + "Signature_Date_vod__c", + "Signature_vod__c", + "Territory_vod__c", + "Submitted_By_Mobile_vod__c", + "Call_Type_vod__c", + "Add_Key_Message_vod__c", + "Address_vod__c", + "Attendees_vod__c", + "Attendee_Type_vod__c", + "Call_Date_vod__c", + "Detailed_Products_vod__c", + "No_Disbursement_vod__c", + "Parent_Call_vod__c", + "User_vod__c", + "Contact_vod__c", + "zvod_Entity_vod__c", + "Medical_Event_vod__c", + "Mobile_Created_Datetime_vod__c", + "Mobile_Last_Modified_Datetime_vod__c", + "License_vod__c", + "Is_Parent_Call_vod__c", + "Entity_Display_Name_vod__c", + "Override_Lock_vod__c", + "Last_Device_vod__c", + "Ship_Address_Line_1_vod__c", + "Ship_Address_Line_2_vod__c", + "Ship_City_vod__c", + "Ship_Country_vod__c", + "Ship_License_Expiration_Date_vod__c", + "Ship_License_Status_vod__c", + "Ship_License_vod__c", + "Ship_State_vod__c", + "Ship_To_Address_vod__c", + "Ship_Zip_vod__c", + "Ship_To_Address_Text_vod__c", + "CLM_vod__c", + "zvod_CLMDetails_vod__c", + "Is_Sampled_Call_vod__c", + "zvod_Surveys_vod__c", + "Presentations_vod__c", + "Entity_Reference_Id_vod__c", + "Error_Reference_Call_vod__c", + "Duration_vod__c", + "Color_vod__c", + "Allowed_Products_vod__c", + "zvod_Attachments_vod__c", + "Sample_Card_Reason_vod__c", + "ASSMCA_vod__c", + "Address_Line_1_vod__c", + "Address_Line_2_vod__c", + "City_vod__c", + "DEA_Address_Line_1_vod__c", + "DEA_Address_Line_2_vod__c", + "DEA_Address_vod__c", + "DEA_City_vod__c", + "DEA_Expiration_Date_vod__c", + "DEA_State_vod__c", + "DEA_Zip_4_vod__c", + "DEA_Zip_vod__c", + "DEA_vod__c", + "Ship_Zip_4_vod__c", + "State_vod__c", + "Zip_4_vod__c", + "Zip_vod__c", + "Sample_Send_Card_vod__c", + "zvod_Address_vod_c_DEA_Status_vod_c_vod__c", + "Signature_Page_Image_vod__c", + "Credentials_vod__c", + "Salutation_vod__c", + "zvod_Account_Call_Reminder_vod_c_vod__c", + "MSJ_Meeting_Duration__c", + "MSJ_Double_Visit_AM__c", + "zvod_Business_Account_vod__c", + "Product_Priority_1_vod__c", + "Product_Priority_2_vod__c", + "Product_Priority_3_vod__c", + "Product_Priority_4_vod__c", + "Product_Priority_5_vod__c", + "zvod_More_Actions_vod__c", + "zvod_Call_Conflict_Status_vod__c", + "Signature_Timestamp_vod__c", + "Expense_Amount_vod__c", + "Total_Expense_Attendees_Count_vod__c", + "Attendee_list_vod__c", + "Expense_Post_Status_vod__c", + "Attendee_Post_Status_vod__c", + "Expense_System_External_ID_vod__c", + "Incurred_Expense_vod__c", + "Assigner_vod__c", + "Assignment_Datetime_vod__c", + "zvod_Call_Objective_vod__c", + "Signature_Location_Longitude_vod__c", + "Signature_Location_Latitude_vod__c", + "Location_Services_Status_vod__c", + "MSJ_Double_Visit_Other__c", + "MSJ_Comment__c", + "MSJ_For_Reporting__c", + "MSJ_Number_of_Attendees__c", + "MSJ_Main_Dept__c", + "Planned_Type_vjh__c", + "Cobrowse_URL_Participant_vod__c", + "MSJ_Activity_Method_Text__c", + "MSJ_Activity_Method__c", + "MSJ_Classification__c", + "MSJ_Double_Visit_MSL__c", + "MSJ_MSL_Comment_for_MR__c", + "MSJ_APD__c", + "Medical_Inquiry_vod__c", + "MSJ_Call_Type_MSJ__c", + "MSJ_Prescription_Request__c", + "MSJ_Patient_Follow__c", + "Child_Account_Id_vod__c", + "Child_Account_vod__c", + "Location_Id_vod__c", + "Location_Name_vod__c", + "MSJ_Comments_about_technology__c", + "Remote_Meeting_vod__c", + "Veeva_Remote_Meeting_Id_vod__c", + "MSJ_Activity_Type_Report__c", + "MSJ_Activity_Type__c", + "MSJ_Activity__c", + "MSJ_Comments__c", + "MSJ_Therapy__c", + "MSJ_Time_Hrs__c", + "EMDS_CO_Reference__c", + "EMDS_Call_Sub_Type__c", + "EMDS_Call_Type__c", + "EMDS_Call_Unsuccessful__c", + "EMDS_Congress_Type__c", + "EMDS_Date_of_Service__c", + "EMDS_Fertility_DisInterest__c", + "EMDS_Fertility_Interest__c", + "EMDS_Installed_Equipment__c", + "EMDS_Pipeline_Stage_Value__c", + "EMDS_Pipeline_Stage__c", + "EMDS_Pipeline__c", + "EMDS_Reason_for_Call__c", + "EMDS_Training_Completed__c", + "MSJ_BrainStorming__c", + "MSJ_SIPAGL_1A__c", + "MSJ_SIPAGL_1B__c", + "MSJ_SIPAGL_2__c", + "MSJ_SIPAGL_3__c", + "MSJ_SIPAGL_4A__c", + "MSJ_SIPAGL_5A__c", + "MSJ_SIPAGL_comment__c", + "MSJ_SIPAGL_4B__c", + "MSJ_SIPAGL_5B__c", + "Location_Text_vod__c", + "Call_Channel_vod__c", + "MSJ_Scientific_Interaction__c", + "MSJ_Activity_Email_Reply__c", + "MSJ_Interaction_Duration__c", + "MSJ_SIPAGL_1A_date__c", + "MSJ_CoPromotion__c", + "Call_Channel_Formula_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Detail_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Is_Parent_Call_vod__c", + "Call2_vod__c", + "Product_vod__c", + "Detail_Priority_vod__c", + "Mobile_ID_vod__c", + "Override_Lock_vod__c", + "Type_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Key_Message_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Call2_vod__c", + "Reaction_vod__c", + "Product_vod__c", + "Key_Message_vod__c", + "Mobile_ID_vod__c", + "Contact_vod__c", + "Call_Date_vod__c", + "User_vod__c", + "Category_vod__c", + "Vehicle_vod__c", + "Is_Parent_Call_vod__c", + "Override_Lock_vod__c", + "CLM_ID_vod__c", + "Slide_Version_vod__c", + "Duration_vod__c", + "Presentation_ID_vod__c", + "Start_Time_vod__c", + "Attendee_Type_vod__c", + "Entity_Reference_Id_vod__c", + "Segment_vod__c", + "Display_Order_vod__c", + "Clm_Presentation_Name_vod__c", + "Clm_Presentation_Version_vod__c", + "Clm_Presentation_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call_Clickstream_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Answer_vod__c", + "Call_vod__c", + "Key_Message_vod__c", + "Mobile_ID_vod__c", + "Popup_Opened_vod__c", + "Possible_Answers_vod__c", + "Presentation_ID_vod__c", + "Product_vod__c", + "Range_Value_vod__c", + "Rollover_Entered_vod__c", + "Selected_Items_vod__c", + "CLM_ID_vod__c", + "Question_vod__c", + "Survey_Type_vod__c", + "Text_Entered_vod__c", + "Toggle_Button_On_vod__c", + "Track_Element_Description_vod__c", + "Track_Element_Id_vod__c", + "Track_Element_Type_vod__c", + "Usage_Duration_vod__c", + "Usage_Start_Time_vod__c", + "AuxillaryId_vod__c", + "ParentId_vod__c", + "Revision_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Discussion_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Call2_vod__c", + "Activity__c", + "Comments__c", + "Contact_vod__c", + "Call_Date_vod__c", + "Product_Strategy_vod__c", + "Product_Tactic_vod__c", + "Restricted_Comments__c", + "Product_vod__c", + "Presentation__c", + "Discussion_Topics__c", + "Slides__c", + "User_vod__c", + "Indication__c", + "Mobile_ID_vod__c", + "Medical_Event_vod__c", + "Is_Parent_Call_vod__c", + "Override_Lock_vod__c", + "zvod_Product_Map_vod__c", + "Attendee_Type_vod__c", + "Entity_Reference_Id_vod__c", + "Account_Tactic_vod__c", + "MSJ_Material_Type__c", + "MSJ_Discussion_Contents__c", + "MSJ_IST_Minutes__c", + "MSJ_Off_Label_Minutes__c", + "MSJ_Discussion_Objectives__c", + "MSJ_Insight__c", + "EMDS_Materials__c", + "EMDS_Topic__c", + "MSJ_Visit_Purpose__c", + "MSJ_Insight_Count__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Time_Off_Territory_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Reason_vod__c", + "Territory_vod__c", + "Date_vod__c", + "Status_vod__c", + "Time_vod__c", + "Hours_vod__c", + "Mobile_ID_vod__c", + "Hours_off_vod__c", + "Start_Time_vod__c", + "MSJ_Day__c", + "MSJ_Comment__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Dynamic_Attribute_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Active_vod__c", + "Dynamic_Attribute_Configuration_vod__c", + "Dynamic_Attribute_Description_vod__c", + "Dynamic_Attribute_Help_Text_vod__c", + "Dynamic_Attribute_Label_vod__c", + "Dynamic_Attribute_Name_vod__c", + "Dynamic_Attribute_Record_Type_vod__c", + "Dynamic_Attribute_Value_Checkbox_vod__c", + "Dynamic_Attribute_Value_Date_Time_vod__c", + "Dynamic_Attribute_Value_Date_vod__c", + "Dynamic_Attribute_Value_Number_vod__c", + "Dynamic_Attribute_Value_Text_Area_vod__c", + "Dynamic_Attribute_Value_Text_vod__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Dynamic_Attribute_Configuration_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Applies_To_vod__c", + "Attribute_Label_vod__c", + "Attribute_Name_vod__c", + "Available_Values_vod__c", + "Description_vod__c", + "Detail_Group_vod__c", + "Display_Order_vod__c", + "External_ID_vod__c", + "Help_Text_vod__c", + "Product_vod__c", + "Read_Only_vod__c", + "Section_Name_vod__c", + "Sharing_Group_vod__c", + "Status_vod__c", + "Track_Changes_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Profile", + "columns": [ + "Id", + "Name", + "PermissionsEmailSingle", + "PermissionsEmailMass", + "PermissionsEditTask", + "PermissionsEditEvent", + "PermissionsExportReport", + "PermissionsImportPersonal", + "PermissionsDataExport", + "PermissionsManageUsers", + "PermissionsEditPublicFilters", + "PermissionsEditPublicTemplates", + "PermissionsModifyAllData", + "PermissionsManageCases", + "PermissionsManageSolutions", + "PermissionsCustomizeApplication", + "PermissionsEditReadonlyFields", + "PermissionsRunReports", + "PermissionsViewSetup", + "PermissionsTransferAnyEntity", + "PermissionsNewReportBuilder", + "PermissionsManageSelfService", + "PermissionsManageCssUsers", + "PermissionsActivateContract", + "PermissionsApproveContract", + "PermissionsImportLeads", + "PermissionsManageLeads", + "PermissionsTransferAnyLead", + "PermissionsViewAllData", + "PermissionsEditPublicDocuments", + "PermissionsViewEncryptedData", + "PermissionsEditBrandTemplates", + "PermissionsEditHtmlTemplates", + "PermissionsManageTranslation", + "PermissionsDeleteActivatedContract", + "PermissionsSendSitRequests", + "PermissionsApiUserOnly", + "PermissionsManageRemoteAccess", + "PermissionsCanUseNewDashboardBuilder", + "PermissionsManageCategories", + "PermissionsConvertLeads", + "PermissionsTestInstanceCreate", + "PermissionsPasswordNeverExpires", + "PermissionsUseTeamReassignWizards", + "PermissionsInstallMultiforce", + "PermissionsPublishMultiforce", + "PermissionsEditOppLineItemUnitPrice", + "PermissionsManageTerritories", + "PermissionsCreateMultiforce", + "PermissionsBulkApiHardDelete", + "PermissionsInboundMigrationToolsUser", + "PermissionsSolutionImport", + "PermissionsManageCallCenters", + "PermissionsManageSynonyms", + "PermissionsOutboundMigrationToolsUser", + "PermissionsViewContent", + "PermissionsManageEmailClientConfig", + "PermissionsEnableNotifications", + "PermissionsManageDataIntegrations", + "PermissionsDistributeFromPersWksp", + "PermissionsViewDataCategories", + "PermissionsManageDataCategories", + "PermissionsAuthorApex", + "PermissionsManageMobile", + "PermissionsApiEnabled", + "PermissionsManageCustomReportTypes", + "PermissionsEditCaseComments", + "PermissionsTransferAnyCase", + "PermissionsContentAdministrator", + "PermissionsCreateWorkspaces", + "PermissionsManageContentPermissions", + "PermissionsManageContentProperties", + "PermissionsManageContentTypes", + "PermissionsScheduleJob", + "PermissionsManageExchangeConfig", + "PermissionsManageAnalyticSnapshots", + "PermissionsScheduleReports", + "PermissionsManageBusinessHourHolidays", + "PermissionsManageDynamicDashboards", + "PermissionsManageInteraction", + "PermissionsViewMyTeamsDashboards", + "PermissionsResetPasswords", + "PermissionsFlowUFLRequired", + "PermissionsActivitiesAccess", + "PermissionsEmailTemplateManagement", + "PermissionsEmailAdministration", + "PermissionsChatterFileLink", + "PermissionsForceTwoFactor", + "PermissionsViewEventLogFiles", + "PermissionsManageNetworks", + "PermissionsManageAuthProviders", + "PermissionsRunFlow", + "PermissionsCreateCustomizeDashboards", + "PermissionsCreateDashboardFolders", + "PermissionsViewPublicDashboards", + "PermissionsManageDashbdsInPubFolders", + "PermissionsCreateCustomizeReports", + "PermissionsCreateReportFolders", + "PermissionsViewPublicReports", + "PermissionsManageReportsInPubFolders", + "PermissionsEditMyDashboards", + "PermissionsEditMyReports", + "PermissionsViewAllUsers", + "PermissionsConnectOrgToEnvironmentHub", + "PermissionsCreateCustomizeFilters", + "PermissionsContentHubUser", + "PermissionsGovernNetworks", + "PermissionsSalesConsole", + "PermissionsTwoFactorApi", + "PermissionsDeleteTopics", + "PermissionsEditTopics", + "PermissionsCreateTopics", + "PermissionsAssignTopics", + "PermissionsIdentityEnabled", + "PermissionsIdentityConnect", + "PermissionsContentWorkspaces", + "PermissionsCustomMobileAppsAccess", + "PermissionsViewHelpLink", + "PermissionsManageProfilesPermissionsets", + "PermissionsAssignPermissionSets", + "PermissionsManageRoles", + "PermissionsManageIpAddresses", + "PermissionsManageSharing", + "PermissionsManageInternalUsers", + "PermissionsManagePasswordPolicies", + "PermissionsManageLoginAccessPolicies", + "PermissionsManageCustomPermissions", + "PermissionsStdAutomaticActivityCapture", + "PermissionsManageTwoFactor", + "PermissionsDebugApex", + "PermissionsLightningExperienceUser", + "PermissionsConfigCustomRecs", + "PermissionsSubmitMacrosAllowed", + "PermissionsBulkMacrosAllowed", + "PermissionsManageSessionPermissionSets", + "PermissionsCreateAuditFields", + "PermissionsUpdateWithInactiveOwner", + "PermissionsManageSandboxes", + "PermissionsAutomaticActivityCapture", + "PermissionsImportCustomObjects", + "PermissionsDelegatedTwoFactor", + "PermissionsSelectFilesFromSalesforce", + "PermissionsModerateNetworkUsers", + "PermissionsMergeTopics", + "PermissionsSubscribeToLightningReports", + "PermissionsManagePvtRptsAndDashbds", + "PermissionsAllowLightningLogin", + "PermissionsCampaignInfluence2", + "PermissionsViewDataAssessment", + "PermissionsCanApproveFeedPost", + "PermissionsAllowViewEditConvertedLeads", + "PermissionsShowCompanyNameAsUserBadge", + "PermissionsAccessCMC", + "PermissionsViewHealthCheck", + "PermissionsManageHealthCheck", + "PermissionsPackaging2", + "PermissionsManageCertificates", + "PermissionsCreateReportInLightning", + "PermissionsPreventClassicExperience", + "PermissionsListEmailSend", + "PermissionsChangeDashboardColors", + "PermissionsManageRecommendationStrategies", + "PermissionsManagePropositions", + "PermissionsSubscribeReportRolesGrps", + "PermissionsSubscribeDashboardRolesGrps", + "PermissionsUseWebLink", + "PermissionsHasUnlimitedNBAExecutions", + "PermissionsViewOnlyEmbeddedAppUser", + "PermissionsViewAllActivities", + "PermissionsSubscribeReportToOtherUsers", + "PermissionsLightningConsoleAllowedForUser", + "PermissionsSubscribeReportsRunAsUser", + "PermissionsSubscribeToLightningDashboards", + "PermissionsSubscribeDashboardToOtherUsers", + "PermissionsCreateLtngTempInPub", + "PermissionsTransactionalEmailSend", + "PermissionsViewPrivateStaticResources", + "PermissionsCreateLtngTempFolder", + "PermissionsApexRestServices", + "PermissionsEnableCommunityAppLauncher", + "PermissionsGiveRecognitionBadge", + "PermissionsUseMySearch", + "PermissionsLtngPromoReserved01UserPerm", + "PermissionsManageSubscriptions", + "PermissionsManageSurveys", + "PermissionsUseAssistantDialog", + "PermissionsUseQuerySuggestions", + "PermissionsViewRoles", + "PermissionsLMOutboundMessagingUserPerm", + "PermissionsModifyDataClassification", + "PermissionsPrivacyDataAccess", + "PermissionsQueryAllFiles", + "PermissionsModifyMetadata", + "PermissionsManageCMS", + "PermissionsSandboxTestingInCommunityApp", + "PermissionsCanEditPrompts", + "PermissionsViewUserPII", + "PermissionsManageHubConnections", + "PermissionsB2BMarketingAnalyticsUser", + "PermissionsTraceXdsQueries", + "PermissionsViewAllCustomSettings", + "PermissionsViewAllForeignKeyNames", + "PermissionsHeadlessCMSAccess", + "PermissionsLMEndMessagingSessionUserPerm", + "PermissionsConsentApiUpdate", + "PermissionsAccessContentBuilder", + "PermissionsAccountSwitcherUser", + "PermissionsManageC360AConnections", + "PermissionsManageReleaseUpdates", + "PermissionsViewAllProfiles", + "PermissionsSkipIdentityConfirmation", + "PermissionsSendCustomNotifications", + "PermissionsPackaging2Delete", + "PermissionsFSCComprehensiveUserAccess", + "PermissionsManageTrustMeasures", + "PermissionsViewTrustMeasures", + "PermissionsIsotopeCToCUser", + "PermissionsIsotopeAccess", + "PermissionsIsotopeLEX", + "PermissionsQuipMetricsAccess", + "PermissionsQuipUserEngagementMetrics", + "PermissionsManageExternalConnections", + "PermissionsAIViewInsightObjects", + "PermissionsAICreateInsightObjects", + "PermissionsNativeWebviewScrolling", + "PermissionsViewDeveloperName", + "Type", + "UserLicenseId", + "UserType", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "Description", + "LastViewedDate", + "LastReferencedDate" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "My_Setup_Products_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Product_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Activity_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_External_ID_Map_vod__c", + "Account_vod__c", + "Call_vod__c", + "City_vod__c", + "Client_Name_vod__c", + "Client_OS_vod__c", + "Client_Type_vod__c", + "Country_vod__c", + "Debug_vod__c", + "Device_vod__c", + "IP_Address_vod__c", + "Multichannel_Activity_vod__c", + "Referring_Site_vod__c", + "Region_vod__c", + "Sent_Email_vod__c", + "Session_Id_vod__c", + "Site_vod__c", + "Start_DateTime_vod__c", + "Total_Duration_vod__c", + "URL_vod__c", + "User_Agent_vod__c", + "VExternal_Id_vod__c", + "Viewport_Height_vod__c", + "Viewport_Width_vod__c", + "Color_vod__c", + "Icon_vod__c", + "MCD_Primary_Key_vod__c", + "Record_Type_Name_vod__c", + "MSJ_Date_Opened__c", + "MSJ_Sent_Date__c", + "MSJ_Email_Subject__c", + "MSJ_Opens__c", + "MSJ_Email_Status__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Activity_Line_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Multichannel_Activity_vod__c", + "Call_vod__c", + "Custom_vod__c", + "DateTime_vod__c", + "Debug_vod__c", + "Detail_Group_VExternal_Id_vod__c", + "Detail_Group_vod__c", + "Duration_vod__c", + "Event_Subtype_vod__c", + "Event_Type_vod__c", + "Key_Message_VExternal_Id_vod__c", + "Key_Message_vod__c", + "Multichannel_Content_Asset_Id_vod__c", + "Multichannel_Content_Asset_Version_vod__c", + "Multichannel_Content_Asset_vod__c", + "Multichannel_Content_vod__c", + "Product_VExternal_Id_vod__c", + "Product_vod__c", + "Sent_Email_vod__c", + "VExternal_Id_vod__c", + "Video_Last_Viewed_Time_vod__c", + "Video_Length_vod__c", + "Video_Total_Time_Spent_vod__c", + "View_Order_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Consent_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Capture_Datetime_vod__c", + "Channel_Value_vod__c", + "Detail_Group_vod__c", + "External_ID_vod__c", + "Last_Device_vod__c", + "Mobile_ID_vod__c", + "Opt_Expiration_Date_vod__c", + "Opt_Type_vod__c", + "Optout_Event_Type_vod__c", + "Product_vod__c", + "Signature_Datetime_vod__c", + "Signature_ID_vod__c", + "Signature_vod__c", + "Sample_Consent_Template_Data_vod__c", + "Sample_Consent_Template_vod__c", + "Consent_Line_vod__c", + "Consent_Type_vod__c", + "Default_Consent_Text_vod__c", + "Disclaimer_Text_vod__c", + "Sub_Channel_Key_vod__c", + "Consent_Confirm_Datetime_vod__c", + "Related_Transaction_Id_vod__c", + "Sent_Email_vod__c", + "Content_Type_vod__c", + "Receipt_Email_vod__c", + "Receipt_Sent_Email_Transaction_Id_vod__c", + "Receipt_Sent_Email_vod__c", + "Captured_By_vod__c", + "Opt_Out_Disclaimer_Text_vod__c", + "Channel_Source_vod__c", + "Union_Id_vod__c", + "User_Last_Notified_vod__c", + "Sub_Channel_Display_Name__c", + "MSJ_Consent_Source__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Inquiry_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Address_Line_1_vod__c", + "Address_Line_2_vod__c", + "City_vod__c", + "Delivery_Method_vod__c", + "Email_vod__c", + "Fax_Number_vod__c", + "Inquiry_Text__c", + "Lock_vod__c", + "Mobile_ID_vod__c", + "Phone_Number_vod__c", + "Product__c", + "Rush_Delivery__c", + "Signature_Date_vod__c", + "Signature_vod__c", + "State_vod__c", + "Status_vod__c", + "Zip_vod__c", + "zvod_Delivery_Method_vod__c", + "zvod_Disclaimer_vod__c", + "Submitted_By_Mobile_vod__c", + "Disclaimer_vod__c", + "Entity_Reference_Id_vod__c", + "Call2_vod__c", + "Country_vod__c", + "Override_Lock_vod__c", + "MSJ_Department__c", + "MSJ_Doctor_Name__c", + "MSJ_Hospital_Name__c", + "MSJ_Indication__c", + "MSJ_Inquiry_Assignment__c", + "MSJ_Inquiry_Date__c", + "MSJ_Inquiry_Input_Manager__c", + "MSJ_Inquiry_Input_User__c", + "MSJ_MSL_Manager__c", + "MSJ_Notice_to_MR__c", + "MSJ_Person_in_charge_1__c", + "MSJ_Person_in_charge_2__c", + "MSJ_Product_for_MEC__c", + "MSJ_Product_for_MR__c", + "MSJ_Reply_Date__c", + "MSJ_Reply_User__c", + "MSJ_Reply__c", + "MSJ_Title__c", + "MSJ_AE_Infomation__c", + "MSJ_FAQ_Number_Report__c", + "MSJ_Return_Call_Report__c", + "MSJ_Inquiry_Origin_Report__c", + "MSJ_AE_Report__c", + "MSJ_Background__c", + "MSJ_MSL_Support__c", + "MSJ_Material_Requirement__c", + "MSJ_Hospital_Name_Disp__c", + "MSJ_Hospital__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Email_Activity_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Sent_Email_vod__c", + "Activity_DateTime_vod__c", + "City_vod__c", + "Click_URL_vod__c", + "Client_Name_vod__c", + "Client_OS_vod__c", + "Client_Type_vod__c", + "Country_vod__c", + "Device_Type_vod__c", + "Event_Msg_vod__c", + "Event_type_vod__c", + "IP_Address_vod__c", + "Region_vod__c", + "User_Agent_vod__c", + "Vault_Doc_ID_vod__c", + "Vault_Doc_Name_vod__c", + "Vault_Document_Major_Version_vod__c", + "Vault_Document_Minor_Version_vod__c", + "Vault_Document_Number_vod__c", + "Vault_Document_Title_vod__c", + "Vault_Instance_ID_vod__c", + "Preference_Modification_vod__c", + "Approved_Document_vod__c", + "Link_Name_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "User", + "columns": [ + "Id", + "Username", + "LastName", + "FirstName", + "Name", + "CompanyName", + "Division", + "Department", + "Title", + "Street", + "City", + "State", + "PostalCode", + "Country", + "Latitude", + "Longitude", + "GeocodeAccuracy", + "Address", + "Email", + "EmailPreferencesAutoBcc", + "EmailPreferencesAutoBccStayInTouch", + "EmailPreferencesStayInTouchReminder", + "SenderEmail", + "SenderName", + "Signature", + "StayInTouchSubject", + "StayInTouchSignature", + "StayInTouchNote", + "Phone", + "Fax", + "MobilePhone", + "Alias", + "CommunityNickname", + "BadgeText", + "IsActive", + "TimeZoneSidKey", + "UserRoleId", + "LocaleSidKey", + "ReceivesInfoEmails", + "ReceivesAdminInfoEmails", + "EmailEncodingKey", + "ProfileId", + "UserType", + "LanguageLocaleKey", + "EmployeeNumber", + "DelegatedApproverId", + "ManagerId", + "LastLoginDate", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "OfflineTrialExpirationDate", + "OfflinePdaTrialExpirationDate", + "UserPermissionsMarketingUser", + "UserPermissionsOfflineUser", + "UserPermissionsWirelessUser", + "UserPermissionsAvantgoUser", + "UserPermissionsCallCenterAutoLogin", + "UserPermissionsSFContentUser", + "UserPermissionsInteractionUser", + "UserPermissionsSupportUser", + "UserPermissionsChatterAnswersUser", + "ForecastEnabled", + "UserPreferencesActivityRemindersPopup", + "UserPreferencesEventRemindersCheckboxDefault", + "UserPreferencesTaskRemindersCheckboxDefault", + "UserPreferencesReminderSoundOff", + "UserPreferencesDisableAllFeedsEmail", + "UserPreferencesApexPagesDeveloperMode", + "UserPreferencesReceiveNoNotificationsAsApprover", + "UserPreferencesReceiveNotificationsAsDelegatedApprover", + "UserPreferencesHideCSNGetChatterMobileTask", + "UserPreferencesHideCSNDesktopTask", + "UserPreferencesHideChatterOnboardingSplash", + "UserPreferencesHideSecondChatterOnboardingSplash", + "UserPreferencesShowTitleToExternalUsers", + "UserPreferencesShowManagerToExternalUsers", + "UserPreferencesShowEmailToExternalUsers", + "UserPreferencesShowWorkPhoneToExternalUsers", + "UserPreferencesShowMobilePhoneToExternalUsers", + "UserPreferencesShowFaxToExternalUsers", + "UserPreferencesShowStreetAddressToExternalUsers", + "UserPreferencesShowCityToExternalUsers", + "UserPreferencesShowStateToExternalUsers", + "UserPreferencesShowPostalCodeToExternalUsers", + "UserPreferencesShowCountryToExternalUsers", + "UserPreferencesShowProfilePicToGuestUsers", + "UserPreferencesShowTitleToGuestUsers", + "UserPreferencesShowCityToGuestUsers", + "UserPreferencesShowStateToGuestUsers", + "UserPreferencesShowPostalCodeToGuestUsers", + "UserPreferencesShowCountryToGuestUsers", + "UserPreferencesHideInvoicesRedirectConfirmation", + "UserPreferencesHideStatementsRedirectConfirmation", + "UserPreferencesPathAssistantCollapsed", + "UserPreferencesCacheDiagnostics", + "UserPreferencesShowEmailToGuestUsers", + "UserPreferencesShowManagerToGuestUsers", + "UserPreferencesShowWorkPhoneToGuestUsers", + "UserPreferencesShowMobilePhoneToGuestUsers", + "UserPreferencesShowFaxToGuestUsers", + "UserPreferencesShowStreetAddressToGuestUsers", + "UserPreferencesLightningExperiencePreferred", + "UserPreferencesPreviewLightning", + "UserPreferencesHideEndUserOnboardingAssistantModal", + "UserPreferencesHideLightningMigrationModal", + "UserPreferencesHideSfxWelcomeMat", + "UserPreferencesHideBiggerPhotoCallout", + "UserPreferencesGlobalNavBarWTShown", + "UserPreferencesGlobalNavGridMenuWTShown", + "UserPreferencesCreateLEXAppsWTShown", + "UserPreferencesFavoritesWTShown", + "UserPreferencesRecordHomeSectionCollapseWTShown", + "UserPreferencesRecordHomeReservedWTShown", + "UserPreferencesFavoritesShowTopFavorites", + "UserPreferencesExcludeMailAppAttachments", + "UserPreferencesSuppressTaskSFXReminders", + "UserPreferencesSuppressEventSFXReminders", + "UserPreferencesPreviewCustomTheme", + "UserPreferencesHasCelebrationBadge", + "UserPreferencesUserDebugModePref", + "UserPreferencesSRHOverrideActivities", + "UserPreferencesNewLightningReportRunPageEnabled", + "UserPreferencesReverseOpenActivitiesView", + "UserPreferencesHideBrowseProductRedirectConfirmation", + "UserPreferencesHideOnlineSalesAppWelcomeMat", + "ContactId", + "AccountId", + "CallCenterId", + "Extension", + "FederationIdentifier", + "AboutMe", + "FullPhotoUrl", + "SmallPhotoUrl", + "IsExtIndicatorVisible", + "OutOfOfficeMessage", + "MediumPhotoUrl", + "DigestFrequency", + "DefaultGroupNotificationFrequency", + "LastViewedDate", + "LastReferencedDate", + "BannerPhotoUrl", + "SmallBannerPhotoUrl", + "MediumBannerPhotoUrl", + "IsProfilePhotoActive", + "IndividualId", + "Last_Mobile_Connect_vod__c", + "Last_Tablet_Connect_vod__c", + "Last_Mobile_Connect_Version_vod__c", + "Last_Tablet_Connect_Version_vod__c", + "Last_Mobile_Sync_vod__c", + "Last_Tablet_Sync_vod__c", + "RaiseLoggingLevel_vod__c", + "SendDetailedLog_vod__c", + "Last_Blackberry_Connect_vod__c", + "Last_Blackberry_Connect_Version_vod__c", + "Last_Blackberry_Sync_vod__c", + "Force_Full_Refresh_vod__c", + "Override_SystemModstamp_Timestamp_vod__c", + "Facetime_Email_vod__c", + "Facetime_Phone_vod__c", + "Product_Expertise_vod__c", + "Available_vod__c", + "Available_Last_Update_vod__c", + "Last_iPad_Connect_Version_vod__c", + "Last_iPad_Connect_vod__c", + "Last_iPad_Sync_vod__c", + "Inventory_Order_Allocation_Group_vod__c", + "Concur_User_Id_vod__c", + "Last_iPad_iOS_Version_vod__c", + "Approved_Email_Admin_vod__c", + "Last_WinModern_Connect_Version_vod__c", + "Last_WinModern_Connect_vod__c", + "Last_WinModern_Sync_vod__c", + "Primary_Territory_vod__c", + "Analytics_Admin_vod__c", + "Content_Admin_vod__c", + "Last_WinModern_Windows_Version_vod__c", + "Upload_VTrans_vod__c", + "Sync_Frequency_vjh__c", + "Clear_Client_Sync_Errors_vod__c", + "Remote_Meeting_Host_Id_vod__c", + "Remote_Meeting_Host_Token_vod__c", + "Last_iPhone_Connect_Version_vod__c", + "Last_iPhone_Connect_vod__c", + "Last_iPhone_Sync_vod__c", + "Last_iPhone_iOS_Version_vod__c", + "Remote_Meeting_Start_From_CRM_Online_vod__c", + "Country_Code_vod__c", + "User_Type_vod__c", + "Engage_Group_Provisioning_Status_vod__c", + "Engage_Group_Request_vod__c", + "Engage_Group_vod__c", + "Last_CRMDesktop_Mac_Sync_vod__c", + "Last_CRMDesktop_Mac_Version_vod__c", + "Last_CRMDesktop_Windows_Sync_vod__c", + "Last_CRMDesktop_Windows_Version_vod__c", + "MSJ_Test_User__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "UserTerritory2Association", + "columns": [ + "Id", + "UserId", + "Territory2Id", + "IsActive", + "RoleInTerritory2", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Remote_Meeting_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Meeting_Id_vod__c", + "Meeting_Name_vod__c", + "Mobile_ID_vod__c", + "Scheduled_DateTime_vod__c", + "Scheduled_vod__c", + "Attendance_Report_Process_Status_vod__c", + "Latest_Meeting_Start_Datetime_vod__c", + "Meeting_Password_vod__c", + "Meeting_Outcome_Status_vod__c", + "Allow_for_Joining_via_Zoom_vod__c", + "Zoom_Join_Token_vod__c", + "VExternal_Id_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "RecordType", + "columns": [ + "Id", + "Name", + "DeveloperName", + "NamespacePrefix", + "Description", + "BusinessProcessId", + "SobjectType", + "IsActive", + "IsPersonType", + "CreatedById", + "CreatedDate", + "LastModifiedById", + "LastModifiedDate", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "UserRole", + "columns": [ + "Id", + "Name", + "ParentRoleId", + "RollupDescription", + "OpportunityAccessForAccountOwner", + "CaseAccessForAccountOwner", + "ContactAccessForAccountOwner", + "ForecastUserId", + "MayForecastManagerShare", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "PortalAccountId", + "PortalType", + "PortalAccountOwnerId" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Account", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "Name", + "LastName", + "FirstName", + "Salutation", + "RecordTypeId", + "Phone", + "Fax", + "Website", + "PhotoUrl", + "NumberOfEmployees", + "Ownership", + "OwnerId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "IsExcludedFromRealign", + "PersonContactId", + "IsPersonAccount", + "PersonMailingStreet", + "PersonMailingCity", + "PersonMailingState", + "PersonMailingPostalCode", + "PersonMailingCountry", + "PersonMailingLatitude", + "PersonMailingLongitude", + "PersonMailingGeocodeAccuracy", + "PersonMailingAddress", + "PersonOtherStreet", + "PersonOtherCity", + "PersonOtherState", + "PersonOtherPostalCode", + "PersonOtherCountry", + "PersonOtherLatitude", + "PersonOtherLongitude", + "PersonOtherGeocodeAccuracy", + "PersonOtherAddress", + "PersonMobilePhone", + "PersonHomePhone", + "PersonOtherPhone", + "PersonAssistantPhone", + "PersonEmail", + "PersonTitle", + "PersonDepartment", + "PersonAssistantName", + "PersonBirthdate", + "PersonHasOptedOutOfEmail", + "PersonHasOptedOutOfFax", + "PersonDoNotCall", + "PersonLastCURequestDate", + "PersonLastCUUpdateDate", + "PersonEmailBouncedReason", + "PersonEmailBouncedDate", + "PersonIndividualId", + "Jigsaw", + "JigsawCompanyId", + "AccountSource", + "SicDesc", + "External_ID_vod__c", + "Credentials_vod__c", + "Territory_vod__c", + "Exclude_from_Zip_to_Terr_Processing_vod__c", + "Group_Specialty_1_vod__c", + "Group_Specialty_2_vod__c", + "Specialty_1_vod__c", + "Specialty_2_vod__c", + "Formatted_Name_vod__c", + "Territory_Test_vod__c", + "Mobile_ID_vod__c", + "Gender_vod__c", + "ID_vod__c", + "Do_Not_Sync_Sales_Data_vod__c", + "ID2_vod__c", + "Preferred_Name_vod__c", + "Sample_Default_vod__c", + "Segmentations_vod__c", + "Restricted_Products_vod__c", + "Payer_Id_vod__c", + "Alternate_Name_vod__c", + "Do_Not_Call_vod__c", + "MSJ_Beds__c", + "Spend_Amount__c", + "PDRP_Opt_Out_vod__c", + "Spend_Status_Value_vod__c", + "PDRP_Opt_Out_Date_vod__c", + "Spend_Status_vod__c", + "Enable_Restricted_Products_vod__c", + "Call_Reminder_vod__c", + "Account_Group_vod__c", + "Primary_Parent_vod__c", + "Color_vod__c", + "Middle_vod__c", + "Suffix_vod__c", + "MSJ_Type__c", + "No_Orders_vod__c", + "MSJ_BU_ONC__c", + "MSJ_BU_FE__c", + "Account_Search_FirstLast_vod__c", + "Account_Search_LastFirst_vod__c", + "MSJ_Operation__c", + "Practice_at_Hospital_vod__c", + "Practice_Near_Hospital_vod__c", + "Do_Not_Create_Child_Account_vod__c", + "Total_MDs_DOs__c", + "AHA__c", + "Order_Type_vod__c", + "NPI_vod__c", + "ME__c", + "Speaker__c", + "Investigator_vod__c", + "Default_Order_Type_vod__c", + "Tax_Status__c", + "Model__c", + "Offerings__c", + "Departments__c", + "Account_Type__c", + "MSJ_ONC_Tier__c", + "Account_Search_Business_vod__c", + "Business_Professional_Person_vod__c", + "Hospital_Type_vod__c", + "Account_Class_vod__c", + "Furigana_vod__c", + "MSJ_JISART__c", + "Total_Revenue_000__c", + "Net_Income_Loss_000__c", + "PMPM_Income_Loss_000__c", + "Commercial_Premiums_PMPM__c", + "Medical_Loss_Ratio__c", + "Medical_Expenses_PMPM__c", + "Commercial_Patient_Days_1000__c", + "HMO_Market_Shr__c", + "HMO__c", + "HMO_POS__c", + "PPO__c", + "PPO_POS__c", + "Medicare__c", + "Medicaid__c", + "MSJ_HP_Name_E__c", + "MSJ_Department__c", + "MSJ_Date_Of_Birth__c", + "MSJ_FE_GF_Potential__c", + "MSJ_FE_SZ_Potential__c", + "MSJ_EB_CRC_Ladder__c", + "MSJ_EB_CRC_Segment__c", + "MSJ_EB_HN_Segment__c", + "Business_Description__c", + "Regional_Strategy__c", + "Contracts_Process__c", + "MSJ_GF_segment__c", + "MSJ_DCF_DR_Code__c", + "MSJ_SZ_Segment__c", + "MSJ_Remark__c", + "MSJ_Title__c", + "MSJ_Role__c", + "MSJ_Kana__c", + "MSJ_Specialism__c", + "MSJ_Graduated_from__c", + "MSJ_Year_Graduation__c", + "Target__c", + "KOL_vod__c", + "MSJ_EPPV_Code__c", + "MSJ_DCF_HP_Code__c", + "Total_Lives__c", + "Total_Physicians_Enrolled__c", + "MSJ_Delete__c", + "MSJ_KOL_LOL__c", + "MSJ_ONC_Status__c", + "Account_Identifier_vod__c", + "Approved_Email_Opt_Type_vod__c", + "Language_vod__c", + "MSJ_KRAS_Routine_Date__c", + "MSJ_KRAS_Routine__c", + "MSJ_DRP_Target__c", + "MSJ_Fertility_Evaluation_Score__c", + "MSJ_Fertility_Tracking_Last_Modify_Date__c", + "Total_Pharmacists__c", + "MSJ_Number_of_Gonadotropin__c", + "MSJ_Number_of_IUI_cycle__c", + "MSJ_Number_of_OI_monthly_cycle__c", + "MSJ_OI_Protocol_learning_level__c", + "MSJ_H_N_Tier__c", + "MSJ_XLK_Segment__c", + "MSJ_XLK_Tier__c", + "Career_Status_vod__c", + "Photo_vod__c", + "MSJ_EB_H_N_LA_Segment__c", + "MSJ_EB_H_N_RM_Segment__c", + "MSJ_FE_CE_Potential__c", + "MSJ_FE_1C_potential__c", + "MSJ_FE_OV_potential__c", + "MSJ_FE_Tech_potential__c", + "MSJ_CE_segment__c", + "MSJ_1C_segment__c", + "MSJ_OV_segment__c", + "MSJ_Tech_segment__c", + "MSJ_Target_Call_Num__c", + "MSJ_DR_Change_Log__c", + "MSJ_Global_scientific_exposure__c", + "MSJ_H_index__c", + "MSJ_Num_of_Article_3Y__c", + "MSJ_Num_of_Article__c", + "MSJ_Num_of_Article_as_1st_Author_3Y__c", + "MSJ_Num_of_article_growth_rate_3Y__c", + "MSJ_Num_of_cited_3Y__c", + "MSJ_Num_of_impact_factor_3Y__c", + "MSJ_impact_factor_as_1st_Author_3Y__c", + "EMDS_Has_Pipeline_Opportunity__c", + "EMDS_Pipeline_Count__c", + "MSJ_BVC_Segment__c", + "MSJ_BVC_Tier__c", + "MSJ_BVC_AcctOpen__c", + "MSJ_BVC_MCC_Patients__c", + "MSJ_ONC_HP_Segment__c", + "MSJ_AE_Department__c", + "MSJ_AE_Facility__c", + "MSJ_AE_Name__c", + "MSJ_AE_Title__c", + "MSJ_Email__c", + "MSJ_FE_GF2_Potential__c", + "MSJ_FE_Location_potential__c", + "MSJ_GF2_segment__c", + "MSJ_OPTIN_target__c", + "MSJ_Merck_Specialty1__c", + "MSJ_Merck_Specialty2__c", + "MSJ_Marketing_Cloud_Integration__c", + "MSJ_Marketing_Cloud1__c", + "MSJ_Marketing_Cloud2__c", + "MSJ_Marketing_Cloud3__c", + "MSJ_Marketing_Cloud4__c", + "MSJ_Medical_Department__c", + "MSJ_Marketing_Cloud0__c", + "Mobile_ID_vod__pc", + "H1Insights__H1_NPI_Value_for_Testing__pc", + "H1Insights__H1_Person_ID__pc", + "H1Insights__H1_Request_Status__pc", + "H1Insights__H1_URL__pc", + "H1Insights__NPI_Number__pc", + "H1Insights__NPI_Number_for_H1_Insights__pc", + "MSJ_Marketing_Cloud_Integration__pc" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true, + "datetime_column": "LastModifiedDate" + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Type_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Header_vod__c", + "Channel_Label_vod__c", + "Channel_Source_vod__c", + "Consent_Expires_In_vod__c", + "Default_Consent_Type_vod__c", + "Disclaimer_Text_vod__c", + "Display_Order_vod__c", + "Product_Preference_vod__c", + "zvod_Consent_Default_Consent_Text_vod__c", + "zvod_Consent_Line_vod__c", + "zvod_Signature_Capture_vod__c", + "Double_Opt_In_vod__c", + "zvod_Consent_Activity_Tracking_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Header_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Header_Help_Text_vod__c", + "Country_vod__c", + "Inactive_Datetime_vod__c", + "Language_vod__c", + "Status_vod__c", + "Signature_Required_On_Opt_Out_vod__c", + "Request_Receipt_vod__c", + "Subscription_Option_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Line_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Type_vod__c", + "Detail_Group_Display_Name_vod__c", + "Detail_Group_vod__c", + "Display_Order_vod__c", + "End_Date_vod__c", + "Group_By_vod__c", + "Product_Display_Name_vod__c", + "Product_vod__c", + "Start_Date_vod__c", + "Sub_Channel_Description_vod__c", + "Sub_Channel_Display_Name_vod__c", + "Sub_Channel_Key_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Inquiry_Assignment__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Medical_Inquiry__c", + "MSJ_Close__c", + "MSJ_Doctor_Name__c", + "MSJ_Hospital_Name__c", + "MSJ_Indication__c", + "MSJ_Inquiry_Text__c", + "MSJ_MEC_User__c", + "MSJ_MSL_Manager__c", + "MSJ_MSL_User__c", + "MSJ_Notice_to_MR__c", + "MSJ_Product_for_MEC__c", + "MSJ_Product_for_MR__c", + "MSJ_Reply_Date__c", + "MSJ_Reply__c", + "MSJ_AE_Infomation__c", + "MSJ_Cancel__c", + "MSJ_FAQ_number_c__c", + "MSJ_Return_Call__c", + "MSJ_Inquiry_Origin__c", + "First_Response__c", + "Inquiry_Created_Date__c", + "Inquiry_Type_1__c", + "Inquiry_Type_2__c", + "MSJ_First_User__c", + "MSJ_MEC_Comment__c", + "MSJ_Send_Email__c", + "MSJ_Temp_Aggregated_Info__c", + "MSJ_AE_Report__c", + "MSJ_Background__c", + "MSJ_Inquiry_Date__c", + "MSJ_MSL_Support__c", + "MSJ_Handover_Comment__c", + "MSJ_Handover_Email__c", + "MSJ_Material_Requirement__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Approved_Document_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Detail_Group_vod__c", + "Document_Description_vod__c", + "Document_Host_URL_vod__c", + "Document_ID_vod__c", + "Document_Last_Mod_DateTime_vod__c", + "Email_Allows_Documents_vod__c", + "Email_Domain_vod__c", + "Email_Fragment_HTML_vod__c", + "Email_From_Address_vod__c", + "Email_From_Name_vod__c", + "Email_HTML_1_vod__c", + "Email_HTML_2_vod__c", + "Email_ReplyTo_Address_vod__c", + "Email_ReplyTo_Name_vod__c", + "Email_Subject_vod__c", + "Email_Template_Fragment_Document_ID_vod__c", + "Email_Template_Fragment_HTML_vod__c", + "ISI_Document_ID_vod__c", + "Language_vod__c", + "Other_Document_ID_List_vod__c", + "PI_Document_ID_vod__c", + "Piece_Document_ID_vod__c", + "Product_vod__c", + "Status_vod__c", + "Territory_vod__c", + "Vault_Instance_ID_vod__c", + "Allow_Any_Product_Fragment_vod__c", + "Allowed_Document_IDs_vod__c", + "Engage_Document_Id_vod__c", + "Vault_Document_ID_vod__c", + "Key_Message_vod__c", + "Events_Management_Subtype_vod__c", + "Survey_vod__c", + "Content_Type_vod__c", + "Bcc_vod__c", + "Audience_vod__c", + "WeChat_Template_ID_vod__c", + "Check_Consent_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Child_Account_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Parent_Account_vod__c", + "Child_Account_vod__c", + "External_ID_vod__c", + "Mobile_ID_vod__c", + "Primary_vod__c", + "Copy_Address_vod__c", + "Child_Name_vod__c", + "Parent_Name_vod__c", + "Parent_Child_Name_vod__c", + "Account_Code__c", + "Child_Department__c", + "Child_Role__c", + "Child_Title__c", + "Child_Remark__c", + "MSJ_1C_segment__c", + "MSJ_BU_FE__c", + "MSJ_BU_ONC__c", + "MSJ_BVC_Segment__c", + "MSJ_CE_segment__c", + "MSJ_Child_Account_Link__c", + "MSJ_DCF_DR_Code__c", + "MSJ_DCF_HP_Code__c", + "MSJ_DR_Change_Log__c", + "MSJ_Delete__c", + "MSJ_Department__c", + "MSJ_EB_CRC_Segment__c", + "MSJ_EB_HN_Segment__c", + "MSJ_EB_H_N_LA_Segment__c", + "MSJ_EB_H_N_RM_Segment__c", + "MSJ_External_ID__c", + "MSJ_Fax__c", + "MSJ_GF2_segment__c", + "MSJ_GF_segment__c", + "MSJ_KOL_LOL__c", + "MSJ_KOL__c", + "MSJ_ONC_HP_Segment__c", + "MSJ_OPTIN_target__c", + "MSJ_OV_segment__c", + "MSJ_Parent_Child_Name__c", + "MSJ_Phone__c", + "MSJ_Remark__c", + "MSJ_Target_Call_Num__c", + "MSJ_Tech_segment__c", + "MSJ_Title__c", + "MSJ_XLK_Segment__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Hospital_Medical_Regimen__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Account_Name__c", + "MSJ_Delete_Date__c", + "MSJ_Delete_Flag__c", + "MSJ_Indication__c", + "MSJ_Line__c", + "MSJ_Medical_Regimen__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Medical_Regimen__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Delete_Date__c", + "MSJ_Delete_Flag__c", + "MSJ_Indication__c", + "MSJ_Remark__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Patient__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Account_Name__c", + "MSJ_CRC_Group__c", + "MSJ_Casus_or_Transfer_Point__c", + "MSJ_Entry_Date__c", + "MSJ_IST_Name__c", + "MSJ_Indication__c", + "MSJ_Line__c", + "MSJ_MR_Comments__c", + "MSJ_MUID__c", + "MSJ_Medical_Regimen__c", + "MSJ_Month__c", + "MSJ_Report_Comments__c", + "MSJ_Start_Date_Of_Administration__c", + "MSJ_Year__c", + "Mobile_ID_vod__c", + "MSJ_CRC_RAS_KRAS__c", + "MSJ_End_Date_Of_Administration__c", + "MSJ_End_Date_of_Stop_Administration__c", + "MSJ_HN_Hospitalized_Type__c", + "MSJ_Start_Date_of_Stop_Administration__c", + "MSJ_Patient_Status__c", + "MSJ_Patient_TA__c", + "MSJ_Child_Account_Name__c", + "MSJ_Child_Account__c", + "MSJ_Parent_Account_Name__c", + "MSJ_Parent_Child_Name__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consumer_site__c", + "Product_info__c", + "Therapeutic_Class_vod__c", + "Parent_Product_vod__c", + "Therapeutic_Area_vod__c", + "Product_Type_vod__c", + "Require_Key_Message_vod__c", + "Cost_vod__c", + "External_ID_vod__c", + "Manufacturer_vod__c", + "Company_Product_vod__c", + "Controlled_Substance_vod__c", + "Description_vod__c", + "Sample_Quantity_Picklist_vod__c", + "Display_Order_vod__c", + "No_Metrics_vod__c", + "Distributor_vod__c", + "Sample_Quantity_Bound_vod__c", + "Sample_U_M_vod__c", + "No_Details_vod__c", + "Quantity_Per_Case_vod__c", + "Schedule_vod__c", + "Restricted_vod__c", + "Pricing_Rule_Quantity_Bound_vod__c", + "No_Promo_Items_vod__c", + "User_Aligned_vod__c", + "Restricted_States_vod__c", + "Sort_Code_vod__c", + "No_Cycle_Plans_vod__c", + "Inventory_Order_UOM_vod__c", + "Inventory_Quantity_Per_Case_vod__c", + "VExternal_Id_vod__c", + "Country__c", + "MSJ_Product_Classification__c", + "MSJ_Indication__c", + "MSJ_Therapeutic_Area__c", + "MSJ_Global_Brand__c", + "MSJ_Global_Business_Unit__c", + "MSJ_Molecules__c", + "MSJ_SBU__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_Group_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Description_vod__c", + "Product_vod__c", + "Product_Catalog_vod__c", + "Start_Date_vod__c", + "End_Date_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_Metrics_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Awareness__c", + "Selling_Stage__c", + "Formulary_Status__c", + "Movement__c", + "Products_vod__c", + "Segment__c", + "X12_mo_trx_chg__c", + "Speaker_Skills__c", + "Investigator_Readiness__c", + "Engagements__c", + "Mobile_ID_vod__c", + "External_ID_vod__c", + "MSJ_Patient__c", + "Detail_Group_vod__c", + "MSJ_EB_1st_Line_Liver_Meta__c", + "MSJ_EB_1st_Line_Multi_Meta__c", + "MSJ_EB_2nd_Line_Mono__c", + "MSJ_EB_2nd_Line_Combination__c", + "MSJ_EB_3rd_Line_Mono__c", + "MSJ_EB_3rd_Line_Combination__c", + "EMDS_Ability__c", + "EMDS_Brand_Loyalty__c", + "EMDS_Decision_Maker__c", + "EMDS_Early_Tech_Adopter__c", + "EMDS_Influence__c", + "EMDS_Main_Driver__c", + "EMDS_Priority__c", + "EMDS_Willingness__c", + "MSJ_KTL_Type__c", + "MSJ_KTL_Tier__c", + "MSJ_Publications__c", + "MSJ_Clinical_Trials__c", + "MSJ_Speaker_for_Medical_Events__c", + "MSJ_Advisor_to_Medical_Affairs__c", + "MSJ_Guidelines_Treatment_Standards__c", + "MSJ_Therapeutic_Area_Expertise__c", + "MSJ_MAP_GAP__c", + "MSJ_Associations__c", + "MSJ_Tier_Score__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Assignment_Type_vod__c", + "Channels_vod__c", + "End_Date_vod__c", + "Expired_vod__c", + "External_ID_vod__c", + "Language_vod__c", + "Lock_vod__c", + "Open_vod__c", + "Product_vod__c", + "Region_vod__c", + "Segment_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "Territory_vod__c", + "zvod_Questions_vod__c", + "zvod_Segments_vod__c", + "zvod_Targets_vod__c", + "Max_Score_vod__c", + "Min_Score_vod__c", + "Autotarget_vod__c", + "Territories_vod__c", + "Target_Type_vod__c", + "MSJ_External_ID__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_Target_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_Display_Name_vod__c", + "Account_vod__c", + "Channels_vod__c", + "End_Date_vod__c", + "Entity_Reference_Id_vod__c", + "External_ID_vod__c", + "Language_vod__c", + "Lock_vod__c", + "Mobile_ID_vod__c", + "No_Autoassign_vod__c", + "Not_Completed_vod__c", + "Region_vod__c", + "Segment_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "Survey_vod__c", + "Territory_vod__c", + "zvod_Address_vod__c", + "zvod_Specialty_vod__c", + "Score_vod__c", + "User_vod__c", + "Child_Account_vod__c", + "Location_Entity_Reference_Id_vod__c", + "Location_vod__c", + "Target_Type_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_Question_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Survey_vod__c", + "Answer_Choice_vod__c", + "External_ID_vod__c", + "Max_Score_vod__c", + "Min_Score_vod__c", + "Order_vod__c", + "Question_vod__c", + "Required_vod__c", + "Text_vod__c", + "Condition_vod__c", + "Source_ID_vod__c", + "MSJ_External_ID__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Question_Response_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Survey_Target_vod__c", + "Answer_Choice_vod__c", + "Date_vod__c", + "Datetime_vod__c", + "External_ID_vod__c", + "Mobile_ID_vod__c", + "Number_vod__c", + "Order_vod__c", + "Question_Text_vod__c", + "Required_vod__c", + "Response_Hash_vod__c", + "Response_vod__c", + "Score_vod__c", + "Survey_Question_vod__c", + "Text_vod__c", + "Type_vod__c", + "Condition_vod__c", + "Inactive_Condition_vod__c", + "Source_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Sent_Fragment_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Sent_Email_vod__c", + "Account_vod__c", + "Email_Template_vod__c", + "Sent_Fragment_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Sent_Email_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Account_Email_vod__c", + "Account_vod__c", + "Approved_Email_Template_vod__c", + "Capture_Datetime_vod__c", + "Detail_Group_vod__c", + "Email_Config_Values_vod__c", + "Email_Content2_vod__c", + "Email_Content_vod__c", + "Email_Fragments_vod__c", + "Email_Sent_Date_vod__c", + "Failure_Msg_vod__c", + "Last_Activity_Date_vod__c", + "Last_Device_vod__c", + "MC_Capture_Datetime_vod__c", + "Mobile_ID_vod__c", + "Opened_vod__c", + "Product_Display_vod__c", + "Product_vod__c", + "Sender_Email_vod__c", + "Status_vod__c", + "Valid_Consent_Exists_vod__c", + "Approved_Document_Views_vod__c", + "Click_Count_vod__c", + "Last_Click_Date_vod__c", + "Last_Open_Date_vod__c", + "Open_Count_vod__c", + "Receipt_Entity_Type_vod__c", + "Receipt_Record_Id_vod__c", + "Territory_vod__c", + "Call2_vod__c", + "Medical_Inquiry_vod__c", + "Parent_Email_vod__c", + "Related_Transaction_ID_vod__c", + "Case_vod__c", + "Key_Message_vod__c", + "Suggestion_vod__c", + "EM_Attendee_vod__c", + "EM_Event_Speaker_vod__c", + "EM_Event_Team_Member_vod__c", + "Event_Attendee_vod__c", + "Event_vod__c", + "Medical_Event_vod__c", + "Scheduled_Send_Datetime_vod__c", + "User_vod__c", + "Content_Type_vod__c", + "Bcc_vod__c", + "Event_Attendee_Mobile_Id_vod__c", + "Event_Mobile_Id_vod__c", + "Activity_Tracking_Mode_vod__c", + "Email_Source_vod__c", + "Subject_vod__c", + "User_Input_Text_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + } + ] } \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_User.txt b/s3/data/crm/settings/CRM_User.txt index a0d5c519..e75fabb8 100644 --- a/s3/data/crm/settings/CRM_User.txt +++ b/s3/data/crm/settings/CRM_User.txt @@ -1,13 +1,13 @@ -CRM -, -utf-8 -" -CRLF -1 -203 -Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,LastPasswordChangeDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,NumberOfFailedLogins,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesNativeEmailClient,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c -id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,last_password_change_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,number_of_failed_logins,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_native_email_client,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c -src02.crm_user -org02.crm_user - - +CRM +, +utf-8 +" +CRLF +1 +200 +Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c +id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c +src02.crm_user +org02.crm_user + + From e5352d086145107ccbe9e9f896e9501cb080def2 Mon Sep 17 00:00:00 2001 From: "shibata.r" Date: Tue, 23 Aug 2022 17:17:52 +0900 Subject: [PATCH 254/275] =?UTF-8?q?style:=E6=94=B9=E8=A1=8C=E3=82=B3?= =?UTF-8?q?=E3=83=BC=E3=83=89=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../crm/object_info/crm_object_list_diff.json | 6012 ++++++++--------- 1 file changed, 3006 insertions(+), 3006 deletions(-) diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index ea846d62..916ecab9 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -1,3007 +1,3007 @@ -{ - "objects": [ - { - "object_name": "Clm_Presentation_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Mobile_ID_vod__c", - "Presentation_Id_vod__c", - "Product_vod__c", - "Default_Presentation_vod__c", - "Training_vod__c", - "ParentId_vod__c", - "Hidden_vod__c", - "Type_vod__c", - "Approved_vod__c", - "Copied_From_vod__c", - "Copy_Date_vod__c", - "Survey_vod__c", - "Original_Record_ID_vod__c", - "Directory_vod__c", - "End_Date_vod__c", - "Start_Date_vod__c", - "Status_vod__c", - "VExternal_Id_vod__c", - "Vault_DNS_vod__c", - "Vault_Doc_Id_vod__c", - "Vault_External_Id_vod__c", - "Vault_GUID_vod__c", - "Vault_Last_Modified_Date_Time_vod__c", - "Version_vod__c", - "Enable_Survey_Overlay_vod__c", - "Description_vod__c", - "Keywords_vod__c", - "Content_Channel_vod__c", - "original_material_approved_in_veritas__c", - "keywords__c", - "trade_team__c", - "ewizard_link__c", - "business_function__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Clm_Presentation_Slide_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Clm_Presentation_vod__c", - "Key_Message_vod__c", - "Display_Order_vod__c", - "Sub_Presentation_vod__c", - "Mobile_ID_vod__c", - "External_ID_vod__c", - "VExternal_Id_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Medical_Insight_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_vod__c", - "Clinical_Trial_vod__c", - "Date_vod__c", - "Description_vod__c", - "Entity_Reference_Id_vod__c", - "Interaction_vod__c", - "Medical_Event_vod__c", - "Mobile_ID_vod__c", - "Other_Source_vod__c", - "Override_Lock_vod__c", - "Publication_vod__c", - "Status_vod__c", - "Summary_vod__c", - "Unlock_vod__c", - "Commercial_Medical__c", - "MSJ_Level_1A__c", - "MSJ_Level_1B__c", - "MSJ_Level_2A__c", - "MSJ_Level_2B__c", - "MSJ_Level_3A__c", - "MSJ_Level_3B__c", - "MSJ_Level_4A__c", - "MSJ_Level_4B__c", - "MSJ_SubStatus__c", - "MSJ_Type_A__c", - "MSJ_Type_B__c", - "MSJ_Description_Backup__c", - "MSJ_Country__c", - "MSJ_Received_at_Boomi__c", - "MSJ_Level_1A_Value__c", - "MSJ_Level_1B_Value__c", - "MSJ_Level_2A_Value__c", - "MSJ_Level_2B_Value__c", - "MSJ_Level_3A_Value__c", - "MSJ_Level_3B_Value__c", - "MSJ_Level_4A_Value__c", - "MSJ_Level_4B_Value__c", - "MSJ_Hospital_ID__c", - "MSJ_Hospital_Name__c", - "MSJ_Hospital__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "MSJ_MR_Weekly_Report__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "MSJ_Account_Name__c", - "MSJ_Activity_Results_Summary__c", - "MSJ_MUID__c", - "MSJ_Next_Week_Action_What__c", - "MSJ_Next_Week_Action_When__c", - "MSJ_Next_Week_Action_Where__c", - "MSJ_Next_Week_Action_Who__c", - "MSJ_Report_Week__c", - "MSJ_Target_Patient_Count__c", - "Mobile_ID_vod__c", - "MSJ_Activity_Results_Summary_HN__c", - "MSJ_Next_Week_Action_Where_HN__c", - "MSJ_Next_Week_Action_Who_HN__c", - "MSJ_Next_Week_Action_What_HN__c", - "MSJ_Next_Week_Action_When_HN__c", - "MSJ_Target_Patient_Count_HN__c", - "MSJ_Activity_Results_Summary_MCC__c", - "MSJ_Next_Week_Action_Where_MCC__c", - "MSJ_Next_Week_Action_Who_MCC__c", - "MSJ_Next_Week_Action_What_MCC__c", - "MSJ_Next_Week_Action_When_MCC__c", - "MSJ_Target_Patient_Count_MCC__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Account_Territory_Loader_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_vod__c", - "External_ID_vod__c", - "Territory_vod__c", - "Mobile_ID_vod__c", - "Territory_To_Add_vod__c", - "Territory_to_Drop_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Event_Attendee_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Attendee_vod__c", - "User_vod__c", - "Medical_Event_vod__c", - "Attendee_Type_vod__c", - "Status_vod__c", - "Contact_vod__c", - "Attendee_Name_vod__c", - "Account_vod__c", - "Start_Date_vod__c", - "Signature_vod__c", - "Signature_Datetime_vod__c", - "MSJ_Copy_Account_Type__c", - "MSJ_Evaluation__c", - "MSJ_Hospital__c", - "MSJ_Role__c", - "Mobile_ID_vod__c", - "MSJ_Evaluation_Comment__c", - "Position_vod__c", - "Talk_Title_vod__c", - "MSJ_Attendee_Reaction__c", - "MSJ_Registration__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "ObjectTerritory2Association", - "columns": [ - "Id", - "ObjectId", - "Territory2Id", - "AssociationCause", - "SobjectType", - "IsDeleted", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Key_Message_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Description_vod__c", - "Product_vod__c", - "Product_Strategy_vod__c", - "Display_Order_vod__c", - "Active_vod__c", - "Category_vod__c", - "Vehicle_vod__c", - "CLM_ID_vod__c", - "Custom_Reaction_vod__c", - "Slide_Version_vod__c", - "Language_vod__c", - "Media_File_CRC_vod__c", - "Media_File_Name_vod__c", - "Media_File_Size_vod__c", - "Segment_vod__c", - "Disable_Actions_vod__c", - "VExternal_Id_vod__c", - "CDN_Path_vod__c", - "Status_vod__c", - "Vault_DNS_vod__c", - "Vault_Doc_Id_vod__c", - "Vault_External_Id_vod__c", - "Vault_GUID_vod__c", - "Vault_Last_Modified_Date_Time_vod__c", - "Is_Shared_Resource_vod__c", - "iOS_Viewer_vod__c", - "iOS_Resolution_vod__c", - "approved_for_distribution_date__c", - "approved_for_use_date__c", - "ewizard_link__c", - "expiration_date__c", - "keywords__c", - "trade_team__c", - "business_function__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Group", - "columns": [ - "Id", - "Name", - "DeveloperName", - "RelatedId", - "Type", - "Email", - "OwnerId", - "DoesSendEmailToMembers", - "DoesIncludeBosses", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Medical_Event_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Primary_Product__c", - "Description_vod__c", - "Start_Date_vod__c", - "Location__c", - "End_Date_vod__c", - "Secondary_Product__c", - "Website__c", - "Active_vod__c", - "Event_Type__c", - "MSJ_Area__c", - "MSJ_Business_Unit__c", - "MSJ_Comment__c", - "MSJ_Company__c", - "MSJ_Expense_App_No__c", - "MSJ_Form__c", - "MSJ_HQ_Area__c", - "MSJ_Location__c", - "MSJ_MR__c", - "MSJ_Number_of_Attendee__c", - "MSJ_Product__c", - "MSJ_Site__c", - "MSJ_Type__c", - "MSJ_Number_of_Attendee_Auto_Calc__c", - "MSJ_Number_of_Attendee_Invited__c", - "Account_vod__c", - "MSJ_MUID__c", - "Country_Name_vod__c", - "MSJ_CE_SIPAGL_Updater__c", - "MSJ_CE_SIPAGL_1A__c", - "MSJ_CE_SIPAGL_1B__c", - "MSJ_CE_SIPAGL_2__c", - "MSJ_CE_SIPAGL_3__c", - "MSJ_CE_SIPAGL_4__c", - "MSJ_CE_SIPAGL_5__c", - "MSJ_CE_SIPAGL_Comment__c", - "MSJ_CE_SIPAGL_1A_date__c", - "MSJ_CE_SIPAGL_6__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "MSJ_Medical_Event_Evaluation__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "MSJ_Medical_Event__c", - "MSJ_Evaluation_Comment__c", - "MSJ_Evaluation__c", - "Mobile_ID_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Coaching_Report_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Mobile_ID_vod__c", - "Manager_vod__c", - "Employee_vod__c", - "Review_Date__c", - "Review_Period__c", - "Status__c", - "Comments__c", - "Strategic_Planning__c", - "Customer_Focus__c", - "Knowledge_Expertise__c", - "Business_Account_Planning__c", - "Call_Productivity__c", - "Overall_Rating__c", - "MSJ_A01__c", - "MSJ_A02__c", - "MSJ_A03__c", - "MSJ_AM_Memo__c", - "MSJ_Aid_Total__c", - "MSJ_C0_GC__c", - "MSJ_C1_GC__c", - "MSJ_C2_GC__c", - "MSJ_Countermeasure__c", - "MSJ_Deadline__c", - "MSJ_Double_Visit_Time__c", - "MSJ_Hospital__c", - "MSJ_K01_FE__c", - "MSJ_K01_ONC__c", - "MSJ_K02_FE__c", - "MSJ_K02_ONC__c", - "MSJ_K03_FE__c", - "MSJ_K03_ONC__c", - "MSJ_K04_FE__c", - "MSJ_K04_ONC__c", - "MSJ_K05_FE__c", - "MSJ_K05_ONC__c", - "MSJ_K06_FE__c", - "MSJ_K06_ONC__c", - "MSJ_K0_GC__c", - "MSJ_K1_GC__c", - "MSJ_K2_GC__c", - "MSJ_Knowledge_Total__c", - "MSJ_L0_GC__c", - "MSJ_L1_GC__c", - "MSJ_L2_GC__c", - "MSJ_MR_GC__c", - "MSJ_MR_Problems__c", - "MSJ_N0_GC__c", - "MSJ_N1_GC__c", - "MSJ_N2_GC__c", - "MSJ_Num_of_DTL__c", - "MSJ_P01__c", - "MSJ_P02__c", - "MSJ_P03__c", - "MSJ_P04__c", - "MSJ_P05__c", - "MSJ_P0_GC__c", - "MSJ_P1_GC__c", - "MSJ_P2_GC__c", - "MSJ_PlanningTotal__c", - "MSJ_R0_GC__c", - "MSJ_R1_GC__c", - "MSJ_R2_GC__c", - "MSJ_S01__c", - "MSJ_S02__c", - "MSJ_S03__c", - "MSJ_S04__c", - "MSJ_S05__c", - "MSJ_S06__c", - "MSJ_S07__c", - "MSJ_S08__c", - "MSJ_S09__c", - "MSJ_S10__c", - "MSJ_S11__c", - "MSJ_S12__c", - "MSJ_Skill_Total__c", - "MSJ_After_Call_01__c", - "MSJ_After_Call_02__c", - "MSJ_After_Call_03__c", - "MSJ_After_Call_04__c", - "MSJ_Closing__c", - "MSJ_Comment_by_MR__c", - "MSJ_Confirmed_by_MR__c", - "MSJ_Createdby__c", - "MSJ_FT_AM_Name__c", - "MSJ_Interview_Preparation__c", - "MSJ_Interview_Reflection__c", - "MSJ_Notify_To_MR__c", - "MSJ_Opening__c", - "MSJ_Others_01_Result__c", - "MSJ_Others_01__c", - "MSJ_Others_02_Result__c", - "MSJ_Others_02__c", - "MSJ_Patient_Thinking__c", - "MSJ_Probing__c", - "MSJ_Supporting__c", - "MSJ_Patient_Thinking_for_FE__c", - "MSJ_After_Call_05__c", - "MSJ_After_Call_06__c", - "MSJ_After_Call_07__c", - "MSJ_After_Call_08__c", - "MSJ_Createdby_FE__c", - "MSJ_Createdby_ONC__c", - "MSJ_Development_Level__c", - "MSJ_Interview_Prep_01__c", - "MSJ_Interview_Prep_02__c", - "MSJ_Leadership_Style__c", - "MSJ_Overcome_01__c", - "MSJ_Overcome_02__c", - "MSJ_Overcome_03__c", - "MSJ_Overcome_04__c", - "MSJ_Review_01__c", - "MSJ_Review_02__c", - "MSJ_SK_01__c", - "MSJ_SK_02__c", - "MSJ_SK_03__c", - "MSJ_SK_04__c", - "MSJ_SK_05__c", - "MSJ_SK_06__c", - "MSJ_SK_07__c", - "MSJ_SK_08__c", - "MSJ_SK_09__c", - "MSJ_SK_10__c", - "MSJ_Specific_Action__c", - "MSJ_Training_Point__c", - "MSJ_Efforts_of_Year__c", - "MSJ_Efforts_of_Month__c", - "MSJ_Skill_Task__c", - "MSJ_Action_of_This_Month__c", - "MSJ_Achievement_of_This_Month__c", - "MSJ_Comment_from_AM__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Call2_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Call_Comments_vod__c", - "Sample_Card_vod__c", - "Add_Detail_vod__c", - "Property_vod__c", - "Account_vod__c", - "zvod_Product_Discussion_vod__c", - "Status_vod__c", - "Parent_Address_vod__c", - "Account_Plan_vod__c", - "zvod_SaveNew_vod__c", - "Next_Call_Notes_vod__c", - "Pre_Call_Notes_vod__c", - "Mobile_ID_vod__c", - "zvod_Account_Credentials_vod_c_vod__c", - "zvod_Account_Preferred_Name_vod_c_vod__c", - "zvod_Account_Sample_Status_vod_c_vod__c", - "zvod_Attendees_vod__c", - "zvod_Key_Messages_vod__c", - "zvod_Detailing_vod__c", - "zvod_Expenses_vod__c", - "zvod_Followup_vod__c", - "zvod_Samples_vod__c", - "zvod_Save_vod__c", - "zvod_Submit_vod__c", - "zvod_Delete_vod__c", - "Activity_Type__c", - "Significant_Event__c", - "Location_vod__c", - "Subject_vod__c", - "Unlock_vod__c", - "Call_Datetime_vod__c", - "Disbursed_To_vod__c", - "Disclaimer_vod__c", - "Request_Receipt_vod__c", - "Signature_Date_vod__c", - "Signature_vod__c", - "Territory_vod__c", - "Submitted_By_Mobile_vod__c", - "Call_Type_vod__c", - "Add_Key_Message_vod__c", - "Address_vod__c", - "Attendees_vod__c", - "Attendee_Type_vod__c", - "Call_Date_vod__c", - "Detailed_Products_vod__c", - "No_Disbursement_vod__c", - "Parent_Call_vod__c", - "User_vod__c", - "Contact_vod__c", - "zvod_Entity_vod__c", - "Medical_Event_vod__c", - "Mobile_Created_Datetime_vod__c", - "Mobile_Last_Modified_Datetime_vod__c", - "License_vod__c", - "Is_Parent_Call_vod__c", - "Entity_Display_Name_vod__c", - "Override_Lock_vod__c", - "Last_Device_vod__c", - "Ship_Address_Line_1_vod__c", - "Ship_Address_Line_2_vod__c", - "Ship_City_vod__c", - "Ship_Country_vod__c", - "Ship_License_Expiration_Date_vod__c", - "Ship_License_Status_vod__c", - "Ship_License_vod__c", - "Ship_State_vod__c", - "Ship_To_Address_vod__c", - "Ship_Zip_vod__c", - "Ship_To_Address_Text_vod__c", - "CLM_vod__c", - "zvod_CLMDetails_vod__c", - "Is_Sampled_Call_vod__c", - "zvod_Surveys_vod__c", - "Presentations_vod__c", - "Entity_Reference_Id_vod__c", - "Error_Reference_Call_vod__c", - "Duration_vod__c", - "Color_vod__c", - "Allowed_Products_vod__c", - "zvod_Attachments_vod__c", - "Sample_Card_Reason_vod__c", - "ASSMCA_vod__c", - "Address_Line_1_vod__c", - "Address_Line_2_vod__c", - "City_vod__c", - "DEA_Address_Line_1_vod__c", - "DEA_Address_Line_2_vod__c", - "DEA_Address_vod__c", - "DEA_City_vod__c", - "DEA_Expiration_Date_vod__c", - "DEA_State_vod__c", - "DEA_Zip_4_vod__c", - "DEA_Zip_vod__c", - "DEA_vod__c", - "Ship_Zip_4_vod__c", - "State_vod__c", - "Zip_4_vod__c", - "Zip_vod__c", - "Sample_Send_Card_vod__c", - "zvod_Address_vod_c_DEA_Status_vod_c_vod__c", - "Signature_Page_Image_vod__c", - "Credentials_vod__c", - "Salutation_vod__c", - "zvod_Account_Call_Reminder_vod_c_vod__c", - "MSJ_Meeting_Duration__c", - "MSJ_Double_Visit_AM__c", - "zvod_Business_Account_vod__c", - "Product_Priority_1_vod__c", - "Product_Priority_2_vod__c", - "Product_Priority_3_vod__c", - "Product_Priority_4_vod__c", - "Product_Priority_5_vod__c", - "zvod_More_Actions_vod__c", - "zvod_Call_Conflict_Status_vod__c", - "Signature_Timestamp_vod__c", - "Expense_Amount_vod__c", - "Total_Expense_Attendees_Count_vod__c", - "Attendee_list_vod__c", - "Expense_Post_Status_vod__c", - "Attendee_Post_Status_vod__c", - "Expense_System_External_ID_vod__c", - "Incurred_Expense_vod__c", - "Assigner_vod__c", - "Assignment_Datetime_vod__c", - "zvod_Call_Objective_vod__c", - "Signature_Location_Longitude_vod__c", - "Signature_Location_Latitude_vod__c", - "Location_Services_Status_vod__c", - "MSJ_Double_Visit_Other__c", - "MSJ_Comment__c", - "MSJ_For_Reporting__c", - "MSJ_Number_of_Attendees__c", - "MSJ_Main_Dept__c", - "Planned_Type_vjh__c", - "Cobrowse_URL_Participant_vod__c", - "MSJ_Activity_Method_Text__c", - "MSJ_Activity_Method__c", - "MSJ_Classification__c", - "MSJ_Double_Visit_MSL__c", - "MSJ_MSL_Comment_for_MR__c", - "MSJ_APD__c", - "Medical_Inquiry_vod__c", - "MSJ_Call_Type_MSJ__c", - "MSJ_Prescription_Request__c", - "MSJ_Patient_Follow__c", - "Child_Account_Id_vod__c", - "Child_Account_vod__c", - "Location_Id_vod__c", - "Location_Name_vod__c", - "MSJ_Comments_about_technology__c", - "Remote_Meeting_vod__c", - "Veeva_Remote_Meeting_Id_vod__c", - "MSJ_Activity_Type_Report__c", - "MSJ_Activity_Type__c", - "MSJ_Activity__c", - "MSJ_Comments__c", - "MSJ_Therapy__c", - "MSJ_Time_Hrs__c", - "EMDS_CO_Reference__c", - "EMDS_Call_Sub_Type__c", - "EMDS_Call_Type__c", - "EMDS_Call_Unsuccessful__c", - "EMDS_Congress_Type__c", - "EMDS_Date_of_Service__c", - "EMDS_Fertility_DisInterest__c", - "EMDS_Fertility_Interest__c", - "EMDS_Installed_Equipment__c", - "EMDS_Pipeline_Stage_Value__c", - "EMDS_Pipeline_Stage__c", - "EMDS_Pipeline__c", - "EMDS_Reason_for_Call__c", - "EMDS_Training_Completed__c", - "MSJ_BrainStorming__c", - "MSJ_SIPAGL_1A__c", - "MSJ_SIPAGL_1B__c", - "MSJ_SIPAGL_2__c", - "MSJ_SIPAGL_3__c", - "MSJ_SIPAGL_4A__c", - "MSJ_SIPAGL_5A__c", - "MSJ_SIPAGL_comment__c", - "MSJ_SIPAGL_4B__c", - "MSJ_SIPAGL_5B__c", - "Location_Text_vod__c", - "Call_Channel_vod__c", - "MSJ_Scientific_Interaction__c", - "MSJ_Activity_Email_Reply__c", - "MSJ_Interaction_Duration__c", - "MSJ_SIPAGL_1A_date__c", - "MSJ_CoPromotion__c", - "Call_Channel_Formula_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Call2_Detail_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Is_Parent_Call_vod__c", - "Call2_vod__c", - "Product_vod__c", - "Detail_Priority_vod__c", - "Mobile_ID_vod__c", - "Override_Lock_vod__c", - "Type_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Call2_Key_Message_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Account_vod__c", - "Call2_vod__c", - "Reaction_vod__c", - "Product_vod__c", - "Key_Message_vod__c", - "Mobile_ID_vod__c", - "Contact_vod__c", - "Call_Date_vod__c", - "User_vod__c", - "Category_vod__c", - "Vehicle_vod__c", - "Is_Parent_Call_vod__c", - "Override_Lock_vod__c", - "CLM_ID_vod__c", - "Slide_Version_vod__c", - "Duration_vod__c", - "Presentation_ID_vod__c", - "Start_Time_vod__c", - "Attendee_Type_vod__c", - "Entity_Reference_Id_vod__c", - "Segment_vod__c", - "Display_Order_vod__c", - "Clm_Presentation_Name_vod__c", - "Clm_Presentation_Version_vod__c", - "Clm_Presentation_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Call_Clickstream_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Answer_vod__c", - "Call_vod__c", - "Key_Message_vod__c", - "Mobile_ID_vod__c", - "Popup_Opened_vod__c", - "Possible_Answers_vod__c", - "Presentation_ID_vod__c", - "Product_vod__c", - "Range_Value_vod__c", - "Rollover_Entered_vod__c", - "Selected_Items_vod__c", - "CLM_ID_vod__c", - "Question_vod__c", - "Survey_Type_vod__c", - "Text_Entered_vod__c", - "Toggle_Button_On_vod__c", - "Track_Element_Description_vod__c", - "Track_Element_Id_vod__c", - "Track_Element_Type_vod__c", - "Usage_Duration_vod__c", - "Usage_Start_Time_vod__c", - "AuxillaryId_vod__c", - "ParentId_vod__c", - "Revision_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Call2_Discussion_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Account_vod__c", - "Call2_vod__c", - "Activity__c", - "Comments__c", - "Contact_vod__c", - "Call_Date_vod__c", - "Product_Strategy_vod__c", - "Product_Tactic_vod__c", - "Restricted_Comments__c", - "Product_vod__c", - "Presentation__c", - "Discussion_Topics__c", - "Slides__c", - "User_vod__c", - "Indication__c", - "Mobile_ID_vod__c", - "Medical_Event_vod__c", - "Is_Parent_Call_vod__c", - "Override_Lock_vod__c", - "zvod_Product_Map_vod__c", - "Attendee_Type_vod__c", - "Entity_Reference_Id_vod__c", - "Account_Tactic_vod__c", - "MSJ_Material_Type__c", - "MSJ_Discussion_Contents__c", - "MSJ_IST_Minutes__c", - "MSJ_Off_Label_Minutes__c", - "MSJ_Discussion_Objectives__c", - "MSJ_Insight__c", - "EMDS_Materials__c", - "EMDS_Topic__c", - "MSJ_Visit_Purpose__c", - "MSJ_Insight_Count__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Time_Off_Territory_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Reason_vod__c", - "Territory_vod__c", - "Date_vod__c", - "Status_vod__c", - "Time_vod__c", - "Hours_vod__c", - "Mobile_ID_vod__c", - "Hours_off_vod__c", - "Start_Time_vod__c", - "MSJ_Day__c", - "MSJ_Comment__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Dynamic_Attribute_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Account_vod__c", - "Active_vod__c", - "Dynamic_Attribute_Configuration_vod__c", - "Dynamic_Attribute_Description_vod__c", - "Dynamic_Attribute_Help_Text_vod__c", - "Dynamic_Attribute_Label_vod__c", - "Dynamic_Attribute_Name_vod__c", - "Dynamic_Attribute_Record_Type_vod__c", - "Dynamic_Attribute_Value_Checkbox_vod__c", - "Dynamic_Attribute_Value_Date_Time_vod__c", - "Dynamic_Attribute_Value_Date_vod__c", - "Dynamic_Attribute_Value_Number_vod__c", - "Dynamic_Attribute_Value_Text_Area_vod__c", - "Dynamic_Attribute_Value_Text_vod__c", - "Mobile_ID_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Dynamic_Attribute_Configuration_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Applies_To_vod__c", - "Attribute_Label_vod__c", - "Attribute_Name_vod__c", - "Available_Values_vod__c", - "Description_vod__c", - "Detail_Group_vod__c", - "Display_Order_vod__c", - "External_ID_vod__c", - "Help_Text_vod__c", - "Product_vod__c", - "Read_Only_vod__c", - "Section_Name_vod__c", - "Sharing_Group_vod__c", - "Status_vod__c", - "Track_Changes_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Territory2", - "columns": [ - "Id", - "Name", - "Territory2TypeId", - "Territory2ModelId", - "ParentTerritory2Id", - "Description", - "ForecastUserId", - "AccountAccessLevel", - "OpportunityAccessLevel", - "CaseAccessLevel", - "ContactAccessLevel", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "DeveloperName", - "MSJ_Territory_Type__c", - "MSJ_Level__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Profile", - "columns": [ - "Id", - "Name", - "PermissionsEmailSingle", - "PermissionsEmailMass", - "PermissionsEditTask", - "PermissionsEditEvent", - "PermissionsExportReport", - "PermissionsImportPersonal", - "PermissionsDataExport", - "PermissionsManageUsers", - "PermissionsEditPublicFilters", - "PermissionsEditPublicTemplates", - "PermissionsModifyAllData", - "PermissionsManageCases", - "PermissionsManageSolutions", - "PermissionsCustomizeApplication", - "PermissionsEditReadonlyFields", - "PermissionsRunReports", - "PermissionsViewSetup", - "PermissionsTransferAnyEntity", - "PermissionsNewReportBuilder", - "PermissionsManageSelfService", - "PermissionsManageCssUsers", - "PermissionsActivateContract", - "PermissionsApproveContract", - "PermissionsImportLeads", - "PermissionsManageLeads", - "PermissionsTransferAnyLead", - "PermissionsViewAllData", - "PermissionsEditPublicDocuments", - "PermissionsViewEncryptedData", - "PermissionsEditBrandTemplates", - "PermissionsEditHtmlTemplates", - "PermissionsManageTranslation", - "PermissionsDeleteActivatedContract", - "PermissionsSendSitRequests", - "PermissionsApiUserOnly", - "PermissionsManageRemoteAccess", - "PermissionsCanUseNewDashboardBuilder", - "PermissionsManageCategories", - "PermissionsConvertLeads", - "PermissionsTestInstanceCreate", - "PermissionsPasswordNeverExpires", - "PermissionsUseTeamReassignWizards", - "PermissionsInstallMultiforce", - "PermissionsPublishMultiforce", - "PermissionsEditOppLineItemUnitPrice", - "PermissionsManageTerritories", - "PermissionsCreateMultiforce", - "PermissionsBulkApiHardDelete", - "PermissionsInboundMigrationToolsUser", - "PermissionsSolutionImport", - "PermissionsManageCallCenters", - "PermissionsManageSynonyms", - "PermissionsOutboundMigrationToolsUser", - "PermissionsViewContent", - "PermissionsManageEmailClientConfig", - "PermissionsEnableNotifications", - "PermissionsManageDataIntegrations", - "PermissionsDistributeFromPersWksp", - "PermissionsViewDataCategories", - "PermissionsManageDataCategories", - "PermissionsAuthorApex", - "PermissionsManageMobile", - "PermissionsApiEnabled", - "PermissionsManageCustomReportTypes", - "PermissionsEditCaseComments", - "PermissionsTransferAnyCase", - "PermissionsContentAdministrator", - "PermissionsCreateWorkspaces", - "PermissionsManageContentPermissions", - "PermissionsManageContentProperties", - "PermissionsManageContentTypes", - "PermissionsScheduleJob", - "PermissionsManageExchangeConfig", - "PermissionsManageAnalyticSnapshots", - "PermissionsScheduleReports", - "PermissionsManageBusinessHourHolidays", - "PermissionsManageDynamicDashboards", - "PermissionsManageInteraction", - "PermissionsViewMyTeamsDashboards", - "PermissionsResetPasswords", - "PermissionsFlowUFLRequired", - "PermissionsActivitiesAccess", - "PermissionsEmailTemplateManagement", - "PermissionsEmailAdministration", - "PermissionsChatterFileLink", - "PermissionsForceTwoFactor", - "PermissionsViewEventLogFiles", - "PermissionsManageNetworks", - "PermissionsManageAuthProviders", - "PermissionsRunFlow", - "PermissionsCreateCustomizeDashboards", - "PermissionsCreateDashboardFolders", - "PermissionsViewPublicDashboards", - "PermissionsManageDashbdsInPubFolders", - "PermissionsCreateCustomizeReports", - "PermissionsCreateReportFolders", - "PermissionsViewPublicReports", - "PermissionsManageReportsInPubFolders", - "PermissionsEditMyDashboards", - "PermissionsEditMyReports", - "PermissionsViewAllUsers", - "PermissionsConnectOrgToEnvironmentHub", - "PermissionsCreateCustomizeFilters", - "PermissionsContentHubUser", - "PermissionsGovernNetworks", - "PermissionsSalesConsole", - "PermissionsTwoFactorApi", - "PermissionsDeleteTopics", - "PermissionsEditTopics", - "PermissionsCreateTopics", - "PermissionsAssignTopics", - "PermissionsIdentityEnabled", - "PermissionsIdentityConnect", - "PermissionsContentWorkspaces", - "PermissionsCustomMobileAppsAccess", - "PermissionsViewHelpLink", - "PermissionsManageProfilesPermissionsets", - "PermissionsAssignPermissionSets", - "PermissionsManageRoles", - "PermissionsManageIpAddresses", - "PermissionsManageSharing", - "PermissionsManageInternalUsers", - "PermissionsManagePasswordPolicies", - "PermissionsManageLoginAccessPolicies", - "PermissionsManageCustomPermissions", - "PermissionsStdAutomaticActivityCapture", - "PermissionsManageTwoFactor", - "PermissionsDebugApex", - "PermissionsLightningExperienceUser", - "PermissionsConfigCustomRecs", - "PermissionsSubmitMacrosAllowed", - "PermissionsBulkMacrosAllowed", - "PermissionsManageSessionPermissionSets", - "PermissionsCreateAuditFields", - "PermissionsUpdateWithInactiveOwner", - "PermissionsManageSandboxes", - "PermissionsAutomaticActivityCapture", - "PermissionsImportCustomObjects", - "PermissionsDelegatedTwoFactor", - "PermissionsSelectFilesFromSalesforce", - "PermissionsModerateNetworkUsers", - "PermissionsMergeTopics", - "PermissionsSubscribeToLightningReports", - "PermissionsManagePvtRptsAndDashbds", - "PermissionsAllowLightningLogin", - "PermissionsCampaignInfluence2", - "PermissionsViewDataAssessment", - "PermissionsCanApproveFeedPost", - "PermissionsAllowViewEditConvertedLeads", - "PermissionsShowCompanyNameAsUserBadge", - "PermissionsAccessCMC", - "PermissionsViewHealthCheck", - "PermissionsManageHealthCheck", - "PermissionsPackaging2", - "PermissionsManageCertificates", - "PermissionsCreateReportInLightning", - "PermissionsPreventClassicExperience", - "PermissionsListEmailSend", - "PermissionsChangeDashboardColors", - "PermissionsManageRecommendationStrategies", - "PermissionsManagePropositions", - "PermissionsSubscribeReportRolesGrps", - "PermissionsSubscribeDashboardRolesGrps", - "PermissionsUseWebLink", - "PermissionsHasUnlimitedNBAExecutions", - "PermissionsViewOnlyEmbeddedAppUser", - "PermissionsViewAllActivities", - "PermissionsSubscribeReportToOtherUsers", - "PermissionsLightningConsoleAllowedForUser", - "PermissionsSubscribeReportsRunAsUser", - "PermissionsSubscribeToLightningDashboards", - "PermissionsSubscribeDashboardToOtherUsers", - "PermissionsCreateLtngTempInPub", - "PermissionsTransactionalEmailSend", - "PermissionsViewPrivateStaticResources", - "PermissionsCreateLtngTempFolder", - "PermissionsApexRestServices", - "PermissionsEnableCommunityAppLauncher", - "PermissionsGiveRecognitionBadge", - "PermissionsUseMySearch", - "PermissionsLtngPromoReserved01UserPerm", - "PermissionsManageSubscriptions", - "PermissionsManageSurveys", - "PermissionsUseAssistantDialog", - "PermissionsUseQuerySuggestions", - "PermissionsViewRoles", - "PermissionsLMOutboundMessagingUserPerm", - "PermissionsModifyDataClassification", - "PermissionsPrivacyDataAccess", - "PermissionsQueryAllFiles", - "PermissionsModifyMetadata", - "PermissionsManageCMS", - "PermissionsSandboxTestingInCommunityApp", - "PermissionsCanEditPrompts", - "PermissionsViewUserPII", - "PermissionsManageHubConnections", - "PermissionsB2BMarketingAnalyticsUser", - "PermissionsTraceXdsQueries", - "PermissionsViewAllCustomSettings", - "PermissionsViewAllForeignKeyNames", - "PermissionsHeadlessCMSAccess", - "PermissionsLMEndMessagingSessionUserPerm", - "PermissionsConsentApiUpdate", - "PermissionsAccessContentBuilder", - "PermissionsAccountSwitcherUser", - "PermissionsManageC360AConnections", - "PermissionsManageReleaseUpdates", - "PermissionsViewAllProfiles", - "PermissionsSkipIdentityConfirmation", - "PermissionsSendCustomNotifications", - "PermissionsPackaging2Delete", - "PermissionsFSCComprehensiveUserAccess", - "PermissionsManageTrustMeasures", - "PermissionsViewTrustMeasures", - "PermissionsIsotopeCToCUser", - "PermissionsIsotopeAccess", - "PermissionsIsotopeLEX", - "PermissionsQuipMetricsAccess", - "PermissionsQuipUserEngagementMetrics", - "PermissionsManageExternalConnections", - "PermissionsAIViewInsightObjects", - "PermissionsAICreateInsightObjects", - "PermissionsNativeWebviewScrolling", - "PermissionsViewDeveloperName", - "Type", - "UserLicenseId", - "UserType", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "Description", - "LastViewedDate", - "LastReferencedDate" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "My_Setup_Products_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Product_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Multichannel_Activity_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_External_ID_Map_vod__c", - "Account_vod__c", - "Call_vod__c", - "City_vod__c", - "Client_Name_vod__c", - "Client_OS_vod__c", - "Client_Type_vod__c", - "Country_vod__c", - "Debug_vod__c", - "Device_vod__c", - "IP_Address_vod__c", - "Multichannel_Activity_vod__c", - "Referring_Site_vod__c", - "Region_vod__c", - "Sent_Email_vod__c", - "Session_Id_vod__c", - "Site_vod__c", - "Start_DateTime_vod__c", - "Total_Duration_vod__c", - "URL_vod__c", - "User_Agent_vod__c", - "VExternal_Id_vod__c", - "Viewport_Height_vod__c", - "Viewport_Width_vod__c", - "Color_vod__c", - "Icon_vod__c", - "MCD_Primary_Key_vod__c", - "Record_Type_Name_vod__c", - "MSJ_Date_Opened__c", - "MSJ_Sent_Date__c", - "MSJ_Email_Subject__c", - "MSJ_Opens__c", - "MSJ_Email_Status__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Multichannel_Activity_Line_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Multichannel_Activity_vod__c", - "Call_vod__c", - "Custom_vod__c", - "DateTime_vod__c", - "Debug_vod__c", - "Detail_Group_VExternal_Id_vod__c", - "Detail_Group_vod__c", - "Duration_vod__c", - "Event_Subtype_vod__c", - "Event_Type_vod__c", - "Key_Message_VExternal_Id_vod__c", - "Key_Message_vod__c", - "Multichannel_Content_Asset_Id_vod__c", - "Multichannel_Content_Asset_Version_vod__c", - "Multichannel_Content_Asset_vod__c", - "Multichannel_Content_vod__c", - "Product_VExternal_Id_vod__c", - "Product_vod__c", - "Sent_Email_vod__c", - "VExternal_Id_vod__c", - "Video_Last_Viewed_Time_vod__c", - "Video_Length_vod__c", - "Video_Total_Time_Spent_vod__c", - "View_Order_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Multichannel_Consent_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_vod__c", - "Capture_Datetime_vod__c", - "Channel_Value_vod__c", - "Detail_Group_vod__c", - "External_ID_vod__c", - "Last_Device_vod__c", - "Mobile_ID_vod__c", - "Opt_Expiration_Date_vod__c", - "Opt_Type_vod__c", - "Optout_Event_Type_vod__c", - "Product_vod__c", - "Signature_Datetime_vod__c", - "Signature_ID_vod__c", - "Signature_vod__c", - "Sample_Consent_Template_Data_vod__c", - "Sample_Consent_Template_vod__c", - "Consent_Line_vod__c", - "Consent_Type_vod__c", - "Default_Consent_Text_vod__c", - "Disclaimer_Text_vod__c", - "Sub_Channel_Key_vod__c", - "Consent_Confirm_Datetime_vod__c", - "Related_Transaction_Id_vod__c", - "Sent_Email_vod__c", - "Content_Type_vod__c", - "Receipt_Email_vod__c", - "Receipt_Sent_Email_Transaction_Id_vod__c", - "Receipt_Sent_Email_vod__c", - "Captured_By_vod__c", - "Opt_Out_Disclaimer_Text_vod__c", - "Channel_Source_vod__c", - "Union_Id_vod__c", - "User_Last_Notified_vod__c", - "Sub_Channel_Display_Name__c", - "MSJ_Consent_Source__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Medical_Inquiry_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_vod__c", - "Address_Line_1_vod__c", - "Address_Line_2_vod__c", - "City_vod__c", - "Delivery_Method_vod__c", - "Email_vod__c", - "Fax_Number_vod__c", - "Inquiry_Text__c", - "Lock_vod__c", - "Mobile_ID_vod__c", - "Phone_Number_vod__c", - "Product__c", - "Rush_Delivery__c", - "Signature_Date_vod__c", - "Signature_vod__c", - "State_vod__c", - "Status_vod__c", - "Zip_vod__c", - "zvod_Delivery_Method_vod__c", - "zvod_Disclaimer_vod__c", - "Submitted_By_Mobile_vod__c", - "Disclaimer_vod__c", - "Entity_Reference_Id_vod__c", - "Call2_vod__c", - "Country_vod__c", - "Override_Lock_vod__c", - "MSJ_Department__c", - "MSJ_Doctor_Name__c", - "MSJ_Hospital_Name__c", - "MSJ_Indication__c", - "MSJ_Inquiry_Assignment__c", - "MSJ_Inquiry_Date__c", - "MSJ_Inquiry_Input_Manager__c", - "MSJ_Inquiry_Input_User__c", - "MSJ_MSL_Manager__c", - "MSJ_Notice_to_MR__c", - "MSJ_Person_in_charge_1__c", - "MSJ_Person_in_charge_2__c", - "MSJ_Product_for_MEC__c", - "MSJ_Product_for_MR__c", - "MSJ_Reply_Date__c", - "MSJ_Reply_User__c", - "MSJ_Reply__c", - "MSJ_Title__c", - "MSJ_AE_Infomation__c", - "MSJ_FAQ_Number_Report__c", - "MSJ_Return_Call_Report__c", - "MSJ_Inquiry_Origin_Report__c", - "MSJ_AE_Report__c", - "MSJ_Background__c", - "MSJ_MSL_Support__c", - "MSJ_Material_Requirement__c", - "MSJ_Hospital_Name_Disp__c", - "MSJ_Hospital__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Email_Activity_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "Sent_Email_vod__c", - "Activity_DateTime_vod__c", - "City_vod__c", - "Click_URL_vod__c", - "Client_Name_vod__c", - "Client_OS_vod__c", - "Client_Type_vod__c", - "Country_vod__c", - "Device_Type_vod__c", - "Event_Msg_vod__c", - "Event_type_vod__c", - "IP_Address_vod__c", - "Region_vod__c", - "User_Agent_vod__c", - "Vault_Doc_ID_vod__c", - "Vault_Doc_Name_vod__c", - "Vault_Document_Major_Version_vod__c", - "Vault_Document_Minor_Version_vod__c", - "Vault_Document_Number_vod__c", - "Vault_Document_Title_vod__c", - "Vault_Instance_ID_vod__c", - "Preference_Modification_vod__c", - "Approved_Document_vod__c", - "Link_Name_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "User", - "columns": [ - "Id", - "Username", - "LastName", - "FirstName", - "Name", - "CompanyName", - "Division", - "Department", - "Title", - "Street", - "City", - "State", - "PostalCode", - "Country", - "Latitude", - "Longitude", - "GeocodeAccuracy", - "Address", - "Email", - "EmailPreferencesAutoBcc", - "EmailPreferencesAutoBccStayInTouch", - "EmailPreferencesStayInTouchReminder", - "SenderEmail", - "SenderName", - "Signature", - "StayInTouchSubject", - "StayInTouchSignature", - "StayInTouchNote", - "Phone", - "Fax", - "MobilePhone", - "Alias", - "CommunityNickname", - "BadgeText", - "IsActive", - "TimeZoneSidKey", - "UserRoleId", - "LocaleSidKey", - "ReceivesInfoEmails", - "ReceivesAdminInfoEmails", - "EmailEncodingKey", - "ProfileId", - "UserType", - "LanguageLocaleKey", - "EmployeeNumber", - "DelegatedApproverId", - "ManagerId", - "LastLoginDate", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "OfflineTrialExpirationDate", - "OfflinePdaTrialExpirationDate", - "UserPermissionsMarketingUser", - "UserPermissionsOfflineUser", - "UserPermissionsWirelessUser", - "UserPermissionsAvantgoUser", - "UserPermissionsCallCenterAutoLogin", - "UserPermissionsSFContentUser", - "UserPermissionsInteractionUser", - "UserPermissionsSupportUser", - "UserPermissionsChatterAnswersUser", - "ForecastEnabled", - "UserPreferencesActivityRemindersPopup", - "UserPreferencesEventRemindersCheckboxDefault", - "UserPreferencesTaskRemindersCheckboxDefault", - "UserPreferencesReminderSoundOff", - "UserPreferencesDisableAllFeedsEmail", - "UserPreferencesApexPagesDeveloperMode", - "UserPreferencesReceiveNoNotificationsAsApprover", - "UserPreferencesReceiveNotificationsAsDelegatedApprover", - "UserPreferencesHideCSNGetChatterMobileTask", - "UserPreferencesHideCSNDesktopTask", - "UserPreferencesHideChatterOnboardingSplash", - "UserPreferencesHideSecondChatterOnboardingSplash", - "UserPreferencesShowTitleToExternalUsers", - "UserPreferencesShowManagerToExternalUsers", - "UserPreferencesShowEmailToExternalUsers", - "UserPreferencesShowWorkPhoneToExternalUsers", - "UserPreferencesShowMobilePhoneToExternalUsers", - "UserPreferencesShowFaxToExternalUsers", - "UserPreferencesShowStreetAddressToExternalUsers", - "UserPreferencesShowCityToExternalUsers", - "UserPreferencesShowStateToExternalUsers", - "UserPreferencesShowPostalCodeToExternalUsers", - "UserPreferencesShowCountryToExternalUsers", - "UserPreferencesShowProfilePicToGuestUsers", - "UserPreferencesShowTitleToGuestUsers", - "UserPreferencesShowCityToGuestUsers", - "UserPreferencesShowStateToGuestUsers", - "UserPreferencesShowPostalCodeToGuestUsers", - "UserPreferencesShowCountryToGuestUsers", - "UserPreferencesHideInvoicesRedirectConfirmation", - "UserPreferencesHideStatementsRedirectConfirmation", - "UserPreferencesPathAssistantCollapsed", - "UserPreferencesCacheDiagnostics", - "UserPreferencesShowEmailToGuestUsers", - "UserPreferencesShowManagerToGuestUsers", - "UserPreferencesShowWorkPhoneToGuestUsers", - "UserPreferencesShowMobilePhoneToGuestUsers", - "UserPreferencesShowFaxToGuestUsers", - "UserPreferencesShowStreetAddressToGuestUsers", - "UserPreferencesLightningExperiencePreferred", - "UserPreferencesPreviewLightning", - "UserPreferencesHideEndUserOnboardingAssistantModal", - "UserPreferencesHideLightningMigrationModal", - "UserPreferencesHideSfxWelcomeMat", - "UserPreferencesHideBiggerPhotoCallout", - "UserPreferencesGlobalNavBarWTShown", - "UserPreferencesGlobalNavGridMenuWTShown", - "UserPreferencesCreateLEXAppsWTShown", - "UserPreferencesFavoritesWTShown", - "UserPreferencesRecordHomeSectionCollapseWTShown", - "UserPreferencesRecordHomeReservedWTShown", - "UserPreferencesFavoritesShowTopFavorites", - "UserPreferencesExcludeMailAppAttachments", - "UserPreferencesSuppressTaskSFXReminders", - "UserPreferencesSuppressEventSFXReminders", - "UserPreferencesPreviewCustomTheme", - "UserPreferencesHasCelebrationBadge", - "UserPreferencesUserDebugModePref", - "UserPreferencesSRHOverrideActivities", - "UserPreferencesNewLightningReportRunPageEnabled", - "UserPreferencesReverseOpenActivitiesView", - "UserPreferencesHideBrowseProductRedirectConfirmation", - "UserPreferencesHideOnlineSalesAppWelcomeMat", - "ContactId", - "AccountId", - "CallCenterId", - "Extension", - "FederationIdentifier", - "AboutMe", - "FullPhotoUrl", - "SmallPhotoUrl", - "IsExtIndicatorVisible", - "OutOfOfficeMessage", - "MediumPhotoUrl", - "DigestFrequency", - "DefaultGroupNotificationFrequency", - "LastViewedDate", - "LastReferencedDate", - "BannerPhotoUrl", - "SmallBannerPhotoUrl", - "MediumBannerPhotoUrl", - "IsProfilePhotoActive", - "IndividualId", - "Last_Mobile_Connect_vod__c", - "Last_Tablet_Connect_vod__c", - "Last_Mobile_Connect_Version_vod__c", - "Last_Tablet_Connect_Version_vod__c", - "Last_Mobile_Sync_vod__c", - "Last_Tablet_Sync_vod__c", - "RaiseLoggingLevel_vod__c", - "SendDetailedLog_vod__c", - "Last_Blackberry_Connect_vod__c", - "Last_Blackberry_Connect_Version_vod__c", - "Last_Blackberry_Sync_vod__c", - "Force_Full_Refresh_vod__c", - "Override_SystemModstamp_Timestamp_vod__c", - "Facetime_Email_vod__c", - "Facetime_Phone_vod__c", - "Product_Expertise_vod__c", - "Available_vod__c", - "Available_Last_Update_vod__c", - "Last_iPad_Connect_Version_vod__c", - "Last_iPad_Connect_vod__c", - "Last_iPad_Sync_vod__c", - "Inventory_Order_Allocation_Group_vod__c", - "Concur_User_Id_vod__c", - "Last_iPad_iOS_Version_vod__c", - "Approved_Email_Admin_vod__c", - "Last_WinModern_Connect_Version_vod__c", - "Last_WinModern_Connect_vod__c", - "Last_WinModern_Sync_vod__c", - "Primary_Territory_vod__c", - "Analytics_Admin_vod__c", - "Content_Admin_vod__c", - "Last_WinModern_Windows_Version_vod__c", - "Upload_VTrans_vod__c", - "Sync_Frequency_vjh__c", - "Clear_Client_Sync_Errors_vod__c", - "Remote_Meeting_Host_Id_vod__c", - "Remote_Meeting_Host_Token_vod__c", - "Last_iPhone_Connect_Version_vod__c", - "Last_iPhone_Connect_vod__c", - "Last_iPhone_Sync_vod__c", - "Last_iPhone_iOS_Version_vod__c", - "Remote_Meeting_Start_From_CRM_Online_vod__c", - "Country_Code_vod__c", - "User_Type_vod__c", - "Engage_Group_Provisioning_Status_vod__c", - "Engage_Group_Request_vod__c", - "Engage_Group_vod__c", - "Last_CRMDesktop_Mac_Sync_vod__c", - "Last_CRMDesktop_Mac_Version_vod__c", - "Last_CRMDesktop_Windows_Sync_vod__c", - "Last_CRMDesktop_Windows_Version_vod__c", - "MSJ_Test_User__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "UserTerritory2Association", - "columns": [ - "Id", - "UserId", - "Territory2Id", - "IsActive", - "RoleInTerritory2", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Remote_Meeting_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Meeting_Id_vod__c", - "Meeting_Name_vod__c", - "Mobile_ID_vod__c", - "Scheduled_DateTime_vod__c", - "Scheduled_vod__c", - "Attendance_Report_Process_Status_vod__c", - "Latest_Meeting_Start_Datetime_vod__c", - "Meeting_Password_vod__c", - "Meeting_Outcome_Status_vod__c", - "Allow_for_Joining_via_Zoom_vod__c", - "Zoom_Join_Token_vod__c", - "VExternal_Id_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "RecordType", - "columns": [ - "Id", - "Name", - "DeveloperName", - "NamespacePrefix", - "Description", - "BusinessProcessId", - "SobjectType", - "IsActive", - "IsPersonType", - "CreatedById", - "CreatedDate", - "LastModifiedById", - "LastModifiedDate", - "SystemModstamp" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "UserRole", - "columns": [ - "Id", - "Name", - "ParentRoleId", - "RollupDescription", - "OpportunityAccessForAccountOwner", - "CaseAccessForAccountOwner", - "ContactAccessForAccountOwner", - "ForecastUserId", - "MayForecastManagerShare", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "DeveloperName", - "PortalAccountId", - "PortalType", - "PortalAccountOwnerId" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Account", - "columns": [ - "Id", - "IsDeleted", - "MasterRecordId", - "Name", - "LastName", - "FirstName", - "Salutation", - "RecordTypeId", - "Phone", - "Fax", - "Website", - "PhotoUrl", - "NumberOfEmployees", - "Ownership", - "OwnerId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "IsExcludedFromRealign", - "PersonContactId", - "IsPersonAccount", - "PersonMailingStreet", - "PersonMailingCity", - "PersonMailingState", - "PersonMailingPostalCode", - "PersonMailingCountry", - "PersonMailingLatitude", - "PersonMailingLongitude", - "PersonMailingGeocodeAccuracy", - "PersonMailingAddress", - "PersonOtherStreet", - "PersonOtherCity", - "PersonOtherState", - "PersonOtherPostalCode", - "PersonOtherCountry", - "PersonOtherLatitude", - "PersonOtherLongitude", - "PersonOtherGeocodeAccuracy", - "PersonOtherAddress", - "PersonMobilePhone", - "PersonHomePhone", - "PersonOtherPhone", - "PersonAssistantPhone", - "PersonEmail", - "PersonTitle", - "PersonDepartment", - "PersonAssistantName", - "PersonBirthdate", - "PersonHasOptedOutOfEmail", - "PersonHasOptedOutOfFax", - "PersonDoNotCall", - "PersonLastCURequestDate", - "PersonLastCUUpdateDate", - "PersonEmailBouncedReason", - "PersonEmailBouncedDate", - "PersonIndividualId", - "Jigsaw", - "JigsawCompanyId", - "AccountSource", - "SicDesc", - "External_ID_vod__c", - "Credentials_vod__c", - "Territory_vod__c", - "Exclude_from_Zip_to_Terr_Processing_vod__c", - "Group_Specialty_1_vod__c", - "Group_Specialty_2_vod__c", - "Specialty_1_vod__c", - "Specialty_2_vod__c", - "Formatted_Name_vod__c", - "Territory_Test_vod__c", - "Mobile_ID_vod__c", - "Gender_vod__c", - "ID_vod__c", - "Do_Not_Sync_Sales_Data_vod__c", - "ID2_vod__c", - "Preferred_Name_vod__c", - "Sample_Default_vod__c", - "Segmentations_vod__c", - "Restricted_Products_vod__c", - "Payer_Id_vod__c", - "Alternate_Name_vod__c", - "Do_Not_Call_vod__c", - "MSJ_Beds__c", - "Spend_Amount__c", - "PDRP_Opt_Out_vod__c", - "Spend_Status_Value_vod__c", - "PDRP_Opt_Out_Date_vod__c", - "Spend_Status_vod__c", - "Enable_Restricted_Products_vod__c", - "Call_Reminder_vod__c", - "Account_Group_vod__c", - "Primary_Parent_vod__c", - "Color_vod__c", - "Middle_vod__c", - "Suffix_vod__c", - "MSJ_Type__c", - "No_Orders_vod__c", - "MSJ_BU_ONC__c", - "MSJ_BU_FE__c", - "Account_Search_FirstLast_vod__c", - "Account_Search_LastFirst_vod__c", - "MSJ_Operation__c", - "Practice_at_Hospital_vod__c", - "Practice_Near_Hospital_vod__c", - "Do_Not_Create_Child_Account_vod__c", - "Total_MDs_DOs__c", - "AHA__c", - "Order_Type_vod__c", - "NPI_vod__c", - "ME__c", - "Speaker__c", - "Investigator_vod__c", - "Default_Order_Type_vod__c", - "Tax_Status__c", - "Model__c", - "Offerings__c", - "Departments__c", - "Account_Type__c", - "MSJ_ONC_Tier__c", - "Account_Search_Business_vod__c", - "Business_Professional_Person_vod__c", - "Hospital_Type_vod__c", - "Account_Class_vod__c", - "Furigana_vod__c", - "MSJ_JISART__c", - "Total_Revenue_000__c", - "Net_Income_Loss_000__c", - "PMPM_Income_Loss_000__c", - "Commercial_Premiums_PMPM__c", - "Medical_Loss_Ratio__c", - "Medical_Expenses_PMPM__c", - "Commercial_Patient_Days_1000__c", - "HMO_Market_Shr__c", - "HMO__c", - "HMO_POS__c", - "PPO__c", - "PPO_POS__c", - "Medicare__c", - "Medicaid__c", - "MSJ_HP_Name_E__c", - "MSJ_Department__c", - "MSJ_Date_Of_Birth__c", - "MSJ_FE_GF_Potential__c", - "MSJ_FE_SZ_Potential__c", - "MSJ_EB_CRC_Ladder__c", - "MSJ_EB_CRC_Segment__c", - "MSJ_EB_HN_Segment__c", - "Business_Description__c", - "Regional_Strategy__c", - "Contracts_Process__c", - "MSJ_GF_segment__c", - "MSJ_DCF_DR_Code__c", - "MSJ_SZ_Segment__c", - "MSJ_Remark__c", - "MSJ_Title__c", - "MSJ_Role__c", - "MSJ_Kana__c", - "MSJ_Specialism__c", - "MSJ_Graduated_from__c", - "MSJ_Year_Graduation__c", - "Target__c", - "KOL_vod__c", - "MSJ_EPPV_Code__c", - "MSJ_DCF_HP_Code__c", - "Total_Lives__c", - "Total_Physicians_Enrolled__c", - "MSJ_Delete__c", - "MSJ_KOL_LOL__c", - "MSJ_ONC_Status__c", - "Account_Identifier_vod__c", - "Approved_Email_Opt_Type_vod__c", - "Language_vod__c", - "MSJ_KRAS_Routine_Date__c", - "MSJ_KRAS_Routine__c", - "MSJ_DRP_Target__c", - "MSJ_Fertility_Evaluation_Score__c", - "MSJ_Fertility_Tracking_Last_Modify_Date__c", - "Total_Pharmacists__c", - "MSJ_Number_of_Gonadotropin__c", - "MSJ_Number_of_IUI_cycle__c", - "MSJ_Number_of_OI_monthly_cycle__c", - "MSJ_OI_Protocol_learning_level__c", - "MSJ_H_N_Tier__c", - "MSJ_XLK_Segment__c", - "MSJ_XLK_Tier__c", - "Career_Status_vod__c", - "Photo_vod__c", - "MSJ_EB_H_N_LA_Segment__c", - "MSJ_EB_H_N_RM_Segment__c", - "MSJ_FE_CE_Potential__c", - "MSJ_FE_1C_potential__c", - "MSJ_FE_OV_potential__c", - "MSJ_FE_Tech_potential__c", - "MSJ_CE_segment__c", - "MSJ_1C_segment__c", - "MSJ_OV_segment__c", - "MSJ_Tech_segment__c", - "MSJ_Target_Call_Num__c", - "MSJ_DR_Change_Log__c", - "MSJ_Global_scientific_exposure__c", - "MSJ_H_index__c", - "MSJ_Num_of_Article_3Y__c", - "MSJ_Num_of_Article__c", - "MSJ_Num_of_Article_as_1st_Author_3Y__c", - "MSJ_Num_of_article_growth_rate_3Y__c", - "MSJ_Num_of_cited_3Y__c", - "MSJ_Num_of_impact_factor_3Y__c", - "MSJ_impact_factor_as_1st_Author_3Y__c", - "EMDS_Has_Pipeline_Opportunity__c", - "EMDS_Pipeline_Count__c", - "MSJ_BVC_Segment__c", - "MSJ_BVC_Tier__c", - "MSJ_BVC_AcctOpen__c", - "MSJ_BVC_MCC_Patients__c", - "MSJ_ONC_HP_Segment__c", - "MSJ_AE_Department__c", - "MSJ_AE_Facility__c", - "MSJ_AE_Name__c", - "MSJ_AE_Title__c", - "MSJ_Email__c", - "MSJ_FE_GF2_Potential__c", - "MSJ_FE_Location_potential__c", - "MSJ_GF2_segment__c", - "MSJ_OPTIN_target__c", - "MSJ_Merck_Specialty1__c", - "MSJ_Merck_Specialty2__c", - "MSJ_Marketing_Cloud_Integration__c", - "MSJ_Marketing_Cloud1__c", - "MSJ_Marketing_Cloud2__c", - "MSJ_Marketing_Cloud3__c", - "MSJ_Marketing_Cloud4__c", - "MSJ_Medical_Department__c", - "MSJ_Marketing_Cloud0__c", - "Mobile_ID_vod__pc", - "H1Insights__H1_NPI_Value_for_Testing__pc", - "H1Insights__H1_Person_ID__pc", - "H1Insights__H1_Request_Status__pc", - "H1Insights__H1_URL__pc", - "H1Insights__NPI_Number__pc", - "H1Insights__NPI_Number_for_H1_Insights__pc", - "MSJ_Marketing_Cloud_Integration__pc" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "AccountShare", - "columns": [ - "Id", - "AccountId", - "UserOrGroupId", - "AccountAccessLevel", - "OpportunityAccessLevel", - "CaseAccessLevel", - "ContactAccessLevel", - "RowCause", - "LastModifiedDate", - "LastModifiedById", - "IsDeleted" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true, - "datetime_column": "LastModifiedDate" - }, - { - "object_name": "Contact", - "columns": [ - "Id", - "IsDeleted", - "MasterRecordId", - "AccountId", - "IsPersonAccount", - "LastName", - "FirstName", - "Salutation", - "Name", - "OtherStreet", - "OtherCity", - "OtherState", - "OtherPostalCode", - "OtherCountry", - "OtherLatitude", - "OtherLongitude", - "OtherGeocodeAccuracy", - "OtherAddress", - "MailingStreet", - "MailingCity", - "MailingState", - "MailingPostalCode", - "MailingCountry", - "MailingLatitude", - "MailingLongitude", - "MailingGeocodeAccuracy", - "MailingAddress", - "Phone", - "Fax", - "MobilePhone", - "HomePhone", - "OtherPhone", - "AssistantPhone", - "ReportsToId", - "Email", - "Title", - "Department", - "AssistantName", - "Birthdate", - "Description", - "OwnerId", - "HasOptedOutOfEmail", - "HasOptedOutOfFax", - "DoNotCall", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "LastCURequestDate", - "LastCUUpdateDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "EmailBouncedReason", - "EmailBouncedDate", - "IsEmailBounced", - "PhotoUrl", - "Jigsaw", - "JigsawContactId", - "IndividualId", - "Mobile_ID_vod__c", - "H1Insights__H1_NPI_Value_for_Testing__c", - "H1Insights__H1_Person_ID__c", - "H1Insights__H1_Request_Status__c", - "H1Insights__H1_URL__c", - "H1Insights__NPI_Number__c", - "H1Insights__NPI_Number_for_H1_Insights__c", - "MSJ_Marketing_Cloud_Integration__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Consent_Type_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Consent_Header_vod__c", - "Channel_Label_vod__c", - "Channel_Source_vod__c", - "Consent_Expires_In_vod__c", - "Default_Consent_Type_vod__c", - "Disclaimer_Text_vod__c", - "Display_Order_vod__c", - "Product_Preference_vod__c", - "zvod_Consent_Default_Consent_Text_vod__c", - "zvod_Consent_Line_vod__c", - "zvod_Signature_Capture_vod__c", - "Double_Opt_In_vod__c", - "zvod_Consent_Activity_Tracking_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Consent_Header_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Consent_Header_Help_Text_vod__c", - "Country_vod__c", - "Inactive_Datetime_vod__c", - "Language_vod__c", - "Status_vod__c", - "Signature_Required_On_Opt_Out_vod__c", - "Request_Receipt_vod__c", - "Subscription_Option_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Consent_Line_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Consent_Type_vod__c", - "Detail_Group_Display_Name_vod__c", - "Detail_Group_vod__c", - "Display_Order_vod__c", - "End_Date_vod__c", - "Group_By_vod__c", - "Product_Display_Name_vod__c", - "Product_vod__c", - "Start_Date_vod__c", - "Sub_Channel_Description_vod__c", - "Sub_Channel_Display_Name_vod__c", - "Sub_Channel_Key_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "MSJ_Inquiry_Assignment__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "MSJ_Medical_Inquiry__c", - "MSJ_Close__c", - "MSJ_Doctor_Name__c", - "MSJ_Hospital_Name__c", - "MSJ_Indication__c", - "MSJ_Inquiry_Text__c", - "MSJ_MEC_User__c", - "MSJ_MSL_Manager__c", - "MSJ_MSL_User__c", - "MSJ_Notice_to_MR__c", - "MSJ_Product_for_MEC__c", - "MSJ_Product_for_MR__c", - "MSJ_Reply_Date__c", - "MSJ_Reply__c", - "MSJ_AE_Infomation__c", - "MSJ_Cancel__c", - "MSJ_FAQ_number_c__c", - "MSJ_Return_Call__c", - "MSJ_Inquiry_Origin__c", - "First_Response__c", - "Inquiry_Created_Date__c", - "Inquiry_Type_1__c", - "Inquiry_Type_2__c", - "MSJ_First_User__c", - "MSJ_MEC_Comment__c", - "MSJ_Send_Email__c", - "MSJ_Temp_Aggregated_Info__c", - "MSJ_AE_Report__c", - "MSJ_Background__c", - "MSJ_Inquiry_Date__c", - "MSJ_MSL_Support__c", - "MSJ_Handover_Comment__c", - "MSJ_Handover_Email__c", - "MSJ_Material_Requirement__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Approved_Document_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Detail_Group_vod__c", - "Document_Description_vod__c", - "Document_Host_URL_vod__c", - "Document_ID_vod__c", - "Document_Last_Mod_DateTime_vod__c", - "Email_Allows_Documents_vod__c", - "Email_Domain_vod__c", - "Email_Fragment_HTML_vod__c", - "Email_From_Address_vod__c", - "Email_From_Name_vod__c", - "Email_HTML_1_vod__c", - "Email_HTML_2_vod__c", - "Email_ReplyTo_Address_vod__c", - "Email_ReplyTo_Name_vod__c", - "Email_Subject_vod__c", - "Email_Template_Fragment_Document_ID_vod__c", - "Email_Template_Fragment_HTML_vod__c", - "ISI_Document_ID_vod__c", - "Language_vod__c", - "Other_Document_ID_List_vod__c", - "PI_Document_ID_vod__c", - "Piece_Document_ID_vod__c", - "Product_vod__c", - "Status_vod__c", - "Territory_vod__c", - "Vault_Instance_ID_vod__c", - "Allow_Any_Product_Fragment_vod__c", - "Allowed_Document_IDs_vod__c", - "Engage_Document_Id_vod__c", - "Vault_Document_ID_vod__c", - "Key_Message_vod__c", - "Events_Management_Subtype_vod__c", - "Survey_vod__c", - "Content_Type_vod__c", - "Bcc_vod__c", - "Audience_vod__c", - "WeChat_Template_ID_vod__c", - "Check_Consent_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Child_Account_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Parent_Account_vod__c", - "Child_Account_vod__c", - "External_ID_vod__c", - "Mobile_ID_vod__c", - "Primary_vod__c", - "Copy_Address_vod__c", - "Child_Name_vod__c", - "Parent_Name_vod__c", - "Parent_Child_Name_vod__c", - "Account_Code__c", - "Child_Department__c", - "Child_Role__c", - "Child_Title__c", - "Child_Remark__c", - "MSJ_1C_segment__c", - "MSJ_BU_FE__c", - "MSJ_BU_ONC__c", - "MSJ_BVC_Segment__c", - "MSJ_CE_segment__c", - "MSJ_Child_Account_Link__c", - "MSJ_DCF_DR_Code__c", - "MSJ_DCF_HP_Code__c", - "MSJ_DR_Change_Log__c", - "MSJ_Delete__c", - "MSJ_Department__c", - "MSJ_EB_CRC_Segment__c", - "MSJ_EB_HN_Segment__c", - "MSJ_EB_H_N_LA_Segment__c", - "MSJ_EB_H_N_RM_Segment__c", - "MSJ_External_ID__c", - "MSJ_Fax__c", - "MSJ_GF2_segment__c", - "MSJ_GF_segment__c", - "MSJ_KOL_LOL__c", - "MSJ_KOL__c", - "MSJ_ONC_HP_Segment__c", - "MSJ_OPTIN_target__c", - "MSJ_OV_segment__c", - "MSJ_Parent_Child_Name__c", - "MSJ_Phone__c", - "MSJ_Remark__c", - "MSJ_Target_Call_Num__c", - "MSJ_Tech_segment__c", - "MSJ_Title__c", - "MSJ_XLK_Segment__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "MSJ_Hospital_Medical_Regimen__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "MSJ_Account_Name__c", - "MSJ_Delete_Date__c", - "MSJ_Delete_Flag__c", - "MSJ_Indication__c", - "MSJ_Line__c", - "MSJ_Medical_Regimen__c", - "Mobile_ID_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "MSJ_Medical_Regimen__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "MSJ_Delete_Date__c", - "MSJ_Delete_Flag__c", - "MSJ_Indication__c", - "MSJ_Remark__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "MSJ_Patient__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "MSJ_Account_Name__c", - "MSJ_CRC_Group__c", - "MSJ_Casus_or_Transfer_Point__c", - "MSJ_Entry_Date__c", - "MSJ_IST_Name__c", - "MSJ_Indication__c", - "MSJ_Line__c", - "MSJ_MR_Comments__c", - "MSJ_MUID__c", - "MSJ_Medical_Regimen__c", - "MSJ_Month__c", - "MSJ_Report_Comments__c", - "MSJ_Start_Date_Of_Administration__c", - "MSJ_Year__c", - "Mobile_ID_vod__c", - "MSJ_CRC_RAS_KRAS__c", - "MSJ_End_Date_Of_Administration__c", - "MSJ_End_Date_of_Stop_Administration__c", - "MSJ_HN_Hospitalized_Type__c", - "MSJ_Start_Date_of_Stop_Administration__c", - "MSJ_Patient_Status__c", - "MSJ_Patient_TA__c", - "MSJ_Child_Account_Name__c", - "MSJ_Child_Account__c", - "MSJ_Parent_Account_Name__c", - "MSJ_Parent_Child_Name__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Product_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Consumer_site__c", - "Product_info__c", - "Therapeutic_Class_vod__c", - "Parent_Product_vod__c", - "Therapeutic_Area_vod__c", - "Product_Type_vod__c", - "Require_Key_Message_vod__c", - "Cost_vod__c", - "External_ID_vod__c", - "Manufacturer_vod__c", - "Company_Product_vod__c", - "Controlled_Substance_vod__c", - "Description_vod__c", - "Sample_Quantity_Picklist_vod__c", - "Display_Order_vod__c", - "No_Metrics_vod__c", - "Distributor_vod__c", - "Sample_Quantity_Bound_vod__c", - "Sample_U_M_vod__c", - "No_Details_vod__c", - "Quantity_Per_Case_vod__c", - "Schedule_vod__c", - "Restricted_vod__c", - "Pricing_Rule_Quantity_Bound_vod__c", - "No_Promo_Items_vod__c", - "User_Aligned_vod__c", - "Restricted_States_vod__c", - "Sort_Code_vod__c", - "No_Cycle_Plans_vod__c", - "Inventory_Order_UOM_vod__c", - "Inventory_Quantity_Per_Case_vod__c", - "VExternal_Id_vod__c", - "Country__c", - "MSJ_Product_Classification__c", - "MSJ_Indication__c", - "MSJ_Therapeutic_Area__c", - "MSJ_Global_Brand__c", - "MSJ_Global_Business_Unit__c", - "MSJ_Molecules__c", - "MSJ_SBU__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Product_Group_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Description_vod__c", - "Product_vod__c", - "Product_Catalog_vod__c", - "Start_Date_vod__c", - "End_Date_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Product_Metrics_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_vod__c", - "Awareness__c", - "Selling_Stage__c", - "Formulary_Status__c", - "Movement__c", - "Products_vod__c", - "Segment__c", - "X12_mo_trx_chg__c", - "Speaker_Skills__c", - "Investigator_Readiness__c", - "Engagements__c", - "Mobile_ID_vod__c", - "External_ID_vod__c", - "MSJ_Patient__c", - "Detail_Group_vod__c", - "MSJ_EB_1st_Line_Liver_Meta__c", - "MSJ_EB_1st_Line_Multi_Meta__c", - "MSJ_EB_2nd_Line_Mono__c", - "MSJ_EB_2nd_Line_Combination__c", - "MSJ_EB_3rd_Line_Mono__c", - "MSJ_EB_3rd_Line_Combination__c", - "EMDS_Ability__c", - "EMDS_Brand_Loyalty__c", - "EMDS_Decision_Maker__c", - "EMDS_Early_Tech_Adopter__c", - "EMDS_Influence__c", - "EMDS_Main_Driver__c", - "EMDS_Priority__c", - "EMDS_Willingness__c", - "MSJ_KTL_Type__c", - "MSJ_KTL_Tier__c", - "MSJ_Publications__c", - "MSJ_Clinical_Trials__c", - "MSJ_Speaker_for_Medical_Events__c", - "MSJ_Advisor_to_Medical_Affairs__c", - "MSJ_Guidelines_Treatment_Standards__c", - "MSJ_Therapeutic_Area_Expertise__c", - "MSJ_MAP_GAP__c", - "MSJ_Associations__c", - "MSJ_Tier_Score__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Survey_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Assignment_Type_vod__c", - "Channels_vod__c", - "End_Date_vod__c", - "Expired_vod__c", - "External_ID_vod__c", - "Language_vod__c", - "Lock_vod__c", - "Open_vod__c", - "Product_vod__c", - "Region_vod__c", - "Segment_vod__c", - "Start_Date_vod__c", - "Status_vod__c", - "Territory_vod__c", - "zvod_Questions_vod__c", - "zvod_Segments_vod__c", - "zvod_Targets_vod__c", - "Max_Score_vod__c", - "Min_Score_vod__c", - "Autotarget_vod__c", - "Territories_vod__c", - "Target_Type_vod__c", - "MSJ_External_ID__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Survey_Target_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "LastViewedDate", - "LastReferencedDate", - "Account_Display_Name_vod__c", - "Account_vod__c", - "Channels_vod__c", - "End_Date_vod__c", - "Entity_Reference_Id_vod__c", - "External_ID_vod__c", - "Language_vod__c", - "Lock_vod__c", - "Mobile_ID_vod__c", - "No_Autoassign_vod__c", - "Not_Completed_vod__c", - "Region_vod__c", - "Segment_vod__c", - "Start_Date_vod__c", - "Status_vod__c", - "Survey_vod__c", - "Territory_vod__c", - "zvod_Address_vod__c", - "zvod_Specialty_vod__c", - "Score_vod__c", - "User_vod__c", - "Child_Account_vod__c", - "Location_Entity_Reference_Id_vod__c", - "Location_vod__c", - "Target_Type_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Survey_Question_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Survey_vod__c", - "Answer_Choice_vod__c", - "External_ID_vod__c", - "Max_Score_vod__c", - "Min_Score_vod__c", - "Order_vod__c", - "Question_vod__c", - "Required_vod__c", - "Text_vod__c", - "Condition_vod__c", - "Source_ID_vod__c", - "MSJ_External_ID__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Question_Response_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "MayEdit", - "IsLocked", - "Survey_Target_vod__c", - "Answer_Choice_vod__c", - "Date_vod__c", - "Datetime_vod__c", - "External_ID_vod__c", - "Mobile_ID_vod__c", - "Number_vod__c", - "Order_vod__c", - "Question_Text_vod__c", - "Required_vod__c", - "Response_Hash_vod__c", - "Response_vod__c", - "Score_vod__c", - "Survey_Question_vod__c", - "Text_vod__c", - "Type_vod__c", - "Condition_vod__c", - "Inactive_Condition_vod__c", - "Source_ID_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Sent_Fragment_vod__c", - "columns": [ - "Id", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "Sent_Email_vod__c", - "Account_vod__c", - "Email_Template_vod__c", - "Sent_Fragment_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - }, - { - "object_name": "Sent_Email_vod__c", - "columns": [ - "Id", - "OwnerId", - "IsDeleted", - "Name", - "RecordTypeId", - "CreatedDate", - "CreatedById", - "LastModifiedDate", - "LastModifiedById", - "SystemModstamp", - "LastActivityDate", - "MayEdit", - "IsLocked", - "Account_Email_vod__c", - "Account_vod__c", - "Approved_Email_Template_vod__c", - "Capture_Datetime_vod__c", - "Detail_Group_vod__c", - "Email_Config_Values_vod__c", - "Email_Content2_vod__c", - "Email_Content_vod__c", - "Email_Fragments_vod__c", - "Email_Sent_Date_vod__c", - "Failure_Msg_vod__c", - "Last_Activity_Date_vod__c", - "Last_Device_vod__c", - "MC_Capture_Datetime_vod__c", - "Mobile_ID_vod__c", - "Opened_vod__c", - "Product_Display_vod__c", - "Product_vod__c", - "Sender_Email_vod__c", - "Status_vod__c", - "Valid_Consent_Exists_vod__c", - "Approved_Document_Views_vod__c", - "Click_Count_vod__c", - "Last_Click_Date_vod__c", - "Last_Open_Date_vod__c", - "Open_Count_vod__c", - "Receipt_Entity_Type_vod__c", - "Receipt_Record_Id_vod__c", - "Territory_vod__c", - "Call2_vod__c", - "Medical_Inquiry_vod__c", - "Parent_Email_vod__c", - "Related_Transaction_ID_vod__c", - "Case_vod__c", - "Key_Message_vod__c", - "Suggestion_vod__c", - "EM_Attendee_vod__c", - "EM_Event_Speaker_vod__c", - "EM_Event_Team_Member_vod__c", - "Event_Attendee_vod__c", - "Event_vod__c", - "Medical_Event_vod__c", - "Scheduled_Send_Datetime_vod__c", - "User_vod__c", - "Content_Type_vod__c", - "Bcc_vod__c", - "Event_Attendee_Mobile_Id_vod__c", - "Event_Mobile_Id_vod__c", - "Activity_Tracking_Mode_vod__c", - "Email_Source_vod__c", - "Subject_vod__c", - "User_Input_Text_vod__c" - ], - "is_skip": false, - "is_update_last_fetch_datetime": true - } - ] +{ + "objects": [ + { + "object_name": "Clm_Presentation_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Mobile_ID_vod__c", + "Presentation_Id_vod__c", + "Product_vod__c", + "Default_Presentation_vod__c", + "Training_vod__c", + "ParentId_vod__c", + "Hidden_vod__c", + "Type_vod__c", + "Approved_vod__c", + "Copied_From_vod__c", + "Copy_Date_vod__c", + "Survey_vod__c", + "Original_Record_ID_vod__c", + "Directory_vod__c", + "End_Date_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "VExternal_Id_vod__c", + "Vault_DNS_vod__c", + "Vault_Doc_Id_vod__c", + "Vault_External_Id_vod__c", + "Vault_GUID_vod__c", + "Vault_Last_Modified_Date_Time_vod__c", + "Version_vod__c", + "Enable_Survey_Overlay_vod__c", + "Description_vod__c", + "Keywords_vod__c", + "Content_Channel_vod__c", + "original_material_approved_in_veritas__c", + "keywords__c", + "trade_team__c", + "ewizard_link__c", + "business_function__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Clm_Presentation_Slide_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Clm_Presentation_vod__c", + "Key_Message_vod__c", + "Display_Order_vod__c", + "Sub_Presentation_vod__c", + "Mobile_ID_vod__c", + "External_ID_vod__c", + "VExternal_Id_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Insight_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Clinical_Trial_vod__c", + "Date_vod__c", + "Description_vod__c", + "Entity_Reference_Id_vod__c", + "Interaction_vod__c", + "Medical_Event_vod__c", + "Mobile_ID_vod__c", + "Other_Source_vod__c", + "Override_Lock_vod__c", + "Publication_vod__c", + "Status_vod__c", + "Summary_vod__c", + "Unlock_vod__c", + "Commercial_Medical__c", + "MSJ_Level_1A__c", + "MSJ_Level_1B__c", + "MSJ_Level_2A__c", + "MSJ_Level_2B__c", + "MSJ_Level_3A__c", + "MSJ_Level_3B__c", + "MSJ_Level_4A__c", + "MSJ_Level_4B__c", + "MSJ_SubStatus__c", + "MSJ_Type_A__c", + "MSJ_Type_B__c", + "MSJ_Description_Backup__c", + "MSJ_Country__c", + "MSJ_Received_at_Boomi__c", + "MSJ_Level_1A_Value__c", + "MSJ_Level_1B_Value__c", + "MSJ_Level_2A_Value__c", + "MSJ_Level_2B_Value__c", + "MSJ_Level_3A_Value__c", + "MSJ_Level_3B_Value__c", + "MSJ_Level_4A_Value__c", + "MSJ_Level_4B_Value__c", + "MSJ_Hospital_ID__c", + "MSJ_Hospital_Name__c", + "MSJ_Hospital__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_MR_Weekly_Report__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Account_Name__c", + "MSJ_Activity_Results_Summary__c", + "MSJ_MUID__c", + "MSJ_Next_Week_Action_What__c", + "MSJ_Next_Week_Action_When__c", + "MSJ_Next_Week_Action_Where__c", + "MSJ_Next_Week_Action_Who__c", + "MSJ_Report_Week__c", + "MSJ_Target_Patient_Count__c", + "Mobile_ID_vod__c", + "MSJ_Activity_Results_Summary_HN__c", + "MSJ_Next_Week_Action_Where_HN__c", + "MSJ_Next_Week_Action_Who_HN__c", + "MSJ_Next_Week_Action_What_HN__c", + "MSJ_Next_Week_Action_When_HN__c", + "MSJ_Target_Patient_Count_HN__c", + "MSJ_Activity_Results_Summary_MCC__c", + "MSJ_Next_Week_Action_Where_MCC__c", + "MSJ_Next_Week_Action_Who_MCC__c", + "MSJ_Next_Week_Action_What_MCC__c", + "MSJ_Next_Week_Action_When_MCC__c", + "MSJ_Target_Patient_Count_MCC__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Account_Territory_Loader_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "External_ID_vod__c", + "Territory_vod__c", + "Mobile_ID_vod__c", + "Territory_To_Add_vod__c", + "Territory_to_Drop_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Event_Attendee_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Attendee_vod__c", + "User_vod__c", + "Medical_Event_vod__c", + "Attendee_Type_vod__c", + "Status_vod__c", + "Contact_vod__c", + "Attendee_Name_vod__c", + "Account_vod__c", + "Start_Date_vod__c", + "Signature_vod__c", + "Signature_Datetime_vod__c", + "MSJ_Copy_Account_Type__c", + "MSJ_Evaluation__c", + "MSJ_Hospital__c", + "MSJ_Role__c", + "Mobile_ID_vod__c", + "MSJ_Evaluation_Comment__c", + "Position_vod__c", + "Talk_Title_vod__c", + "MSJ_Attendee_Reaction__c", + "MSJ_Registration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "ObjectTerritory2Association", + "columns": [ + "Id", + "ObjectId", + "Territory2Id", + "AssociationCause", + "SobjectType", + "IsDeleted", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Key_Message_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Description_vod__c", + "Product_vod__c", + "Product_Strategy_vod__c", + "Display_Order_vod__c", + "Active_vod__c", + "Category_vod__c", + "Vehicle_vod__c", + "CLM_ID_vod__c", + "Custom_Reaction_vod__c", + "Slide_Version_vod__c", + "Language_vod__c", + "Media_File_CRC_vod__c", + "Media_File_Name_vod__c", + "Media_File_Size_vod__c", + "Segment_vod__c", + "Disable_Actions_vod__c", + "VExternal_Id_vod__c", + "CDN_Path_vod__c", + "Status_vod__c", + "Vault_DNS_vod__c", + "Vault_Doc_Id_vod__c", + "Vault_External_Id_vod__c", + "Vault_GUID_vod__c", + "Vault_Last_Modified_Date_Time_vod__c", + "Is_Shared_Resource_vod__c", + "iOS_Viewer_vod__c", + "iOS_Resolution_vod__c", + "approved_for_distribution_date__c", + "approved_for_use_date__c", + "ewizard_link__c", + "expiration_date__c", + "keywords__c", + "trade_team__c", + "business_function__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Group", + "columns": [ + "Id", + "Name", + "DeveloperName", + "RelatedId", + "Type", + "Email", + "OwnerId", + "DoesSendEmailToMembers", + "DoesIncludeBosses", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Event_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Primary_Product__c", + "Description_vod__c", + "Start_Date_vod__c", + "Location__c", + "End_Date_vod__c", + "Secondary_Product__c", + "Website__c", + "Active_vod__c", + "Event_Type__c", + "MSJ_Area__c", + "MSJ_Business_Unit__c", + "MSJ_Comment__c", + "MSJ_Company__c", + "MSJ_Expense_App_No__c", + "MSJ_Form__c", + "MSJ_HQ_Area__c", + "MSJ_Location__c", + "MSJ_MR__c", + "MSJ_Number_of_Attendee__c", + "MSJ_Product__c", + "MSJ_Site__c", + "MSJ_Type__c", + "MSJ_Number_of_Attendee_Auto_Calc__c", + "MSJ_Number_of_Attendee_Invited__c", + "Account_vod__c", + "MSJ_MUID__c", + "Country_Name_vod__c", + "MSJ_CE_SIPAGL_Updater__c", + "MSJ_CE_SIPAGL_1A__c", + "MSJ_CE_SIPAGL_1B__c", + "MSJ_CE_SIPAGL_2__c", + "MSJ_CE_SIPAGL_3__c", + "MSJ_CE_SIPAGL_4__c", + "MSJ_CE_SIPAGL_5__c", + "MSJ_CE_SIPAGL_Comment__c", + "MSJ_CE_SIPAGL_1A_date__c", + "MSJ_CE_SIPAGL_6__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Medical_Event_Evaluation__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Medical_Event__c", + "MSJ_Evaluation_Comment__c", + "MSJ_Evaluation__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Coaching_Report_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Mobile_ID_vod__c", + "Manager_vod__c", + "Employee_vod__c", + "Review_Date__c", + "Review_Period__c", + "Status__c", + "Comments__c", + "Strategic_Planning__c", + "Customer_Focus__c", + "Knowledge_Expertise__c", + "Business_Account_Planning__c", + "Call_Productivity__c", + "Overall_Rating__c", + "MSJ_A01__c", + "MSJ_A02__c", + "MSJ_A03__c", + "MSJ_AM_Memo__c", + "MSJ_Aid_Total__c", + "MSJ_C0_GC__c", + "MSJ_C1_GC__c", + "MSJ_C2_GC__c", + "MSJ_Countermeasure__c", + "MSJ_Deadline__c", + "MSJ_Double_Visit_Time__c", + "MSJ_Hospital__c", + "MSJ_K01_FE__c", + "MSJ_K01_ONC__c", + "MSJ_K02_FE__c", + "MSJ_K02_ONC__c", + "MSJ_K03_FE__c", + "MSJ_K03_ONC__c", + "MSJ_K04_FE__c", + "MSJ_K04_ONC__c", + "MSJ_K05_FE__c", + "MSJ_K05_ONC__c", + "MSJ_K06_FE__c", + "MSJ_K06_ONC__c", + "MSJ_K0_GC__c", + "MSJ_K1_GC__c", + "MSJ_K2_GC__c", + "MSJ_Knowledge_Total__c", + "MSJ_L0_GC__c", + "MSJ_L1_GC__c", + "MSJ_L2_GC__c", + "MSJ_MR_GC__c", + "MSJ_MR_Problems__c", + "MSJ_N0_GC__c", + "MSJ_N1_GC__c", + "MSJ_N2_GC__c", + "MSJ_Num_of_DTL__c", + "MSJ_P01__c", + "MSJ_P02__c", + "MSJ_P03__c", + "MSJ_P04__c", + "MSJ_P05__c", + "MSJ_P0_GC__c", + "MSJ_P1_GC__c", + "MSJ_P2_GC__c", + "MSJ_PlanningTotal__c", + "MSJ_R0_GC__c", + "MSJ_R1_GC__c", + "MSJ_R2_GC__c", + "MSJ_S01__c", + "MSJ_S02__c", + "MSJ_S03__c", + "MSJ_S04__c", + "MSJ_S05__c", + "MSJ_S06__c", + "MSJ_S07__c", + "MSJ_S08__c", + "MSJ_S09__c", + "MSJ_S10__c", + "MSJ_S11__c", + "MSJ_S12__c", + "MSJ_Skill_Total__c", + "MSJ_After_Call_01__c", + "MSJ_After_Call_02__c", + "MSJ_After_Call_03__c", + "MSJ_After_Call_04__c", + "MSJ_Closing__c", + "MSJ_Comment_by_MR__c", + "MSJ_Confirmed_by_MR__c", + "MSJ_Createdby__c", + "MSJ_FT_AM_Name__c", + "MSJ_Interview_Preparation__c", + "MSJ_Interview_Reflection__c", + "MSJ_Notify_To_MR__c", + "MSJ_Opening__c", + "MSJ_Others_01_Result__c", + "MSJ_Others_01__c", + "MSJ_Others_02_Result__c", + "MSJ_Others_02__c", + "MSJ_Patient_Thinking__c", + "MSJ_Probing__c", + "MSJ_Supporting__c", + "MSJ_Patient_Thinking_for_FE__c", + "MSJ_After_Call_05__c", + "MSJ_After_Call_06__c", + "MSJ_After_Call_07__c", + "MSJ_After_Call_08__c", + "MSJ_Createdby_FE__c", + "MSJ_Createdby_ONC__c", + "MSJ_Development_Level__c", + "MSJ_Interview_Prep_01__c", + "MSJ_Interview_Prep_02__c", + "MSJ_Leadership_Style__c", + "MSJ_Overcome_01__c", + "MSJ_Overcome_02__c", + "MSJ_Overcome_03__c", + "MSJ_Overcome_04__c", + "MSJ_Review_01__c", + "MSJ_Review_02__c", + "MSJ_SK_01__c", + "MSJ_SK_02__c", + "MSJ_SK_03__c", + "MSJ_SK_04__c", + "MSJ_SK_05__c", + "MSJ_SK_06__c", + "MSJ_SK_07__c", + "MSJ_SK_08__c", + "MSJ_SK_09__c", + "MSJ_SK_10__c", + "MSJ_Specific_Action__c", + "MSJ_Training_Point__c", + "MSJ_Efforts_of_Year__c", + "MSJ_Efforts_of_Month__c", + "MSJ_Skill_Task__c", + "MSJ_Action_of_This_Month__c", + "MSJ_Achievement_of_This_Month__c", + "MSJ_Comment_from_AM__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Call_Comments_vod__c", + "Sample_Card_vod__c", + "Add_Detail_vod__c", + "Property_vod__c", + "Account_vod__c", + "zvod_Product_Discussion_vod__c", + "Status_vod__c", + "Parent_Address_vod__c", + "Account_Plan_vod__c", + "zvod_SaveNew_vod__c", + "Next_Call_Notes_vod__c", + "Pre_Call_Notes_vod__c", + "Mobile_ID_vod__c", + "zvod_Account_Credentials_vod_c_vod__c", + "zvod_Account_Preferred_Name_vod_c_vod__c", + "zvod_Account_Sample_Status_vod_c_vod__c", + "zvod_Attendees_vod__c", + "zvod_Key_Messages_vod__c", + "zvod_Detailing_vod__c", + "zvod_Expenses_vod__c", + "zvod_Followup_vod__c", + "zvod_Samples_vod__c", + "zvod_Save_vod__c", + "zvod_Submit_vod__c", + "zvod_Delete_vod__c", + "Activity_Type__c", + "Significant_Event__c", + "Location_vod__c", + "Subject_vod__c", + "Unlock_vod__c", + "Call_Datetime_vod__c", + "Disbursed_To_vod__c", + "Disclaimer_vod__c", + "Request_Receipt_vod__c", + "Signature_Date_vod__c", + "Signature_vod__c", + "Territory_vod__c", + "Submitted_By_Mobile_vod__c", + "Call_Type_vod__c", + "Add_Key_Message_vod__c", + "Address_vod__c", + "Attendees_vod__c", + "Attendee_Type_vod__c", + "Call_Date_vod__c", + "Detailed_Products_vod__c", + "No_Disbursement_vod__c", + "Parent_Call_vod__c", + "User_vod__c", + "Contact_vod__c", + "zvod_Entity_vod__c", + "Medical_Event_vod__c", + "Mobile_Created_Datetime_vod__c", + "Mobile_Last_Modified_Datetime_vod__c", + "License_vod__c", + "Is_Parent_Call_vod__c", + "Entity_Display_Name_vod__c", + "Override_Lock_vod__c", + "Last_Device_vod__c", + "Ship_Address_Line_1_vod__c", + "Ship_Address_Line_2_vod__c", + "Ship_City_vod__c", + "Ship_Country_vod__c", + "Ship_License_Expiration_Date_vod__c", + "Ship_License_Status_vod__c", + "Ship_License_vod__c", + "Ship_State_vod__c", + "Ship_To_Address_vod__c", + "Ship_Zip_vod__c", + "Ship_To_Address_Text_vod__c", + "CLM_vod__c", + "zvod_CLMDetails_vod__c", + "Is_Sampled_Call_vod__c", + "zvod_Surveys_vod__c", + "Presentations_vod__c", + "Entity_Reference_Id_vod__c", + "Error_Reference_Call_vod__c", + "Duration_vod__c", + "Color_vod__c", + "Allowed_Products_vod__c", + "zvod_Attachments_vod__c", + "Sample_Card_Reason_vod__c", + "ASSMCA_vod__c", + "Address_Line_1_vod__c", + "Address_Line_2_vod__c", + "City_vod__c", + "DEA_Address_Line_1_vod__c", + "DEA_Address_Line_2_vod__c", + "DEA_Address_vod__c", + "DEA_City_vod__c", + "DEA_Expiration_Date_vod__c", + "DEA_State_vod__c", + "DEA_Zip_4_vod__c", + "DEA_Zip_vod__c", + "DEA_vod__c", + "Ship_Zip_4_vod__c", + "State_vod__c", + "Zip_4_vod__c", + "Zip_vod__c", + "Sample_Send_Card_vod__c", + "zvod_Address_vod_c_DEA_Status_vod_c_vod__c", + "Signature_Page_Image_vod__c", + "Credentials_vod__c", + "Salutation_vod__c", + "zvod_Account_Call_Reminder_vod_c_vod__c", + "MSJ_Meeting_Duration__c", + "MSJ_Double_Visit_AM__c", + "zvod_Business_Account_vod__c", + "Product_Priority_1_vod__c", + "Product_Priority_2_vod__c", + "Product_Priority_3_vod__c", + "Product_Priority_4_vod__c", + "Product_Priority_5_vod__c", + "zvod_More_Actions_vod__c", + "zvod_Call_Conflict_Status_vod__c", + "Signature_Timestamp_vod__c", + "Expense_Amount_vod__c", + "Total_Expense_Attendees_Count_vod__c", + "Attendee_list_vod__c", + "Expense_Post_Status_vod__c", + "Attendee_Post_Status_vod__c", + "Expense_System_External_ID_vod__c", + "Incurred_Expense_vod__c", + "Assigner_vod__c", + "Assignment_Datetime_vod__c", + "zvod_Call_Objective_vod__c", + "Signature_Location_Longitude_vod__c", + "Signature_Location_Latitude_vod__c", + "Location_Services_Status_vod__c", + "MSJ_Double_Visit_Other__c", + "MSJ_Comment__c", + "MSJ_For_Reporting__c", + "MSJ_Number_of_Attendees__c", + "MSJ_Main_Dept__c", + "Planned_Type_vjh__c", + "Cobrowse_URL_Participant_vod__c", + "MSJ_Activity_Method_Text__c", + "MSJ_Activity_Method__c", + "MSJ_Classification__c", + "MSJ_Double_Visit_MSL__c", + "MSJ_MSL_Comment_for_MR__c", + "MSJ_APD__c", + "Medical_Inquiry_vod__c", + "MSJ_Call_Type_MSJ__c", + "MSJ_Prescription_Request__c", + "MSJ_Patient_Follow__c", + "Child_Account_Id_vod__c", + "Child_Account_vod__c", + "Location_Id_vod__c", + "Location_Name_vod__c", + "MSJ_Comments_about_technology__c", + "Remote_Meeting_vod__c", + "Veeva_Remote_Meeting_Id_vod__c", + "MSJ_Activity_Type_Report__c", + "MSJ_Activity_Type__c", + "MSJ_Activity__c", + "MSJ_Comments__c", + "MSJ_Therapy__c", + "MSJ_Time_Hrs__c", + "EMDS_CO_Reference__c", + "EMDS_Call_Sub_Type__c", + "EMDS_Call_Type__c", + "EMDS_Call_Unsuccessful__c", + "EMDS_Congress_Type__c", + "EMDS_Date_of_Service__c", + "EMDS_Fertility_DisInterest__c", + "EMDS_Fertility_Interest__c", + "EMDS_Installed_Equipment__c", + "EMDS_Pipeline_Stage_Value__c", + "EMDS_Pipeline_Stage__c", + "EMDS_Pipeline__c", + "EMDS_Reason_for_Call__c", + "EMDS_Training_Completed__c", + "MSJ_BrainStorming__c", + "MSJ_SIPAGL_1A__c", + "MSJ_SIPAGL_1B__c", + "MSJ_SIPAGL_2__c", + "MSJ_SIPAGL_3__c", + "MSJ_SIPAGL_4A__c", + "MSJ_SIPAGL_5A__c", + "MSJ_SIPAGL_comment__c", + "MSJ_SIPAGL_4B__c", + "MSJ_SIPAGL_5B__c", + "Location_Text_vod__c", + "Call_Channel_vod__c", + "MSJ_Scientific_Interaction__c", + "MSJ_Activity_Email_Reply__c", + "MSJ_Interaction_Duration__c", + "MSJ_SIPAGL_1A_date__c", + "MSJ_CoPromotion__c", + "Call_Channel_Formula_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Detail_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Is_Parent_Call_vod__c", + "Call2_vod__c", + "Product_vod__c", + "Detail_Priority_vod__c", + "Mobile_ID_vod__c", + "Override_Lock_vod__c", + "Type_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Key_Message_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Call2_vod__c", + "Reaction_vod__c", + "Product_vod__c", + "Key_Message_vod__c", + "Mobile_ID_vod__c", + "Contact_vod__c", + "Call_Date_vod__c", + "User_vod__c", + "Category_vod__c", + "Vehicle_vod__c", + "Is_Parent_Call_vod__c", + "Override_Lock_vod__c", + "CLM_ID_vod__c", + "Slide_Version_vod__c", + "Duration_vod__c", + "Presentation_ID_vod__c", + "Start_Time_vod__c", + "Attendee_Type_vod__c", + "Entity_Reference_Id_vod__c", + "Segment_vod__c", + "Display_Order_vod__c", + "Clm_Presentation_Name_vod__c", + "Clm_Presentation_Version_vod__c", + "Clm_Presentation_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call_Clickstream_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Answer_vod__c", + "Call_vod__c", + "Key_Message_vod__c", + "Mobile_ID_vod__c", + "Popup_Opened_vod__c", + "Possible_Answers_vod__c", + "Presentation_ID_vod__c", + "Product_vod__c", + "Range_Value_vod__c", + "Rollover_Entered_vod__c", + "Selected_Items_vod__c", + "CLM_ID_vod__c", + "Question_vod__c", + "Survey_Type_vod__c", + "Text_Entered_vod__c", + "Toggle_Button_On_vod__c", + "Track_Element_Description_vod__c", + "Track_Element_Id_vod__c", + "Track_Element_Type_vod__c", + "Usage_Duration_vod__c", + "Usage_Start_Time_vod__c", + "AuxillaryId_vod__c", + "ParentId_vod__c", + "Revision_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Discussion_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Call2_vod__c", + "Activity__c", + "Comments__c", + "Contact_vod__c", + "Call_Date_vod__c", + "Product_Strategy_vod__c", + "Product_Tactic_vod__c", + "Restricted_Comments__c", + "Product_vod__c", + "Presentation__c", + "Discussion_Topics__c", + "Slides__c", + "User_vod__c", + "Indication__c", + "Mobile_ID_vod__c", + "Medical_Event_vod__c", + "Is_Parent_Call_vod__c", + "Override_Lock_vod__c", + "zvod_Product_Map_vod__c", + "Attendee_Type_vod__c", + "Entity_Reference_Id_vod__c", + "Account_Tactic_vod__c", + "MSJ_Material_Type__c", + "MSJ_Discussion_Contents__c", + "MSJ_IST_Minutes__c", + "MSJ_Off_Label_Minutes__c", + "MSJ_Discussion_Objectives__c", + "MSJ_Insight__c", + "EMDS_Materials__c", + "EMDS_Topic__c", + "MSJ_Visit_Purpose__c", + "MSJ_Insight_Count__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Time_Off_Territory_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Reason_vod__c", + "Territory_vod__c", + "Date_vod__c", + "Status_vod__c", + "Time_vod__c", + "Hours_vod__c", + "Mobile_ID_vod__c", + "Hours_off_vod__c", + "Start_Time_vod__c", + "MSJ_Day__c", + "MSJ_Comment__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Dynamic_Attribute_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Active_vod__c", + "Dynamic_Attribute_Configuration_vod__c", + "Dynamic_Attribute_Description_vod__c", + "Dynamic_Attribute_Help_Text_vod__c", + "Dynamic_Attribute_Label_vod__c", + "Dynamic_Attribute_Name_vod__c", + "Dynamic_Attribute_Record_Type_vod__c", + "Dynamic_Attribute_Value_Checkbox_vod__c", + "Dynamic_Attribute_Value_Date_Time_vod__c", + "Dynamic_Attribute_Value_Date_vod__c", + "Dynamic_Attribute_Value_Number_vod__c", + "Dynamic_Attribute_Value_Text_Area_vod__c", + "Dynamic_Attribute_Value_Text_vod__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Dynamic_Attribute_Configuration_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Applies_To_vod__c", + "Attribute_Label_vod__c", + "Attribute_Name_vod__c", + "Available_Values_vod__c", + "Description_vod__c", + "Detail_Group_vod__c", + "Display_Order_vod__c", + "External_ID_vod__c", + "Help_Text_vod__c", + "Product_vod__c", + "Read_Only_vod__c", + "Section_Name_vod__c", + "Sharing_Group_vod__c", + "Status_vod__c", + "Track_Changes_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Profile", + "columns": [ + "Id", + "Name", + "PermissionsEmailSingle", + "PermissionsEmailMass", + "PermissionsEditTask", + "PermissionsEditEvent", + "PermissionsExportReport", + "PermissionsImportPersonal", + "PermissionsDataExport", + "PermissionsManageUsers", + "PermissionsEditPublicFilters", + "PermissionsEditPublicTemplates", + "PermissionsModifyAllData", + "PermissionsManageCases", + "PermissionsManageSolutions", + "PermissionsCustomizeApplication", + "PermissionsEditReadonlyFields", + "PermissionsRunReports", + "PermissionsViewSetup", + "PermissionsTransferAnyEntity", + "PermissionsNewReportBuilder", + "PermissionsManageSelfService", + "PermissionsManageCssUsers", + "PermissionsActivateContract", + "PermissionsApproveContract", + "PermissionsImportLeads", + "PermissionsManageLeads", + "PermissionsTransferAnyLead", + "PermissionsViewAllData", + "PermissionsEditPublicDocuments", + "PermissionsViewEncryptedData", + "PermissionsEditBrandTemplates", + "PermissionsEditHtmlTemplates", + "PermissionsManageTranslation", + "PermissionsDeleteActivatedContract", + "PermissionsSendSitRequests", + "PermissionsApiUserOnly", + "PermissionsManageRemoteAccess", + "PermissionsCanUseNewDashboardBuilder", + "PermissionsManageCategories", + "PermissionsConvertLeads", + "PermissionsTestInstanceCreate", + "PermissionsPasswordNeverExpires", + "PermissionsUseTeamReassignWizards", + "PermissionsInstallMultiforce", + "PermissionsPublishMultiforce", + "PermissionsEditOppLineItemUnitPrice", + "PermissionsManageTerritories", + "PermissionsCreateMultiforce", + "PermissionsBulkApiHardDelete", + "PermissionsInboundMigrationToolsUser", + "PermissionsSolutionImport", + "PermissionsManageCallCenters", + "PermissionsManageSynonyms", + "PermissionsOutboundMigrationToolsUser", + "PermissionsViewContent", + "PermissionsManageEmailClientConfig", + "PermissionsEnableNotifications", + "PermissionsManageDataIntegrations", + "PermissionsDistributeFromPersWksp", + "PermissionsViewDataCategories", + "PermissionsManageDataCategories", + "PermissionsAuthorApex", + "PermissionsManageMobile", + "PermissionsApiEnabled", + "PermissionsManageCustomReportTypes", + "PermissionsEditCaseComments", + "PermissionsTransferAnyCase", + "PermissionsContentAdministrator", + "PermissionsCreateWorkspaces", + "PermissionsManageContentPermissions", + "PermissionsManageContentProperties", + "PermissionsManageContentTypes", + "PermissionsScheduleJob", + "PermissionsManageExchangeConfig", + "PermissionsManageAnalyticSnapshots", + "PermissionsScheduleReports", + "PermissionsManageBusinessHourHolidays", + "PermissionsManageDynamicDashboards", + "PermissionsManageInteraction", + "PermissionsViewMyTeamsDashboards", + "PermissionsResetPasswords", + "PermissionsFlowUFLRequired", + "PermissionsActivitiesAccess", + "PermissionsEmailTemplateManagement", + "PermissionsEmailAdministration", + "PermissionsChatterFileLink", + "PermissionsForceTwoFactor", + "PermissionsViewEventLogFiles", + "PermissionsManageNetworks", + "PermissionsManageAuthProviders", + "PermissionsRunFlow", + "PermissionsCreateCustomizeDashboards", + "PermissionsCreateDashboardFolders", + "PermissionsViewPublicDashboards", + "PermissionsManageDashbdsInPubFolders", + "PermissionsCreateCustomizeReports", + "PermissionsCreateReportFolders", + "PermissionsViewPublicReports", + "PermissionsManageReportsInPubFolders", + "PermissionsEditMyDashboards", + "PermissionsEditMyReports", + "PermissionsViewAllUsers", + "PermissionsConnectOrgToEnvironmentHub", + "PermissionsCreateCustomizeFilters", + "PermissionsContentHubUser", + "PermissionsGovernNetworks", + "PermissionsSalesConsole", + "PermissionsTwoFactorApi", + "PermissionsDeleteTopics", + "PermissionsEditTopics", + "PermissionsCreateTopics", + "PermissionsAssignTopics", + "PermissionsIdentityEnabled", + "PermissionsIdentityConnect", + "PermissionsContentWorkspaces", + "PermissionsCustomMobileAppsAccess", + "PermissionsViewHelpLink", + "PermissionsManageProfilesPermissionsets", + "PermissionsAssignPermissionSets", + "PermissionsManageRoles", + "PermissionsManageIpAddresses", + "PermissionsManageSharing", + "PermissionsManageInternalUsers", + "PermissionsManagePasswordPolicies", + "PermissionsManageLoginAccessPolicies", + "PermissionsManageCustomPermissions", + "PermissionsStdAutomaticActivityCapture", + "PermissionsManageTwoFactor", + "PermissionsDebugApex", + "PermissionsLightningExperienceUser", + "PermissionsConfigCustomRecs", + "PermissionsSubmitMacrosAllowed", + "PermissionsBulkMacrosAllowed", + "PermissionsManageSessionPermissionSets", + "PermissionsCreateAuditFields", + "PermissionsUpdateWithInactiveOwner", + "PermissionsManageSandboxes", + "PermissionsAutomaticActivityCapture", + "PermissionsImportCustomObjects", + "PermissionsDelegatedTwoFactor", + "PermissionsSelectFilesFromSalesforce", + "PermissionsModerateNetworkUsers", + "PermissionsMergeTopics", + "PermissionsSubscribeToLightningReports", + "PermissionsManagePvtRptsAndDashbds", + "PermissionsAllowLightningLogin", + "PermissionsCampaignInfluence2", + "PermissionsViewDataAssessment", + "PermissionsCanApproveFeedPost", + "PermissionsAllowViewEditConvertedLeads", + "PermissionsShowCompanyNameAsUserBadge", + "PermissionsAccessCMC", + "PermissionsViewHealthCheck", + "PermissionsManageHealthCheck", + "PermissionsPackaging2", + "PermissionsManageCertificates", + "PermissionsCreateReportInLightning", + "PermissionsPreventClassicExperience", + "PermissionsListEmailSend", + "PermissionsChangeDashboardColors", + "PermissionsManageRecommendationStrategies", + "PermissionsManagePropositions", + "PermissionsSubscribeReportRolesGrps", + "PermissionsSubscribeDashboardRolesGrps", + "PermissionsUseWebLink", + "PermissionsHasUnlimitedNBAExecutions", + "PermissionsViewOnlyEmbeddedAppUser", + "PermissionsViewAllActivities", + "PermissionsSubscribeReportToOtherUsers", + "PermissionsLightningConsoleAllowedForUser", + "PermissionsSubscribeReportsRunAsUser", + "PermissionsSubscribeToLightningDashboards", + "PermissionsSubscribeDashboardToOtherUsers", + "PermissionsCreateLtngTempInPub", + "PermissionsTransactionalEmailSend", + "PermissionsViewPrivateStaticResources", + "PermissionsCreateLtngTempFolder", + "PermissionsApexRestServices", + "PermissionsEnableCommunityAppLauncher", + "PermissionsGiveRecognitionBadge", + "PermissionsUseMySearch", + "PermissionsLtngPromoReserved01UserPerm", + "PermissionsManageSubscriptions", + "PermissionsManageSurveys", + "PermissionsUseAssistantDialog", + "PermissionsUseQuerySuggestions", + "PermissionsViewRoles", + "PermissionsLMOutboundMessagingUserPerm", + "PermissionsModifyDataClassification", + "PermissionsPrivacyDataAccess", + "PermissionsQueryAllFiles", + "PermissionsModifyMetadata", + "PermissionsManageCMS", + "PermissionsSandboxTestingInCommunityApp", + "PermissionsCanEditPrompts", + "PermissionsViewUserPII", + "PermissionsManageHubConnections", + "PermissionsB2BMarketingAnalyticsUser", + "PermissionsTraceXdsQueries", + "PermissionsViewAllCustomSettings", + "PermissionsViewAllForeignKeyNames", + "PermissionsHeadlessCMSAccess", + "PermissionsLMEndMessagingSessionUserPerm", + "PermissionsConsentApiUpdate", + "PermissionsAccessContentBuilder", + "PermissionsAccountSwitcherUser", + "PermissionsManageC360AConnections", + "PermissionsManageReleaseUpdates", + "PermissionsViewAllProfiles", + "PermissionsSkipIdentityConfirmation", + "PermissionsSendCustomNotifications", + "PermissionsPackaging2Delete", + "PermissionsFSCComprehensiveUserAccess", + "PermissionsManageTrustMeasures", + "PermissionsViewTrustMeasures", + "PermissionsIsotopeCToCUser", + "PermissionsIsotopeAccess", + "PermissionsIsotopeLEX", + "PermissionsQuipMetricsAccess", + "PermissionsQuipUserEngagementMetrics", + "PermissionsManageExternalConnections", + "PermissionsAIViewInsightObjects", + "PermissionsAICreateInsightObjects", + "PermissionsNativeWebviewScrolling", + "PermissionsViewDeveloperName", + "Type", + "UserLicenseId", + "UserType", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "Description", + "LastViewedDate", + "LastReferencedDate" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "My_Setup_Products_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Product_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Activity_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_External_ID_Map_vod__c", + "Account_vod__c", + "Call_vod__c", + "City_vod__c", + "Client_Name_vod__c", + "Client_OS_vod__c", + "Client_Type_vod__c", + "Country_vod__c", + "Debug_vod__c", + "Device_vod__c", + "IP_Address_vod__c", + "Multichannel_Activity_vod__c", + "Referring_Site_vod__c", + "Region_vod__c", + "Sent_Email_vod__c", + "Session_Id_vod__c", + "Site_vod__c", + "Start_DateTime_vod__c", + "Total_Duration_vod__c", + "URL_vod__c", + "User_Agent_vod__c", + "VExternal_Id_vod__c", + "Viewport_Height_vod__c", + "Viewport_Width_vod__c", + "Color_vod__c", + "Icon_vod__c", + "MCD_Primary_Key_vod__c", + "Record_Type_Name_vod__c", + "MSJ_Date_Opened__c", + "MSJ_Sent_Date__c", + "MSJ_Email_Subject__c", + "MSJ_Opens__c", + "MSJ_Email_Status__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Activity_Line_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Multichannel_Activity_vod__c", + "Call_vod__c", + "Custom_vod__c", + "DateTime_vod__c", + "Debug_vod__c", + "Detail_Group_VExternal_Id_vod__c", + "Detail_Group_vod__c", + "Duration_vod__c", + "Event_Subtype_vod__c", + "Event_Type_vod__c", + "Key_Message_VExternal_Id_vod__c", + "Key_Message_vod__c", + "Multichannel_Content_Asset_Id_vod__c", + "Multichannel_Content_Asset_Version_vod__c", + "Multichannel_Content_Asset_vod__c", + "Multichannel_Content_vod__c", + "Product_VExternal_Id_vod__c", + "Product_vod__c", + "Sent_Email_vod__c", + "VExternal_Id_vod__c", + "Video_Last_Viewed_Time_vod__c", + "Video_Length_vod__c", + "Video_Total_Time_Spent_vod__c", + "View_Order_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Consent_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Capture_Datetime_vod__c", + "Channel_Value_vod__c", + "Detail_Group_vod__c", + "External_ID_vod__c", + "Last_Device_vod__c", + "Mobile_ID_vod__c", + "Opt_Expiration_Date_vod__c", + "Opt_Type_vod__c", + "Optout_Event_Type_vod__c", + "Product_vod__c", + "Signature_Datetime_vod__c", + "Signature_ID_vod__c", + "Signature_vod__c", + "Sample_Consent_Template_Data_vod__c", + "Sample_Consent_Template_vod__c", + "Consent_Line_vod__c", + "Consent_Type_vod__c", + "Default_Consent_Text_vod__c", + "Disclaimer_Text_vod__c", + "Sub_Channel_Key_vod__c", + "Consent_Confirm_Datetime_vod__c", + "Related_Transaction_Id_vod__c", + "Sent_Email_vod__c", + "Content_Type_vod__c", + "Receipt_Email_vod__c", + "Receipt_Sent_Email_Transaction_Id_vod__c", + "Receipt_Sent_Email_vod__c", + "Captured_By_vod__c", + "Opt_Out_Disclaimer_Text_vod__c", + "Channel_Source_vod__c", + "Union_Id_vod__c", + "User_Last_Notified_vod__c", + "Sub_Channel_Display_Name__c", + "MSJ_Consent_Source__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Inquiry_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Address_Line_1_vod__c", + "Address_Line_2_vod__c", + "City_vod__c", + "Delivery_Method_vod__c", + "Email_vod__c", + "Fax_Number_vod__c", + "Inquiry_Text__c", + "Lock_vod__c", + "Mobile_ID_vod__c", + "Phone_Number_vod__c", + "Product__c", + "Rush_Delivery__c", + "Signature_Date_vod__c", + "Signature_vod__c", + "State_vod__c", + "Status_vod__c", + "Zip_vod__c", + "zvod_Delivery_Method_vod__c", + "zvod_Disclaimer_vod__c", + "Submitted_By_Mobile_vod__c", + "Disclaimer_vod__c", + "Entity_Reference_Id_vod__c", + "Call2_vod__c", + "Country_vod__c", + "Override_Lock_vod__c", + "MSJ_Department__c", + "MSJ_Doctor_Name__c", + "MSJ_Hospital_Name__c", + "MSJ_Indication__c", + "MSJ_Inquiry_Assignment__c", + "MSJ_Inquiry_Date__c", + "MSJ_Inquiry_Input_Manager__c", + "MSJ_Inquiry_Input_User__c", + "MSJ_MSL_Manager__c", + "MSJ_Notice_to_MR__c", + "MSJ_Person_in_charge_1__c", + "MSJ_Person_in_charge_2__c", + "MSJ_Product_for_MEC__c", + "MSJ_Product_for_MR__c", + "MSJ_Reply_Date__c", + "MSJ_Reply_User__c", + "MSJ_Reply__c", + "MSJ_Title__c", + "MSJ_AE_Infomation__c", + "MSJ_FAQ_Number_Report__c", + "MSJ_Return_Call_Report__c", + "MSJ_Inquiry_Origin_Report__c", + "MSJ_AE_Report__c", + "MSJ_Background__c", + "MSJ_MSL_Support__c", + "MSJ_Material_Requirement__c", + "MSJ_Hospital_Name_Disp__c", + "MSJ_Hospital__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Email_Activity_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Sent_Email_vod__c", + "Activity_DateTime_vod__c", + "City_vod__c", + "Click_URL_vod__c", + "Client_Name_vod__c", + "Client_OS_vod__c", + "Client_Type_vod__c", + "Country_vod__c", + "Device_Type_vod__c", + "Event_Msg_vod__c", + "Event_type_vod__c", + "IP_Address_vod__c", + "Region_vod__c", + "User_Agent_vod__c", + "Vault_Doc_ID_vod__c", + "Vault_Doc_Name_vod__c", + "Vault_Document_Major_Version_vod__c", + "Vault_Document_Minor_Version_vod__c", + "Vault_Document_Number_vod__c", + "Vault_Document_Title_vod__c", + "Vault_Instance_ID_vod__c", + "Preference_Modification_vod__c", + "Approved_Document_vod__c", + "Link_Name_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "User", + "columns": [ + "Id", + "Username", + "LastName", + "FirstName", + "Name", + "CompanyName", + "Division", + "Department", + "Title", + "Street", + "City", + "State", + "PostalCode", + "Country", + "Latitude", + "Longitude", + "GeocodeAccuracy", + "Address", + "Email", + "EmailPreferencesAutoBcc", + "EmailPreferencesAutoBccStayInTouch", + "EmailPreferencesStayInTouchReminder", + "SenderEmail", + "SenderName", + "Signature", + "StayInTouchSubject", + "StayInTouchSignature", + "StayInTouchNote", + "Phone", + "Fax", + "MobilePhone", + "Alias", + "CommunityNickname", + "BadgeText", + "IsActive", + "TimeZoneSidKey", + "UserRoleId", + "LocaleSidKey", + "ReceivesInfoEmails", + "ReceivesAdminInfoEmails", + "EmailEncodingKey", + "ProfileId", + "UserType", + "LanguageLocaleKey", + "EmployeeNumber", + "DelegatedApproverId", + "ManagerId", + "LastLoginDate", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "OfflineTrialExpirationDate", + "OfflinePdaTrialExpirationDate", + "UserPermissionsMarketingUser", + "UserPermissionsOfflineUser", + "UserPermissionsWirelessUser", + "UserPermissionsAvantgoUser", + "UserPermissionsCallCenterAutoLogin", + "UserPermissionsSFContentUser", + "UserPermissionsInteractionUser", + "UserPermissionsSupportUser", + "UserPermissionsChatterAnswersUser", + "ForecastEnabled", + "UserPreferencesActivityRemindersPopup", + "UserPreferencesEventRemindersCheckboxDefault", + "UserPreferencesTaskRemindersCheckboxDefault", + "UserPreferencesReminderSoundOff", + "UserPreferencesDisableAllFeedsEmail", + "UserPreferencesApexPagesDeveloperMode", + "UserPreferencesReceiveNoNotificationsAsApprover", + "UserPreferencesReceiveNotificationsAsDelegatedApprover", + "UserPreferencesHideCSNGetChatterMobileTask", + "UserPreferencesHideCSNDesktopTask", + "UserPreferencesHideChatterOnboardingSplash", + "UserPreferencesHideSecondChatterOnboardingSplash", + "UserPreferencesShowTitleToExternalUsers", + "UserPreferencesShowManagerToExternalUsers", + "UserPreferencesShowEmailToExternalUsers", + "UserPreferencesShowWorkPhoneToExternalUsers", + "UserPreferencesShowMobilePhoneToExternalUsers", + "UserPreferencesShowFaxToExternalUsers", + "UserPreferencesShowStreetAddressToExternalUsers", + "UserPreferencesShowCityToExternalUsers", + "UserPreferencesShowStateToExternalUsers", + "UserPreferencesShowPostalCodeToExternalUsers", + "UserPreferencesShowCountryToExternalUsers", + "UserPreferencesShowProfilePicToGuestUsers", + "UserPreferencesShowTitleToGuestUsers", + "UserPreferencesShowCityToGuestUsers", + "UserPreferencesShowStateToGuestUsers", + "UserPreferencesShowPostalCodeToGuestUsers", + "UserPreferencesShowCountryToGuestUsers", + "UserPreferencesHideInvoicesRedirectConfirmation", + "UserPreferencesHideStatementsRedirectConfirmation", + "UserPreferencesPathAssistantCollapsed", + "UserPreferencesCacheDiagnostics", + "UserPreferencesShowEmailToGuestUsers", + "UserPreferencesShowManagerToGuestUsers", + "UserPreferencesShowWorkPhoneToGuestUsers", + "UserPreferencesShowMobilePhoneToGuestUsers", + "UserPreferencesShowFaxToGuestUsers", + "UserPreferencesShowStreetAddressToGuestUsers", + "UserPreferencesLightningExperiencePreferred", + "UserPreferencesPreviewLightning", + "UserPreferencesHideEndUserOnboardingAssistantModal", + "UserPreferencesHideLightningMigrationModal", + "UserPreferencesHideSfxWelcomeMat", + "UserPreferencesHideBiggerPhotoCallout", + "UserPreferencesGlobalNavBarWTShown", + "UserPreferencesGlobalNavGridMenuWTShown", + "UserPreferencesCreateLEXAppsWTShown", + "UserPreferencesFavoritesWTShown", + "UserPreferencesRecordHomeSectionCollapseWTShown", + "UserPreferencesRecordHomeReservedWTShown", + "UserPreferencesFavoritesShowTopFavorites", + "UserPreferencesExcludeMailAppAttachments", + "UserPreferencesSuppressTaskSFXReminders", + "UserPreferencesSuppressEventSFXReminders", + "UserPreferencesPreviewCustomTheme", + "UserPreferencesHasCelebrationBadge", + "UserPreferencesUserDebugModePref", + "UserPreferencesSRHOverrideActivities", + "UserPreferencesNewLightningReportRunPageEnabled", + "UserPreferencesReverseOpenActivitiesView", + "UserPreferencesHideBrowseProductRedirectConfirmation", + "UserPreferencesHideOnlineSalesAppWelcomeMat", + "ContactId", + "AccountId", + "CallCenterId", + "Extension", + "FederationIdentifier", + "AboutMe", + "FullPhotoUrl", + "SmallPhotoUrl", + "IsExtIndicatorVisible", + "OutOfOfficeMessage", + "MediumPhotoUrl", + "DigestFrequency", + "DefaultGroupNotificationFrequency", + "LastViewedDate", + "LastReferencedDate", + "BannerPhotoUrl", + "SmallBannerPhotoUrl", + "MediumBannerPhotoUrl", + "IsProfilePhotoActive", + "IndividualId", + "Last_Mobile_Connect_vod__c", + "Last_Tablet_Connect_vod__c", + "Last_Mobile_Connect_Version_vod__c", + "Last_Tablet_Connect_Version_vod__c", + "Last_Mobile_Sync_vod__c", + "Last_Tablet_Sync_vod__c", + "RaiseLoggingLevel_vod__c", + "SendDetailedLog_vod__c", + "Last_Blackberry_Connect_vod__c", + "Last_Blackberry_Connect_Version_vod__c", + "Last_Blackberry_Sync_vod__c", + "Force_Full_Refresh_vod__c", + "Override_SystemModstamp_Timestamp_vod__c", + "Facetime_Email_vod__c", + "Facetime_Phone_vod__c", + "Product_Expertise_vod__c", + "Available_vod__c", + "Available_Last_Update_vod__c", + "Last_iPad_Connect_Version_vod__c", + "Last_iPad_Connect_vod__c", + "Last_iPad_Sync_vod__c", + "Inventory_Order_Allocation_Group_vod__c", + "Concur_User_Id_vod__c", + "Last_iPad_iOS_Version_vod__c", + "Approved_Email_Admin_vod__c", + "Last_WinModern_Connect_Version_vod__c", + "Last_WinModern_Connect_vod__c", + "Last_WinModern_Sync_vod__c", + "Primary_Territory_vod__c", + "Analytics_Admin_vod__c", + "Content_Admin_vod__c", + "Last_WinModern_Windows_Version_vod__c", + "Upload_VTrans_vod__c", + "Sync_Frequency_vjh__c", + "Clear_Client_Sync_Errors_vod__c", + "Remote_Meeting_Host_Id_vod__c", + "Remote_Meeting_Host_Token_vod__c", + "Last_iPhone_Connect_Version_vod__c", + "Last_iPhone_Connect_vod__c", + "Last_iPhone_Sync_vod__c", + "Last_iPhone_iOS_Version_vod__c", + "Remote_Meeting_Start_From_CRM_Online_vod__c", + "Country_Code_vod__c", + "User_Type_vod__c", + "Engage_Group_Provisioning_Status_vod__c", + "Engage_Group_Request_vod__c", + "Engage_Group_vod__c", + "Last_CRMDesktop_Mac_Sync_vod__c", + "Last_CRMDesktop_Mac_Version_vod__c", + "Last_CRMDesktop_Windows_Sync_vod__c", + "Last_CRMDesktop_Windows_Version_vod__c", + "MSJ_Test_User__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "UserTerritory2Association", + "columns": [ + "Id", + "UserId", + "Territory2Id", + "IsActive", + "RoleInTerritory2", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Remote_Meeting_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Meeting_Id_vod__c", + "Meeting_Name_vod__c", + "Mobile_ID_vod__c", + "Scheduled_DateTime_vod__c", + "Scheduled_vod__c", + "Attendance_Report_Process_Status_vod__c", + "Latest_Meeting_Start_Datetime_vod__c", + "Meeting_Password_vod__c", + "Meeting_Outcome_Status_vod__c", + "Allow_for_Joining_via_Zoom_vod__c", + "Zoom_Join_Token_vod__c", + "VExternal_Id_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "RecordType", + "columns": [ + "Id", + "Name", + "DeveloperName", + "NamespacePrefix", + "Description", + "BusinessProcessId", + "SobjectType", + "IsActive", + "IsPersonType", + "CreatedById", + "CreatedDate", + "LastModifiedById", + "LastModifiedDate", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "UserRole", + "columns": [ + "Id", + "Name", + "ParentRoleId", + "RollupDescription", + "OpportunityAccessForAccountOwner", + "CaseAccessForAccountOwner", + "ContactAccessForAccountOwner", + "ForecastUserId", + "MayForecastManagerShare", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "PortalAccountId", + "PortalType", + "PortalAccountOwnerId" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Account", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "Name", + "LastName", + "FirstName", + "Salutation", + "RecordTypeId", + "Phone", + "Fax", + "Website", + "PhotoUrl", + "NumberOfEmployees", + "Ownership", + "OwnerId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "IsExcludedFromRealign", + "PersonContactId", + "IsPersonAccount", + "PersonMailingStreet", + "PersonMailingCity", + "PersonMailingState", + "PersonMailingPostalCode", + "PersonMailingCountry", + "PersonMailingLatitude", + "PersonMailingLongitude", + "PersonMailingGeocodeAccuracy", + "PersonMailingAddress", + "PersonOtherStreet", + "PersonOtherCity", + "PersonOtherState", + "PersonOtherPostalCode", + "PersonOtherCountry", + "PersonOtherLatitude", + "PersonOtherLongitude", + "PersonOtherGeocodeAccuracy", + "PersonOtherAddress", + "PersonMobilePhone", + "PersonHomePhone", + "PersonOtherPhone", + "PersonAssistantPhone", + "PersonEmail", + "PersonTitle", + "PersonDepartment", + "PersonAssistantName", + "PersonBirthdate", + "PersonHasOptedOutOfEmail", + "PersonHasOptedOutOfFax", + "PersonDoNotCall", + "PersonLastCURequestDate", + "PersonLastCUUpdateDate", + "PersonEmailBouncedReason", + "PersonEmailBouncedDate", + "PersonIndividualId", + "Jigsaw", + "JigsawCompanyId", + "AccountSource", + "SicDesc", + "External_ID_vod__c", + "Credentials_vod__c", + "Territory_vod__c", + "Exclude_from_Zip_to_Terr_Processing_vod__c", + "Group_Specialty_1_vod__c", + "Group_Specialty_2_vod__c", + "Specialty_1_vod__c", + "Specialty_2_vod__c", + "Formatted_Name_vod__c", + "Territory_Test_vod__c", + "Mobile_ID_vod__c", + "Gender_vod__c", + "ID_vod__c", + "Do_Not_Sync_Sales_Data_vod__c", + "ID2_vod__c", + "Preferred_Name_vod__c", + "Sample_Default_vod__c", + "Segmentations_vod__c", + "Restricted_Products_vod__c", + "Payer_Id_vod__c", + "Alternate_Name_vod__c", + "Do_Not_Call_vod__c", + "MSJ_Beds__c", + "Spend_Amount__c", + "PDRP_Opt_Out_vod__c", + "Spend_Status_Value_vod__c", + "PDRP_Opt_Out_Date_vod__c", + "Spend_Status_vod__c", + "Enable_Restricted_Products_vod__c", + "Call_Reminder_vod__c", + "Account_Group_vod__c", + "Primary_Parent_vod__c", + "Color_vod__c", + "Middle_vod__c", + "Suffix_vod__c", + "MSJ_Type__c", + "No_Orders_vod__c", + "MSJ_BU_ONC__c", + "MSJ_BU_FE__c", + "Account_Search_FirstLast_vod__c", + "Account_Search_LastFirst_vod__c", + "MSJ_Operation__c", + "Practice_at_Hospital_vod__c", + "Practice_Near_Hospital_vod__c", + "Do_Not_Create_Child_Account_vod__c", + "Total_MDs_DOs__c", + "AHA__c", + "Order_Type_vod__c", + "NPI_vod__c", + "ME__c", + "Speaker__c", + "Investigator_vod__c", + "Default_Order_Type_vod__c", + "Tax_Status__c", + "Model__c", + "Offerings__c", + "Departments__c", + "Account_Type__c", + "MSJ_ONC_Tier__c", + "Account_Search_Business_vod__c", + "Business_Professional_Person_vod__c", + "Hospital_Type_vod__c", + "Account_Class_vod__c", + "Furigana_vod__c", + "MSJ_JISART__c", + "Total_Revenue_000__c", + "Net_Income_Loss_000__c", + "PMPM_Income_Loss_000__c", + "Commercial_Premiums_PMPM__c", + "Medical_Loss_Ratio__c", + "Medical_Expenses_PMPM__c", + "Commercial_Patient_Days_1000__c", + "HMO_Market_Shr__c", + "HMO__c", + "HMO_POS__c", + "PPO__c", + "PPO_POS__c", + "Medicare__c", + "Medicaid__c", + "MSJ_HP_Name_E__c", + "MSJ_Department__c", + "MSJ_Date_Of_Birth__c", + "MSJ_FE_GF_Potential__c", + "MSJ_FE_SZ_Potential__c", + "MSJ_EB_CRC_Ladder__c", + "MSJ_EB_CRC_Segment__c", + "MSJ_EB_HN_Segment__c", + "Business_Description__c", + "Regional_Strategy__c", + "Contracts_Process__c", + "MSJ_GF_segment__c", + "MSJ_DCF_DR_Code__c", + "MSJ_SZ_Segment__c", + "MSJ_Remark__c", + "MSJ_Title__c", + "MSJ_Role__c", + "MSJ_Kana__c", + "MSJ_Specialism__c", + "MSJ_Graduated_from__c", + "MSJ_Year_Graduation__c", + "Target__c", + "KOL_vod__c", + "MSJ_EPPV_Code__c", + "MSJ_DCF_HP_Code__c", + "Total_Lives__c", + "Total_Physicians_Enrolled__c", + "MSJ_Delete__c", + "MSJ_KOL_LOL__c", + "MSJ_ONC_Status__c", + "Account_Identifier_vod__c", + "Approved_Email_Opt_Type_vod__c", + "Language_vod__c", + "MSJ_KRAS_Routine_Date__c", + "MSJ_KRAS_Routine__c", + "MSJ_DRP_Target__c", + "MSJ_Fertility_Evaluation_Score__c", + "MSJ_Fertility_Tracking_Last_Modify_Date__c", + "Total_Pharmacists__c", + "MSJ_Number_of_Gonadotropin__c", + "MSJ_Number_of_IUI_cycle__c", + "MSJ_Number_of_OI_monthly_cycle__c", + "MSJ_OI_Protocol_learning_level__c", + "MSJ_H_N_Tier__c", + "MSJ_XLK_Segment__c", + "MSJ_XLK_Tier__c", + "Career_Status_vod__c", + "Photo_vod__c", + "MSJ_EB_H_N_LA_Segment__c", + "MSJ_EB_H_N_RM_Segment__c", + "MSJ_FE_CE_Potential__c", + "MSJ_FE_1C_potential__c", + "MSJ_FE_OV_potential__c", + "MSJ_FE_Tech_potential__c", + "MSJ_CE_segment__c", + "MSJ_1C_segment__c", + "MSJ_OV_segment__c", + "MSJ_Tech_segment__c", + "MSJ_Target_Call_Num__c", + "MSJ_DR_Change_Log__c", + "MSJ_Global_scientific_exposure__c", + "MSJ_H_index__c", + "MSJ_Num_of_Article_3Y__c", + "MSJ_Num_of_Article__c", + "MSJ_Num_of_Article_as_1st_Author_3Y__c", + "MSJ_Num_of_article_growth_rate_3Y__c", + "MSJ_Num_of_cited_3Y__c", + "MSJ_Num_of_impact_factor_3Y__c", + "MSJ_impact_factor_as_1st_Author_3Y__c", + "EMDS_Has_Pipeline_Opportunity__c", + "EMDS_Pipeline_Count__c", + "MSJ_BVC_Segment__c", + "MSJ_BVC_Tier__c", + "MSJ_BVC_AcctOpen__c", + "MSJ_BVC_MCC_Patients__c", + "MSJ_ONC_HP_Segment__c", + "MSJ_AE_Department__c", + "MSJ_AE_Facility__c", + "MSJ_AE_Name__c", + "MSJ_AE_Title__c", + "MSJ_Email__c", + "MSJ_FE_GF2_Potential__c", + "MSJ_FE_Location_potential__c", + "MSJ_GF2_segment__c", + "MSJ_OPTIN_target__c", + "MSJ_Merck_Specialty1__c", + "MSJ_Merck_Specialty2__c", + "MSJ_Marketing_Cloud_Integration__c", + "MSJ_Marketing_Cloud1__c", + "MSJ_Marketing_Cloud2__c", + "MSJ_Marketing_Cloud3__c", + "MSJ_Marketing_Cloud4__c", + "MSJ_Medical_Department__c", + "MSJ_Marketing_Cloud0__c", + "Mobile_ID_vod__pc", + "H1Insights__H1_NPI_Value_for_Testing__pc", + "H1Insights__H1_Person_ID__pc", + "H1Insights__H1_Request_Status__pc", + "H1Insights__H1_URL__pc", + "H1Insights__NPI_Number__pc", + "H1Insights__NPI_Number_for_H1_Insights__pc", + "MSJ_Marketing_Cloud_Integration__pc" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true, + "datetime_column": "LastModifiedDate" + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Type_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Header_vod__c", + "Channel_Label_vod__c", + "Channel_Source_vod__c", + "Consent_Expires_In_vod__c", + "Default_Consent_Type_vod__c", + "Disclaimer_Text_vod__c", + "Display_Order_vod__c", + "Product_Preference_vod__c", + "zvod_Consent_Default_Consent_Text_vod__c", + "zvod_Consent_Line_vod__c", + "zvod_Signature_Capture_vod__c", + "Double_Opt_In_vod__c", + "zvod_Consent_Activity_Tracking_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Header_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Header_Help_Text_vod__c", + "Country_vod__c", + "Inactive_Datetime_vod__c", + "Language_vod__c", + "Status_vod__c", + "Signature_Required_On_Opt_Out_vod__c", + "Request_Receipt_vod__c", + "Subscription_Option_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Line_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Type_vod__c", + "Detail_Group_Display_Name_vod__c", + "Detail_Group_vod__c", + "Display_Order_vod__c", + "End_Date_vod__c", + "Group_By_vod__c", + "Product_Display_Name_vod__c", + "Product_vod__c", + "Start_Date_vod__c", + "Sub_Channel_Description_vod__c", + "Sub_Channel_Display_Name_vod__c", + "Sub_Channel_Key_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Inquiry_Assignment__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Medical_Inquiry__c", + "MSJ_Close__c", + "MSJ_Doctor_Name__c", + "MSJ_Hospital_Name__c", + "MSJ_Indication__c", + "MSJ_Inquiry_Text__c", + "MSJ_MEC_User__c", + "MSJ_MSL_Manager__c", + "MSJ_MSL_User__c", + "MSJ_Notice_to_MR__c", + "MSJ_Product_for_MEC__c", + "MSJ_Product_for_MR__c", + "MSJ_Reply_Date__c", + "MSJ_Reply__c", + "MSJ_AE_Infomation__c", + "MSJ_Cancel__c", + "MSJ_FAQ_number_c__c", + "MSJ_Return_Call__c", + "MSJ_Inquiry_Origin__c", + "First_Response__c", + "Inquiry_Created_Date__c", + "Inquiry_Type_1__c", + "Inquiry_Type_2__c", + "MSJ_First_User__c", + "MSJ_MEC_Comment__c", + "MSJ_Send_Email__c", + "MSJ_Temp_Aggregated_Info__c", + "MSJ_AE_Report__c", + "MSJ_Background__c", + "MSJ_Inquiry_Date__c", + "MSJ_MSL_Support__c", + "MSJ_Handover_Comment__c", + "MSJ_Handover_Email__c", + "MSJ_Material_Requirement__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Approved_Document_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Detail_Group_vod__c", + "Document_Description_vod__c", + "Document_Host_URL_vod__c", + "Document_ID_vod__c", + "Document_Last_Mod_DateTime_vod__c", + "Email_Allows_Documents_vod__c", + "Email_Domain_vod__c", + "Email_Fragment_HTML_vod__c", + "Email_From_Address_vod__c", + "Email_From_Name_vod__c", + "Email_HTML_1_vod__c", + "Email_HTML_2_vod__c", + "Email_ReplyTo_Address_vod__c", + "Email_ReplyTo_Name_vod__c", + "Email_Subject_vod__c", + "Email_Template_Fragment_Document_ID_vod__c", + "Email_Template_Fragment_HTML_vod__c", + "ISI_Document_ID_vod__c", + "Language_vod__c", + "Other_Document_ID_List_vod__c", + "PI_Document_ID_vod__c", + "Piece_Document_ID_vod__c", + "Product_vod__c", + "Status_vod__c", + "Territory_vod__c", + "Vault_Instance_ID_vod__c", + "Allow_Any_Product_Fragment_vod__c", + "Allowed_Document_IDs_vod__c", + "Engage_Document_Id_vod__c", + "Vault_Document_ID_vod__c", + "Key_Message_vod__c", + "Events_Management_Subtype_vod__c", + "Survey_vod__c", + "Content_Type_vod__c", + "Bcc_vod__c", + "Audience_vod__c", + "WeChat_Template_ID_vod__c", + "Check_Consent_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Child_Account_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Parent_Account_vod__c", + "Child_Account_vod__c", + "External_ID_vod__c", + "Mobile_ID_vod__c", + "Primary_vod__c", + "Copy_Address_vod__c", + "Child_Name_vod__c", + "Parent_Name_vod__c", + "Parent_Child_Name_vod__c", + "Account_Code__c", + "Child_Department__c", + "Child_Role__c", + "Child_Title__c", + "Child_Remark__c", + "MSJ_1C_segment__c", + "MSJ_BU_FE__c", + "MSJ_BU_ONC__c", + "MSJ_BVC_Segment__c", + "MSJ_CE_segment__c", + "MSJ_Child_Account_Link__c", + "MSJ_DCF_DR_Code__c", + "MSJ_DCF_HP_Code__c", + "MSJ_DR_Change_Log__c", + "MSJ_Delete__c", + "MSJ_Department__c", + "MSJ_EB_CRC_Segment__c", + "MSJ_EB_HN_Segment__c", + "MSJ_EB_H_N_LA_Segment__c", + "MSJ_EB_H_N_RM_Segment__c", + "MSJ_External_ID__c", + "MSJ_Fax__c", + "MSJ_GF2_segment__c", + "MSJ_GF_segment__c", + "MSJ_KOL_LOL__c", + "MSJ_KOL__c", + "MSJ_ONC_HP_Segment__c", + "MSJ_OPTIN_target__c", + "MSJ_OV_segment__c", + "MSJ_Parent_Child_Name__c", + "MSJ_Phone__c", + "MSJ_Remark__c", + "MSJ_Target_Call_Num__c", + "MSJ_Tech_segment__c", + "MSJ_Title__c", + "MSJ_XLK_Segment__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Hospital_Medical_Regimen__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Account_Name__c", + "MSJ_Delete_Date__c", + "MSJ_Delete_Flag__c", + "MSJ_Indication__c", + "MSJ_Line__c", + "MSJ_Medical_Regimen__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Medical_Regimen__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Delete_Date__c", + "MSJ_Delete_Flag__c", + "MSJ_Indication__c", + "MSJ_Remark__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Patient__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Account_Name__c", + "MSJ_CRC_Group__c", + "MSJ_Casus_or_Transfer_Point__c", + "MSJ_Entry_Date__c", + "MSJ_IST_Name__c", + "MSJ_Indication__c", + "MSJ_Line__c", + "MSJ_MR_Comments__c", + "MSJ_MUID__c", + "MSJ_Medical_Regimen__c", + "MSJ_Month__c", + "MSJ_Report_Comments__c", + "MSJ_Start_Date_Of_Administration__c", + "MSJ_Year__c", + "Mobile_ID_vod__c", + "MSJ_CRC_RAS_KRAS__c", + "MSJ_End_Date_Of_Administration__c", + "MSJ_End_Date_of_Stop_Administration__c", + "MSJ_HN_Hospitalized_Type__c", + "MSJ_Start_Date_of_Stop_Administration__c", + "MSJ_Patient_Status__c", + "MSJ_Patient_TA__c", + "MSJ_Child_Account_Name__c", + "MSJ_Child_Account__c", + "MSJ_Parent_Account_Name__c", + "MSJ_Parent_Child_Name__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consumer_site__c", + "Product_info__c", + "Therapeutic_Class_vod__c", + "Parent_Product_vod__c", + "Therapeutic_Area_vod__c", + "Product_Type_vod__c", + "Require_Key_Message_vod__c", + "Cost_vod__c", + "External_ID_vod__c", + "Manufacturer_vod__c", + "Company_Product_vod__c", + "Controlled_Substance_vod__c", + "Description_vod__c", + "Sample_Quantity_Picklist_vod__c", + "Display_Order_vod__c", + "No_Metrics_vod__c", + "Distributor_vod__c", + "Sample_Quantity_Bound_vod__c", + "Sample_U_M_vod__c", + "No_Details_vod__c", + "Quantity_Per_Case_vod__c", + "Schedule_vod__c", + "Restricted_vod__c", + "Pricing_Rule_Quantity_Bound_vod__c", + "No_Promo_Items_vod__c", + "User_Aligned_vod__c", + "Restricted_States_vod__c", + "Sort_Code_vod__c", + "No_Cycle_Plans_vod__c", + "Inventory_Order_UOM_vod__c", + "Inventory_Quantity_Per_Case_vod__c", + "VExternal_Id_vod__c", + "Country__c", + "MSJ_Product_Classification__c", + "MSJ_Indication__c", + "MSJ_Therapeutic_Area__c", + "MSJ_Global_Brand__c", + "MSJ_Global_Business_Unit__c", + "MSJ_Molecules__c", + "MSJ_SBU__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_Group_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Description_vod__c", + "Product_vod__c", + "Product_Catalog_vod__c", + "Start_Date_vod__c", + "End_Date_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_Metrics_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Awareness__c", + "Selling_Stage__c", + "Formulary_Status__c", + "Movement__c", + "Products_vod__c", + "Segment__c", + "X12_mo_trx_chg__c", + "Speaker_Skills__c", + "Investigator_Readiness__c", + "Engagements__c", + "Mobile_ID_vod__c", + "External_ID_vod__c", + "MSJ_Patient__c", + "Detail_Group_vod__c", + "MSJ_EB_1st_Line_Liver_Meta__c", + "MSJ_EB_1st_Line_Multi_Meta__c", + "MSJ_EB_2nd_Line_Mono__c", + "MSJ_EB_2nd_Line_Combination__c", + "MSJ_EB_3rd_Line_Mono__c", + "MSJ_EB_3rd_Line_Combination__c", + "EMDS_Ability__c", + "EMDS_Brand_Loyalty__c", + "EMDS_Decision_Maker__c", + "EMDS_Early_Tech_Adopter__c", + "EMDS_Influence__c", + "EMDS_Main_Driver__c", + "EMDS_Priority__c", + "EMDS_Willingness__c", + "MSJ_KTL_Type__c", + "MSJ_KTL_Tier__c", + "MSJ_Publications__c", + "MSJ_Clinical_Trials__c", + "MSJ_Speaker_for_Medical_Events__c", + "MSJ_Advisor_to_Medical_Affairs__c", + "MSJ_Guidelines_Treatment_Standards__c", + "MSJ_Therapeutic_Area_Expertise__c", + "MSJ_MAP_GAP__c", + "MSJ_Associations__c", + "MSJ_Tier_Score__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Assignment_Type_vod__c", + "Channels_vod__c", + "End_Date_vod__c", + "Expired_vod__c", + "External_ID_vod__c", + "Language_vod__c", + "Lock_vod__c", + "Open_vod__c", + "Product_vod__c", + "Region_vod__c", + "Segment_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "Territory_vod__c", + "zvod_Questions_vod__c", + "zvod_Segments_vod__c", + "zvod_Targets_vod__c", + "Max_Score_vod__c", + "Min_Score_vod__c", + "Autotarget_vod__c", + "Territories_vod__c", + "Target_Type_vod__c", + "MSJ_External_ID__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_Target_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_Display_Name_vod__c", + "Account_vod__c", + "Channels_vod__c", + "End_Date_vod__c", + "Entity_Reference_Id_vod__c", + "External_ID_vod__c", + "Language_vod__c", + "Lock_vod__c", + "Mobile_ID_vod__c", + "No_Autoassign_vod__c", + "Not_Completed_vod__c", + "Region_vod__c", + "Segment_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "Survey_vod__c", + "Territory_vod__c", + "zvod_Address_vod__c", + "zvod_Specialty_vod__c", + "Score_vod__c", + "User_vod__c", + "Child_Account_vod__c", + "Location_Entity_Reference_Id_vod__c", + "Location_vod__c", + "Target_Type_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_Question_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Survey_vod__c", + "Answer_Choice_vod__c", + "External_ID_vod__c", + "Max_Score_vod__c", + "Min_Score_vod__c", + "Order_vod__c", + "Question_vod__c", + "Required_vod__c", + "Text_vod__c", + "Condition_vod__c", + "Source_ID_vod__c", + "MSJ_External_ID__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Question_Response_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Survey_Target_vod__c", + "Answer_Choice_vod__c", + "Date_vod__c", + "Datetime_vod__c", + "External_ID_vod__c", + "Mobile_ID_vod__c", + "Number_vod__c", + "Order_vod__c", + "Question_Text_vod__c", + "Required_vod__c", + "Response_Hash_vod__c", + "Response_vod__c", + "Score_vod__c", + "Survey_Question_vod__c", + "Text_vod__c", + "Type_vod__c", + "Condition_vod__c", + "Inactive_Condition_vod__c", + "Source_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Sent_Fragment_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Sent_Email_vod__c", + "Account_vod__c", + "Email_Template_vod__c", + "Sent_Fragment_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Sent_Email_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Account_Email_vod__c", + "Account_vod__c", + "Approved_Email_Template_vod__c", + "Capture_Datetime_vod__c", + "Detail_Group_vod__c", + "Email_Config_Values_vod__c", + "Email_Content2_vod__c", + "Email_Content_vod__c", + "Email_Fragments_vod__c", + "Email_Sent_Date_vod__c", + "Failure_Msg_vod__c", + "Last_Activity_Date_vod__c", + "Last_Device_vod__c", + "MC_Capture_Datetime_vod__c", + "Mobile_ID_vod__c", + "Opened_vod__c", + "Product_Display_vod__c", + "Product_vod__c", + "Sender_Email_vod__c", + "Status_vod__c", + "Valid_Consent_Exists_vod__c", + "Approved_Document_Views_vod__c", + "Click_Count_vod__c", + "Last_Click_Date_vod__c", + "Last_Open_Date_vod__c", + "Open_Count_vod__c", + "Receipt_Entity_Type_vod__c", + "Receipt_Record_Id_vod__c", + "Territory_vod__c", + "Call2_vod__c", + "Medical_Inquiry_vod__c", + "Parent_Email_vod__c", + "Related_Transaction_ID_vod__c", + "Case_vod__c", + "Key_Message_vod__c", + "Suggestion_vod__c", + "EM_Attendee_vod__c", + "EM_Event_Speaker_vod__c", + "EM_Event_Team_Member_vod__c", + "Event_Attendee_vod__c", + "Event_vod__c", + "Medical_Event_vod__c", + "Scheduled_Send_Datetime_vod__c", + "User_vod__c", + "Content_Type_vod__c", + "Bcc_vod__c", + "Event_Attendee_Mobile_Id_vod__c", + "Event_Mobile_Id_vod__c", + "Activity_Tracking_Mode_vod__c", + "Email_Source_vod__c", + "Subject_vod__c", + "User_Input_Text_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + } + ] } \ No newline at end of file From 735200849f980298cb5938cfd0a6bdf50f6d0f5d Mon Sep 17 00:00:00 2001 From: "shibata.r" Date: Tue, 23 Aug 2022 17:21:09 +0900 Subject: [PATCH 255/275] =?UTF-8?q?style:=E6=94=B9=E8=A1=8C=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/config/crm/object_info/crm_object_list_diff.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index 916ecab9..dda641e2 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -1649,7 +1649,7 @@ "CreatedById", "LastModifiedDate", "LastModifiedById", - "SystemModstamp", + "SystemModstamp", "OfflineTrialExpirationDate", "OfflinePdaTrialExpirationDate", "UserPermissionsMarketingUser", @@ -1722,7 +1722,7 @@ "UserPreferencesUserDebugModePref", "UserPreferencesSRHOverrideActivities", "UserPreferencesNewLightningReportRunPageEnabled", - "UserPreferencesReverseOpenActivitiesView", + "UserPreferencesReverseOpenActivitiesView", "UserPreferencesHideBrowseProductRedirectConfirmation", "UserPreferencesHideOnlineSalesAppWelcomeMat", "ContactId", From a4474984a533a0ccd608f2daff665c5e55c35ba5 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 23 Aug 2022 19:18:28 +0900 Subject: [PATCH 256/275] =?UTF-8?q?fix:=20=E3=83=A2=E3=82=B8=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E3=83=AB=E3=82=B3=E3=83=94=E3=83=BC=E3=81=AE=E9=9A=8E?= =?UTF-8?q?=E5=B1=A4=E8=AA=A4=E3=82=8A=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/Dockerfile b/ecs/crm-datafetch/Dockerfile index 56087bf0..2d6318e7 100644 --- a/ecs/crm-datafetch/Dockerfile +++ b/ecs/crm-datafetch/Dockerfile @@ -14,6 +14,6 @@ RUN \ pip uninstall -y pipenv virtualenv-clone virtualenv COPY main.py ./ -COPY src ./ +COPY src ./src CMD [ "python", "./main.py" ] From 52f4e513929874f65f4dd975706942bdf0e7cc48 Mon Sep 17 00:00:00 2001 From: "shibata.r" Date: Wed, 24 Aug 2022 09:29:58 +0900 Subject: [PATCH 257/275] =?UTF-8?q?style:=E6=94=B9=E8=A1=8C=E3=82=B3?= =?UTF-8?q?=E3=83=BC=E3=83=89=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/crm/settings/CRM_User.txt | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/s3/data/crm/settings/CRM_User.txt b/s3/data/crm/settings/CRM_User.txt index e75fabb8..727d82e0 100644 --- a/s3/data/crm/settings/CRM_User.txt +++ b/s3/data/crm/settings/CRM_User.txt @@ -1,13 +1,13 @@ -CRM -, -utf-8 -" -CRLF -1 -200 -Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c -id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c -src02.crm_user -org02.crm_user - - +CRM +, +utf-8 +" +CRLF +1 +200 +Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c +id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c +src02.crm_user +org02.crm_user + + From 96b4aac794e5e45983c872612d5465557d38f2f7 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 24 Aug 2022 13:24:34 +0900 Subject: [PATCH 258/275] =?UTF-8?q?fix:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E4=BF=AE=E6=AD=A3=20https://nds-tyo?= =?UTF-8?q?.backlog.com/git/NEWDWH2021/newsdwh2021/pullRequests/84#comment?= =?UTF-8?q?-1336807?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/converter/test_convert_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py index 4b4552a8..bfbe5052 100644 --- a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -55,7 +55,7 @@ class TestConvertStrategyFactory: # Act sut = ConvertStrategyFactory() - actual = sut.create(1.2345678E7) + actual = sut.create(1.2345678) # Expects # 変換しない From baee5b4142ff1687c6f90aeac106922fa59f003d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 25 Aug 2022 11:50:47 +0900 Subject: [PATCH 259/275] =?UTF-8?q?fix:=20User=E3=82=AA=E3=83=96=E3=82=B8?= =?UTF-8?q?=E3=82=A7=E3=82=AF=E3=83=88=E3=81=AE=E9=80=A3=E6=90=BA=E5=AF=BE?= =?UTF-8?q?=E8=B1=A1=E5=A4=96=E3=82=AB=E3=83=A9=E3=83=A0=E3=81=8C=E6=AD=A3?= =?UTF-8?q?=E5=BC=8F=E5=AF=BE=E5=BF=9C=E3=81=95=E3=82=8C=E3=81=9F=E3=81=AE?= =?UTF-8?q?=E3=81=A7=E4=BF=AE=E6=AD=A3=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index e8c3b6e4..dabd8a02 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -73,7 +73,6 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): object_info_list = get_object_config_list('object_info') for object_info in object_info_list: json_file = read_json(object_info) - json_file = to_upload_json(json_file) upload_json(json_file, s3_client, CONFIG_BUCKET, f'{OBJECT_INFO_FOLDER}/{path.basename(object_info)}') object_info_files.append(json_file) last_fetch_datetime_list = get_object_config_list('last_fetch_datetime') @@ -295,21 +294,6 @@ def upload_json(json_file, s3_client, bucket, folder): s3_client.put_object(Bucket=bucket, Key=folder, Body=json_str) -def to_upload_json(json_file): - """Userオブジェクトの取得できないプロパティを取り除く - TODO: Userオブジェクトの恒久対応が確定したらこのメソッドは消す - """ - for object_info in json_file['objects']: - if object_info['object_name'] != 'User': - continue - columns: list = object_info['columns'] - columns.remove('LastPasswordChangeDate') - columns.remove('NumberOfFailedLogins') - columns.remove('UserPreferencesNativeEmailClient') - - return json_file - - def set_environment(monkeypatch): # 環境変数を設定(CRMの認証情報は別途設定しておくこと) monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET) From eed3c8a60cd8fd3a180586fd6eab454cde428d4d Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 25 Aug 2022 15:05:58 +0900 Subject: [PATCH 260/275] =?UTF-8?q?fix:=20=E3=82=AA=E3=83=96=E3=82=B8?= =?UTF-8?q?=E3=82=A7=E3=82=AF=E3=83=88=E6=83=85=E5=A0=B1=E5=AE=9A=E7=BE=A9?= =?UTF-8?q?=E3=82=92=E7=92=B0=E5=A2=83=E5=A4=89=E6=95=B0=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=A4=E3=83=AB=E3=81=AB=E6=8C=81=E3=81=9F=E3=81=9B=E3=82=8B?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/config/crm/task_settings/task_settings.env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env index 580bf564..72d5ea8c 100644 --- a/s3/config/crm/task_settings/task_settings.env +++ b/s3/config/crm/task_settings/task_settings.env @@ -10,4 +10,5 @@ CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3 CRM_FETCH_RECORD_RETRY_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 -CONVERT_TZ='Asia/Tokyo' \ No newline at end of file +CONVERT_TZ='Asia/Tokyo' +OBJECT_INFO_FILENAME='crm_object_list_diff.json' \ No newline at end of file From 564335f6de0d277002e7f80d7b43020c2c48fe26 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Fri, 26 Aug 2022 01:44:08 +0900 Subject: [PATCH 261/275] =?UTF-8?q?fix:=20=E3=83=87=E3=83=90=E3=83=83?= =?UTF-8?q?=E3=82=B0=E3=83=AD=E3=82=B0=E3=81=AES3=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=A4=E3=83=AB=E3=83=91=E3=82=B9=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/prepare_data_fetch_process.py | 2 +- ecs/crm-datafetch/tests/test_walk_through.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index d546c7b8..5f5a1c4d 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -34,7 +34,7 @@ def prepare_data_fetch_process(): try: # ③ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する - object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' + object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' logger.debug( f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index dabd8a02..981219a9 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -104,7 +104,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # Act logger = logging.getLogger() - logger.setLevel(logging.INFO) + logger.setLevel(logging.DEBUG) logger.info(f'##########################') logger.info(f'# 差分データ取得処理:実行開始 #') logger.info(f'##########################') @@ -308,3 +308,4 @@ def set_environment(monkeypatch): monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) + monkeypatch.setattr('src.prepare_data_fetch_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET) From 629d927b4989878db2c8eb3e43e5e205740a017e Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Fri, 26 Aug 2022 01:48:14 +0900 Subject: [PATCH 262/275] =?UTF-8?q?fix:=20=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=AE=E5=80=A4?= =?UTF-8?q?=E6=8C=87=E5=AE=9A=E6=96=B9=E6=B3=95=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/config/crm/task_settings/task_settings.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env index 72d5ea8c..445aaa73 100644 --- a/s3/config/crm/task_settings/task_settings.env +++ b/s3/config/crm/task_settings/task_settings.env @@ -10,5 +10,5 @@ CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3 CRM_FETCH_RECORD_RETRY_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 -CONVERT_TZ='Asia/Tokyo' -OBJECT_INFO_FILENAME='crm_object_list_diff.json' \ No newline at end of file +CONVERT_TZ=Asia/Tokyo +OBJECT_INFO_FILENAME=crm_object_list_diff.json \ No newline at end of file From e2e36f5b98f46137d07664d8bda36fceb7d384b5 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Fri, 26 Aug 2022 13:28:31 +0900 Subject: [PATCH 263/275] =?UTF-8?q?refactor:=20=E4=B8=80=E9=83=A8=E3=83=AD?= =?UTF-8?q?=E3=82=B0=E3=81=AE=E3=83=AC=E3=83=99=E3=83=AB=E3=82=92=E5=A4=89?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/prepare_data_fetch_process.py | 4 ++-- ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index 5f5a1c4d..f14e85b8 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -35,8 +35,8 @@ def prepare_data_fetch_process(): try: # ③ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' - logger.debug( - f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') + logger.info( + f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') config_bucket = ConfigBucket() object_info_file_str = config_bucket.get_object_info_file() diff --git a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py index 9471946f..8eb3d476 100644 --- a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py @@ -7,6 +7,9 @@ from src.config.objects import FetchTargetObjects from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.prepare_data_fetch_process import prepare_data_fetch_process from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ +from src.system_var.environments import (CRM_CONFIG_BUCKET, + OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER) from .test_utils.log_message import generate_log_message_tuple @@ -80,6 +83,8 @@ class TestPrepareDataFetchProcess: # ログの確認 assert generate_log_message_tuple(log_message='I-PRE-01 データ取得準備処理を開始します') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-PRE-02 データ取得処理開始日時：{str(actual_execute_datetime)}') in caplog.record_tuples + assert generate_log_message_tuple( + log_message=f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}]') in caplog.record_tuples assert generate_log_message_tuple(log_message='I-PRE-09 データ取得準備処理を終了します') in caplog.record_tuples def test_call_depended_modules(self): From e6389bbd1b6dfcabd4282326b056fffa35e6e067 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Fri, 26 Aug 2022 16:05:47 +0900 Subject: [PATCH 264/275] =?UTF-8?q?refactor:=20walk=5Fthrough=E3=81=AB?= =?UTF-8?q?=E3=83=AC=E3=83=99=E3=83=AB=E5=A4=89=E6=9B=B4=E3=81=97=E3=81=9F?= =?UTF-8?q?=E3=83=AD=E3=82=B0=E3=83=A1=E3=83=83=E3=82=BB=E3=83=BC=E3=82=B8?= =?UTF-8?q?=E3=81=AE=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/tests/test_walk_through.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index 981219a9..e2be6670 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -117,7 +117,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # ループ前のログ確認 assert 'I-CTRL-01 CRMデータ取得処理を開始します' in log_messages assert 'I-CTRL-02 データ取得準備処理呼び出し' in log_messages - assert_prepare_process_log(log_messages, now) + assert_prepare_process_log(log_messages, now, path.basename(object_info_list[0])) assert 'I-CTRL-03 取得対象オブジェクトのループ処理開始' in log_messages # オブジェクト情報を取得する(diff) object_info_list = object_info_files[0] @@ -206,9 +206,10 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): """ -def assert_prepare_process_log(log_messages, now): +def assert_prepare_process_log(log_messages, now, object_info_list_filename): assert 'I-PRE-01 データ取得準備処理を開始します' in log_messages assert f'I-PRE-02 データ取得処理開始日時：{now}' in log_messages + assert f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{object_info_list_filename}]' in log_messages assert 'I-PRE-09 データ取得準備処理を終了します' in log_messages From d1cc31b1e803eb51245b05378f21b207d13365d2 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 1 Sep 2022 11:42:14 +0900 Subject: [PATCH 265/275] =?UTF-8?q?fix:=20=E3=83=AA=E3=83=88=E3=83=A9?= =?UTF-8?q?=E3=82=A4=E5=9B=9E=E6=95=B0=E3=82=922=E5=9B=9E=E3=81=8B?= =?UTF-8?q?=E3=82=893=E5=9B=9E=E3=81=AB=E3=81=99=E3=82=8B=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/system_var/environments.py | 8 +-- .../tests/test_fetch_crm_data_process.py | 60 +++++++++---------- s3/config/crm/task_settings/task_settings.env | 4 +- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index 297cfb2b..c457386f 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -9,8 +9,8 @@ LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)) # CRMのレコード件数取得処理のタイムアウト秒数 CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)) -# CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3)) +# CRMのレコード件数取得処理の最大リトライ試行回数(処理の実施総回数) +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 4)) # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 CRM_GET_RECORD_COUNT_RETRY_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5)) # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 @@ -19,8 +19,8 @@ CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_GET_R CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50)) # CRMのレコード取得処理のタイムアウト秒数 CRM_FETCH_RECORD_TIMEOUT = int(os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300)) -# CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3)) +# CRMのレコード取得処理の最大リトライ試行回数(処理の実施総回数) +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 4)) # CRMのレコード取得処理のリトライ時の初回待ち秒数 CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最小待ち秒数 diff --git a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py index 66f5577e..6b17477f 100644 --- a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py @@ -217,7 +217,7 @@ class TestFetchCrmDataProcess: 2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること 3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること Arranges: - - データ件数取得処理の最大リトライ試行回数を3に設定する + - データ件数取得処理の最大リトライ試行回数を4に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - データ件数取得処理の初回に接続タイムアウト例外が発生するようにする Expects: @@ -225,7 +225,7 @@ class TestFetchCrmDataProcess: - データ件数取得に失敗した旨のエラーが出力されない """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -236,8 +236,8 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_counter_inst.describe.side_effect = [1, 2, 3, 4] + mock_counter_inst.increment.side_effect = [2, 3, 4, 5] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' @@ -271,15 +271,15 @@ class TestFetchCrmDataProcess: 2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること 3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること Arranges: - - データ件数取得処理の最大リトライ試行回数を3に設定する + - データ件数取得処理の最大リトライ試行回数を4に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - - データ件数取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする + - データ件数取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする Expects: - 異常終了する - データ件数取得に失敗した旨のエラーが出力される """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -290,13 +290,13 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_counter_inst.describe.side_effect = [1, 2, 3, 4] + mock_counter_inst.increment.side_effect = [2, 3, 4, 5] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' mock_client_inst = mock_api_client.return_value - mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception] + mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception, exception] mock_client_inst.fetch_sf_data.return_value = common_expect # Act with pytest.raises(SalesforceAPIException) as e: @@ -304,16 +304,16 @@ class TestFetchCrmDataProcess: # Assert - # 取得は3回行われる - assert mock_counter_inst.describe.call_count == 3 - # 足し込みは2回のみ - assert mock_counter_inst.increment.call_count == 2 + # 取得は4回行われる + assert mock_counter_inst.describe.call_count == 4 + # 足し込みは3回のみ + assert mock_counter_inst.increment.call_count == 3 assert generate_log_message_tuple( log_level=logging.WARNING, log_message=expect_message) in caplog.record_tuples called_log_counts = len([log for log in caplog.messages if log == expect_message]) - assert called_log_counts == 2 + assert called_log_counts == 3 assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples assert e.value.error_id == 'E-FETCH-01' @@ -333,7 +333,7 @@ class TestFetchCrmDataProcess: 2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること 3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること Arranges: - - レコード取得処理の最大リトライ試行回数を3に設定する + - レコード取得処理の最大リトライ試行回数を4に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - レコード取得処理の初回に接続タイムアウト例外が発生するようにする Expects: @@ -341,7 +341,7 @@ class TestFetchCrmDataProcess: - データレコード取得に失敗した旨のエラーが出力されない """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -352,8 +352,8 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_counter_inst.describe.side_effect = [1, 2, 3, 4] + mock_counter_inst.increment.side_effect = [2, 3, 4, 5] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' @@ -387,15 +387,15 @@ class TestFetchCrmDataProcess: 2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること 3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること Arranges: - - レコード取得処理の最大リトライ試行回数を3に設定する + - レコード取得処理の最大リトライ試行回数を4に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - - レコード取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする + - レコード取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする Expects: - 異常終了する - データレコード取得に失敗した旨のエラーが出力される """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 4) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_INTERVAL', 1) @@ -406,30 +406,30 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_counter_inst.describe.side_effect = [1, 2, 3, 4] + mock_counter_inst.increment.side_effect = [2, 3, 4, 5] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' mock_client_inst = mock_api_client.return_value mock_client_inst.fetch_sf_count.return_value = 1 - mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception] + mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception, exception] # Act with pytest.raises(SalesforceAPIException) as e: fetch_crm_data_process(common_target_object, common_last_fetch_datetime) # Assert - # 取得は3回行われる - assert mock_counter_inst.describe.call_count == 3 - # 足し込みは2回のみ - assert mock_counter_inst.increment.call_count == 2 + # 取得は4回行われる + assert mock_counter_inst.describe.call_count == 4 + # 足し込みは3回のみ + assert mock_counter_inst.increment.call_count == 3 assert generate_log_message_tuple( log_level=logging.WARNING, log_message=expect_message) in caplog.record_tuples called_log_counts = len([log for log in caplog.messages if log == expect_message]) - assert called_log_counts == 2 + assert called_log_counts == 3 assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples assert e.value.error_id == 'E-FETCH-02' diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env index 445aaa73..3ae4d809 100644 --- a/s3/config/crm/task_settings/task_settings.env +++ b/s3/config/crm/task_settings/task_settings.env @@ -1,12 +1,12 @@ LOG_LEVEL=INFO CRM_AUTH_TIMEOUT=100 CRM_GET_RECORD_COUNT_TIMEOUT=300 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=3 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=4 CRM_GET_RECORD_COUNT_RETRY_INTERVAL=5 CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL=5 CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL=50 CRM_FETCH_RECORD_TIMEOUT=300 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=4 CRM_FETCH_RECORD_RETRY_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 From 831de315e59bb89aa1bc4801abeacc4a5ad7a30f Mon Sep 17 00:00:00 2001 From: "shibata.r" Date: Fri, 2 Sep 2022 15:14:44 +0900 Subject: [PATCH 266/275] =?UTF-8?q?fix:=E3=82=B9=E3=82=AD=E3=83=BC?= =?UTF-8?q?=E3=83=9E=E3=83=95=E3=83=AB=E6=8C=87=E5=AE=9A=E3=81=A7=E5=91=BC?= =?UTF-8?q?=E3=81=B3=E5=87=BA=E3=81=99=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/crm/settings/CRM_AccountShare_ex.sql | 2 +- s3/data/crm/settings/CRM_Account_ex.sql | 2 +- s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql | 2 +- s3/data/crm/settings/CRM_Contact_ex.sql | 2 +- s3/data/crm/settings/CRM_Group_ex.sql | 2 +- s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql | 2 +- s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql | 2 +- s3/data/crm/settings/CRM_Profile_ex.sql | 2 +- s3/data/crm/settings/CRM_Territory2_ex.sql | 4 ++-- s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql | 4 ++-- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/s3/data/crm/settings/CRM_AccountShare_ex.sql b/s3/data/crm/settings/CRM_AccountShare_ex.sql index d18e68db..de5e6dac 100644 --- a/s3/data/crm/settings/CRM_AccountShare_ex.sql +++ b/s3/data/crm/settings/CRM_AccountShare_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_account_share', 'last_modified_date'); \ No newline at end of file +CALL src02.crm_history('src02.crm_account_share', 'last_modified_date'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Account_ex.sql b/s3/data/crm/settings/CRM_Account_ex.sql index 792d7a28..1511c5b1 100644 --- a/s3/data/crm/settings/CRM_Account_ex.sql +++ b/s3/data/crm/settings/CRM_Account_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_account', 'system_modstamp'); +CALL src02.crm_history('src02.crm_account', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql b/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql index cac33f47..4c2bf134 100644 --- a/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql +++ b/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_child_account_vod__c', 'system_modstamp'); \ No newline at end of file +CALL src02.crm_history('src02.crm_child_account_vod__c', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Contact_ex.sql b/s3/data/crm/settings/CRM_Contact_ex.sql index 4b916e23..31fc59a6 100644 --- a/s3/data/crm/settings/CRM_Contact_ex.sql +++ b/s3/data/crm/settings/CRM_Contact_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_contact', 'system_modstamp'); +CALL src02.crm_history('src02.crm_contact', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Group_ex.sql b/s3/data/crm/settings/CRM_Group_ex.sql index 5e50dab6..f0667ff9 100644 --- a/s3/data/crm/settings/CRM_Group_ex.sql +++ b/s3/data/crm/settings/CRM_Group_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_group', 'system_modstamp'); \ No newline at end of file +CALL src02.crm_history('src02.crm_group', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql index d142c071..2e1a751e 100644 --- a/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql +++ b/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_object_territory2_association', 'system_modstamp'); +CALL src02.crm_history('src02.crm_object_territory2_association', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql b/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql index af2e678e..76134c74 100644 --- a/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql +++ b/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_product_metrics_vod__c', 'system_modstamp'); \ No newline at end of file +CALL src02.crm_history('src02.crm_product_metrics_vod__c', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Profile_ex.sql b/s3/data/crm/settings/CRM_Profile_ex.sql index b858ab02..80a780f0 100644 --- a/s3/data/crm/settings/CRM_Profile_ex.sql +++ b/s3/data/crm/settings/CRM_Profile_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_profile', 'system_modstamp'); +CALL src02.crm_history('src02.crm_profile', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Territory2_ex.sql b/s3/data/crm/settings/CRM_Territory2_ex.sql index a30ccd91..ffc2befe 100644 --- a/s3/data/crm/settings/CRM_Territory2_ex.sql +++ b/s3/data/crm/settings/CRM_Territory2_ex.sql @@ -1,2 +1,2 @@ -CALL crm_data_sync('src02.crm_territory2', 'src02.crm_territory2_all', 'system_modstamp'); -CALL crm_history('src02.crm_territory2', 'system_modstamp'); +CALL src02.crm_data_sync('src02.crm_territory2', 'src02.crm_territory2_all', 'system_modstamp'); +CALL src02.crm_history('src02.crm_territory2', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql index 6a9aaece..41b6531d 100644 --- a/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql +++ b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql @@ -1,2 +1,2 @@ -CALL crm_data_sync('src02.crm_user_territory2_association', 'src02.crm_user_territory2_association_all', 'system_modstamp'); -CALL crm_history('src02.crm_user_territory2_association', 'system_modstamp'); +CALL src02.crm_data_sync('src02.crm_user_territory2_association', 'src02.crm_user_territory2_association_all', 'system_modstamp'); +CALL src02.crm_history('src02.crm_user_territory2_association', 'system_modstamp'); From bae8447658a03ff8a0bd84948d493f332724bebb Mon Sep 17 00:00:00 2001 From: "shibata.r" Date: Fri, 2 Sep 2022 15:33:15 +0900 Subject: [PATCH 267/275] Revert "Merge branch 'develop-6crm' of nds-tyo.git.backlog.com:/NEWDWH2021/newsdwh2021 into develop-6crm" This reverts commit 820491adc06bb8bbd5c4bf0ec676176a6b5820eb, reversing changes made to 831de315e59bb89aa1bc4801abeacc4a5ad7a30f. --- ecs/crm-datafetch/Dockerfile | 2 +- ecs/crm-datafetch/src/aws/s3.py | 4 +- .../src/converter/convert_strategy.py | 21 ++- .../src/prepare_data_fetch_process.py | 6 +- .../src/system_var/environments.py | 8 +- .../tests/converter/test_convert_strategy.py | 145 ++++-------------- .../tests/converter/test_converter.py | 10 +- .../tests/salesforce/test_salesforce.py | 110 +++---------- .../tests/test_backup_crm_data_process.py | 11 +- .../test_convert_crm_csv_data_process.py | 51 ++---- .../tests/test_fetch_crm_data_process.py | 60 ++++---- .../tests/test_prepare_data_fetch_process.py | 5 - ecs/crm-datafetch/tests/test_walk_through.py | 24 ++- .../crm/object_info/crm_object_list_diff.json | 3 + s3/config/crm/task_settings/task_settings.env | 7 +- .../settings/CRM_Event_Attendee_vod__c.txt | 6 +- s3/data/crm/settings/CRM_User.txt | 6 +- 17 files changed, 149 insertions(+), 330 deletions(-) diff --git a/ecs/crm-datafetch/Dockerfile b/ecs/crm-datafetch/Dockerfile index 2d6318e7..56087bf0 100644 --- a/ecs/crm-datafetch/Dockerfile +++ b/ecs/crm-datafetch/Dockerfile @@ -14,6 +14,6 @@ RUN \ pip uninstall -y pipenv virtualenv-clone virtualenv COPY main.py ./ -COPY src ./src +COPY src ./ CMD [ "python", "./main.py" ] diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index 7b63861f..d5bd03f0 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -87,7 +87,7 @@ class BackupBucket: def put_response_json(self, file_path: str, data: dict) -> None: object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_path}' - self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False)) + self.__s3_resource.put_object(object_key, json.dumps(data)) return def put_csv(self, file_path: str, data: str) -> None: @@ -97,5 +97,5 @@ class BackupBucket: def put_result_json(self, file_path: str, data: dict) -> None: object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}' - self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False)) + self.__s3_resource.put_object(object_key, json.dumps(data)) return diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 7c123d39..22fca8fc 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -1,6 +1,4 @@ -import json import re -from collections import OrderedDict from datetime import datetime from dateutil.tz import gettz @@ -14,17 +12,20 @@ from src.system_var.environments import CONVERT_TZ class ConvertStrategyFactory: def __init__(self) -> None: self.__none_value_convert_strategy = NoneValueConvertStrategy() + self.__float_convert_strategy = FloatConvertStrategy() self.__boolean_convert_strategy = BooleanConvertStrategy() self.__datetime_convert_strategy = DatetimeConvertStrategy() self.__int_convert_strategy = IntConvertStrategy() self.__string_convert_strategy = StringConvertStrategy() - self.__dict_convert_strategy = DictConvertStrategy() def create(self, value): if value is None: convert_strategy = self.__none_value_convert_strategy + elif type(value) == float: + convert_strategy = self.__float_convert_strategy + elif type(value) == bool: convert_strategy = self.__boolean_convert_strategy @@ -34,8 +35,6 @@ class ConvertStrategyFactory: elif type(value) == int: convert_strategy = self.__int_convert_strategy - elif type(value) == dict or type(value) == OrderedDict: - convert_strategy = self.__dict_convert_strategy else: convert_strategy = self.__string_convert_strategy @@ -61,6 +60,12 @@ class DatetimeConvertStrategy: return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS) +class FloatConvertStrategy: + def convert_value(self, convert_value: str) -> int: + """float型をint型に変換する処理""" + return int(convert_value) + + class IntConvertStrategy: def convert_value(self, convert_value: int): """int型を変換せずに返す処理""" @@ -73,9 +78,3 @@ class StringConvertStrategy: """string型を変換せずに返す処理""" # ConvertStrategyFactoryにて型チェックを行っているため値を変換せずに返す return convert_value - - -class DictConvertStrategy: - def convert_value(self, convert_value: dict): - """dict型の項目を文字列に変換して返す処理""" - return json.dumps(convert_value, ensure_ascii=False) diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index f14e85b8..d546c7b8 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -34,9 +34,9 @@ def prepare_data_fetch_process(): try: # ③ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する - object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' - logger.info( - f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') + object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' + logger.debug( + f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') config_bucket = ConfigBucket() object_info_file_str = config_bucket.get_object_info_file() diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index c457386f..297cfb2b 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -9,8 +9,8 @@ LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)) # CRMのレコード件数取得処理のタイムアウト秒数 CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)) -# CRMのレコード件数取得処理の最大リトライ試行回数(処理の実施総回数) -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 4)) +# CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3)) # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 CRM_GET_RECORD_COUNT_RETRY_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5)) # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 @@ -19,8 +19,8 @@ CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_GET_R CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50)) # CRMのレコード取得処理のタイムアウト秒数 CRM_FETCH_RECORD_TIMEOUT = int(os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300)) -# CRMのレコード取得処理の最大リトライ試行回数(処理の実施総回数) -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 4)) +# CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3)) # CRMのレコード取得処理のリトライ時の初回待ち秒数 CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最小待ち秒数 diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py index 129b3980..0e268fc4 100644 --- a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -1,9 +1,7 @@ -from collections import OrderedDict - from src.converter.convert_strategy import (BooleanConvertStrategy, ConvertStrategyFactory, DatetimeConvertStrategy, - DictConvertStrategy, + FloatConvertStrategy, IntConvertStrategy, NoneValueConvertStrategy, StringConvertStrategy) @@ -31,7 +29,7 @@ class TestConvertStrategyFactory: def test_create_float(self): """ Cases: - 引数に指数表記を指定した場合、StringConvertStrategyインスタンスが返ってくること + 引数に指数表記を指定した場合、FloatConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -43,26 +41,7 @@ class TestConvertStrategyFactory: actual = sut.create(1.2345678E7) # Expects - # 変換しない - assert type(actual) == StringConvertStrategy - - def test_create_float_scale(self): - """ - Cases: - 引数に少数を指定した場合、StringConvertStrategyインスタンスが返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = ConvertStrategyFactory() - actual = sut.create(1.2345678) - - # Expects - # 変換しない - assert type(actual) == StringConvertStrategy + assert type(actual) == FloatConvertStrategy def test_create_bool_true(self): """ @@ -115,10 +94,10 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == DatetimeConvertStrategy - def test_create_str(self): + def test_create_other_str(self): """ Cases: - 引数にSalesforce日付型以外の文字列を指定した場合、StringConvertStrategyインスタンスが返ってくること + 引数にSalesforce日付型以外の文字列を指定した場合、NonConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -132,10 +111,10 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == StringConvertStrategy - def test_create_int(self): + def test_create_other_int(self): """ Cases: - 引数に整数を指定した場合、IntConvertStrategyインスタンスが返ってくること + 引数に整数を指定した場合、NonConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -149,40 +128,6 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == IntConvertStrategy - def test_create_dict(self): - """ - Cases: - 引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = ConvertStrategyFactory() - actual = sut.create({'key': 'value'}) - - # Expects - assert type(actual) == DictConvertStrategy - - def test_create_ordered_dict_dict(self): - """ - Cases: - 引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = ConvertStrategyFactory() - actual = sut.create(OrderedDict([('key', 'value')])) - - # Expects - assert type(actual) == DictConvertStrategy - class TestNoneValueConvertStrategy: @@ -261,6 +206,26 @@ class TestDatetimeConvertStrategy: assert actual == "2022-06-14 05:15:32" +class TestFloatConvertStrategy: + + def test_convert_value(self) -> int: + """ + Cases: + 引数に指数表記を指定した場合、整数で返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = FloatConvertStrategy() + actual = sut.convert_value(1.2345678E7) + + # Expects + assert actual == 12345678 + + class TestIntConvertStrategy: def test_convert_value(self): @@ -299,59 +264,3 @@ class TestStringConvertStrategy: # Expects assert actual == 'テストデータ' - - -class TestDictConvertStrategy: - - def test_convert_value_dict(self): - """ - Cases: - 引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = DictConvertStrategy() - actual = sut.convert_value({'テストデータキー': 'テストデータバリュー'}) - - # Expects - assert actual == '{"テストデータキー": "テストデータバリュー"}' - - def test_convert_value_dict_in_line_break(self): - """ - Cases: - 引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること(バリューに改行を含む) - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = DictConvertStrategy() - actual = sut.convert_value({'テストデータキー': 'テスト\nデータ\nバリュー'}) - - # Expects - assert actual == '{"テストデータキー": "テスト\\nデータ\\nバリュー"}' - - def test_convert_value_ordered_dict(self): - """ - Cases: - 引数に整列された辞書型のデータを指定した場合、JSONの文字列が返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = DictConvertStrategy() - actual = sut.convert_value(OrderedDict( - [('テストデータキー', 'テストデータバリュー')] - )) - - # Expects - assert actual == '{"テストデータキー": "テストデータバリュー"}' diff --git a/ecs/crm-datafetch/tests/converter/test_converter.py b/ecs/crm-datafetch/tests/converter/test_converter.py index da80982b..0300381d 100644 --- a/ecs/crm-datafetch/tests/converter/test_converter.py +++ b/ecs/crm-datafetch/tests/converter/test_converter.py @@ -71,7 +71,7 @@ class TestCSVStringConverter: ('ContactAccessLevel', 8), ('RowCause', 'テストのため2'), ('LastModifiedDate', '2022-06-02T16:30:30.000+0000'), - ('LastModifiedById', 2.23E+0), + ('LastModifiedById', 2.234567E+6), ('IsDeleted', True) ]), OrderedDict([ @@ -85,7 +85,7 @@ class TestCSVStringConverter: ('ContactAccessLevel', 12), ('RowCause', 'テストのため3'), ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), - ('LastModifiedById', 3.234567), + ('LastModifiedById', 3.234567E+6), ('IsDeleted', False) ]) ] @@ -100,9 +100,9 @@ class TestCSVStringConverter: # Expects expected_value = '''\ "Id","AccountId","UserOrGroupId","AccountAccessLevel","OpportunityAccessLevel","CaseAccessLevel","ContactAccessLevel","RowCause","LastModifiedDate","LastModifiedById","IsDeleted"\r\n\ - "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567.0","0"\r\n\ - "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2.23","1"\r\n\ - "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3.234567","0"\r\n\ + "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567","0"\r\n\ + "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2234567","1"\r\n\ + "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3234567","0"\r\n\ ''' # expected_valueのインデントが半角スペースと認識されてしまうため、`textwrap.dedent`にて補正 diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index 64530071..b815e772 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -9,8 +9,6 @@ Accountオブジェクトの下記SFIDのレコードはいじらないように 変更してしまった場合は各SOQLの取得日付とデータを修正してください """ -from typing import OrderedDict - import pytest from requests.exceptions import ConnectTimeout, ReadTimeout from src.config.objects import LastFetchDatetime, TargetObject @@ -35,8 +33,8 @@ class TestSalesforceApiClient: FROM Account WHERE - SystemModstamp > 2022-08-23T01:56:39.000Z AND - SystemModstamp <= 2022-08-24T00:00:00.000Z + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-06T00:00:00.000Z """ sut = SalesforceApiClient() @@ -65,7 +63,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual >= 0 - def test_fetch_sf_count_by_soql_builder_system_modstamp_to_ge(self): + def test_fetch_sf_count_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること @@ -81,8 +79,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-23T02:38:59.000Z', - 'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z', + 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -108,7 +106,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual == 1 - def test_fetch_sf_count_by_soql_builder_system_modstamp_to_lt(self): + def test_fetch_sf_count_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること @@ -124,8 +122,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z', - 'last_fetch_datetime_to': '2022-08-23T02:39:01.000Z', + 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -162,7 +160,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得件数が17になる + 取得件数が16になる """ execute_datetime = ExecuteDateTime() @@ -192,7 +190,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_count(soql) - assert actual == 17 + assert actual == 16 def test_fetch_sf_data_one_record(self): """ @@ -219,8 +217,8 @@ class TestSalesforceApiClient: FROM Account WHERE - SystemModstamp > 2022-08-23T01:56:39.000Z AND - SystemModstamp <= 2022-08-24T00:00:00.000Z + SystemModstamp > 2022-08-04T00:00:00.000Z AND + SystemModstamp <= 2022-08-06T00:00:00.000Z """ sut = SalesforceApiClient() @@ -286,7 +284,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_data(soql) assert len(actual) >= 0 - def test_fetch_sf_data_by_soql_builder_system_modstamp_to_ge(self): + def test_fetch_sf_data_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること @@ -302,8 +300,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-23T03:48:39.000Z', - 'last_fetch_datetime_to': '2022-08-23T03:48:40.000Z', + 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -355,7 +353,7 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect - def test_fetch_sf_data_by_soql_builder_system_modstamp_to_gt(self): + def test_fetch_sf_data_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること @@ -363,16 +361,16 @@ class TestSalesforceApiClient: - SystemModStampのToが指定日付未満のレコードは取得できないこと Arranges: - SalesforceのAccountオブジェクトに、レコードを作成する - - LastFetchDatetimeのFromがSystemModstampより小さくなるように指定する（UTC指定） - - LastFetchDatetimeのToがSystemModstampより大きくなるように指定する（UTC指定） + - LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定） Expects: 取得できたオブジェクト1件が期待値どおりであること """ execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-23T03:42:24.000Z', - 'last_fetch_datetime_to': '2022-08-23T03:42:26.000Z', + 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', + 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -424,69 +422,6 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect - def test_fetch_sf_data_by_soql_builder_address_item_check(self): - """ - Cases: - - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること - - できること - Arranges: - - SalesforceのAccountオブジェクトに、住所項目を持つレコードを作成する - - 住所項目を持つレコードだけが取れるよう日付を設定する - Expects: - 取得できたオブジェクト件数が1件になる - 住所項目(BillingAddress)が想定通りの値になっていること - """ - - execute_datetime = ExecuteDateTime() - last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z', - 'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z', - }, execute_datetime) - target_object = TargetObject({ - 'object_name': 'Account', - 'columns': [ - 'Id', - 'Name', - 'SystemModstamp', - 'LastModifiedDate', - 'BillingStreet', - 'BillingCity', - 'BillingState', - 'BillingPostalCode', - 'BillingCountry', - 'BillingLatitude', - 'BillingLongitude', - 'BillingGeocodeAccuracy', - 'BillingAddress', - ] - }, execute_datetime) - soql_builder = SOQLBuilder(target_object, last_fetch_datetime) - soql = soql_builder.create_fetch_soql() - sut = SalesforceApiClient() - - actual = sut.fetch_sf_data(soql) - assert len(actual) == 1 - expect_address = OrderedDict([ - ("city", '〇〇区'), - ("country", "日本"), - ("geocodeAccuracy", None), - ("latitude", None), - ("longitude", None), - ("postalCode", '999-9999'), - ("state", '東京都'), - ("street", '△△-✗✗'), - ]) - - assert actual[0]['BillingAddress'] == expect_address - assert actual[0]['BillingCity'] == '〇〇区' - assert actual[0]['BillingCountry'] == '日本' - assert actual[0]['BillingGeocodeAccuracy'] is None - assert actual[0]['BillingLatitude'] is None - assert actual[0]['BillingLongitude'] is None - assert actual[0]['BillingPostalCode'] == '999-9999' - assert actual[0]['BillingState'] == '東京都' - assert actual[0]['BillingStreet'] == '△△-✗✗' - def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self): """ Cases: @@ -498,7 +433,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得できたオブジェクト件数が17件になる + 取得できたオブジェクト件数が16件になる """ execute_datetime = ExecuteDateTime() @@ -513,7 +448,6 @@ class TestSalesforceApiClient: 'Name', 'SystemModstamp', 'LastModifiedDate', - 'BillingAddress', 'CustomItem1__c', 'CustomItem2__c', 'CustomItem3__c', @@ -529,7 +463,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_data(soql) - assert len(actual) == 17 + assert len(actual) == 16 # 内容の確認は別のケースで行っているため省略 def test_raise_create_instance_cause_auth_failed(self, monkeypatch): diff --git a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py index 4e48c419..9ad35d52 100644 --- a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py @@ -44,8 +44,7 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False), - ('Name', 'テスト取引先1') + ('IsDeleted', False) ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -55,8 +54,7 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False), - ('Name', 'テスト取引先2') + ('IsDeleted', False) ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -66,8 +64,7 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False), - ('Name', 'テスト取引先3') + ('IsDeleted', False) ]), ] @@ -99,7 +96,7 @@ class TestBackupCrmDataProcess: actual = s3_client.get_object( Bucket=bucket_name, Key=f'response_json/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.json') - assert actual['Body'].read().decode('utf-8') == json.dumps(response_json, ensure_ascii=False) + assert actual['Body'].read().decode('utf-8') == json.dumps(response_json) # ログの確認 assert generate_log_message_tuple( diff --git a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py index bf439dac..49d960c0 100644 --- a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py +++ b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py @@ -34,17 +34,7 @@ class TestConvertCrmCsvDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', True), - ('PersonMailingAddress', OrderedDict([ - ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 - ('PersonMailingCity', 'New york city'), - ('PersonMailingState', 'Ohaio'), - ('PersonMailingPostalCode', '999-9999'), - ('PersonMailingCountry', 'US'), - ('PersonMailingLatitude', 50.1234567), - ('PersonMailingLongitude', 103.1234567), - ('PersonMailingGeocodeAccuracy', 'Address'), - ])), + ('IsDeleted', True) ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -52,19 +42,9 @@ class TestConvertCrmCsvDataProcess: ('Id', 'TEST002'), ('AccountNumber', 'test002'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), - ('LastModifiedById', 1.23E+0), + ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False), - ('PersonMailingAddress', OrderedDict([ - ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 - ('PersonMailingCity', 'New york city'), - ('PersonMailingState', 'Ohaio'), - ('PersonMailingPostalCode', '999-9999'), - ('PersonMailingCountry', 'US'), - ('PersonMailingLatitude', 50.1234567), - ('PersonMailingLongitude', 103.1234567), - ('PersonMailingGeocodeAccuracy', 'Address'), - ])), + ('IsDeleted', False) ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -72,19 +52,9 @@ class TestConvertCrmCsvDataProcess: ('Id', 'TEST003'), ('AccountNumber', 'test003'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), - ('LastModifiedById', 1.234567), + ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False), - ('PersonMailingAddress', OrderedDict([ - ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 - ('PersonMailingCity', 'New york city'), - ('PersonMailingState', 'Ohaio'), - ('PersonMailingPostalCode', '999-9999'), - ('PersonMailingCountry', 'US'), - ('PersonMailingLatitude', 50.1234567), - ('PersonMailingLongitude', 103.1234567), - ('PersonMailingGeocodeAccuracy', 'Address'), - ])), + ('IsDeleted', False) ]), ] @@ -96,8 +66,7 @@ class TestConvertCrmCsvDataProcess: 'LastModifiedDate', 'LastModifiedById', 'SystemModstamp', - 'IsDeleted', - 'PersonMailingAddress' + 'IsDeleted' ] } @@ -110,10 +79,10 @@ class TestConvertCrmCsvDataProcess: # Assert expect_csv_string = """\ - "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted","PersonMailingAddress"\r\n\ - "TEST001","test001","2022-06-01 09:00:00","1234567.0","2022-06-01 09:00:00","1","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ - "TEST002","test002","2022-06-01 09:00:00","1.23","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ - "TEST003","test003","2022-06-01 09:00:00","1.234567","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ + "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\ + "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\ + "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ + "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ """ # 返り値の期待値チェック assert isinstance(actual_csv_string, str), 'CSV文字列が返却される' diff --git a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py index 6b17477f..66f5577e 100644 --- a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py @@ -217,7 +217,7 @@ class TestFetchCrmDataProcess: 2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること 3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること Arranges: - - データ件数取得処理の最大リトライ試行回数を4に設定する + - データ件数取得処理の最大リトライ試行回数を3に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - データ件数取得処理の初回に接続タイムアウト例外が発生するようにする Expects: @@ -225,7 +225,7 @@ class TestFetchCrmDataProcess: - データ件数取得に失敗した旨のエラーが出力されない """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -236,8 +236,8 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3, 4] - mock_counter_inst.increment.side_effect = [2, 3, 4, 5] + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' @@ -271,15 +271,15 @@ class TestFetchCrmDataProcess: 2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること 3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること Arranges: - - データ件数取得処理の最大リトライ試行回数を4に設定する + - データ件数取得処理の最大リトライ試行回数を3に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - - データ件数取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする + - データ件数取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする Expects: - 異常終了する - データ件数取得に失敗した旨のエラーが出力される """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -290,13 +290,13 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3, 4] - mock_counter_inst.increment.side_effect = [2, 3, 4, 5] + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' mock_client_inst = mock_api_client.return_value - mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception, exception] + mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception] mock_client_inst.fetch_sf_data.return_value = common_expect # Act with pytest.raises(SalesforceAPIException) as e: @@ -304,16 +304,16 @@ class TestFetchCrmDataProcess: # Assert - # 取得は4回行われる - assert mock_counter_inst.describe.call_count == 4 - # 足し込みは3回のみ - assert mock_counter_inst.increment.call_count == 3 + # 取得は3回行われる + assert mock_counter_inst.describe.call_count == 3 + # 足し込みは2回のみ + assert mock_counter_inst.increment.call_count == 2 assert generate_log_message_tuple( log_level=logging.WARNING, log_message=expect_message) in caplog.record_tuples called_log_counts = len([log for log in caplog.messages if log == expect_message]) - assert called_log_counts == 3 + assert called_log_counts == 2 assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples assert e.value.error_id == 'E-FETCH-01' @@ -333,7 +333,7 @@ class TestFetchCrmDataProcess: 2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること 3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること Arranges: - - レコード取得処理の最大リトライ試行回数を4に設定する + - レコード取得処理の最大リトライ試行回数を3に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - レコード取得処理の初回に接続タイムアウト例外が発生するようにする Expects: @@ -341,7 +341,7 @@ class TestFetchCrmDataProcess: - データレコード取得に失敗した旨のエラーが出力されない """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -352,8 +352,8 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3, 4] - mock_counter_inst.increment.side_effect = [2, 3, 4, 5] + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' @@ -387,15 +387,15 @@ class TestFetchCrmDataProcess: 2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること 3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること Arranges: - - レコード取得処理の最大リトライ試行回数を4に設定する + - レコード取得処理の最大リトライ試行回数を3に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - - レコード取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする + - レコード取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする Expects: - 異常終了する - データレコード取得に失敗した旨のエラーが出力される """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 4) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 3) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_INTERVAL', 1) @@ -406,30 +406,30 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3, 4] - mock_counter_inst.increment.side_effect = [2, 3, 4, 5] + mock_counter_inst.describe.side_effect = [1, 2, 3] + mock_counter_inst.increment.side_effect = [2, 3, 4] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' mock_client_inst = mock_api_client.return_value mock_client_inst.fetch_sf_count.return_value = 1 - mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception, exception] + mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception] # Act with pytest.raises(SalesforceAPIException) as e: fetch_crm_data_process(common_target_object, common_last_fetch_datetime) # Assert - # 取得は4回行われる - assert mock_counter_inst.describe.call_count == 4 - # 足し込みは3回のみ - assert mock_counter_inst.increment.call_count == 3 + # 取得は3回行われる + assert mock_counter_inst.describe.call_count == 3 + # 足し込みは2回のみ + assert mock_counter_inst.increment.call_count == 2 assert generate_log_message_tuple( log_level=logging.WARNING, log_message=expect_message) in caplog.record_tuples called_log_counts = len([log for log in caplog.messages if log == expect_message]) - assert called_log_counts == 3 + assert called_log_counts == 2 assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples assert e.value.error_id == 'E-FETCH-02' diff --git a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py index 8eb3d476..9471946f 100644 --- a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py @@ -7,9 +7,6 @@ from src.config.objects import FetchTargetObjects from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.prepare_data_fetch_process import prepare_data_fetch_process from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ -from src.system_var.environments import (CRM_CONFIG_BUCKET, - OBJECT_INFO_FILENAME, - OBJECT_INFO_FOLDER) from .test_utils.log_message import generate_log_message_tuple @@ -83,8 +80,6 @@ class TestPrepareDataFetchProcess: # ログの確認 assert generate_log_message_tuple(log_message='I-PRE-01 データ取得準備処理を開始します') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-PRE-02 データ取得処理開始日時：{str(actual_execute_datetime)}') in caplog.record_tuples - assert generate_log_message_tuple( - log_message=f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}]') in caplog.record_tuples assert generate_log_message_tuple(log_message='I-PRE-09 データ取得準備処理を終了します') in caplog.record_tuples def test_call_depended_modules(self): diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index e2be6670..e8c3b6e4 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -73,6 +73,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): object_info_list = get_object_config_list('object_info') for object_info in object_info_list: json_file = read_json(object_info) + json_file = to_upload_json(json_file) upload_json(json_file, s3_client, CONFIG_BUCKET, f'{OBJECT_INFO_FOLDER}/{path.basename(object_info)}') object_info_files.append(json_file) last_fetch_datetime_list = get_object_config_list('last_fetch_datetime') @@ -104,7 +105,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # Act logger = logging.getLogger() - logger.setLevel(logging.DEBUG) + logger.setLevel(logging.INFO) logger.info(f'##########################') logger.info(f'# 差分データ取得処理:実行開始 #') logger.info(f'##########################') @@ -117,7 +118,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # ループ前のログ確認 assert 'I-CTRL-01 CRMデータ取得処理を開始します' in log_messages assert 'I-CTRL-02 データ取得準備処理呼び出し' in log_messages - assert_prepare_process_log(log_messages, now, path.basename(object_info_list[0])) + assert_prepare_process_log(log_messages, now) assert 'I-CTRL-03 取得対象オブジェクトのループ処理開始' in log_messages # オブジェクト情報を取得する(diff) object_info_list = object_info_files[0] @@ -206,10 +207,9 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): """ -def assert_prepare_process_log(log_messages, now, object_info_list_filename): +def assert_prepare_process_log(log_messages, now): assert 'I-PRE-01 データ取得準備処理を開始します' in log_messages assert f'I-PRE-02 データ取得処理開始日時：{now}' in log_messages - assert f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{object_info_list_filename}]' in log_messages assert 'I-PRE-09 データ取得準備処理を終了します' in log_messages @@ -295,6 +295,21 @@ def upload_json(json_file, s3_client, bucket, folder): s3_client.put_object(Bucket=bucket, Key=folder, Body=json_str) +def to_upload_json(json_file): + """Userオブジェクトの取得できないプロパティを取り除く + TODO: Userオブジェクトの恒久対応が確定したらこのメソッドは消す + """ + for object_info in json_file['objects']: + if object_info['object_name'] != 'User': + continue + columns: list = object_info['columns'] + columns.remove('LastPasswordChangeDate') + columns.remove('NumberOfFailedLogins') + columns.remove('UserPreferencesNativeEmailClient') + + return json_file + + def set_environment(monkeypatch): # 環境変数を設定(CRMの認証情報は別途設定しておくこと) monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET) @@ -309,4 +324,3 @@ def set_environment(monkeypatch): monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) - monkeypatch.setattr('src.prepare_data_fetch_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET) diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index dda641e2..a5f8da67 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -1645,11 +1645,13 @@ "DelegatedApproverId", "ManagerId", "LastLoginDate", + "LastPasswordChangeDate", "CreatedDate", "CreatedById", "LastModifiedDate", "LastModifiedById", "SystemModstamp", + "NumberOfFailedLogins", "OfflineTrialExpirationDate", "OfflinePdaTrialExpirationDate", "UserPermissionsMarketingUser", @@ -1723,6 +1725,7 @@ "UserPreferencesSRHOverrideActivities", "UserPreferencesNewLightningReportRunPageEnabled", "UserPreferencesReverseOpenActivitiesView", + "UserPreferencesNativeEmailClient", "UserPreferencesHideBrowseProductRedirectConfirmation", "UserPreferencesHideOnlineSalesAppWelcomeMat", "ContactId", diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env index 3ae4d809..580bf564 100644 --- a/s3/config/crm/task_settings/task_settings.env +++ b/s3/config/crm/task_settings/task_settings.env @@ -1,14 +1,13 @@ LOG_LEVEL=INFO CRM_AUTH_TIMEOUT=100 CRM_GET_RECORD_COUNT_TIMEOUT=300 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=4 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=3 CRM_GET_RECORD_COUNT_RETRY_INTERVAL=5 CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL=5 CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL=50 CRM_FETCH_RECORD_TIMEOUT=300 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=4 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3 CRM_FETCH_RECORD_RETRY_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 -CONVERT_TZ=Asia/Tokyo -OBJECT_INFO_FILENAME=crm_object_list_diff.json \ No newline at end of file +CONVERT_TZ='Asia/Tokyo' \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt index 813f86ee..d5d6c0dc 100644 --- a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt +++ b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt @@ -4,9 +4,9 @@ utf-8 " CRLF 1 -32 -Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c,MSJ_Registration__c -id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c,msj_registration__c +31 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c src02.crm_event_attendee_vod__c org02.crm_event_attendee_vod__c diff --git a/s3/data/crm/settings/CRM_User.txt b/s3/data/crm/settings/CRM_User.txt index 727d82e0..a0d5c519 100644 --- a/s3/data/crm/settings/CRM_User.txt +++ b/s3/data/crm/settings/CRM_User.txt @@ -4,9 +4,9 @@ utf-8 " CRLF 1 -200 -Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c -id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c +203 +Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,LastPasswordChangeDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,NumberOfFailedLogins,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesNativeEmailClient,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c +id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,last_password_change_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,number_of_failed_logins,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_native_email_client,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c src02.crm_user org02.crm_user From 0247ea5e574d4f706574769617c62148ca6fefd5 Mon Sep 17 00:00:00 2001 From: "shibata.r" Date: Fri, 2 Sep 2022 15:33:31 +0900 Subject: [PATCH 268/275] =?UTF-8?q?Revert=20"fix:=E3=82=B9=E3=82=AD?= =?UTF-8?q?=E3=83=BC=E3=83=9E=E3=83=95=E3=83=AB=E6=8C=87=E5=AE=9A=E3=81=A7?= =?UTF-8?q?=E5=91=BC=E3=81=B3=E5=87=BA=E3=81=99=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 831de315e59bb89aa1bc4801abeacc4a5ad7a30f. --- s3/data/crm/settings/CRM_AccountShare_ex.sql | 2 +- s3/data/crm/settings/CRM_Account_ex.sql | 2 +- s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql | 2 +- s3/data/crm/settings/CRM_Contact_ex.sql | 2 +- s3/data/crm/settings/CRM_Group_ex.sql | 2 +- s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql | 2 +- s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql | 2 +- s3/data/crm/settings/CRM_Profile_ex.sql | 2 +- s3/data/crm/settings/CRM_Territory2_ex.sql | 4 ++-- s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql | 4 ++-- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/s3/data/crm/settings/CRM_AccountShare_ex.sql b/s3/data/crm/settings/CRM_AccountShare_ex.sql index de5e6dac..d18e68db 100644 --- a/s3/data/crm/settings/CRM_AccountShare_ex.sql +++ b/s3/data/crm/settings/CRM_AccountShare_ex.sql @@ -1 +1 @@ -CALL src02.crm_history('src02.crm_account_share', 'last_modified_date'); \ No newline at end of file +CALL crm_history('src02.crm_account_share', 'last_modified_date'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Account_ex.sql b/s3/data/crm/settings/CRM_Account_ex.sql index 1511c5b1..792d7a28 100644 --- a/s3/data/crm/settings/CRM_Account_ex.sql +++ b/s3/data/crm/settings/CRM_Account_ex.sql @@ -1 +1 @@ -CALL src02.crm_history('src02.crm_account', 'system_modstamp'); +CALL crm_history('src02.crm_account', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql b/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql index 4c2bf134..cac33f47 100644 --- a/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql +++ b/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql @@ -1 +1 @@ -CALL src02.crm_history('src02.crm_child_account_vod__c', 'system_modstamp'); \ No newline at end of file +CALL crm_history('src02.crm_child_account_vod__c', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Contact_ex.sql b/s3/data/crm/settings/CRM_Contact_ex.sql index 31fc59a6..4b916e23 100644 --- a/s3/data/crm/settings/CRM_Contact_ex.sql +++ b/s3/data/crm/settings/CRM_Contact_ex.sql @@ -1 +1 @@ -CALL src02.crm_history('src02.crm_contact', 'system_modstamp'); +CALL crm_history('src02.crm_contact', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Group_ex.sql b/s3/data/crm/settings/CRM_Group_ex.sql index f0667ff9..5e50dab6 100644 --- a/s3/data/crm/settings/CRM_Group_ex.sql +++ b/s3/data/crm/settings/CRM_Group_ex.sql @@ -1 +1 @@ -CALL src02.crm_history('src02.crm_group', 'system_modstamp'); \ No newline at end of file +CALL crm_history('src02.crm_group', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql index 2e1a751e..d142c071 100644 --- a/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql +++ b/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql @@ -1 +1 @@ -CALL src02.crm_history('src02.crm_object_territory2_association', 'system_modstamp'); +CALL crm_history('src02.crm_object_territory2_association', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql b/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql index 76134c74..af2e678e 100644 --- a/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql +++ b/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql @@ -1 +1 @@ -CALL src02.crm_history('src02.crm_product_metrics_vod__c', 'system_modstamp'); \ No newline at end of file +CALL crm_history('src02.crm_product_metrics_vod__c', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Profile_ex.sql b/s3/data/crm/settings/CRM_Profile_ex.sql index 80a780f0..b858ab02 100644 --- a/s3/data/crm/settings/CRM_Profile_ex.sql +++ b/s3/data/crm/settings/CRM_Profile_ex.sql @@ -1 +1 @@ -CALL src02.crm_history('src02.crm_profile', 'system_modstamp'); +CALL crm_history('src02.crm_profile', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Territory2_ex.sql b/s3/data/crm/settings/CRM_Territory2_ex.sql index ffc2befe..a30ccd91 100644 --- a/s3/data/crm/settings/CRM_Territory2_ex.sql +++ b/s3/data/crm/settings/CRM_Territory2_ex.sql @@ -1,2 +1,2 @@ -CALL src02.crm_data_sync('src02.crm_territory2', 'src02.crm_territory2_all', 'system_modstamp'); -CALL src02.crm_history('src02.crm_territory2', 'system_modstamp'); +CALL crm_data_sync('src02.crm_territory2', 'src02.crm_territory2_all', 'system_modstamp'); +CALL crm_history('src02.crm_territory2', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql index 41b6531d..6a9aaece 100644 --- a/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql +++ b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql @@ -1,2 +1,2 @@ -CALL src02.crm_data_sync('src02.crm_user_territory2_association', 'src02.crm_user_territory2_association_all', 'system_modstamp'); -CALL src02.crm_history('src02.crm_user_territory2_association', 'system_modstamp'); +CALL crm_data_sync('src02.crm_user_territory2_association', 'src02.crm_user_territory2_association_all', 'system_modstamp'); +CALL crm_history('src02.crm_user_territory2_association', 'system_modstamp'); From 5625ebeac528267e74456a5fd393ad01c75d3d33 Mon Sep 17 00:00:00 2001 From: "shibata.r" Date: Fri, 2 Sep 2022 16:14:36 +0900 Subject: [PATCH 269/275] =?UTF-8?q?fix:=E3=82=B9=E3=82=AD=E3=83=BC?= =?UTF-8?q?=E3=83=9E=E3=83=95=E3=83=AB=E6=8C=87=E5=AE=9A=E3=81=A7=E5=91=BC?= =?UTF-8?q?=E3=81=B3=E5=87=BA=E3=81=99=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 2 +- rds_mysql/stored_procedure/crm_history.sql | 2 +- s3/data/crm/settings/CRM_AccountShare_ex.sql | 2 +- s3/data/crm/settings/CRM_Account_ex.sql | 2 +- s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql | 2 +- s3/data/crm/settings/CRM_Contact_ex.sql | 2 +- s3/data/crm/settings/CRM_Group_ex.sql | 2 +- s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql | 2 +- s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql | 2 +- s3/data/crm/settings/CRM_Profile_ex.sql | 2 +- s3/data/crm/settings/CRM_Territory2_ex.sql | 4 ++-- s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql | 4 ++-- 12 files changed, 14 insertions(+), 14 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql index e9986c0f..b830265f 100644 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -1,7 +1,7 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること -- $$から始まる文字は後からREPLACEする文字を示す独自ルール -- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない -CREATE PROCEDURE crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) +CREATE PROCEDURE src02.crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) BEGIN -- 例外処理 -- エラーが発生した場合に一時テーブルの削除を実施 diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index 10bb0095..b3247c5f 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,7 +1,7 @@ -- A5M2で実行時にSQL区切り文字を「;」以外にすること -- $$から始まり$$で終わる文字は後からREPLACEする文字を示す独自ルール -- crm_historyストアドプロシージャは、同一セッション内での並列処理を実行することができない -CREATE PROCEDURE crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) +CREATE PROCEDURE src02.crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) BEGIN -- 例外処理 -- エラーが発生した場合に一時テーブルの削除を実施 diff --git a/s3/data/crm/settings/CRM_AccountShare_ex.sql b/s3/data/crm/settings/CRM_AccountShare_ex.sql index d18e68db..de5e6dac 100644 --- a/s3/data/crm/settings/CRM_AccountShare_ex.sql +++ b/s3/data/crm/settings/CRM_AccountShare_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_account_share', 'last_modified_date'); \ No newline at end of file +CALL src02.crm_history('src02.crm_account_share', 'last_modified_date'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Account_ex.sql b/s3/data/crm/settings/CRM_Account_ex.sql index 792d7a28..1511c5b1 100644 --- a/s3/data/crm/settings/CRM_Account_ex.sql +++ b/s3/data/crm/settings/CRM_Account_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_account', 'system_modstamp'); +CALL src02.crm_history('src02.crm_account', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql b/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql index cac33f47..4c2bf134 100644 --- a/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql +++ b/s3/data/crm/settings/CRM_Child_Account_vod__c_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_child_account_vod__c', 'system_modstamp'); \ No newline at end of file +CALL src02.crm_history('src02.crm_child_account_vod__c', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Contact_ex.sql b/s3/data/crm/settings/CRM_Contact_ex.sql index 4b916e23..31fc59a6 100644 --- a/s3/data/crm/settings/CRM_Contact_ex.sql +++ b/s3/data/crm/settings/CRM_Contact_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_contact', 'system_modstamp'); +CALL src02.crm_history('src02.crm_contact', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Group_ex.sql b/s3/data/crm/settings/CRM_Group_ex.sql index 5e50dab6..f0667ff9 100644 --- a/s3/data/crm/settings/CRM_Group_ex.sql +++ b/s3/data/crm/settings/CRM_Group_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_group', 'system_modstamp'); \ No newline at end of file +CALL src02.crm_history('src02.crm_group', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql index d142c071..2e1a751e 100644 --- a/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql +++ b/s3/data/crm/settings/CRM_ObjectTerritory2Association_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_object_territory2_association', 'system_modstamp'); +CALL src02.crm_history('src02.crm_object_territory2_association', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql b/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql index af2e678e..76134c74 100644 --- a/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql +++ b/s3/data/crm/settings/CRM_Product_Metrics_vod__c_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_product_metrics_vod__c', 'system_modstamp'); \ No newline at end of file +CALL src02.crm_history('src02.crm_product_metrics_vod__c', 'system_modstamp'); \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Profile_ex.sql b/s3/data/crm/settings/CRM_Profile_ex.sql index b858ab02..80a780f0 100644 --- a/s3/data/crm/settings/CRM_Profile_ex.sql +++ b/s3/data/crm/settings/CRM_Profile_ex.sql @@ -1 +1 @@ -CALL crm_history('src02.crm_profile', 'system_modstamp'); +CALL src02.crm_history('src02.crm_profile', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Territory2_ex.sql b/s3/data/crm/settings/CRM_Territory2_ex.sql index a30ccd91..ffc2befe 100644 --- a/s3/data/crm/settings/CRM_Territory2_ex.sql +++ b/s3/data/crm/settings/CRM_Territory2_ex.sql @@ -1,2 +1,2 @@ -CALL crm_data_sync('src02.crm_territory2', 'src02.crm_territory2_all', 'system_modstamp'); -CALL crm_history('src02.crm_territory2', 'system_modstamp'); +CALL src02.crm_data_sync('src02.crm_territory2', 'src02.crm_territory2_all', 'system_modstamp'); +CALL src02.crm_history('src02.crm_territory2', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql index 6a9aaece..41b6531d 100644 --- a/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql +++ b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql @@ -1,2 +1,2 @@ -CALL crm_data_sync('src02.crm_user_territory2_association', 'src02.crm_user_territory2_association_all', 'system_modstamp'); -CALL crm_history('src02.crm_user_territory2_association', 'system_modstamp'); +CALL src02.crm_data_sync('src02.crm_user_territory2_association', 'src02.crm_user_territory2_association_all', 'system_modstamp'); +CALL src02.crm_history('src02.crm_user_territory2_association', 'system_modstamp'); From 107bbc00e759fbe2ef5e1180a9b4989c58cf2449 Mon Sep 17 00:00:00 2001 From: "shibata.r" Date: Fri, 2 Sep 2022 17:42:07 +0900 Subject: [PATCH 270/275] =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- rds_mysql/stored_procedure/crm_data_sync.sql | 4 ++-- rds_mysql/stored_procedure/crm_history.sql | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql index b830265f..f04b6f8b 100644 --- a/rds_mysql/stored_procedure/crm_data_sync.sql +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -1,4 +1,4 @@ --- A5M2で実行時にSQL区切り文字を「;」以外にすること +-- A5M2で実行時に[SQL] - [スラッシュ(/)のみの行でSQLを区切る]に変えてから実行する -- $$から始まる文字は後からREPLACEする文字を示す独自ルール -- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない CREATE PROCEDURE src02.crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) @@ -43,4 +43,4 @@ BEGIN EXECUTE update_end_datetime_stmt; COMMIT; -END \ No newline at end of file +END diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql index b3247c5f..1b59cb7a 100644 --- a/rds_mysql/stored_procedure/crm_history.sql +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -1,4 +1,4 @@ --- A5M2で実行時にSQL区切り文字を「;」以外にすること +-- A5M2で実行時に[SQL] - [スラッシュ(/)のみの行でSQLを区切る]に変えてから実行する -- $$から始まり$$で終わる文字は後からREPLACEする文字を示す独自ルール -- crm_historyストアドプロシージャは、同一セッション内での並列処理を実行することができない CREATE PROCEDURE src02.crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) @@ -90,4 +90,4 @@ BEGIN COMMIT; -END \ No newline at end of file +END From 3132be0ff1f5624c2251112a2665387483205bdf Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Wed, 7 Sep 2022 09:55:19 +0900 Subject: [PATCH 271/275] =?UTF-8?q?fix:=E5=90=8C=E6=9C=9F=E3=82=B9?= =?UTF-8?q?=E3=83=88=E3=82=A2=E3=83=89=E3=81=AE=E5=AE=9F=E8=A1=8C=E5=A0=B4?= =?UTF-8?q?=E6=89=80=E3=82=92=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/crm/settings/CRM_Territory2_ALL.txt | 2 +- s3/data/crm/settings/CRM_Territory2__ALL_ex.sql | 1 + s3/data/crm/settings/CRM_Territory2_ex.sql | 1 - s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt | 2 +- s3/data/crm/settings/CRM_UserTerritory2Association_ALL_ex.sql | 1 + s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql | 1 - 6 files changed, 4 insertions(+), 4 deletions(-) create mode 100644 s3/data/crm/settings/CRM_Territory2__ALL_ex.sql create mode 100644 s3/data/crm/settings/CRM_UserTerritory2Association_ALL_ex.sql diff --git a/s3/data/crm/settings/CRM_Territory2_ALL.txt b/s3/data/crm/settings/CRM_Territory2_ALL.txt index fff3fc29..e51caf1e 100644 --- a/s3/data/crm/settings/CRM_Territory2_ALL.txt +++ b/s3/data/crm/settings/CRM_Territory2_ALL.txt @@ -9,6 +9,6 @@ Id,Name,Territory2TypeId,Territory2ModelId,ParentTerritory2Id,Description,Foreca id,name,territory2_type_id,territory2_model_id,parent_territory2_id,description,forecast_user_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,last_modified_date,last_modified_by_id,system_modstamp,developer_name,msj_territory_type__c,msj_level__c src02.crm_territory2_all org02.crm_territory2_all - +CRM_Territory2__ALL_ex.sql truncate_src_table:src02.crm_territory2_all diff --git a/s3/data/crm/settings/CRM_Territory2__ALL_ex.sql b/s3/data/crm/settings/CRM_Territory2__ALL_ex.sql new file mode 100644 index 00000000..0619f478 --- /dev/null +++ b/s3/data/crm/settings/CRM_Territory2__ALL_ex.sql @@ -0,0 +1 @@ +CALL src02.crm_data_sync('src02.crm_territory2', 'src02.crm_territory2_all', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_Territory2_ex.sql b/s3/data/crm/settings/CRM_Territory2_ex.sql index ffc2befe..8431f9e3 100644 --- a/s3/data/crm/settings/CRM_Territory2_ex.sql +++ b/s3/data/crm/settings/CRM_Territory2_ex.sql @@ -1,2 +1 @@ -CALL src02.crm_data_sync('src02.crm_territory2', 'src02.crm_territory2_all', 'system_modstamp'); CALL src02.crm_history('src02.crm_territory2', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt b/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt index 4617829d..f5b579ec 100644 --- a/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt +++ b/s3/data/crm/settings/CRM_UserTerritory2Association_ALL.txt @@ -9,6 +9,6 @@ Id,UserId,Territory2Id,IsActive,RoleInTerritory2,LastModifiedDate,LastModifiedBy id,user_id,territory2_id,is_active,role_in_territory2,last_modified_date,last_modified_by_id,system_modstamp src02.crm_user_territory2_association_all org02.crm_user_territory2_association_all - +CRM_UserTerritory2Association_ALL_ex.sql truncate_src_table:src02.crm_user_territory2_association_all diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association_ALL_ex.sql b/s3/data/crm/settings/CRM_UserTerritory2Association_ALL_ex.sql new file mode 100644 index 00000000..a34c64d2 --- /dev/null +++ b/s3/data/crm/settings/CRM_UserTerritory2Association_ALL_ex.sql @@ -0,0 +1 @@ +CALL src02.crm_data_sync('src02.crm_user_territory2_association', 'src02.crm_user_territory2_association_all', 'system_modstamp'); diff --git a/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql index 41b6531d..4c99e171 100644 --- a/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql +++ b/s3/data/crm/settings/CRM_UserTerritory2Association_ex.sql @@ -1,2 +1 @@ -CALL src02.crm_data_sync('src02.crm_user_territory2_association', 'src02.crm_user_territory2_association_all', 'system_modstamp'); CALL src02.crm_history('src02.crm_user_territory2_association', 'system_modstamp'); From c19c6bf0f8e5ad9b6fe87f3d9fd7e547ba0ecc62 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Wed, 7 Sep 2022 10:18:25 +0900 Subject: [PATCH 272/275] =?UTF-8?q?style:=5F=E3=81=8C=E4=BA=8C=E3=81=A4?= =?UTF-8?q?=E3=81=AB=E3=81=AA=E3=81=A3=E3=81=A6=E3=81=84=E3=81=9F=E3=81=AE?= =?UTF-8?q?=E3=82=92=E4=B8=80=E3=81=A4=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/crm/settings/CRM_Territory2_ALL.txt | 2 +- .../{CRM_Territory2__ALL_ex.sql => CRM_Territory2_ALL_ex.sql} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename s3/data/crm/settings/{CRM_Territory2__ALL_ex.sql => CRM_Territory2_ALL_ex.sql} (100%) diff --git a/s3/data/crm/settings/CRM_Territory2_ALL.txt b/s3/data/crm/settings/CRM_Territory2_ALL.txt index e51caf1e..31eaea73 100644 --- a/s3/data/crm/settings/CRM_Territory2_ALL.txt +++ b/s3/data/crm/settings/CRM_Territory2_ALL.txt @@ -9,6 +9,6 @@ Id,Name,Territory2TypeId,Territory2ModelId,ParentTerritory2Id,Description,Foreca id,name,territory2_type_id,territory2_model_id,parent_territory2_id,description,forecast_user_id,account_access_level,opportunity_access_level,case_access_level,contact_access_level,last_modified_date,last_modified_by_id,system_modstamp,developer_name,msj_territory_type__c,msj_level__c src02.crm_territory2_all org02.crm_territory2_all -CRM_Territory2__ALL_ex.sql +CRM_Territory2_ALL_ex.sql truncate_src_table:src02.crm_territory2_all diff --git a/s3/data/crm/settings/CRM_Territory2__ALL_ex.sql b/s3/data/crm/settings/CRM_Territory2_ALL_ex.sql similarity index 100% rename from s3/data/crm/settings/CRM_Territory2__ALL_ex.sql rename to s3/data/crm/settings/CRM_Territory2_ALL_ex.sql From 6b0d978378b21d3a02f483b1f97bb0c760637dfc Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 7 Sep 2022 18:16:51 +0900 Subject: [PATCH 273/275] Revert "Revert "Merge branch 'develop-6crm' of nds-tyo.git.backlog.com:/NEWDWH2021/newsdwh2021 into develop-6crm"" This reverts commit bae8447658a03ff8a0bd84948d493f332724bebb. --- ecs/crm-datafetch/Dockerfile | 2 +- ecs/crm-datafetch/src/aws/s3.py | 4 +- .../src/converter/convert_strategy.py | 21 +-- .../src/prepare_data_fetch_process.py | 6 +- .../src/system_var/environments.py | 8 +- .../tests/converter/test_convert_strategy.py | 145 ++++++++++++++---- .../tests/converter/test_converter.py | 10 +- .../tests/salesforce/test_salesforce.py | 110 ++++++++++--- .../tests/test_backup_crm_data_process.py | 11 +- .../test_convert_crm_csv_data_process.py | 51 ++++-- .../tests/test_fetch_crm_data_process.py | 60 ++++---- .../tests/test_prepare_data_fetch_process.py | 5 + ecs/crm-datafetch/tests/test_walk_through.py | 24 +-- .../crm/object_info/crm_object_list_diff.json | 3 - s3/config/crm/task_settings/task_settings.env | 7 +- .../settings/CRM_Event_Attendee_vod__c.txt | 6 +- s3/data/crm/settings/CRM_User.txt | 6 +- 17 files changed, 330 insertions(+), 149 deletions(-) diff --git a/ecs/crm-datafetch/Dockerfile b/ecs/crm-datafetch/Dockerfile index 56087bf0..2d6318e7 100644 --- a/ecs/crm-datafetch/Dockerfile +++ b/ecs/crm-datafetch/Dockerfile @@ -14,6 +14,6 @@ RUN \ pip uninstall -y pipenv virtualenv-clone virtualenv COPY main.py ./ -COPY src ./ +COPY src ./src CMD [ "python", "./main.py" ] diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index d5bd03f0..7b63861f 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -87,7 +87,7 @@ class BackupBucket: def put_response_json(self, file_path: str, data: dict) -> None: object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_path}' - self.__s3_resource.put_object(object_key, json.dumps(data)) + self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False)) return def put_csv(self, file_path: str, data: str) -> None: @@ -97,5 +97,5 @@ class BackupBucket: def put_result_json(self, file_path: str, data: dict) -> None: object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}' - self.__s3_resource.put_object(object_key, json.dumps(data)) + self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False)) return diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 22fca8fc..7c123d39 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -1,4 +1,6 @@ +import json import re +from collections import OrderedDict from datetime import datetime from dateutil.tz import gettz @@ -12,20 +14,17 @@ from src.system_var.environments import CONVERT_TZ class ConvertStrategyFactory: def __init__(self) -> None: self.__none_value_convert_strategy = NoneValueConvertStrategy() - self.__float_convert_strategy = FloatConvertStrategy() self.__boolean_convert_strategy = BooleanConvertStrategy() self.__datetime_convert_strategy = DatetimeConvertStrategy() self.__int_convert_strategy = IntConvertStrategy() self.__string_convert_strategy = StringConvertStrategy() + self.__dict_convert_strategy = DictConvertStrategy() def create(self, value): if value is None: convert_strategy = self.__none_value_convert_strategy - elif type(value) == float: - convert_strategy = self.__float_convert_strategy - elif type(value) == bool: convert_strategy = self.__boolean_convert_strategy @@ -35,6 +34,8 @@ class ConvertStrategyFactory: elif type(value) == int: convert_strategy = self.__int_convert_strategy + elif type(value) == dict or type(value) == OrderedDict: + convert_strategy = self.__dict_convert_strategy else: convert_strategy = self.__string_convert_strategy @@ -60,12 +61,6 @@ class DatetimeConvertStrategy: return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS) -class FloatConvertStrategy: - def convert_value(self, convert_value: str) -> int: - """float型をint型に変換する処理""" - return int(convert_value) - - class IntConvertStrategy: def convert_value(self, convert_value: int): """int型を変換せずに返す処理""" @@ -78,3 +73,9 @@ class StringConvertStrategy: """string型を変換せずに返す処理""" # ConvertStrategyFactoryにて型チェックを行っているため値を変換せずに返す return convert_value + + +class DictConvertStrategy: + def convert_value(self, convert_value: dict): + """dict型の項目を文字列に変換して返す処理""" + return json.dumps(convert_value, ensure_ascii=False) diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index d546c7b8..f14e85b8 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -34,9 +34,9 @@ def prepare_data_fetch_process(): try: # ③ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する - object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' - logger.debug( - f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') + object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' + logger.info( + f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') config_bucket = ConfigBucket() object_info_file_str = config_bucket.get_object_info_file() diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index 297cfb2b..c457386f 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -9,8 +9,8 @@ LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)) # CRMのレコード件数取得処理のタイムアウト秒数 CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)) -# CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3)) +# CRMのレコード件数取得処理の最大リトライ試行回数(処理の実施総回数) +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 4)) # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 CRM_GET_RECORD_COUNT_RETRY_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5)) # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 @@ -19,8 +19,8 @@ CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_GET_R CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50)) # CRMのレコード取得処理のタイムアウト秒数 CRM_FETCH_RECORD_TIMEOUT = int(os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300)) -# CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3)) +# CRMのレコード取得処理の最大リトライ試行回数(処理の実施総回数) +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 4)) # CRMのレコード取得処理のリトライ時の初回待ち秒数 CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最小待ち秒数 diff --git a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py index 0e268fc4..129b3980 100644 --- a/ecs/crm-datafetch/tests/converter/test_convert_strategy.py +++ b/ecs/crm-datafetch/tests/converter/test_convert_strategy.py @@ -1,7 +1,9 @@ +from collections import OrderedDict + from src.converter.convert_strategy import (BooleanConvertStrategy, ConvertStrategyFactory, DatetimeConvertStrategy, - FloatConvertStrategy, + DictConvertStrategy, IntConvertStrategy, NoneValueConvertStrategy, StringConvertStrategy) @@ -29,7 +31,7 @@ class TestConvertStrategyFactory: def test_create_float(self): """ Cases: - 引数に指数表記を指定した場合、FloatConvertStrategyインスタンスが返ってくること + 引数に指数表記を指定した場合、StringConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -41,7 +43,26 @@ class TestConvertStrategyFactory: actual = sut.create(1.2345678E7) # Expects - assert type(actual) == FloatConvertStrategy + # 変換しない + assert type(actual) == StringConvertStrategy + + def test_create_float_scale(self): + """ + Cases: + 引数に少数を指定した場合、StringConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(1.2345678) + + # Expects + # 変換しない + assert type(actual) == StringConvertStrategy def test_create_bool_true(self): """ @@ -94,10 +115,10 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == DatetimeConvertStrategy - def test_create_other_str(self): + def test_create_str(self): """ Cases: - 引数にSalesforce日付型以外の文字列を指定した場合、NonConvertStrategyインスタンスが返ってくること + 引数にSalesforce日付型以外の文字列を指定した場合、StringConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -111,10 +132,10 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == StringConvertStrategy - def test_create_other_int(self): + def test_create_int(self): """ Cases: - 引数に整数を指定した場合、NonConvertStrategyインスタンスが返ってくること + 引数に整数を指定した場合、IntConvertStrategyインスタンスが返ってくること Arranges: - なし Expects: @@ -128,6 +149,40 @@ class TestConvertStrategyFactory: # Expects assert type(actual) == IntConvertStrategy + def test_create_dict(self): + """ + Cases: + 引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create({'key': 'value'}) + + # Expects + assert type(actual) == DictConvertStrategy + + def test_create_ordered_dict_dict(self): + """ + Cases: + 引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = ConvertStrategyFactory() + actual = sut.create(OrderedDict([('key', 'value')])) + + # Expects + assert type(actual) == DictConvertStrategy + class TestNoneValueConvertStrategy: @@ -206,26 +261,6 @@ class TestDatetimeConvertStrategy: assert actual == "2022-06-14 05:15:32" -class TestFloatConvertStrategy: - - def test_convert_value(self) -> int: - """ - Cases: - 引数に指数表記を指定した場合、整数で返ってくること - Arranges: - - なし - Expects: - - 戻り値が、期待値と一致する - """ - - # Act - sut = FloatConvertStrategy() - actual = sut.convert_value(1.2345678E7) - - # Expects - assert actual == 12345678 - - class TestIntConvertStrategy: def test_convert_value(self): @@ -264,3 +299,59 @@ class TestStringConvertStrategy: # Expects assert actual == 'テストデータ' + + +class TestDictConvertStrategy: + + def test_convert_value_dict(self): + """ + Cases: + 引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DictConvertStrategy() + actual = sut.convert_value({'テストデータキー': 'テストデータバリュー'}) + + # Expects + assert actual == '{"テストデータキー": "テストデータバリュー"}' + + def test_convert_value_dict_in_line_break(self): + """ + Cases: + 引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること(バリューに改行を含む) + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DictConvertStrategy() + actual = sut.convert_value({'テストデータキー': 'テスト\nデータ\nバリュー'}) + + # Expects + assert actual == '{"テストデータキー": "テスト\\nデータ\\nバリュー"}' + + def test_convert_value_ordered_dict(self): + """ + Cases: + 引数に整列された辞書型のデータを指定した場合、JSONの文字列が返ってくること + Arranges: + - なし + Expects: + - 戻り値が、期待値と一致する + """ + + # Act + sut = DictConvertStrategy() + actual = sut.convert_value(OrderedDict( + [('テストデータキー', 'テストデータバリュー')] + )) + + # Expects + assert actual == '{"テストデータキー": "テストデータバリュー"}' diff --git a/ecs/crm-datafetch/tests/converter/test_converter.py b/ecs/crm-datafetch/tests/converter/test_converter.py index 0300381d..da80982b 100644 --- a/ecs/crm-datafetch/tests/converter/test_converter.py +++ b/ecs/crm-datafetch/tests/converter/test_converter.py @@ -71,7 +71,7 @@ class TestCSVStringConverter: ('ContactAccessLevel', 8), ('RowCause', 'テストのため2'), ('LastModifiedDate', '2022-06-02T16:30:30.000+0000'), - ('LastModifiedById', 2.234567E+6), + ('LastModifiedById', 2.23E+0), ('IsDeleted', True) ]), OrderedDict([ @@ -85,7 +85,7 @@ class TestCSVStringConverter: ('ContactAccessLevel', 12), ('RowCause', 'テストのため3'), ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), - ('LastModifiedById', 3.234567E+6), + ('LastModifiedById', 3.234567), ('IsDeleted', False) ]) ] @@ -100,9 +100,9 @@ class TestCSVStringConverter: # Expects expected_value = '''\ "Id","AccountId","UserOrGroupId","AccountAccessLevel","OpportunityAccessLevel","CaseAccessLevel","ContactAccessLevel","RowCause","LastModifiedDate","LastModifiedById","IsDeleted"\r\n\ - "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567","0"\r\n\ - "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2234567","1"\r\n\ - "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3234567","0"\r\n\ + "TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567.0","0"\r\n\ + "TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2.23","1"\r\n\ + "TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3.234567","0"\r\n\ ''' # expected_valueのインデントが半角スペースと認識されてしまうため、`textwrap.dedent`にて補正 diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index b815e772..64530071 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -9,6 +9,8 @@ Accountオブジェクトの下記SFIDのレコードはいじらないように 変更してしまった場合は各SOQLの取得日付とデータを修正してください """ +from typing import OrderedDict + import pytest from requests.exceptions import ConnectTimeout, ReadTimeout from src.config.objects import LastFetchDatetime, TargetObject @@ -33,8 +35,8 @@ class TestSalesforceApiClient: FROM Account WHERE - SystemModstamp > 2022-08-04T00:00:00.000Z AND - SystemModstamp <= 2022-08-06T00:00:00.000Z + SystemModstamp > 2022-08-23T01:56:39.000Z AND + SystemModstamp <= 2022-08-24T00:00:00.000Z """ sut = SalesforceApiClient() @@ -63,7 +65,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual >= 0 - def test_fetch_sf_count_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): + def test_fetch_sf_count_by_soql_builder_system_modstamp_to_ge(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること @@ -79,8 +81,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', - 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', + 'last_fetch_datetime_from': '2022-08-23T02:38:59.000Z', + 'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -106,7 +108,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_count(soql) assert actual == 1 - def test_fetch_sf_count_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): + def test_fetch_sf_count_by_soql_builder_system_modstamp_to_lt(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること @@ -122,8 +124,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', - 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', + 'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z', + 'last_fetch_datetime_to': '2022-08-23T02:39:01.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -160,7 +162,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得件数が16になる + 取得件数が17になる """ execute_datetime = ExecuteDateTime() @@ -190,7 +192,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_count(soql) - assert actual == 16 + assert actual == 17 def test_fetch_sf_data_one_record(self): """ @@ -217,8 +219,8 @@ class TestSalesforceApiClient: FROM Account WHERE - SystemModstamp > 2022-08-04T00:00:00.000Z AND - SystemModstamp <= 2022-08-06T00:00:00.000Z + SystemModstamp > 2022-08-23T01:56:39.000Z AND + SystemModstamp <= 2022-08-24T00:00:00.000Z """ sut = SalesforceApiClient() @@ -284,7 +286,7 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_data(soql) assert len(actual) >= 0 - def test_fetch_sf_data_by_soql_builder_system_modstamp_lt_from_and_to_ge(self): + def test_fetch_sf_data_by_soql_builder_system_modstamp_to_ge(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること @@ -300,8 +302,8 @@ class TestSalesforceApiClient: execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z', - 'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z', + 'last_fetch_datetime_from': '2022-08-23T03:48:39.000Z', + 'last_fetch_datetime_to': '2022-08-23T03:48:40.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -353,7 +355,7 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect - def test_fetch_sf_data_by_soql_builder_system_modstamp_gt_from_and_to_lt(self): + def test_fetch_sf_data_by_soql_builder_system_modstamp_to_gt(self): """ Cases: - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること @@ -361,16 +363,16 @@ class TestSalesforceApiClient: - SystemModStampのToが指定日付未満のレコードは取得できないこと Arranges: - SalesforceのAccountオブジェクトに、レコードを作成する - - LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定） - - LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定） + - LastFetchDatetimeのFromがSystemModstampより小さくなるように指定する（UTC指定） + - LastFetchDatetimeのToがSystemModstampより大きくなるように指定する（UTC指定） Expects: 取得できたオブジェクト1件が期待値どおりであること """ execute_datetime = ExecuteDateTime() last_fetch_datetime = LastFetchDatetime({ - 'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z', - 'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z', + 'last_fetch_datetime_from': '2022-08-23T03:42:24.000Z', + 'last_fetch_datetime_to': '2022-08-23T03:42:26.000Z', }, execute_datetime) target_object = TargetObject({ 'object_name': 'Account', @@ -422,6 +424,69 @@ class TestSalesforceApiClient: assert dict(actual[0]) == expect + def test_fetch_sf_data_by_soql_builder_address_item_check(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること + - できること + Arranges: + - SalesforceのAccountオブジェクトに、住所項目を持つレコードを作成する + - 住所項目を持つレコードだけが取れるよう日付を設定する + Expects: + 取得できたオブジェクト件数が1件になる + 住所項目(BillingAddress)が想定通りの値になっていること + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z', + 'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'Account', + 'columns': [ + 'Id', + 'Name', + 'SystemModstamp', + 'LastModifiedDate', + 'BillingStreet', + 'BillingCity', + 'BillingState', + 'BillingPostalCode', + 'BillingCountry', + 'BillingLatitude', + 'BillingLongitude', + 'BillingGeocodeAccuracy', + 'BillingAddress', + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_fetch_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) == 1 + expect_address = OrderedDict([ + ("city", '〇〇区'), + ("country", "日本"), + ("geocodeAccuracy", None), + ("latitude", None), + ("longitude", None), + ("postalCode", '999-9999'), + ("state", '東京都'), + ("street", '△△-✗✗'), + ]) + + assert actual[0]['BillingAddress'] == expect_address + assert actual[0]['BillingCity'] == '〇〇区' + assert actual[0]['BillingCountry'] == '日本' + assert actual[0]['BillingGeocodeAccuracy'] is None + assert actual[0]['BillingLatitude'] is None + assert actual[0]['BillingLongitude'] is None + assert actual[0]['BillingPostalCode'] == '999-9999' + assert actual[0]['BillingState'] == '東京都' + assert actual[0]['BillingStreet'] == '△△-✗✗' + def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self): """ Cases: @@ -433,7 +498,7 @@ class TestSalesforceApiClient: - LastFetchDatetimeのFromに2000年1月1日を指定する - LastFetchDatetimeのToに2100年12月31日を指定する Expects: - 取得できたオブジェクト件数が16件になる + 取得できたオブジェクト件数が17件になる """ execute_datetime = ExecuteDateTime() @@ -448,6 +513,7 @@ class TestSalesforceApiClient: 'Name', 'SystemModstamp', 'LastModifiedDate', + 'BillingAddress', 'CustomItem1__c', 'CustomItem2__c', 'CustomItem3__c', @@ -463,7 +529,7 @@ class TestSalesforceApiClient: sut = SalesforceApiClient() actual = sut.fetch_sf_data(soql) - assert len(actual) == 16 + assert len(actual) == 17 # 内容の確認は別のケースで行っているため省略 def test_raise_create_instance_cause_auth_failed(self, monkeypatch): diff --git a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py index 9ad35d52..4e48c419 100644 --- a/ecs/crm-datafetch/tests/test_backup_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_backup_crm_data_process.py @@ -44,7 +44,8 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('Name', 'テスト取引先1') ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -54,7 +55,8 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('Name', 'テスト取引先2') ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -64,7 +66,8 @@ class TestBackupCrmDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('Name', 'テスト取引先3') ]), ] @@ -96,7 +99,7 @@ class TestBackupCrmDataProcess: actual = s3_client.get_object( Bucket=bucket_name, Key=f'response_json/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.json') - assert actual['Body'].read().decode('utf-8') == json.dumps(response_json) + assert actual['Body'].read().decode('utf-8') == json.dumps(response_json, ensure_ascii=False) # ログの確認 assert generate_log_message_tuple( diff --git a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py index 49d960c0..bf439dac 100644 --- a/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py +++ b/ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py @@ -34,7 +34,17 @@ class TestConvertCrmCsvDataProcess: ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', True) + ('IsDeleted', True), + ('PersonMailingAddress', OrderedDict([ + ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 + ('PersonMailingCity', 'New york city'), + ('PersonMailingState', 'Ohaio'), + ('PersonMailingPostalCode', '999-9999'), + ('PersonMailingCountry', 'US'), + ('PersonMailingLatitude', 50.1234567), + ('PersonMailingLongitude', 103.1234567), + ('PersonMailingGeocodeAccuracy', 'Address'), + ])), ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -42,9 +52,19 @@ class TestConvertCrmCsvDataProcess: ('Id', 'TEST002'), ('AccountNumber', 'test002'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), - ('LastModifiedById', 1.234567E+6), + ('LastModifiedById', 1.23E+0), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('PersonMailingAddress', OrderedDict([ + ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 + ('PersonMailingCity', 'New york city'), + ('PersonMailingState', 'Ohaio'), + ('PersonMailingPostalCode', '999-9999'), + ('PersonMailingCountry', 'US'), + ('PersonMailingLatitude', 50.1234567), + ('PersonMailingLongitude', 103.1234567), + ('PersonMailingGeocodeAccuracy', 'Address'), + ])), ]), OrderedDict([ ('attributes', OrderedDict([('type', 'Account'), @@ -52,9 +72,19 @@ class TestConvertCrmCsvDataProcess: ('Id', 'TEST003'), ('AccountNumber', 'test003'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), - ('LastModifiedById', 1.234567E+6), + ('LastModifiedById', 1.234567), ('SystemModstamp', '2022-06-01T00:00:00.000+0000'), - ('IsDeleted', False) + ('IsDeleted', False), + ('PersonMailingAddress', OrderedDict([ + ('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501 + ('PersonMailingCity', 'New york city'), + ('PersonMailingState', 'Ohaio'), + ('PersonMailingPostalCode', '999-9999'), + ('PersonMailingCountry', 'US'), + ('PersonMailingLatitude', 50.1234567), + ('PersonMailingLongitude', 103.1234567), + ('PersonMailingGeocodeAccuracy', 'Address'), + ])), ]), ] @@ -66,7 +96,8 @@ class TestConvertCrmCsvDataProcess: 'LastModifiedDate', 'LastModifiedById', 'SystemModstamp', - 'IsDeleted' + 'IsDeleted', + 'PersonMailingAddress' ] } @@ -79,10 +110,10 @@ class TestConvertCrmCsvDataProcess: # Assert expect_csv_string = """\ - "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\ - "TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\ - "TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ - "TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\ + "Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted","PersonMailingAddress"\r\n\ + "TEST001","test001","2022-06-01 09:00:00","1234567.0","2022-06-01 09:00:00","1","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ + "TEST002","test002","2022-06-01 09:00:00","1.23","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ + "TEST003","test003","2022-06-01 09:00:00","1.234567","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\ """ # 返り値の期待値チェック assert isinstance(actual_csv_string, str), 'CSV文字列が返却される' diff --git a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py index 66f5577e..6b17477f 100644 --- a/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py +++ b/ecs/crm-datafetch/tests/test_fetch_crm_data_process.py @@ -217,7 +217,7 @@ class TestFetchCrmDataProcess: 2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること 3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること Arranges: - - データ件数取得処理の最大リトライ試行回数を3に設定する + - データ件数取得処理の最大リトライ試行回数を4に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - データ件数取得処理の初回に接続タイムアウト例外が発生するようにする Expects: @@ -225,7 +225,7 @@ class TestFetchCrmDataProcess: - データ件数取得に失敗した旨のエラーが出力されない """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -236,8 +236,8 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_counter_inst.describe.side_effect = [1, 2, 3, 4] + mock_counter_inst.increment.side_effect = [2, 3, 4, 5] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' @@ -271,15 +271,15 @@ class TestFetchCrmDataProcess: 2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること 3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること Arranges: - - データ件数取得処理の最大リトライ試行回数を3に設定する + - データ件数取得処理の最大リトライ試行回数を4に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - - データ件数取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする + - データ件数取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする Expects: - 異常終了する - データ件数取得に失敗した旨のエラーが出力される """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -290,13 +290,13 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_counter_inst.describe.side_effect = [1, 2, 3, 4] + mock_counter_inst.increment.side_effect = [2, 3, 4, 5] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' mock_client_inst = mock_api_client.return_value - mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception] + mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception, exception] mock_client_inst.fetch_sf_data.return_value = common_expect # Act with pytest.raises(SalesforceAPIException) as e: @@ -304,16 +304,16 @@ class TestFetchCrmDataProcess: # Assert - # 取得は3回行われる - assert mock_counter_inst.describe.call_count == 3 - # 足し込みは2回のみ - assert mock_counter_inst.increment.call_count == 2 + # 取得は4回行われる + assert mock_counter_inst.describe.call_count == 4 + # 足し込みは3回のみ + assert mock_counter_inst.increment.call_count == 3 assert generate_log_message_tuple( log_level=logging.WARNING, log_message=expect_message) in caplog.record_tuples called_log_counts = len([log for log in caplog.messages if log == expect_message]) - assert called_log_counts == 2 + assert called_log_counts == 3 assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples assert e.value.error_id == 'E-FETCH-01' @@ -333,7 +333,7 @@ class TestFetchCrmDataProcess: 2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること 3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること Arranges: - - レコード取得処理の最大リトライ試行回数を3に設定する + - レコード取得処理の最大リトライ試行回数を4に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - レコード取得処理の初回に接続タイムアウト例外が発生するようにする Expects: @@ -341,7 +341,7 @@ class TestFetchCrmDataProcess: - データレコード取得に失敗した旨のエラーが出力されない """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1) @@ -352,8 +352,8 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_counter_inst.describe.side_effect = [1, 2, 3, 4] + mock_counter_inst.increment.side_effect = [2, 3, 4, 5] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' @@ -387,15 +387,15 @@ class TestFetchCrmDataProcess: 2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること 3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること Arranges: - - レコード取得処理の最大リトライ試行回数を3に設定する + - レコード取得処理の最大リトライ試行回数を4に設定する - timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する - - レコード取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする + - レコード取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする Expects: - 異常終了する - データレコード取得に失敗した旨のエラーが出力される """ - monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 3) + monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 4) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL', 1) monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_INTERVAL', 1) @@ -406,30 +406,30 @@ class TestFetchCrmDataProcess: patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client: # モック化 mock_counter_inst = mock_counter.return_value - mock_counter_inst.describe.side_effect = [1, 2, 3] - mock_counter_inst.increment.side_effect = [2, 3, 4] + mock_counter_inst.describe.side_effect = [1, 2, 3, 4] + mock_counter_inst.increment.side_effect = [2, 3, 4, 5] mock_builder_inst = mock_soql_builder.return_value mock_builder_inst.create_count_soql.return_value = '' mock_builder_inst.create_fetch_soql.return_value = '' mock_client_inst = mock_api_client.return_value mock_client_inst.fetch_sf_count.return_value = 1 - mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception] + mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception, exception] # Act with pytest.raises(SalesforceAPIException) as e: fetch_crm_data_process(common_target_object, common_last_fetch_datetime) # Assert - # 取得は3回行われる - assert mock_counter_inst.describe.call_count == 3 - # 足し込みは2回のみ - assert mock_counter_inst.increment.call_count == 2 + # 取得は4回行われる + assert mock_counter_inst.describe.call_count == 4 + # 足し込みは3回のみ + assert mock_counter_inst.increment.call_count == 3 assert generate_log_message_tuple( log_level=logging.WARNING, log_message=expect_message) in caplog.record_tuples called_log_counts = len([log for log in caplog.messages if log == expect_message]) - assert called_log_counts == 2 + assert called_log_counts == 3 assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples assert e.value.error_id == 'E-FETCH-02' diff --git a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py index 9471946f..8eb3d476 100644 --- a/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py @@ -7,6 +7,9 @@ from src.config.objects import FetchTargetObjects from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.prepare_data_fetch_process import prepare_data_fetch_process from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ +from src.system_var.environments import (CRM_CONFIG_BUCKET, + OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER) from .test_utils.log_message import generate_log_message_tuple @@ -80,6 +83,8 @@ class TestPrepareDataFetchProcess: # ログの確認 assert generate_log_message_tuple(log_message='I-PRE-01 データ取得準備処理を開始します') in caplog.record_tuples assert generate_log_message_tuple(log_message=f'I-PRE-02 データ取得処理開始日時：{str(actual_execute_datetime)}') in caplog.record_tuples + assert generate_log_message_tuple( + log_message=f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}]') in caplog.record_tuples assert generate_log_message_tuple(log_message='I-PRE-09 データ取得準備処理を終了します') in caplog.record_tuples def test_call_depended_modules(self): diff --git a/ecs/crm-datafetch/tests/test_walk_through.py b/ecs/crm-datafetch/tests/test_walk_through.py index e8c3b6e4..e2be6670 100644 --- a/ecs/crm-datafetch/tests/test_walk_through.py +++ b/ecs/crm-datafetch/tests/test_walk_through.py @@ -73,7 +73,6 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): object_info_list = get_object_config_list('object_info') for object_info in object_info_list: json_file = read_json(object_info) - json_file = to_upload_json(json_file) upload_json(json_file, s3_client, CONFIG_BUCKET, f'{OBJECT_INFO_FOLDER}/{path.basename(object_info)}') object_info_files.append(json_file) last_fetch_datetime_list = get_object_config_list('last_fetch_datetime') @@ -105,7 +104,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # Act logger = logging.getLogger() - logger.setLevel(logging.INFO) + logger.setLevel(logging.DEBUG) logger.info(f'##########################') logger.info(f'# 差分データ取得処理:実行開始 #') logger.info(f'##########################') @@ -118,7 +117,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): # ループ前のログ確認 assert 'I-CTRL-01 CRMデータ取得処理を開始します' in log_messages assert 'I-CTRL-02 データ取得準備処理呼び出し' in log_messages - assert_prepare_process_log(log_messages, now) + assert_prepare_process_log(log_messages, now, path.basename(object_info_list[0])) assert 'I-CTRL-03 取得対象オブジェクトのループ処理開始' in log_messages # オブジェクト情報を取得する(diff) object_info_list = object_info_files[0] @@ -207,9 +206,10 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog): """ -def assert_prepare_process_log(log_messages, now): +def assert_prepare_process_log(log_messages, now, object_info_list_filename): assert 'I-PRE-01 データ取得準備処理を開始します' in log_messages assert f'I-PRE-02 データ取得処理開始日時：{now}' in log_messages + assert f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{object_info_list_filename}]' in log_messages assert 'I-PRE-09 データ取得準備処理を終了します' in log_messages @@ -295,21 +295,6 @@ def upload_json(json_file, s3_client, bucket, folder): s3_client.put_object(Bucket=bucket, Key=folder, Body=json_str) -def to_upload_json(json_file): - """Userオブジェクトの取得できないプロパティを取り除く - TODO: Userオブジェクトの恒久対応が確定したらこのメソッドは消す - """ - for object_info in json_file['objects']: - if object_info['object_name'] != 'User': - continue - columns: list = object_info['columns'] - columns.remove('LastPasswordChangeDate') - columns.remove('NumberOfFailedLogins') - columns.remove('UserPreferencesNativeEmailClient') - - return json_file - - def set_environment(monkeypatch): # 環境変数を設定(CRMの認証情報は別途設定しておくこと) monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET) @@ -324,3 +309,4 @@ def set_environment(monkeypatch): monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER) monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER) + monkeypatch.setattr('src.prepare_data_fetch_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET) diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index a5f8da67..dda641e2 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -1645,13 +1645,11 @@ "DelegatedApproverId", "ManagerId", "LastLoginDate", - "LastPasswordChangeDate", "CreatedDate", "CreatedById", "LastModifiedDate", "LastModifiedById", "SystemModstamp", - "NumberOfFailedLogins", "OfflineTrialExpirationDate", "OfflinePdaTrialExpirationDate", "UserPermissionsMarketingUser", @@ -1725,7 +1723,6 @@ "UserPreferencesSRHOverrideActivities", "UserPreferencesNewLightningReportRunPageEnabled", "UserPreferencesReverseOpenActivitiesView", - "UserPreferencesNativeEmailClient", "UserPreferencesHideBrowseProductRedirectConfirmation", "UserPreferencesHideOnlineSalesAppWelcomeMat", "ContactId", diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env index 580bf564..3ae4d809 100644 --- a/s3/config/crm/task_settings/task_settings.env +++ b/s3/config/crm/task_settings/task_settings.env @@ -1,13 +1,14 @@ LOG_LEVEL=INFO CRM_AUTH_TIMEOUT=100 CRM_GET_RECORD_COUNT_TIMEOUT=300 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=3 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=4 CRM_GET_RECORD_COUNT_RETRY_INTERVAL=5 CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL=5 CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL=50 CRM_FETCH_RECORD_TIMEOUT=300 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=4 CRM_FETCH_RECORD_RETRY_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 -CONVERT_TZ='Asia/Tokyo' \ No newline at end of file +CONVERT_TZ=Asia/Tokyo +OBJECT_INFO_FILENAME=crm_object_list_diff.json \ No newline at end of file diff --git a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt index d5d6c0dc..813f86ee 100644 --- a/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt +++ b/s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt @@ -4,9 +4,9 @@ utf-8 " CRLF 1 -31 -Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c -id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c +32 +Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c,MSJ_Registration__c +id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c,msj_registration__c src02.crm_event_attendee_vod__c org02.crm_event_attendee_vod__c diff --git a/s3/data/crm/settings/CRM_User.txt b/s3/data/crm/settings/CRM_User.txt index a0d5c519..727d82e0 100644 --- a/s3/data/crm/settings/CRM_User.txt +++ b/s3/data/crm/settings/CRM_User.txt @@ -4,9 +4,9 @@ utf-8 " CRLF 1 -203 -Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,LastPasswordChangeDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,NumberOfFailedLogins,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesNativeEmailClient,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c -id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,last_password_change_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,number_of_failed_logins,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_native_email_client,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c +200 +Id,Username,LastName,FirstName,Name,CompanyName,Division,Department,Title,Street,City,State,PostalCode,Country,Latitude,Longitude,GeocodeAccuracy,Address,Email,EmailPreferencesAutoBcc,EmailPreferencesAutoBccStayInTouch,EmailPreferencesStayInTouchReminder,SenderEmail,SenderName,Signature,StayInTouchSubject,StayInTouchSignature,StayInTouchNote,Phone,Fax,MobilePhone,Alias,CommunityNickname,BadgeText,IsActive,TimeZoneSidKey,UserRoleId,LocaleSidKey,ReceivesInfoEmails,ReceivesAdminInfoEmails,EmailEncodingKey,ProfileId,UserType,LanguageLocaleKey,EmployeeNumber,DelegatedApproverId,ManagerId,LastLoginDate,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,OfflineTrialExpirationDate,OfflinePdaTrialExpirationDate,UserPermissionsMarketingUser,UserPermissionsOfflineUser,UserPermissionsWirelessUser,UserPermissionsAvantgoUser,UserPermissionsCallCenterAutoLogin,UserPermissionsSFContentUser,UserPermissionsInteractionUser,UserPermissionsSupportUser,UserPermissionsChatterAnswersUser,ForecastEnabled,UserPreferencesActivityRemindersPopup,UserPreferencesEventRemindersCheckboxDefault,UserPreferencesTaskRemindersCheckboxDefault,UserPreferencesReminderSoundOff,UserPreferencesDisableAllFeedsEmail,UserPreferencesApexPagesDeveloperMode,UserPreferencesReceiveNoNotificationsAsApprover,UserPreferencesReceiveNotificationsAsDelegatedApprover,UserPreferencesHideCSNGetChatterMobileTask,UserPreferencesHideCSNDesktopTask,UserPreferencesHideChatterOnboardingSplash,UserPreferencesHideSecondChatterOnboardingSplash,UserPreferencesShowTitleToExternalUsers,UserPreferencesShowManagerToExternalUsers,UserPreferencesShowEmailToExternalUsers,UserPreferencesShowWorkPhoneToExternalUsers,UserPreferencesShowMobilePhoneToExternalUsers,UserPreferencesShowFaxToExternalUsers,UserPreferencesShowStreetAddressToExternalUsers,UserPreferencesShowCityToExternalUsers,UserPreferencesShowStateToExternalUsers,UserPreferencesShowPostalCodeToExternalUsers,UserPreferencesShowCountryToExternalUsers,UserPreferencesShowProfilePicToGuestUsers,UserPreferencesShowTitleToGuestUsers,UserPreferencesShowCityToGuestUsers,UserPreferencesShowStateToGuestUsers,UserPreferencesShowPostalCodeToGuestUsers,UserPreferencesShowCountryToGuestUsers,UserPreferencesHideInvoicesRedirectConfirmation,UserPreferencesHideStatementsRedirectConfirmation,UserPreferencesPathAssistantCollapsed,UserPreferencesCacheDiagnostics,UserPreferencesShowEmailToGuestUsers,UserPreferencesShowManagerToGuestUsers,UserPreferencesShowWorkPhoneToGuestUsers,UserPreferencesShowMobilePhoneToGuestUsers,UserPreferencesShowFaxToGuestUsers,UserPreferencesShowStreetAddressToGuestUsers,UserPreferencesLightningExperiencePreferred,UserPreferencesPreviewLightning,UserPreferencesHideEndUserOnboardingAssistantModal,UserPreferencesHideLightningMigrationModal,UserPreferencesHideSfxWelcomeMat,UserPreferencesHideBiggerPhotoCallout,UserPreferencesGlobalNavBarWTShown,UserPreferencesGlobalNavGridMenuWTShown,UserPreferencesCreateLEXAppsWTShown,UserPreferencesFavoritesWTShown,UserPreferencesRecordHomeSectionCollapseWTShown,UserPreferencesRecordHomeReservedWTShown,UserPreferencesFavoritesShowTopFavorites,UserPreferencesExcludeMailAppAttachments,UserPreferencesSuppressTaskSFXReminders,UserPreferencesSuppressEventSFXReminders,UserPreferencesPreviewCustomTheme,UserPreferencesHasCelebrationBadge,UserPreferencesUserDebugModePref,UserPreferencesSRHOverrideActivities,UserPreferencesNewLightningReportRunPageEnabled,UserPreferencesReverseOpenActivitiesView,UserPreferencesHideBrowseProductRedirectConfirmation,UserPreferencesHideOnlineSalesAppWelcomeMat,ContactId,AccountId,CallCenterId,Extension,FederationIdentifier,AboutMe,FullPhotoUrl,SmallPhotoUrl,IsExtIndicatorVisible,OutOfOfficeMessage,MediumPhotoUrl,DigestFrequency,DefaultGroupNotificationFrequency,LastViewedDate,LastReferencedDate,BannerPhotoUrl,SmallBannerPhotoUrl,MediumBannerPhotoUrl,IsProfilePhotoActive,IndividualId,Last_Mobile_Connect_vod__c,Last_Tablet_Connect_vod__c,Last_Mobile_Connect_Version_vod__c,Last_Tablet_Connect_Version_vod__c,Last_Mobile_Sync_vod__c,Last_Tablet_Sync_vod__c,RaiseLoggingLevel_vod__c,SendDetailedLog_vod__c,Last_Blackberry_Connect_vod__c,Last_Blackberry_Connect_Version_vod__c,Last_Blackberry_Sync_vod__c,Force_Full_Refresh_vod__c,Override_SystemModstamp_Timestamp_vod__c,Facetime_Email_vod__c,Facetime_Phone_vod__c,Product_Expertise_vod__c,Available_vod__c,Available_Last_Update_vod__c,Last_iPad_Connect_Version_vod__c,Last_iPad_Connect_vod__c,Last_iPad_Sync_vod__c,Inventory_Order_Allocation_Group_vod__c,Concur_User_Id_vod__c,Last_iPad_iOS_Version_vod__c,Approved_Email_Admin_vod__c,Last_WinModern_Connect_Version_vod__c,Last_WinModern_Connect_vod__c,Last_WinModern_Sync_vod__c,Primary_Territory_vod__c,Analytics_Admin_vod__c,Content_Admin_vod__c,Last_WinModern_Windows_Version_vod__c,Upload_VTrans_vod__c,Sync_Frequency_vjh__c,Clear_Client_Sync_Errors_vod__c,Remote_Meeting_Host_Id_vod__c,Remote_Meeting_Host_Token_vod__c,Last_iPhone_Connect_Version_vod__c,Last_iPhone_Connect_vod__c,Last_iPhone_Sync_vod__c,Last_iPhone_iOS_Version_vod__c,Remote_Meeting_Start_From_CRM_Online_vod__c,Country_Code_vod__c,User_Type_vod__c,Engage_Group_Provisioning_Status_vod__c,Engage_Group_Request_vod__c,Engage_Group_vod__c,Last_CRMDesktop_Mac_Sync_vod__c,Last_CRMDesktop_Mac_Version_vod__c,Last_CRMDesktop_Windows_Sync_vod__c,Last_CRMDesktop_Windows_Version_vod__c,MSJ_Test_User__c +id,username,last_name,first_name,name,company_name,division,department,title,street,city,state,postal_code,country,latitude,longitude,geocode_accuracy,address,email,email_preferences_auto_bcc,email_preferences_auto_bcc_stay_in_touch,email_preferences_stay_in_touch_reminder,sender_email,sender_name,signature,stay_in_touch_subject,stay_in_touch_signature,stay_in_touch_note,phone,fax,mobile_phone,alias,community_nickname,badge_text,is_active,time_zone_sid_key,user_role_id,locale_sid_key,receives_info_emails,receives_admin_info_emails,email_encoding_key,profile_id,user_type,language_locale_key,employee_number,delegated_approver_id,manager_id,last_login_date,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,offline_trial_expiration_date,offline_pda_trial_expiration_date,user_permissions_marketing_user,user_permissions_offline_user,user_permissions_wireless_user,user_permissions_avantgo_user,user_permissions_call_center_auto_login,user_permissions_sfcontent_user,user_permissions_interaction_user,user_permissions_support_user,user_permissions_chatter_answers_user,forecast_enabled,user_preferences_activity_reminders_popup,user_preferences_event_reminders_checkbox_default,user_preferences_task_reminders_checkbox_default,user_preferences_reminder_sound_off,user_preferences_disable_all_feeds_email,user_preferences_apex_pages_developer_mode,user_preferences_receive_no_notifications_as_approver,user_preferences_receive_notifications_as_delegated_approver,user_preferences_hide_csnget_chatter_mobile_task,user_preferences_hide_csndesktop_task,user_preferences_hide_chatter_onboarding_splash,user_preferences_hide_second_chatter_onboarding_splash,user_preferences_show_title_to_external_users,user_preferences_show_manager_to_external_users,user_preferences_show_email_to_external_users,user_preferences_show_work_phone_to_external_users,user_preferences_show_mobile_phone_to_external_users,user_preferences_show_fax_to_external_users,user_preferences_show_street_address_to_external_users,user_preferences_show_city_to_external_users,user_preferences_show_state_to_external_users,user_preferences_show_postal_code_to_external_users,user_preferences_show_country_to_external_users,user_preferences_show_profile_pic_to_guest_users,user_preferences_show_title_to_guest_users,user_preferences_show_city_to_guest_users,user_preferences_show_state_to_guest_users,user_preferences_show_postal_code_to_guest_users,user_preferences_show_country_to_guest_users,user_preferences_hide_invoices_redirect_confirmation,user_preferences_hide_statements_redirect_confirmation,user_preferences_path_assistant_collapsed,user_preferences_cache_diagnostics,user_preferences_show_email_to_guest_users,user_preferences_show_manager_to_guest_users,user_preferences_show_work_phone_to_guest_users,user_preferences_show_mobile_phone_to_guest_users,user_preferences_show_fax_to_guest_users,user_preferences_show_street_address_to_guest_users,user_preferences_lightning_experience_preferred,user_preferences_preview_lightning,user_preferences_hide_end_user_onboarding_assistant_modal,user_preferences_hide_lightning_migration_modal,user_preferences_hide_sfx_welcome_mat,user_preferences_hide_bigger_photo_callout,user_preferences_global_nav_bar_wtshown,user_preferences_global_nav_grid_menu_wtshown,user_preferences_create_lexapps_wtshown,user_preferences_favorites_wtshown,user_preferences_record_home_section_collapse_wtshown,user_preferences_record_home_reserved_wtshown,user_preferences_favorites_show_top_favorites,user_preferences_exclude_mail_app_attachments,user_preferences_suppress_task_sfxreminders,user_preferences_suppress_event_sfxreminders,user_preferences_preview_custom_theme,user_preferences_has_celebration_badge,user_preferences_user_debug_mode_pref,user_preferences_srhoverride_activities,user_preferences_new_lightning_report_run_page_enabled,user_preferences_reverse_open_activities_view,user_preferences_hide_browse_product_redirect_confirmation,user_preferences_hide_online_sales_app_welcome_mat,contact_id,account_id,call_center_id,extension,federation_identifier,about_me,full_photo_url,small_photo_url,is_ext_indicator_visible,out_of_office_message,medium_photo_url,digest_frequency,default_group_notification_frequency,last_viewed_date,last_referenced_date,banner_photo_url,small_banner_photo_url,medium_banner_photo_url,is_profile_photo_active,individual_id,last_mobile_connect_vod__c,last_tablet_connect_vod__c,last_mobile_connect_version_vod__c,last_tablet_connect_version_vod__c,last_mobile_sync_vod__c,last_tablet_sync_vod__c,raise_logging_level_vod__c,send_detailed_log_vod__c,last_blackberry_connect_vod__c,last_blackberry_connect_version_vod__c,last_blackberry_sync_vod__c,force_full_refresh_vod__c,override_system_modstamp_timestamp_vod__c,facetime_email_vod__c,facetime_phone_vod__c,product_expertise_vod__c,available_vod__c,available_last_update_vod__c,last_i_pad_connect_version_vod__c,last_i_pad_connect_vod__c,last_i_pad_sync_vod__c,inventory_order_allocation_group_vod__c,concur_user_id_vod__c,last_i_pad_i_os_version_vod__c,approved_email_admin_vod__c,last_win_modern_connect_version_vod__c,last_win_modern_connect_vod__c,last_win_modern_sync_vod__c,primary_territory_vod__c,analytics_admin_vod__c,content_admin_vod__c,last_win_modern_windows_version_vod__c,upload_vtrans_vod__c,sync_frequency_vjh__c,clear_client_sync_errors_vod__c,remote_meeting_host_id_vod__c,remote_meeting_host_token_vod__c,last_i_phone_connect_version_vod__c,last_i_phone_connect_vod__c,last_i_phone_sync_vod__c,last_i_phone_i_os_version_vod__c,remote_meeting_start_from_crm_online_vod__c,country_code_vod__c,user_type_vod__c,engage_group_provisioning_status_vod__c,engage_group_request_vod__c,engage_group_vod__c,last_crmdesktop_mac_sync_vod__c,last_crmdesktop_mac_version_vod__c,last_crmdesktop_windows_sync_vod__c,last_crmdesktop_windows_version_vod__c,msj_test_user__c src02.crm_user org02.crm_user From 4349805824188644790e4bec7fc584ceb90a6dbc Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Tue, 13 Sep 2022 16:00:46 +0900 Subject: [PATCH 274/275] =?UTF-8?q?style:=E3=83=97=E3=83=AA=E3=83=9A?= =?UTF-8?q?=E3=82=A2=E3=83=BC=E3=83=89=E3=82=B9=E3=83=86=E3=83=BC=E3=83=88?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E5=90=8C=E7=AD=89=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index d86ee84e..eaf4b7af 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -129,6 +129,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf process_count += 1 # SQL文生成 + sql_arg = [] sql = f'INSERT INTO {settings_list[SETTINGS_ITEM["loadSchemaName"]]} (' for i in range(len(settings_db_columu_list)): sql = f'{sql} {settings_db_columu_list[i]},' @@ -144,9 +145,8 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # データ項目値が0桁より大きいかチェックする if len(line[i]) > 0: # 0桁より大きい場合 - replace_line = line[i].replace('\\', '\\\\') - replace_line = line[i].replace('"', '\\"') - sql = f'{sql} "{replace_line}",' + sql_arg.append(line[i]) + sql = f'{sql} %s,' else: # 上記以外の場合 sql = f'{sql} NULL,' @@ -164,7 +164,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # ロードスキーマのトランザクション開始 with conn.cursor() as cur: - cur.execute(sql) + cur.execute(sql, sql_arg) conn.commit() normal_count += 1 except Exception as e: From a976cadfbbcb5f6f60ae29b55802064abf8d5de7 Mon Sep 17 00:00:00 2001 From: y-ono-r <95060536+y-ono-r@users.noreply.github.com> Date: Wed, 14 Sep 2022 09:18:20 +0900 Subject: [PATCH 275/275] =?UTF-8?q?style:=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index eaf4b7af..281c1728 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -129,7 +129,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf process_count += 1 # SQL文生成 - sql_arg = [] + query_parameter_list = [] sql = f'INSERT INTO {settings_list[SETTINGS_ITEM["loadSchemaName"]]} (' for i in range(len(settings_db_columu_list)): sql = f'{sql} {settings_db_columu_list[i]},' @@ -145,7 +145,8 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # データ項目値が0桁より大きいかチェックする if len(line[i]) > 0: # 0桁より大きい場合 - sql_arg.append(line[i]) + # INSERT文のパラメータとそれに対応するプレースホルダーを設定する + query_parameter_list.append(line[i]) sql = f'{sql} %s,' else: # 上記以外の場合 @@ -164,7 +165,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # ロードスキーマのトランザクション開始 with conn.cursor() as cur: - cur.execute(sql, sql_arg) + cur.execute(sql, query_parameter_list) conn.commit() normal_count += 1 except Exception as e: