From f2650c1ac89f0317f8edcbf7889d4567aabc93d4 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 4 Jul 2022 14:40:57 +0900 Subject: [PATCH 01/21] =?UTF-8?q?feat:=20CRM=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E5=8F=96=E5=BE=97=E3=83=95=E3=82=A1=E3=83=BC=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=82=B3=E3=83=9F=E3=83=83=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Dockerfile | 12 ++ ecs/crm-datafetch/Pipfile | 4 + ecs/crm-datafetch/Pipfile.lock | 0 ecs/crm-datafetch/main.py | 181 ++++++++++++++++++ ecs/crm-datafetch/src/__init__.py | 0 ecs/crm-datafetch/src/chk.py | 0 ecs/crm-datafetch/src/conv.py | 0 ecs/crm-datafetch/src/csvbk.py | 0 ecs/crm-datafetch/src/date.py | 0 ecs/crm-datafetch/src/end.py | 0 ecs/crm-datafetch/src/fetch.py | 0 ecs/crm-datafetch/src/pre.py | 96 ++++++++++ ecs/crm-datafetch/src/resbk.py | 0 ecs/crm-datafetch/src/upd.py | 0 ecs/crm-datafetch/src/upld.py | 0 ecs/crm-datafetch/src/utils/__init__.py | 0 ecs/crm-datafetch/src/utils/data_retention.py | 169 ++++++++++++++++ ecs/crm-datafetch/src/utils/json_parser.py | 28 +++ ecs/crm-datafetch/src/utils/logger.py | 33 ++++ ecs/crm-datafetch/src/utils/s3_access.py | 48 +++++ .../crm/object_info/crm_object_list_diff.json | 41 ++++ s3/config/crm/task_settings/task_settings.env | 16 ++ 22 files changed, 628 insertions(+) create mode 100644 ecs/crm-datafetch/Dockerfile create mode 100644 ecs/crm-datafetch/Pipfile create mode 100644 ecs/crm-datafetch/Pipfile.lock create mode 100644 ecs/crm-datafetch/main.py create mode 100644 ecs/crm-datafetch/src/__init__.py create mode 100644 ecs/crm-datafetch/src/chk.py create mode 100644 ecs/crm-datafetch/src/conv.py create mode 100644 ecs/crm-datafetch/src/csvbk.py create mode 100644 ecs/crm-datafetch/src/date.py create mode 100644 ecs/crm-datafetch/src/end.py create mode 100644 ecs/crm-datafetch/src/fetch.py create mode 100644 ecs/crm-datafetch/src/pre.py create mode 100644 ecs/crm-datafetch/src/resbk.py create mode 100644 ecs/crm-datafetch/src/upd.py create mode 100644 ecs/crm-datafetch/src/upld.py create mode 100644 ecs/crm-datafetch/src/utils/__init__.py create mode 100644 ecs/crm-datafetch/src/utils/data_retention.py create mode 100644 ecs/crm-datafetch/src/utils/json_parser.py create mode 100644 ecs/crm-datafetch/src/utils/logger.py create mode 100644 ecs/crm-datafetch/src/utils/s3_access.py create mode 100644 s3/config/crm/object_info/crm_object_list_diff.json create mode 100644 s3/config/crm/task_settings/task_settings.env diff --git a/ecs/crm-datafetch/Dockerfile b/ecs/crm-datafetch/Dockerfile new file mode 100644 index 00000000..e84b79b3 --- /dev/null +++ b/ecs/crm-datafetch/Dockerfile @@ -0,0 +1,12 @@ +FROM python:3.8 + +ENV TZ="Asia/Tokyo" + +WORKDIR /usr/src/app +COPY Pipfile Pipfile.lock ./ +RUN pip install --no-cache-dir -r requirements.txt && \ + cp /usr/share/zoneinfo/Asia/Tokyo /etc/localtime +COPY main.py ./ +COPY crm-datafetch /usr/src + +CMD [ "python", "./main.py" ] diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile new file mode 100644 index 00000000..ec9569f7 --- /dev/null +++ b/ecs/crm-datafetch/Pipfile @@ -0,0 +1,4 @@ +[packages] +boto3 +simple-salesforce +tenacity \ No newline at end of file diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/main.py b/ecs/crm-datafetch/main.py new file mode 100644 index 00000000..0170f413 --- /dev/null +++ b/ecs/crm-datafetch/main.py @@ -0,0 +1,181 @@ +# モジュールロード +import os +import sys +from datetime import datetime +import src.pre as pre # データ取得準備処理 +import src.chk as chk # オブジェクト情報形式チェック処理 +import src.date as date # データ取得期間設定処理 +import src.fetch as fetch # CRMデータ取得処理 +import src.resbk as resbk # CRM電文データバックアップ処理 +import src.conv as conv # CSV変換処理 +import src.csvbk as csvbk # CSVバックアップ処理 +import src.upld as upld # CSVアップロード処理 +import src.upd as upd # 前回取得日時ファイル更新 +import src.end as end # 取得処理実施結果アップロード処理 + +from src.utils.logger import Logger +import src.utils.data_retention as data_retention + + +# 変数 + +# logger設定 +logger = Logger().get_logger() + + +# 処理 +def main() -> None: + """コントロール処理 + + 各処理を呼び出すコントローラー + + """ + + try: + # ① CRMデータ取得処理開始ログを出力する + logger.info('I-CTRL-01 CRMデータ取得処理を開始します') + + # ② データ取得準備処理を呼び出す + try: + logger.info('I-CTRL-02 データ取得準備処理呼び出し') + crm_object_info_var = data_retention.CrmObjectInfoVar() + crm_object_info_dict, execute_datetime, process_result_per_object = pre( + crm_object_info_var) + + except Exception as e: + logger.error('') + raise CustomException(e) + + # ③ object_infoのobjectsキーの値の件数分ループする + logger.info('I-CTRL-03 取得対象オブジェクトのループ処理開始') + + for object_info in crm_object_info_dict.get('objects'): + try: + + # 1. オブジェクト情報形式チェック処理を呼び出す + logger.info('I-CTRL-04 オブジェクト情報形式チェック処理呼び出し') + + try: + chk() + + except Exception as e: + logger.error('') + + # 2. 処理対象のオブジェクト名をログ出力する + logger.info('I-CTRL-05 ') + + # 3. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する + logger.info('I-CTRL-06 ') + + # 4. データ取得期間設定処理を呼び出す + logger.info('I-CTRL-07 ') + try: + + except Exception as e: + logger.error('') + + # 5. CRMデータ取得処理を呼び出す + logger.info('I-CTRL-08 ') + try: + + except Exception as e: + logger.error('') + + # 6. CRMデータのジェネレータをループする + # 6-1. 出力ファイル名を生成する + logger.info('I-CTRL-10 ') + try: + + except Exception as e: + logger.error('') + + # 6-2. CRM電文データバックアップ処理を呼び出す + logger.info('I-CTRL-11 ') + try: + + except Exception as e: + logger.error('') + + # 6-3. CSV変換処理を呼び出す + logger.info('I-CTRL-12 ') + try: + + except Exception as e: + logger.error('') + + # 6-4. CSVバックアップ処理を呼び出す + logger.info('I-CTRL-13 ') + try: + + except Exception as e: + logger.error('') + + # 6-5. CSVアップロード処理を呼び出す + logger.info('I-CTRL-14 ') + try: + + except Exception as e: + logger.error('') + + # 7. 前回取得日時ファイル更新処理を呼びだす + logger.info('I-CTRL-15 ') + try: + + except Exception as e: + logger.error('') + + # 8. 1オブジェクトのアップロードが完了した旨をログに出力する + logger.info('I-CTRL-16 ') + + except GetObjectException as e: + logger.error( + '[取得対象のオブジェクト名] の[処理名]でエラーが発生しました 次のオブジェクトの処理に移行します') + + except Exception as e: + logger.error( + '[取得対象のオブジェクト名] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します') + + # ④ すべてのオブジェクトの処理が完了したことと、オブジェクト毎の処理結果をログに出力する + try: + logger.info(f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{実行結果JSON}]') + + except Exception as e: + logger.error('') + raise CustomException() + + # ⑤ 取得処理実施結果アップロード処理を呼び出す + try: + logger.info('I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始') + + except Exception as e: + logger.error('') + raise CustomException() + + # ⑥ CRMデータ取得処理終了ログを出力する + try: + logger.info('I-CTRL-19 CRMデータ取得処理を終了します') + + except Exception as e: + logger.error('') + raise + + except CustomException as e: + logger.error('I-ERR-01 [処理名]でエラーが発生したため、処理を終了します') + + except Exception as e: + logger.error('I-ERR-02 予期せぬエラーが発生したため、処理を終了します') + + +# カスタムExceptionクラス +class CustomException(Exception, metaclass=ABCMeta): + def __init__(self, id, arg): + self.arg = arg + self.id = id + + +class GetObjectException(CustomException): + pass + + +if __name__ == '__main__': + main() diff --git a/ecs/crm-datafetch/src/__init__.py b/ecs/crm-datafetch/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/chk.py b/ecs/crm-datafetch/src/chk.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/conv.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/csvbk.py b/ecs/crm-datafetch/src/csvbk.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/date.py b/ecs/crm-datafetch/src/date.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/end.py b/ecs/crm-datafetch/src/end.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/pre.py b/ecs/crm-datafetch/src/pre.py new file mode 100644 index 00000000..546cbc37 --- /dev/null +++ b/ecs/crm-datafetch/src/pre.py @@ -0,0 +1,96 @@ +# モジュールロード +from datetime import datetime +from src.utils.logger import Logger +import src.utils.data_retention as data_retention +import src.utils.s3_access as s3_access +import src.utils.json_parser as json_parser + + +# 変数 +# .000ZはUTCを表す。ミリ秒までの考慮は不要なので固定で指定 +YYYYMMDDTHHMMSSTZ = '%Y-%m-%dT%H:%M:%S.000Z' +CRM_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.000%z' +YYYYMMDDHHMMSS = '%Y-%m-%d %H:%M:%S' + + +# logger設定 +logger = Logger().get_logger() + + +# 処理 + + +def pre(crm_object_info_var): + try: + # ① データ取得準備処理の開始ログを出力する + logger.info('I-PRE-01 データ取得準備処理を開始します') + + # ② 取得処理開始年月日時分秒を控える + execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) + logger.info(f'I-PRE-02 データ取得処理開始日時：{execute_datetime}') + + # ⑤ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する + + try: + crm_object_info_var = data_retention.CrmObjectInfoVar() + s3_fullpath = f's3://{crm_object_info_var.bucket}/{crm_object_info_var.file_name_key}' + logger.debug(f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{s3_fullpath}]') + + get_file_s3 = s3_access.GetFileS3() + crm_object_info_json = get_file_s3.get_file_from_s3(crm_object_info_var.bucket, crm_object_info_var.file_name_key) + + logger.debug('D-PRE-04 CRM_取得オブジェクト情報ファイルの取得成功しました') + + except Exception as e: + logger.error(f'E-PRE-01 CRM_取得オブジェクト情報ファイルが存在しません　ファイル名：[{s3_fullpath}]', e) + raise e + + + # ⑥ CRM_取得オブジェクト情報ファイルをパースし、メモリ上に展開する + try: + logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします') + + json_exclusde_comment = json_parser.JsonExculudeComment() + crm_object_info_dict: dict = json_exclusde_comment(crm_object_info_json) + + logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました') + + except Exception as e: + logger.error(f'E-PRE-02 CRM_取得オブジェクト情報ファイルのパースに失敗しました　エラー内容：[{e}]') + raise e + + # ⑦ メモリ上のCRM_取得オブジェクト情報のキーobjectsの形式チェックを行う + try: + logger.debug('D-PRE-07 CRM_取得オブジェクト情報ファイルの形式チェックを開始します') + + if 'objects' not in crm_object_info_dict: + logger.error(f'E-PRE-03 CRM_取得オブジェクト情報ファイル「objects」キーは必須です　ファイル名：[{s3_fullpath}]') + raise e + + if not isinstance(crm_object_info_dict["objects"], list): + logger.error(f'E-PRE-04 CRM_取得オブジェクト情報ファイル「objects」キーの値は配列でなければなりません　ファイル名：[{s3_fullpath}]') + raise e + + logger.debug('D-PRE-07 CRM_取得オブジェクト情報ファイルの形式チェック 正常終了') + + except Exception as e: + logger.error(f'CRM_取得オブジェクト情報ファイル「objects」キーの値は配列でなければなりません　ファイル名：[{s3_fullpath}]', e) + raise e + + # ⑧ 処理結果出力用のマップを初期化 + process_result_per_object = {} + for sfdc_object in crm_object_info_dict.get('objects'): + object_name = sfdc_object.get('object_name') + process_result_per_object[object_name] = 'fail' + + + + # ⑨ データ取得準備処理の終了ログを出力する + logger.info('I-PRE-09 データ取得準備処理を終了します') + + # ⑩ 次の処理へ移行する + return(crm_object_info_dict, execute_datetime, process_result_per_object) + + except Exception as e: + logger.error(f'データ取得準備処理で想定外のエラーが発生しました　エラー内容：[{e}]', e) + raise e diff --git a/ecs/crm-datafetch/src/resbk.py b/ecs/crm-datafetch/src/resbk.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upd.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/upld.py b/ecs/crm-datafetch/src/upld.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/utils/__init__.py b/ecs/crm-datafetch/src/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/utils/data_retention.py b/ecs/crm-datafetch/src/utils/data_retention.py new file mode 100644 index 00000000..89d7d4c3 --- /dev/null +++ b/ecs/crm-datafetch/src/utils/data_retention.py @@ -0,0 +1,169 @@ +# モジュールロード +import dataclasses +#from fileinput import filename +import os +#from tracemalloc import DomainFilter + +# 変数 +# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +LOG_LEVEL = os.environ["LOG_LEVEL"] + +CRM_AUTH_TIMEOUT = os.environ["CRM_AUTH_TIMEOUT"] # CRMへの認証処理のタイムアウト秒数 +# CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ["CRM_AUTH_MAX_RETRY_ATTEMPT"] +# CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_INTERVAL = os.environ["CRM_AUTH_RETRY_INTERVAL"] +# CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = os.environ["CRM_AUTH_RETRY_MIN_INTERVAL"] +# CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = os.environ["CRM_AUTH_RETRY_MAX_INTERVAL"] + +# CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = os.environ["CRM_GET_RECORD_COUNT_TIMEOUT"] +# CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ["CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT"] +# CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_INTERVAL"] +# CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL"] +# CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL"] + +# CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_TIMEOUT = os.environ["CRM_FETCH_RECORD_TIMEOUT"] +# CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ["CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT"] +# CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_INTERVAL"] +# CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MIN_INTERVAL"] +# CRMのレコード取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MAX_INTERVAL"] + + +CRM_AUTH_DOMAIN = os.environ["CRM_AUTH_DOMAIN"] # CRMのAPI実行のための認証エンドポイントのドメイン +CRM_USER_NAME = os.environ["CRM_USER_NAME"] # CRMのAPI実行用ユーザ名 +CRM_USER_PASSWORD = os.environ["CRM_USER_PASSWORD"] # CRMのAPI実行用ユーザパスワード +# CRMのAPI実行用ユーザのセキュリティトークン +CRM_USER_SECURITY_TOKEN = os.environ["CRM_USER_SECURITY_TOKEN"] + +# CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_CONFIG_BUCKET = os.environ["CRM_CONFIG_BUCKET"] +CRM_BACKUP_BUCKET = os.environ["CRM_BACKUP_BUCKET"] # CRMのバックアップデータを格納するバケット名 +IMPORT_DATA_BUCKET = os.environ["IMPORT_DATA_BUCKET"] # CRMの取込データを格納するバケット名 + +# TASK_SETTING_FOLDER = os.environ["TASK_SETTING_FOLDER"] # CRM取得処理タスクの設定ファイルを格納するフォルダパス +# TASK_SETTING_FILENAME = os.environ["TASK_SETTING_FILENAME"] # CRM取得処理タスクの設定ファイルのファイル名 + +# CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FOLDER = os.environ["OBJECT_INFO_FOLDER"] +# CRM取得対象オブジェクトの情報のファイル名 +OBJECT_INFO_FILENAME = os.environ["OBJECT_INFO_FILENAME"] + +# CRMデータ取得結果を格納するフォルダパス +FETCH_RESULT_FOLDER = os.environ["FETCH_RESULT_FOLDER"] +# CRMデータ取得結果を格納するファイル名 +FETCH_RESULT_FILENAME = os.environ["FETCH_RESULT_FILENAME"] + +# CRMからの最終取得日時ファイルを格納するフォルダパス +LAST_FETCH_DATE_FOLDER = os.environ["LAST_FETCH_DATE_FOLDER"] +# CRMから取得し、取込用に変換したデータを格納するフォルダ +CRM_IMPORT_DATA_FOLDER = os.environ["CRM_IMPORT_DATA_FOLDER"] + +# CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +LAST_FETCH_DATE_BACKUP_FOLDER = os.environ["LAST_FETCH_DATE_BACKUP_FOLDER"] +# CRMから取得した生データのバックアップを格納するフォルダパス +FETCH_DATA_BACKUP_FOLDER = os.environ["FETCH_DATA_BACKUP_FOLDER"] +# CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ +CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ["CRM_IMPORT_DATA_BACKUP_FOLDER"] + + +# 処理 +@dataclasses.dataclass +class LogLevel: + loglevel: str = LOG_LEVEL + + +@dataclasses.dataclass +class CrmAuthVar: + auth_domain: str = CRM_AUTH_DOMAIN + user_name: str = CRM_USER_NAME + user_password: str = CRM_USER_PASSWORD + security_token: str = CRM_USER_SECURITY_TOKEN + timeout: int = CRM_AUTH_TIMEOUT + max_retry_attempt: int = CRM_AUTH_MAX_RETRY_ATTEMPT + retry_interval: int = CRM_AUTH_RETRY_INTERVAL + retry_min_interval: int = CRM_AUTH_RETRY_MIN_INTERVAL + retry_max_interval: int = CRM_AUTH_RETRY_MAX_INTERVAL + + +@dataclasses.dataclass +class CrmGetRecordCountVar: + timeout: int = CRM_GET_RECORD_COUNT_TIMEOUT + max_retry_attempt: int = CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT + retry_interval: int = CRM_GET_RECORD_COUNT_RETRY_INTERVAL + retry_min_interval: int = CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL + retry_max_interval: int = CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL + + +@dataclasses.dataclass +class CrmFetchRecordVar: + timeout: int = CRM_FETCH_RECORD_TIMEOUT + max_retry_attempt: int = CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT + retry_interval: int = CRM_FETCH_RECORD_RETRY_INTERVAL + retry_min_interval: int = CRM_FETCH_RECORD_RETRY_MIN_INTERVAL + retry_max_interval: int = CRM_FETCH_RECORD_RETRY_MAX_INTERVAL + + +@dataclasses.dataclass +class CrmBucketVar: + config_bucket: str = CRM_CONFIG_BUCKET + backup_bucket: str = CRM_BACKUP_BUCKET + data_bucket: str = IMPORT_DATA_BUCKET + + +@dataclasses.dataclass +class CrmObjectInfoVar: + bucket: str = CRM_CONFIG_BUCKET + folder: str = OBJECT_INFO_FOLDER + file_name: str = OBJECT_INFO_FILENAME + file_name_key: str = f'{folder}/{file_name}' + + +@dataclasses.dataclass +class FetchResultVar: + folder: str = FETCH_RESULT_FOLDER + file_name: str = FETCH_RESULT_FILENAME + file_name_key: str = f'{folder}/{file_name}' + + +@dataclasses.dataclass +class FetchResultVar: + folder: str = FETCH_RESULT_FOLDER + file_name: str = FETCH_RESULT_FILENAME + file_name_key: str = f'{folder}/{file_name}' + + +@dataclasses.dataclass +class LastFetchDateVar: + folder: str = LAST_FETCH_DATE_FOLDER + + +@dataclasses.dataclass +class CrmImportDataVar: + folder: str = CRM_IMPORT_DATA_FOLDER + + +@dataclasses.dataclass +class LastFetchDateBackupVar: + folder: str = LAST_FETCH_DATE_BACKUP_FOLDER + + +@dataclasses.dataclass +class FetchDataBackupVar: + folder: str = FETCH_DATA_BACKUP_FOLDER + + +@dataclasses.dataclass +class CrmImportDataBackupVar: + folder: str = CRM_IMPORT_DATA_BACKUP_FOLDER diff --git a/ecs/crm-datafetch/src/utils/json_parser.py b/ecs/crm-datafetch/src/utils/json_parser.py new file mode 100644 index 00000000..b6d288a9 --- /dev/null +++ b/ecs/crm-datafetch/src/utils/json_parser.py @@ -0,0 +1,28 @@ +# モジュールロード +import re +import json + + +# 変数 +SYMBOL = ['#', '/'] + + +# 処理 +class JsonExculudeComment: + def __init__(self) -> None: + pass + + def json_exclude_comment(self, json_str) -> dict: + for symbol in SYMBOL: + # コメントアウトシンボルを含む部分を置き換える正規表現 + replace_comment_regex = rf'\s(?!\"){symbol}[\s\S]*?.*' + + json_str = self.json_regex_parse(replace_comment_regex, json_str) + + result = json.loads(json_str) + + return result + + def json_regex_parse(replace_comment_regex, json_str) -> str: + json_without_comment = re.sub(replace_comment_regex, '', json_str) + return json_without_comment diff --git a/ecs/crm-datafetch/src/utils/logger.py b/ecs/crm-datafetch/src/utils/logger.py new file mode 100644 index 00000000..5ec076fd --- /dev/null +++ b/ecs/crm-datafetch/src/utils/logger.py @@ -0,0 +1,33 @@ +import logging +import os + +LOG_LEVEL = { + 'critical': logging.CRITICAL, + 'error': logging.ERROR, + 'warn': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG +} + + +class Logger(): + __logger: logging.Logger + + def __init__(self): + # logger設定 + self.__logger = logging.getLogger() + formatter = logging.Formatter( + '[%(levelname)s]\t%(asctime)s\t%(message)s\n', + '%Y-%m-%d %H:%M:%S' + ) + handler = logging.StreamHandler() + self.__logger.addHandler(handler) + for handler in self.__logger.handlers: + handler.setFormatter(formatter) + level = logging.getLevelName(os.environ.get('LOG_LEVEL', 'WARN').upper()) + if not isinstance(level, int): + level = logging.INFO + self.__logger.setLevel(level) + + def get_logger(self) -> logging.Logger: + return self.__logger diff --git a/ecs/crm-datafetch/src/utils/s3_access.py b/ecs/crm-datafetch/src/utils/s3_access.py new file mode 100644 index 00000000..7ffb697d --- /dev/null +++ b/ecs/crm-datafetch/src/utils/s3_access.py @@ -0,0 +1,48 @@ +# モジュールロード +import boto3 +from tenacity import retry, stop_after_attempt +from tenacity.wait import wait_exponential + +# 変数 +s3_client = boto3.client('s3') +s3_resource = boto3.resource('s3') + +# 処理 + + +class UploadFileS3: + def __init__(self, bucket_name, local_file_path, file_name_key) -> None: + self.bucket_name: str = bucket_name + self.local_file_path: str = local_file_path + self.file_name_key: str = file_name_key + + def upload_file_to_s3(self) -> None: + bucket = s3_resource.Bucket(self.bucket_name) + bucket.upload_file(self.local_file_path, self.file_name_key) + + +class GetFileS3: + def __init__(self, bucket_name, file_name_key, multiplier=5, min=5, max=50, count=3) -> None: + self.bucket_name: str = bucket_name + self.file_name_key: str = file_name_key + self.multiplier: int = multiplier + self.min: int = min + self.max: int = max + self.count: int = count + + def get_file_from_s3(self) -> str: + bucket = s3_resource.Bucket(self.bucket_name) + response = bucket.Object(self.file_name_key).get() + body = response['Body'].read() + + return body.decode('utf-8') + + def get_file_from_s3_with_retry(self) -> str: + + @retry(wait=wait_exponential(multiplier=self.multiplier, min=self.min, max=self.max), stop=stop_after_attempt(self.count)) + def get_file_from_s3_with_retry_deco(*args, **kwargs) -> str: + return self.get_file_from_s3(self.bucket_name, self.file_name_key) + + response = get_file_from_s3_with_retry_deco(self) + + return response diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json new file mode 100644 index 00000000..78ec31a7 --- /dev/null +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -0,0 +1,41 @@ +{ + "objects": [ + { + # これはコメント行です + "object_name": "Account", + "columns": [ + "Id", + "Name", + "SystemModStamp", + "NumberOfEmployees", + "IsDeleted" + ] + }, + { + "object_name": "Territory2", + "is_skip": false, + "can_update_last_update": false, + "upload_file_name": "CRM_Territory2_All_{execute_datetime}_{split_csv_suffix_number:03}", + "last_fetch_datetime_file_name": "Territory2.json", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "SystemModStamp" + ] + }, + { + "object_name": "AccountShare", + "is_skip": false, + "datetime_column": "LastModifiedDate", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "LastModifiedDate" + ] + } + ] + } + + \ No newline at end of file diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env new file mode 100644 index 00000000..d48ff51a --- /dev/null +++ b/s3/config/crm/task_settings/task_settings.env @@ -0,0 +1,16 @@ +LOG_LEVEL=INFO +CRM_AUTH_TIMEOUT=100 +CRM_AUTH_MAX_RETRY_ATTEMPT=3 +CRM_AUTH_RETRY_INTERVAL=5 +CRM_AUTH_RETRY_MIN_INTERVAL=5 +CRM_AUTH_RETRY_MAX_INTERVAL=50 +CRM_GET_RECORD_COUNT_TIMEOUT=300 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=3 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL=5 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL=5 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL=50 +CRM_FETCH_RECORD_TIMEOUT=300 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3 +CRM_FETCH_RECORD_RETRY_INTERVAL=5 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 \ No newline at end of file From a7426900b73be088bb4905938774894354f90166 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 19 Jul 2022 20:16:40 +0900 Subject: [PATCH 02/21] =?UTF-8?q?feat:=E6=A9=9F=E8=83=BD=E5=85=A8=E8=88=AC?= =?UTF-8?q?=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Dockerfile | 7 + ecs/crm-datafetch/main.py | 181 +--------------- .../src/{utils => aws}/__init__.py | 0 ecs/crm-datafetch/src/aws/s3.py | 107 ++++++++++ ecs/crm-datafetch/src/chk.py | 30 +++ ecs/crm-datafetch/src/config/objects.py | 195 ++++++++++++++++++ ecs/crm-datafetch/src/constants.py | 73 +++++++ ecs/crm-datafetch/src/conv.py | 33 +++ ecs/crm-datafetch/src/converter/converter.py | 138 +++++++++++++ ecs/crm-datafetch/src/csvbk.py | 34 +++ ecs/crm-datafetch/src/date.py | 65 ++++++ ecs/crm-datafetch/src/end.py | 37 ++++ ecs/crm-datafetch/src/environments.py | 73 +++++++ ecs/crm-datafetch/src/error/__init__.py | 0 ecs/crm-datafetch/src/error/exceptions.py | 45 ++++ ecs/crm-datafetch/src/fetch.py | 71 +++++++ ecs/crm-datafetch/src/parser/__init__.py | 0 ecs/crm-datafetch/src/parser/json_parse.py | 15 ++ ecs/crm-datafetch/src/pre.py | 153 +++++++------- ecs/crm-datafetch/src/process.py | 150 ++++++++++++++ ecs/crm-datafetch/src/resbk.py | 35 ++++ ecs/crm-datafetch/src/salesforce/__init__.py | 0 .../src/salesforce/salesforce_api.py | 78 +++++++ ecs/crm-datafetch/src/upd.py | 49 +++++ ecs/crm-datafetch/src/upld.py | 43 ++++ ecs/crm-datafetch/src/util/__init__.py | 0 ecs/crm-datafetch/src/util/dict_checker.py | 64 ++++++ ecs/crm-datafetch/src/util/logger.py | 29 +++ ecs/crm-datafetch/src/utils/data_retention.py | 169 --------------- ecs/crm-datafetch/src/utils/json_parser.py | 28 --- ecs/crm-datafetch/src/utils/logger.py | 33 --- ecs/crm-datafetch/src/utils/s3_access.py | 48 ----- 32 files changed, 1443 insertions(+), 540 deletions(-) rename ecs/crm-datafetch/src/{utils => aws}/__init__.py (100%) create mode 100644 ecs/crm-datafetch/src/aws/s3.py create mode 100644 ecs/crm-datafetch/src/config/objects.py create mode 100644 ecs/crm-datafetch/src/constants.py create mode 100644 ecs/crm-datafetch/src/converter/converter.py create mode 100644 ecs/crm-datafetch/src/environments.py create mode 100644 ecs/crm-datafetch/src/error/__init__.py create mode 100644 ecs/crm-datafetch/src/error/exceptions.py create mode 100644 ecs/crm-datafetch/src/parser/__init__.py create mode 100644 ecs/crm-datafetch/src/parser/json_parse.py create mode 100644 ecs/crm-datafetch/src/process.py create mode 100644 ecs/crm-datafetch/src/salesforce/__init__.py create mode 100644 ecs/crm-datafetch/src/salesforce/salesforce_api.py create mode 100644 ecs/crm-datafetch/src/util/__init__.py create mode 100644 ecs/crm-datafetch/src/util/dict_checker.py create mode 100644 ecs/crm-datafetch/src/util/logger.py delete mode 100644 ecs/crm-datafetch/src/utils/data_retention.py delete mode 100644 ecs/crm-datafetch/src/utils/json_parser.py delete mode 100644 ecs/crm-datafetch/src/utils/logger.py delete mode 100644 ecs/crm-datafetch/src/utils/s3_access.py diff --git a/ecs/crm-datafetch/Dockerfile b/ecs/crm-datafetch/Dockerfile index e84b79b3..88e588e1 100644 --- a/ecs/crm-datafetch/Dockerfile +++ b/ecs/crm-datafetch/Dockerfile @@ -6,6 +6,13 @@ WORKDIR /usr/src/app COPY Pipfile Pipfile.lock ./ RUN pip install --no-cache-dir -r requirements.txt && \ cp /usr/share/zoneinfo/Asia/Tokyo /etc/localtime + +RUN \ + apt update -y && \ + # パッケージのセキュリティアップデートのみを適用するコマンド + apt install -y unattended-upgrades && \ + unattended-upgrades + COPY main.py ./ COPY crm-datafetch /usr/src diff --git a/ecs/crm-datafetch/main.py b/ecs/crm-datafetch/main.py index 0170f413..b0edf545 100644 --- a/ecs/crm-datafetch/main.py +++ b/ecs/crm-datafetch/main.py @@ -1,181 +1,4 @@ -# モジュールロード -import os -import sys -from datetime import datetime -import src.pre as pre # データ取得準備処理 -import src.chk as chk # オブジェクト情報形式チェック処理 -import src.date as date # データ取得期間設定処理 -import src.fetch as fetch # CRMデータ取得処理 -import src.resbk as resbk # CRM電文データバックアップ処理 -import src.conv as conv # CSV変換処理 -import src.csvbk as csvbk # CSVバックアップ処理 -import src.upld as upld # CSVアップロード処理 -import src.upd as upd # 前回取得日時ファイル更新 -import src.end as end # 取得処理実施結果アップロード処理 - -from src.utils.logger import Logger -import src.utils.data_retention as data_retention - - -# 変数 - -# logger設定 -logger = Logger().get_logger() - - -# 処理 -def main() -> None: - """コントロール処理 - - 各処理を呼び出すコントローラー - - """ - - try: - # ① CRMデータ取得処理開始ログを出力する - logger.info('I-CTRL-01 CRMデータ取得処理を開始します') - - # ② データ取得準備処理を呼び出す - try: - logger.info('I-CTRL-02 データ取得準備処理呼び出し') - crm_object_info_var = data_retention.CrmObjectInfoVar() - crm_object_info_dict, execute_datetime, process_result_per_object = pre( - crm_object_info_var) - - except Exception as e: - logger.error('') - raise CustomException(e) - - # ③ object_infoのobjectsキーの値の件数分ループする - logger.info('I-CTRL-03 取得対象オブジェクトのループ処理開始') - - for object_info in crm_object_info_dict.get('objects'): - try: - - # 1. オブジェクト情報形式チェック処理を呼び出す - logger.info('I-CTRL-04 オブジェクト情報形式チェック処理呼び出し') - - try: - chk() - - except Exception as e: - logger.error('') - - # 2. 処理対象のオブジェクト名をログ出力する - logger.info('I-CTRL-05 ') - - # 3. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する - logger.info('I-CTRL-06 ') - - # 4. データ取得期間設定処理を呼び出す - logger.info('I-CTRL-07 ') - try: - - except Exception as e: - logger.error('') - - # 5. CRMデータ取得処理を呼び出す - logger.info('I-CTRL-08 ') - try: - - except Exception as e: - logger.error('') - - # 6. CRMデータのジェネレータをループする - # 6-1. 出力ファイル名を生成する - logger.info('I-CTRL-10 ') - try: - - except Exception as e: - logger.error('') - - # 6-2. CRM電文データバックアップ処理を呼び出す - logger.info('I-CTRL-11 ') - try: - - except Exception as e: - logger.error('') - - # 6-3. CSV変換処理を呼び出す - logger.info('I-CTRL-12 ') - try: - - except Exception as e: - logger.error('') - - # 6-4. CSVバックアップ処理を呼び出す - logger.info('I-CTRL-13 ') - try: - - except Exception as e: - logger.error('') - - # 6-5. CSVアップロード処理を呼び出す - logger.info('I-CTRL-14 ') - try: - - except Exception as e: - logger.error('') - - # 7. 前回取得日時ファイル更新処理を呼びだす - logger.info('I-CTRL-15 ') - try: - - except Exception as e: - logger.error('') - - # 8. 1オブジェクトのアップロードが完了した旨をログに出力する - logger.info('I-CTRL-16 ') - - except GetObjectException as e: - logger.error( - '[取得対象のオブジェクト名] の[処理名]でエラーが発生しました 次のオブジェクトの処理に移行します') - - except Exception as e: - logger.error( - '[取得対象のオブジェクト名] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します') - - # ④ すべてのオブジェクトの処理が完了したことと、オブジェクト毎の処理結果をログに出力する - try: - logger.info(f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{実行結果JSON}]') - - except Exception as e: - logger.error('') - raise CustomException() - - # ⑤ 取得処理実施結果アップロード処理を呼び出す - try: - logger.info('I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始') - - except Exception as e: - logger.error('') - raise CustomException() - - # ⑥ CRMデータ取得処理終了ログを出力する - try: - logger.info('I-CTRL-19 CRMデータ取得処理を終了します') - - except Exception as e: - logger.error('') - raise - - except CustomException as e: - logger.error('I-ERR-01 [処理名]でエラーが発生したため、処理を終了します') - - except Exception as e: - logger.error('I-ERR-02 予期せぬエラーが発生したため、処理を終了します') - - -# カスタムExceptionクラス -class CustomException(Exception, metaclass=ABCMeta): - def __init__(self, id, arg): - self.arg = arg - self.id = id - - -class GetObjectException(CustomException): - pass - +import src.process as process if __name__ == '__main__': - main() + process.main() diff --git a/ecs/crm-datafetch/src/utils/__init__.py b/ecs/crm-datafetch/src/aws/__init__.py similarity index 100% rename from ecs/crm-datafetch/src/utils/__init__.py rename to ecs/crm-datafetch/src/aws/__init__.py diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py new file mode 100644 index 00000000..e5235016 --- /dev/null +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -0,0 +1,107 @@ +import boto3 +import json +from src.environments import ( + CRM_CONFIG_BUCKET, + CRM_BACKUP_BUCKET, + IMPORT_DATA_BUCKET, + OBJECT_INFO_FOLDER, + OBJECT_INFO_FILENAME, + CRM_IMPORT_DATA_FOLDER, + CRM_IMPORT_DATA_BACKUP_FOLDER, + LAST_FETCH_DATE_FOLDER, + RESPONSE_JSON_BACKUP_FOLDER +) +from src.constants import ( + AWS_RESOURCE_S3, + AWS_CLINET_S3, + S3_RESPONSE_BODY +) + + +class S3Clinet: + def __init__(self) -> None: + self.__s3_client = boto3.client(AWS_CLINET_S3) + + def copy_object(self, src_bucket, src_key, dest_bucket, dest_key): + self.__s3_client.copy( + {"Bucket": src_bucket, "key": src_key}, dest_bucket, dest_key) + return + + +class S3Resource: + def __init__(self, bucket_name: str) -> None: + self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) + self.__s3_bucket = self.__s3_resource.Bucket(bucket_name) + + def get_object(self, object_key: str) -> str: + response = self.__s3_bucket.Object(object_key).get() + body = response[S3_RESPONSE_BODY].read() + + return body.decode('utf-8') + + def put_object(self, object_key: str, data) -> str: + s3_object = self.__s3_bucket.Object(object_key) + s3_object.put(Body=data.encode('utf-8'), ContentEncoding='utf-8') + return + + +class S3ResourceNonBucket: + def __init__(self) -> None: + self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) + + def copy(self, src_bucket, src_key, dest_bucket, dest_key): + copy_source = {'Bucket': src_bucket, 'Key': src_key} + self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) + + return + + +class ConfigBucket: + __s3_resource: S3Resource = None + + def __init__(self) -> None: + self.__s3_resource = S3Resource(CRM_CONFIG_BUCKET) + + def get_object_info_file(self): + return self.__s3_resource.get_object(f'{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}') + + def get_last_fetch_datetime_file(self, file_name): + return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_name}') + + def put_last_fetch_datetime_file(self, file_name, data): + self.__s3_resource.put_object( + f'{LAST_FETCH_DATE_FOLDER}/{file_name}', data) + return + + +class DataBucket: + __s3_resource: S3Resource = None + + def __init__(self) -> None: + self.__s3_resource = S3Resource(IMPORT_DATA_BUCKET) + + def put_csv(self, file_name, data): + object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_name}' + self.__s3_resource.put_object(object_key, data) + return + + +class BackupBucket: + __s3_resource: S3Resource = None + + def __init__(self) -> None: + self.__s3_resource = S3Resource(CRM_BACKUP_BUCKET) + + def put_response_json(self, file_name, data): + object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_name}' + self.__s3_resource.put_object(object_key, json.dumps(data)) + return + + def put_csv_bk(self, file_name, data): + object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_name}' + self.__s3_resource.put_object(object_key, data) + return + + def put_result_json(self, file_name, data): + object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_name}' + self.__s3_resource.put_object(object_key, json.dumps(data)) diff --git a/ecs/crm-datafetch/src/chk.py b/ecs/crm-datafetch/src/chk.py index e69de29b..1342243e 100644 --- a/ecs/crm-datafetch/src/chk.py +++ b/ecs/crm-datafetch/src/chk.py @@ -0,0 +1,30 @@ +from src.util.logger import Logger +from src.constants import( + CHK_JP_NAME +) +from src.error.exceptions import( + InvalidConfigException +) +from src.config.objects import TargetObject + + +logger = Logger().get_logger() + + +def check_object_info(object_info, execute_datetime): + # ① オブジェクト情報形式チェック処理開始ログを出力する + logger.info('I-CHK-01 オブジェクト情報形式チェック処理を開始します') + + try: + # ② オブジェクト情報形式チェック + target_object = TargetObject(object_info, execute_datetime) + + except Exception as e: + raise InvalidConfigException( + 'E-CHK-01', CHK_JP_NAME, f'オブジェクト情報形式チェック処理が失敗しました　エラー内容：[{e}]') + + # ③ チェック処理終了ログを出力する + logger.info('I-CHK-02 オブジェクト情報形式チェック処理を終了します') + + # ④ 次の処理へ移行する + return target_object diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py new file mode 100644 index 00000000..edce5b33 --- /dev/null +++ b/ecs/crm-datafetch/src/config/objects.py @@ -0,0 +1,195 @@ +from src.constants import ( + DATE_PATTERN_YYYYMMDDTHHMMSSTZ +) +from src.util.dict_checker import DictCheck + + +class FetchTargetObjects(): + def __init__(self, object_info_file_dict) -> None: + self.__dict_check = DictCheck() + self.__objects = object_info_file_dict + self.__key = 'objects' + self.check_key_objects() + self.__i = 0 + + def __iter__(self): + return self + + def __next__(self): + if self.__i == len(self.__objects[self.__key]): + raise StopIteration() + value = self.__objects['objects'][self.__i] + self.__i += 1 + return value + + def check_key_objects(self) -> None: + __check_key = self.__key + __check_type = list + self.__dict_check.check_key_exist_and_value_type( + self.__objects, __check_key, __check_type) + + +class TargetObject(): + def __init__(self, object_info, execute_datetime) -> None: + self.__dict_check = DictCheck() + self.__object_info = object_info + self.execute_datetime = execute_datetime + self.check_key_object_name() + self.check_key_columns() + self.check_key_is_skip() + self.check_key_is_update_last_fetch_datetime() + self.check_key_last_fetch_datetime_file_name() + self.check_key_upload_file_name() + self.object_name = self.__object_info['object_name'] + self.columns = self.__object_info['columns'] + self.is_skip = self.set_is_skip() + self.is_update_last_fetch_datetime = self.set_is_update_last_fetch_datetime() + self.last_fetch_datetime_file_name = self.set_fetch_datetime_file_name() + self.upload_file_name = self.set_upload_file_name() + + def check_key_object_name(self) -> None: + ''' + オブジェクト名チェック + ''' + __check_key = 'object_name' + __check_type = str + self.__dict_check.check_key_exist_and_value_type( + self.__object_info, __check_key, __check_type) + + return + + def check_key_columns(self) -> None: + ''' + カラム情報チェック + ''' + __key = 'columns' + __type = list + self.__dict_check.check_key_exist_and_value_type( + self.__object_info, __key, __type) + + return + + def check_key_is_skip(self,) -> None: + ''' + スキップフラグ型チェック + ''' + __check_key = 'is_skip' + __check_type = bool + self.__dict_check.check_key_exist_case_value_type( + self.__object_info, __check_key, __check_type) + + return + + def check_key_is_update_last_fetch_datetime(self) -> None: + ''' + 前回取得日時ファイル更新フラグチェック + ''' + __check_key = 'is_update_last_fetch_datetime' + __check_type = bool + self.__dict_check.check_key_exist_case_value_type( + self.__object_info, __check_key, __check_type) + + return + + def check_key_last_fetch_datetime_file_name(self) -> None: + ''' + 前回取得日時ファイル名型チェック + ''' + __check_key = 'last_fetch_datetime_file_name' + __check_type = str + self.__dict_check.check_key_exist_case_value_type( + self.__object_info, __check_key, __check_type) + + return + + def check_key_upload_file_name(self) -> None: + ''' + アップロードファイル名称型チェック + ''' + __check_key = 'upload_file_name' + __check_type = str + self.__dict_check.check_key_exist_case_value_type( + self.__object_info, __check_key, __check_type) + + return + + def set_is_skip(self) -> bool: + ''' + スキップフラグ設定 + ''' + __check_key = 'is_skip' + if self.__dict_check.check_key_exist(self.__object_info, __check_key): + return self.__object_info[__check_key] + else: + return False + + def set_is_update_last_fetch_datetime(self) -> bool: + ''' + 前回取得日時ファイル更新フラグ設定 + ''' + __check_key = 'is_update_last_fetch_datetime' + if self.__dict_check.check_key_exist(self.__object_info, __check_key): + return self.__object_info[__check_key] + else: + return False + + def set_fetch_datetime_file_name(self) -> str: + ''' + 前回取得日時ファイル名設定 + ''' + __check_key = 'last_fetch_datetime_file_name' + if self.__dict_check.check_key_exist(self.__object_info, __check_key): + return self.__object_info[__check_key] + else: + return self.__object_info['object_name'] + '.json' + + def set_upload_file_name(self) -> str: + ''' + アップロードファイル名称設定 + ''' + __check_key = 'upload_file_name' + if self.__dict_check.check_key_exist(self.__object_info, __check_key): + return self.__object_info[__check_key].format(execute_datetime=self.execute_datetime) + else: + return 'CRM_' + self.__object_info['object_name'] + '_' + self.execute_datetime + + +class LastFetchDatetime(): + def __init__(self, last_fetch_datetime_file_name, last_fetch_datetime_file_dict, execute_datetime) -> None: + self.__dict_check = DictCheck() + self.execute_datetime = execute_datetime + self.__last_fetch_datetime_file_dict = last_fetch_datetime_file_dict + self.last_fetch_datetime_file_name = last_fetch_datetime_file_name + self.check_key_last_fetch_datetime_from + self.check_key_last_fetch_datetime_to + self.last_fetch_datetime_from = self.__last_fetch_datetime_file_dict[ + 'last_fetch_datetime_from'] + self.last_fetch_datetime_to = self.set_last_fetch_datetime_to() + + def check_key_last_fetch_datetime_from(self) -> None: + ''' + データ取得開始日時チェック + ''' + __check_key = 'last_fetch_datetime_from' + __regex_str = DATE_PATTERN_YYYYMMDDTHHMMSSTZ + self.__dict_check.check_key_exsit_and_regex( + self.__last_fetch_datetime_file_dict, __check_key, __regex_str) + + def check_key_last_fetch_datetime_to(self) -> None: + ''' + データ取得終了日時チェック + ''' + __check_key = 'last_fetch_datetime_to' + __regex_str = DATE_PATTERN_YYYYMMDDTHHMMSSTZ + self.__dict_check.check_key_exsit_case_regex( + self.__last_fetch_datetime_file_dict, __check_key, __regex_str) + + def set_last_fetch_datetime_to(self) -> str: + ''' + データ取得終了日時設定 + ''' + __check_key = 'last_fetch_datetime_to' + if self.__dict_check.check_key_exist(self.__last_fetch_datetime_file_dict, __check_key): + return self.__last_fetch_datetime_file_dict[__check_key].format(execute_datetime=self.execute_datetime) + else: + return self.execute_datetime diff --git a/ecs/crm-datafetch/src/constants.py b/ecs/crm-datafetch/src/constants.py new file mode 100644 index 00000000..fcd6b4a3 --- /dev/null +++ b/ecs/crm-datafetch/src/constants.py @@ -0,0 +1,73 @@ +# environments(task settings file) +LOG_LEVEL = "LOG_LEVEL" # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 +CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = 'CRM_AUTH_RETRY_MIN_INTERVAL' # CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = 'CRM_AUTH_RETRY_MAX_INTERVAL' # CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = 'CRM_GET_RECORD_COUNT_TIMEOUT' # CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = 'CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT' # CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_INTERVAL' # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL' # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL' # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_TIMEOUT = 'CRM_FETCH_RECORD_TIMEOUT' # CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = 'CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT' # CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_RETRY_INTERVAL = 'CRM_FETCH_RECORD_RETRY_INTERVAL' # CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MIN_INTERVAL' # CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MAX_INTERVAL' # CRMのレコード取得処理のリトライ時の最大待ち秒数 + +# environments(ECS Task Enviroment) +CRM_AUTH_DOMAIN = 'CRM_AUTH_DOMAIN' # CRMのAPI実行のための認証エンドポイントのドメイン +CRM_USER_NAME = 'CRM_USER_NAME' # CRMのAPI実行用ユーザ名 +CRM_USER_PASSWORD = 'CRM_USER_PASSWORD' # CRMのAPI実行用ユーザパスワード +CRM_USER_SECURITY_TOKEN = 'CRM_USER_SECURITY_TOKEN' # CRMのAPI実行用ユーザのセキュリティトークン +CRM_CONFIG_BUCKET = 'CRM_CONFIG_BUCKET' # CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_BACKUP_BUCKET = 'CRM_BACKUP_BUCKET' # CRMのバックアップデータを格納するバケット名 +IMPORT_DATA_BUCKET = 'IMPORT_DATA_BUCKET' # CRMの取込データを格納するバケット名 +OBJECT_INFO_FOLDER = 'OBJECT_INFO_FOLDER' # CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FILENAME = 'OBJECT_INFO_FILENAME' # CRM取得対象オブジェクトの情報のファイル名 +PROCESS_RESULT_FOLDER = 'PROCESS_RESULT_FOLDER' # CRMデータ取得結果を格納するフォルダパス +PROCESS_RESULT_FILENAME = 'PROCESS_RESULT_FILENAME' # CRMデータ取得結果を格納するファイル名 +LAST_FETCH_DATE_FOLDER = 'LAST_FETCH_DATE_FOLDER' # CRMからの最終取得日時ファイルを格納するフォルダパス +CRM_IMPORT_DATA_FOLDER ='CRM_IMPORT_DATA_FOLDER' # CRMから取得し、取込用に変換したデータを格納するフォルダ +LAST_FETCH_DATE_BACKUP_FOLDER = 'LAST_FETCH_DATE_BACKUP_FOLDER' # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +RESPONSE_JSON_BACKUP_FOLDER = 'RESPONSE_JSON_BACKUP_FOLDER' # CRMから取得した生データのバックアップを格納するフォルダパス +CRM_IMPORT_DATA_BACKUP_FOLDER = 'CRM_IMPORT_DATA_BACKUP_FOLDER' # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ + +# 時刻フォーマット +# .000ZはUTCを表す。ミリ秒までの考慮は不要なので固定で指定 +YYYYMMDDTHHMMSSTZ = '%Y-%m-%dT%H:%M:%S.000Z' +CRM_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.000%z' +YYYYMMDDHHMMSS = '%Y-%m-%d %H:%M:%S' + +# aws +AWS_RESOURCE_S3 = 's3' +AWS_CLINET_S3 = 's3' +S3_RESPONSE_BODY = 'Body' + + +# 正規表現チェック +EXCLUDE_SYMBOL = ['#', '/'] +DATE_PATTERN_YYYYMMDDTHHMMSSTZ = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9].000Z' + +# logger +LOG_FORMAT = '[%(levelname)s]\t%(asctime)s\t%(message)s\n' +LOG_DATE_FORMAT = '%Y-%m-%d %H:%M:%S' +LOG_LEVEL_INFO = 'INFO' + +# 処理名 +PROCESS_JP_NAME = 'コントロール処理' +PRE_JP_NAME = 'データ取得準備処理' +CHK_JP_NAME = 'オブジェクト情報形式チェック処理' +DATE_JP_NAME = 'データ取得期間設定処理' +FETCH_JP_NAME = 'CRMデータ取得処理' +RESBK_JP_NAME = 'CRM電文データバックアップ処理' +CONV_JP_NAME = 'CSV変換処理' +CSVBK_JP_NAME = 'CSVバックアップ処理' +UPLD_JP_NAME = 'CSVアップロード処理' +UPD_JP_NAME = '前回取得日時ファイル更新' +END_JP_NAME = '取得処理実施結果アップロード処理' + +# CSVチェック +CSV_TRUE_VALUE = '1' +CSV_FALSE_VALUE = '0' \ No newline at end of file diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/conv.py index e69de29b..88b86ae8 100644 --- a/ecs/crm-datafetch/src/conv.py +++ b/ecs/crm-datafetch/src/conv.py @@ -0,0 +1,33 @@ +from datetime import datetime +from src.util.logger import Logger +from src.constants import ( + CONV_JP_NAME, +) +from src.error.exceptions import( + DataConvertException +) +from src.converter.converter import CSVStringConverter + + +logger = Logger().get_logger() + + +def convert_crm_csvdata(target_object, sf_object_jsons): + # ① CSV変換処理の開始ログを出力する + logger.info(f'I-CONV-01 [{target_object.object_name}] のCSV変換処理を開始します') + + try: + # ② CSV変換 + csv_object = CSVStringConverter(target_object, sf_object_jsons) + + logger.debug(f'D-CONV-02 [{target_object.object_name}] のCSV変換処理 正常終了') + + except Exception as e: + raise DataConvertException( + 'E-CONV-01', CONV_JP_NAME, f'[{target_object.object_name}] CSV変換に失敗しました エラー内容:[{e}]') + + # ③ CSV変換処理の終了ログを出力する + logger.info(f'I-CONV-03 [{target_object.object_name}] のCSV変換処理を終了します') + + # ④ 次の処理へ移行する + return csv_object diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py new file mode 100644 index 00000000..1b785c35 --- /dev/null +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -0,0 +1,138 @@ +import re +import io +import csv +from datetime import datetime +from src.constants import( + CSV_TRUE_VALUE, + CSV_FALSE_VALUE, + CRM_DATETIME_FORMAT, + YYYYMMDDHHMMSS +) + + +class CSVStringConverter: + def __init__(self, target_object, sf_object_jsons) -> None: + self.__target_object = target_object + self.__sf_object_jsons = sf_object_jsons + self.__extracted_sf_object_jsons = self.extract_sf_object_jsons() + self.csv_data = self.convert_to_csv() + self.csv_buffer = self.write_csv() + + def extract_sf_object_jsons(self) -> list: + try: + extracted_sf_object_jsons = [] + for sf_object_json in self.__sf_object_jsons: + extracted_sf_object_jsons.append( + self.extract_necessary_props_from(sf_object_json)) + + return extracted_sf_object_jsons + + except Exception as e: + raise Exception('必要なjsonのデータ抽出に失敗しました') + + def extract_necessary_props_from(self, sf_object_json) -> dict: + try: + clone_sf_object = {**sf_object_json} + + del clone_sf_object['attributes'] + + uppercase_key_sf_object = { + k.upper(): v for k, v in clone_sf_object.items()} + + return uppercase_key_sf_object + + except Exception as e: + raise Exception('必要なjsonのデータ成形に失敗しました') + + def convert_to_csv(self) -> list: + try: + columns = self.__target_object.columns + csv_data = [] + for i, json_object in enumerate(self.__extracted_sf_object_jsons, 1): + csv_row = [] + for column in columns: + v = json_object[column.upper()] + + converted_value = CSVStringConverterFactory( + v).value_convert() + + csv_row.append(converted_value) + + csv_data.append(csv_row) + return csv_data + + except Exception as e: + raise Exception( + f'CSV変換に失敗しました カラム名：[{column}] 行番号: [{i}] エラー内容:[{e}]') + + def write_csv(self) -> str: + try: + with io.StringIO(newline='') as string_stream: + writer = csv.writer(string_stream, delimiter=',', lineterminator='\r\n', + doublequote=True, quotechar='"', quoting=csv.QUOTE_ALL, strict=True) + writer.writerow(self.__target_object.columns) + writer.writerows(self.csv_data) + csv_value = string_stream.getvalue() + + return csv_value + + except Exception as e: + raise Exception('csvデータの取得に失敗しました') + + +class NoneValueConverter: + def __init__(self, convert_value) -> None: + self.__convert_value = convert_value + self.value = self.convert_value() + + def convert_value(self) -> str: + return '' + + +class BooleanConverter: + def __init__(self, convert_value) -> None: + self.__convert_value = convert_value + self.value = self.convert_value() + + def convert_value(self) -> bool: + return CSV_TRUE_VALUE if self.__convert_value is True else CSV_FALSE_VALUE + + +class DatatimeConverter: + def __init__(self, convert_value) -> None: + self.__convert_value = convert_value + self.value = self.convert_value() + + def convert_value(self) -> str: + return datetime.strptime(self.__convert_value, CRM_DATETIME_FORMAT).strftime(YYYYMMDDHHMMSS) + + +class FloatConverter: + def __init__(self, convert_value) -> None: + self.__convert_value = convert_value + self.value = self.convert_value() + + def convert_value(self) -> int: + return int(self.__convert_value) + + +class CSVStringConverterFactory: + def __init__(self, v) -> None: + self.__value = v + + def value_convert(self): + + converted_value = self.__value + + if self.__value is None: + converted_value = NoneValueConverter(self.__value).value + # 指数表記で取得できるパターン。指数表記を整数表記に変換する。 + elif type(self.__value) == float: + converted_value = FloatConverter(self.__value).value + # SQLの真偽値に対応するために変換する + elif type(self.__value) == bool: + converted_value = BooleanConverter(self.__value).value + elif type(self.__value) == str and re.fullmatch(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.000\+0000', self.__value): + converted_value = DatatimeConverter(self.__value).value + + return converted_value diff --git a/ecs/crm-datafetch/src/csvbk.py b/ecs/crm-datafetch/src/csvbk.py index e69de29b..db490c63 100644 --- a/ecs/crm-datafetch/src/csvbk.py +++ b/ecs/crm-datafetch/src/csvbk.py @@ -0,0 +1,34 @@ +from src.util.logger import Logger +from src.aws.s3 import BackupBucket +from src.constants import ( + CSVBK_JP_NAME +) +from src.error.exceptions import( + FileUploadException +) + + +logger = Logger().get_logger() + + +def backup_crm_csvdata(target_object, date_path, csv_object): + # ① CSVバックアップ処理の開始ログを出力する + logger.info( + f'I-CSVBK-01 [{target_object.object_name}] のCSVデータのバックアップ処理を開始します ファイル名：[{target_object.upload_file_name}.csv]') + + try: + # ② CRMバックアップ保管用バケットに、変換後のCSVデータのバックアップを保管する + backup_bucket = BackupBucket() + backup_bucket.put_csv_bk( + f'{date_path}/{target_object.upload_file_name}.csv', csv_object.csv_buffer) + + except Exception as e: + raise FileUploadException( + 'E-CSVBK-01', CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') + + # ③ CSVバックアップ処理の終了ログを出力する + logger.info( + f'I-CSVBK-03 [{target_object.object_name}] のCSVデータのバックアップ処理を終了します') + + # ④ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/date.py b/ecs/crm-datafetch/src/date.py index e69de29b..8fff0d94 100644 --- a/ecs/crm-datafetch/src/date.py +++ b/ecs/crm-datafetch/src/date.py @@ -0,0 +1,65 @@ +from src.util.logger import Logger +from src.constants import( + DATE_JP_NAME +) +from src.environments import ( + CRM_CONFIG_BUCKET, + LAST_FETCH_DATE_FOLDER +) +from src.error.exceptions import( + FileNotFoundException, + InvalidConfigException +) +from src.parser.json_parse import JsonParser +from src.config.objects import LastFetchDatetime +from src.aws.s3 import ConfigBucket + + +logger = Logger().get_logger() + + +def set_datetime_period(target_object, execute_datetime): + # ① データ取得期間設定処理の開始ログを出力する + logger.info( + f'I-DATE-01 [{target_object.object_name}] のデータ取得期間設定処理を開始します') + + try: + # ② S3 設定ファイル保管用バケットから、前回取得日時ファイルを取得する + logger.info( + f'I-DATE-02 前回取得日時ファイルの取得開始します ファイルパス：[s3://{CRM_CONFIG_BUCKET}/{LAST_FETCH_DATE_FOLDER}/{target_object.last_fetch_datetime_file_name}]') + + s3_config_bucket = ConfigBucket() + last_fetch_datetime_file_json = s3_config_bucket.get_last_fetch_datetime_file( + target_object.last_fetch_datetime_file_name) + + logger.info(f'I-DATE-03 前回取得日時ファイルの取得成功しました') + + except Exception as e: + raise FileNotFoundException( + 'E-DATE-01', DATE_JP_NAME, f'前回取得日時ファイルが存在しません　ファイル名：[{target_object.last_fetch_datetime_file_name}] エラー内容：[{e}]') + + try: + # ③ 取得した前回取得日時ファイルの形式チェックを行う + # ④ データの取得期間を設定する + logger.debug(f'D-DATE-04 前回取得日時ファイルの形式チェックを開始します') + + json_parser = JsonParser(last_fetch_datetime_file_json) + last_fetch_datetime_file_dict = json_parser.json_parser() + + last_fetch_datetime = LastFetchDatetime(target_object.last_fetch_datetime_file_name, + last_fetch_datetime_file_dict, execute_datetime) + + logger.debug(f'D-DATE-05 前回取得日時ファイルの形式チェック 正常終了') + logger.info( + f'I-DATE-06 取得範囲 From: [{last_fetch_datetime.last_fetch_datetime_from}] To: [{last_fetch_datetime.last_fetch_datetime_to}]') + + except Exception as e: + raise InvalidConfigException( + 'E-DATE-02', DATE_JP_NAME, f'前回取得日時ファイルの形式チェック処理が失敗しました エラー内容：[{e}]') + + # ⑤ データ取得準備処理の終了ログを出力する + logger.info( + f'I-DATE-07 [{target_object.object_name}] のデータ取得期間設定処理を終了します') + + # ⑥ 次の処理へ移行する + return last_fetch_datetime diff --git a/ecs/crm-datafetch/src/end.py b/ecs/crm-datafetch/src/end.py index e69de29b..e2b274b3 100644 --- a/ecs/crm-datafetch/src/end.py +++ b/ecs/crm-datafetch/src/end.py @@ -0,0 +1,37 @@ +from src.util.logger import Logger +from src.constants import ( + END_JP_NAME +) +from src.environments import( + PROCESS_RESULT_FILENAME +) +from src.error.exceptions import( + FileUploadException +) +from src.aws.s3 import BackupBucket + + +logger = Logger().get_logger() + + +def updload_result_data(process_result, date_path): + # ① 取得処理実施結果アップロード処理のログを出力する + logger.info( + f'I-END-01 取得処理実施結果アップロード処理を開始します') + + try: + # ② CRMバックアップ保管用バケットに、取得処理実施結果のJSONデータを保管する + backup_bucket = BackupBucket() + backup_bucket.put_result_json( + f'{date_path}/{PROCESS_RESULT_FILENAME}', process_result) + + logger.debug(f'D-END-02 取得処理実施結果アップロード 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-END-01', END_JP_NAME, f'取得処理実施結果のアップロードに失敗しました　ファイル名：[{PROCESS_RESULT_FILENAME}] エラー内容：[{e}]') + + # ③ 取得処理実施結果アップロード処理の終了ログを出力する + logger.info(f'I-END-03 取得処理実施結果アップロード処理を終了します') + + return diff --git a/ecs/crm-datafetch/src/environments.py b/ecs/crm-datafetch/src/environments.py new file mode 100644 index 00000000..8d63cc64 --- /dev/null +++ b/ecs/crm-datafetch/src/environments.py @@ -0,0 +1,73 @@ +import os + +from src.constants import ( + LOG_LEVEL, + LOG_LEVEL_INFO, + CRM_AUTH_TIMEOUT, + CRM_AUTH_MAX_RETRY_ATTEMPT, + CRM_AUTH_RETRY_INTERVAL, + CRM_AUTH_RETRY_MIN_INTERVAL, + CRM_AUTH_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_TIMEOUT, + CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, + CRM_AUTH_DOMAIN, + CRM_USER_NAME, + CRM_USER_PASSWORD, + CRM_USER_SECURITY_TOKEN, + CRM_CONFIG_BUCKET, + CRM_BACKUP_BUCKET, + IMPORT_DATA_BUCKET, + OBJECT_INFO_FOLDER, + OBJECT_INFO_FILENAME, + PROCESS_RESULT_FOLDER, + PROCESS_RESULT_FILENAME, + LAST_FETCH_DATE_FOLDER, + CRM_IMPORT_DATA_FOLDER, + LAST_FETCH_DATE_BACKUP_FOLDER, + RESPONSE_JSON_BACKUP_FOLDER, + CRM_IMPORT_DATA_BACKUP_FOLDER +) + +# environments(task settings file) +LOG_LEVEL = os.environ.get(LOG_LEVEL, LOG_LEVEL_INFO) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +CRM_AUTH_TIMEOUT = os.environ["CRM_AUTH_TIMEOUT"] # CRMへの認証処理のタイムアウト秒数 +CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ["CRM_AUTH_MAX_RETRY_ATTEMPT"] # CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_RETRY_INTERVAL = os.environ["CRM_AUTH_RETRY_INTERVAL"] # CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = os.environ["CRM_AUTH_RETRY_MIN_INTERVAL"] # CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = os.environ["CRM_AUTH_RETRY_MAX_INTERVAL"] # CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = os.environ["CRM_GET_RECORD_COUNT_TIMEOUT"] # CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ["CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT"] # CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_INTERVAL"] # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL"] # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL"] # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_TIMEOUT = os.environ["CRM_FETCH_RECORD_TIMEOUT"] # CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ["CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT"] # CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_INTERVAL"] # CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MIN_INTERVAL"] # CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MAX_INTERVAL"] # CRMのレコード取得処理のリトライ時の最大待ち秒数 + +# environments(ECS Task Enviroment) +CRM_AUTH_DOMAIN = os.environ["CRM_AUTH_DOMAIN"] # CRMのAPI実行のための認証エンドポイントのドメイン +CRM_USER_NAME = os.environ["CRM_USER_NAME"] # CRMのAPI実行用ユーザ名 +CRM_USER_PASSWORD = os.environ["CRM_USER_PASSWORD"] # CRMのAPI実行用ユーザパスワード +CRM_USER_SECURITY_TOKEN = os.environ["CRM_USER_SECURITY_TOKEN"] # CRMのAPI実行用ユーザのセキュリティトークン +CRM_CONFIG_BUCKET = os.environ["CRM_CONFIG_BUCKET"] # CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_BACKUP_BUCKET = os.environ["CRM_BACKUP_BUCKET"] # CRMのバックアップデータを格納するバケット名 +IMPORT_DATA_BUCKET = os.environ["IMPORT_DATA_BUCKET"] # CRMの取込データを格納するバケット名 +OBJECT_INFO_FOLDER = os.environ["OBJECT_INFO_FOLDER"] # CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FILENAME = os.environ["OBJECT_INFO_FILENAME"] # CRM取得対象オブジェクトの情報のファイル名 +PROCESS_RESULT_FOLDER = os.environ["PROCESS_RESULT_FOLDER"] # CRMデータ取得結果を格納するフォルダパス +PROCESS_RESULT_FILENAME = os.environ["PROCESS_RESULT_FILENAME"] # CRMデータ取得結果を格納するファイル名 +LAST_FETCH_DATE_FOLDER = os.environ["LAST_FETCH_DATE_FOLDER"] # CRMからの最終取得日時ファイルを格納するフォルダパス +CRM_IMPORT_DATA_FOLDER = os.environ["CRM_IMPORT_DATA_FOLDER"] # CRMから取得し、取込用に変換したデータを格納するフォルダ +LAST_FETCH_DATE_BACKUP_FOLDER = os.environ["LAST_FETCH_DATE_BACKUP_FOLDER"] # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +RESPONSE_JSON_BACKUP_FOLDER = os.environ["FETCH_DATA_BACKUP_FOLDER"] # CRMから取得した生データのバックアップを格納するフォルダパス +CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ["CRM_IMPORT_DATA_BACKUP_FOLDER"] # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ diff --git a/ecs/crm-datafetch/src/error/__init__.py b/ecs/crm-datafetch/src/error/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/error/exceptions.py b/ecs/crm-datafetch/src/error/exceptions.py new file mode 100644 index 00000000..528c262d --- /dev/null +++ b/ecs/crm-datafetch/src/error/exceptions.py @@ -0,0 +1,45 @@ +from abc import ABCMeta + + +class MeDaCaCRMDataFetchException(Exception, metaclass=ABCMeta): + """MeDaCaシステム固有のカスタムエラークラス""" + + def __init__(self, error_id: str, func_name, message) -> None: + super().__init__(message) + self.func_name = func_name + self.error_id = error_id + + +class FileNotFoundException(MeDaCaCRMDataFetchException): + """S3のファイルが見つからない場合の例外""" + pass + + +class FileUploadException(MeDaCaCRMDataFetchException): + """S3のファイルアップロード失敗の例外""" + pass + + +class InvalidConfigException(MeDaCaCRMDataFetchException): + """Configのバリデーションチェック失敗の例外""" + pass + + +class DataConvertException(MeDaCaCRMDataFetchException): + """データ変換が失敗した場合の例外""" + pass + + +class TimeOutException(MeDaCaCRMDataFetchException): + """タイムアウトが発生した場合の例外""" + pass + + +class RetryExceededException(MeDaCaCRMDataFetchException): + """リトライ処理が発生した場合の例外""" + pass + + +class SalesforceAPIException(MeDaCaCRMDataFetchException): + """SalseforceのAPI実行失敗が発生した場合の例外""" + pass diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch.py index e69de29b..bac313fd 100644 --- a/ecs/crm-datafetch/src/fetch.py +++ b/ecs/crm-datafetch/src/fetch.py @@ -0,0 +1,71 @@ +from src.util.logger import Logger +from src.constants import( + FETCH_JP_NAME +) +from src.error.exceptions import( + SalesforceAPIException, + DataConvertException +) +from src.salesforce.salesforce_api import( + SalesForceCount, + SalesForceData +) + + +logger = Logger().get_logger() + + +def fetch_crm_data(target_object, last_fetch_datetime): + # ① CRMデータ取得処理開始ログを出力する + logger.info( + f'I-FETCH-01 [{target_object.object_name}] のCRMからのデータ取得処理を開始します') + + object_name = target_object.object_name + columns = ','.join(target_object.columns) + last_fetch_datetime_from = last_fetch_datetime.last_fetch_datetime_from + last_fetch_datetime_to = last_fetch_datetime.last_fetch_datetime_to + + try: + # ② 取得対象オブジェクトの取得期間内のレコード件数を取得する + logger.info(f'I-FETCH-02 [{object_name}] の件数取得を開始します') + + saleforce_count = SalesForceCount() + record_count = saleforce_count.fetch_sf_count_retry( + object_name, last_fetch_datetime_from, last_fetch_datetime_to) + + logger.info(f'I-FETCH-03 [{object_name}] の件数：[{record_count}]') + + except Exception as e: + raise SalesforceAPIException( + 'E-FETCH-01', FETCH_JP_NAME, f'[{object_name}] の件数取得に失敗しました [{e}]') + + try: + # ③ 取得対象オブジェクトのレコードを取得する + logger.info(f'I-FETCH-04 [{object_name}] のレコード取得を開始します') + + saleforce_data = SalesForceData() + record_generator = saleforce_data.fetch_sf_data_retry( + columns, object_name, last_fetch_datetime_from, last_fetch_datetime_to) + + except Exception as e: + raise SalesforceAPIException( + 'E-FETCH-02', FETCH_JP_NAME, f'[{object_name}] のレコード取得に失敗しました [{e}]') + + try: + # ④ 取得対象オブジェクトをJSONに変換 + logger.info(f'I-FETCH-05 [{object_name}] のレコードをJSONに変換します') + + sf_object_jsons = [] + + for record in record_generator: + sf_object_jsons.append(record) + + except Exception as e: + raise DataConvertException( + 'E-FETCH-03', FETCH_JP_NAME, f'[{object_name}] のレコードのJSON変換に失敗しました [{e}]') + + # ⑤ CRMデータ取得処理終了ログを出力する + logger.info(f'I-FETCH-06 [{object_name}] のレコード取得が成功しました') + + # ⑥ 次の処理へ移行する + return sf_object_jsons diff --git a/ecs/crm-datafetch/src/parser/__init__.py b/ecs/crm-datafetch/src/parser/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parse.py new file mode 100644 index 00000000..4273125d --- /dev/null +++ b/ecs/crm-datafetch/src/parser/json_parse.py @@ -0,0 +1,15 @@ +import re +import json +from src.constants import EXCLUDE_SYMBOL + +class JsonParser(): + def __init__(self, json_str) -> None: + self.__json_str = json_str + + def json_parser(self) -> dict: + for symbol in EXCLUDE_SYMBOL: + # コメントアウトシンボルを含む部分を置き換える正規表現 + replace_comment_regex = rf'\s(?!\"){symbol}[\s\S]*?.*' + self.__json_str = re.sub(replace_comment_regex, '', self.__json_str) + + return json.loads(self.__json_str) diff --git a/ecs/crm-datafetch/src/pre.py b/ecs/crm-datafetch/src/pre.py index 546cbc37..50071fe1 100644 --- a/ecs/crm-datafetch/src/pre.py +++ b/ecs/crm-datafetch/src/pre.py @@ -1,96 +1,83 @@ -# モジュールロード from datetime import datetime -from src.utils.logger import Logger -import src.utils.data_retention as data_retention -import src.utils.s3_access as s3_access -import src.utils.json_parser as json_parser +from src.util.logger import Logger +from src.constants import( + PRE_JP_NAME, + YYYYMMDDTHHMMSSTZ +) +from src.environments import( + CRM_CONFIG_BUCKET, + OBJECT_INFO_FOLDER, + OBJECT_INFO_FILENAME +) +from src.error.exceptions import( + FileNotFoundException, + InvalidConfigException +) +from src.aws.s3 import ConfigBucket +from src.parser.json_parse import JsonParser +from src.config.objects import FetchTargetObjects -# 変数 -# .000ZはUTCを表す。ミリ秒までの考慮は不要なので固定で指定 -YYYYMMDDTHHMMSSTZ = '%Y-%m-%dT%H:%M:%S.000Z' -CRM_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.000%z' -YYYYMMDDHHMMSS = '%Y-%m-%d %H:%M:%S' - - -# logger設定 logger = Logger().get_logger() -# 処理 +def prepare_get_data(): + # ① データ取得準備処理の開始ログを出力する + logger.info('I-PRE-01 データ取得準備処理を開始します') + + # ② 取得処理開始年月日時分秒を控える + execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) + date_path = execute_datetime.rstrip('000Z').translate( + str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) + + logger.info(f'I-PRE-02 データ取得処理開始日時：{execute_datetime}') -def pre(crm_object_info_var): try: - # ① データ取得準備処理の開始ログを出力する - logger.info('I-PRE-01 データ取得準備処理を開始します') + # ③ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する + object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}' + logger.debug( + f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') - # ② 取得処理開始年月日時分秒を控える - execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) - logger.info(f'I-PRE-02 データ取得処理開始日時：{execute_datetime}') + config_bucket = ConfigBucket() + object_info_file_json = config_bucket.get_object_info_file() - # ⑤ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する - - try: - crm_object_info_var = data_retention.CrmObjectInfoVar() - s3_fullpath = f's3://{crm_object_info_var.bucket}/{crm_object_info_var.file_name_key}' - logger.debug(f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{s3_fullpath}]') - - get_file_s3 = s3_access.GetFileS3() - crm_object_info_json = get_file_s3.get_file_from_s3(crm_object_info_var.bucket, crm_object_info_var.file_name_key) - - logger.debug('D-PRE-04 CRM_取得オブジェクト情報ファイルの取得成功しました') - - except Exception as e: - logger.error(f'E-PRE-01 CRM_取得オブジェクト情報ファイルが存在しません　ファイル名：[{s3_fullpath}]', e) - raise e - - - # ⑥ CRM_取得オブジェクト情報ファイルをパースし、メモリ上に展開する - try: - logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします') - - json_exclusde_comment = json_parser.JsonExculudeComment() - crm_object_info_dict: dict = json_exclusde_comment(crm_object_info_json) - - logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました') - - except Exception as e: - logger.error(f'E-PRE-02 CRM_取得オブジェクト情報ファイルのパースに失敗しました　エラー内容：[{e}]') - raise e - - # ⑦ メモリ上のCRM_取得オブジェクト情報のキーobjectsの形式チェックを行う - try: - logger.debug('D-PRE-07 CRM_取得オブジェクト情報ファイルの形式チェックを開始します') - - if 'objects' not in crm_object_info_dict: - logger.error(f'E-PRE-03 CRM_取得オブジェクト情報ファイル「objects」キーは必須です　ファイル名：[{s3_fullpath}]') - raise e - - if not isinstance(crm_object_info_dict["objects"], list): - logger.error(f'E-PRE-04 CRM_取得オブジェクト情報ファイル「objects」キーの値は配列でなければなりません　ファイル名：[{s3_fullpath}]') - raise e - - logger.debug('D-PRE-07 CRM_取得オブジェクト情報ファイルの形式チェック 正常終了') - - except Exception as e: - logger.error(f'CRM_取得オブジェクト情報ファイル「objects」キーの値は配列でなければなりません　ファイル名：[{s3_fullpath}]', e) - raise e - - # ⑧ 処理結果出力用のマップを初期化 - process_result_per_object = {} - for sfdc_object in crm_object_info_dict.get('objects'): - object_name = sfdc_object.get('object_name') - process_result_per_object[object_name] = 'fail' - - - - # ⑨ データ取得準備処理の終了ログを出力する - logger.info('I-PRE-09 データ取得準備処理を終了します') - - # ⑩ 次の処理へ移行する - return(crm_object_info_dict, execute_datetime, process_result_per_object) + logger.debug('D-PRE-04 CRM_取得オブジェクト情報ファイルの取得成功しました') except Exception as e: - logger.error(f'データ取得準備処理で想定外のエラーが発生しました　エラー内容：[{e}]', e) - raise e + raise FileNotFoundException( + 'E-PRE-01', PRE_JP_NAME, f'CRM_取得オブジェクト情報ファイルが存在しません ファイル名：[{OBJECT_INFO_FILENAME}] エラー内容：[{e}]') + + try: + # ④ CRM_取得オブジェクト情報ファイルをパースし、メモリ上に展開する + logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします') + + json_parser = JsonParser(object_info_file_json) + object_info_file_dict = json_parser.json_parser() + + logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました') + + except Exception as e: + raise InvalidConfigException( + 'E-PRE-02', PRE_JP_NAME, f'CRM_取得オブジェクト情報ファイルのパースに失敗しました エラー内容：[{e}]') + + # ⑤ メモリ上のCRM_取得オブジェクト情報のキーobjectsの形式チェックを行う + try: + logger.debug('D-PRE-07 CRM_取得オブジェクト情報ファイルの形式チェックを開始します') + + fetch_target_objects = FetchTargetObjects(object_info_file_dict) + + logger.debug('D-PRE-08 CRM_取得オブジェクト情報ファイルの形式チェック 正常終了') + + except Exception as e: + raise InvalidConfigException( + 'E-PRE-03', PRE_JP_NAME, f'CRM_取得オブジェクト情報ファイルの形式チェックに失敗しました ファイル名：[{OBJECT_INFO_FILENAME}] エラー内容：[{e}]') + + # ⑥ 処理結果出力用のマップを初期化 + process_result = {} + + # ⑦ データ取得準備処理の終了ログを出力する + logger.info('I-PRE-09 データ取得準備処理を終了します') + + # ⑧ 次の処理へ移行する + return(fetch_target_objects, execute_datetime, date_path, process_result) diff --git a/ecs/crm-datafetch/src/process.py b/ecs/crm-datafetch/src/process.py new file mode 100644 index 00000000..5ccbd716 --- /dev/null +++ b/ecs/crm-datafetch/src/process.py @@ -0,0 +1,150 @@ +from src.pre import prepare_get_data # データ取得準備処理 +from src.chk import check_object_info # オブジェクト情報形式チェック処理 +from src.date import set_datetime_period # データ取得期間設定処理 +from src.fetch import fetch_crm_data # CRMデータ取得処理 +from src.resbk import backup_crm_data # CRM電文データバックアップ処理 +from src.conv import convert_crm_csvdata # CSV変換処理 +from src.csvbk import backup_crm_csvdata # CSVバックアップ処理 +from src.upld import copy_crm_csvdata # CSVアップロード処理 +from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 +from src.end import updload_result_data # 取得処理実施結果アップロード処理 +from src.util.logger import Logger +from src.error.exceptions import( + MeDaCaCRMDataFetchException +) + +logger = Logger().get_logger() + + +def main() -> None: + try: + # ① CRMデータ取得処理開始ログを出力する + logger.info('I-CTRL-01 CRMデータ取得処理を開始します') + + fetch_target_objects = None # オブジェクト情報ファイル用オブジェクト + execute_datetime = None # 実行日次文字列 + date_path = None # 実行日次のパス文字列 + process_result = None # オブジェクトごとの実行結果JSON + + # ② データ取得準備処理を呼び出す + logger.info('I-CTRL-02 データ取得準備処理呼び出し') + + fetch_target_objects, execute_datetime, date_path, process_result = prepare_get_data() + + # ③ object_infoのobjectsキーの値の件数分ループする + logger.info('I-CTRL-03 取得対象オブジェクトのループ処理開始') + + for object_info in fetch_target_objects: + try: + # 1. オブジェクト処理結果の初期化 + target_object = None # オブジェクトごとの情報格納用オブジェクト + last_fetch_datetime = None # オブジェクトごとの取得日付格納用オブジェクト + sf_object_jsons = None # オブジェクトごとのSalesforce取得変数JSON + csv_object = None # CSVオブジェクト + process_result[object_info.get('object_name')] = 'fail' # オブジェクト処理結果 + + logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：{object_info}') + + # 2. オブジェクト情報形式チェック処理を呼び出す + logger.info('I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') + + target_object = check_object_info(object_info, execute_datetime) + + # 3. 処理対象のオブジェクト名をログ出力する + logger.info( + f'I-CTRL-06 [{target_object.object_name}]のデータ取得を開始します') + + # 4. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する + if target_object.is_skip is True: + logger.info( + f'I-CTRL-07 [{target_object.object_name}]のデータ取得処理をスキップします') + continue + + # 5. データ取得期間設定処理を呼び出す + logger.info( + f'I-CTRL-08 [{target_object.object_name}]のデータ取得期間設定処理呼び出し') + + last_fetch_datetime = set_datetime_period(target_object, execute_datetime) + + # 6. CRMデータ取得処理を呼び出す + logger.info( + f'I-CTRL-09 [{target_object.object_name}]のデータ取得処理呼び出し') + + sf_object_jsons = fetch_crm_data(target_object, last_fetch_datetime) + + # 7. 出力ファイル名をログ出力する + logger.info( + f'I-CTRL-10 [{target_object.object_name}] の出力ファイル名は [{target_object.upload_file_name}]となります') + + # 8. CRM電文データバックアップ処理を呼び出す + logger.info( + f'I-CTRL-11 [{target_object.object_name}] CRM電文データバックアップ処理呼び出し') + backup_crm_data(target_object.object_name, sf_object_jsons, date_path) + + # 9. CSV変換処理を呼び出す + logger.info( + f'I-CTRL-12 [{target_object.object_name}] CSV変換処理呼び出し') + csv_object = convert_crm_csvdata(target_object, sf_object_jsons) + + # 10. CSVバックアップ処理を呼び出す + logger.info( + f'I-CTRL-13 [{target_object.object_name}] CSVデータバックアップ処理呼び出し') + backup_crm_csvdata(target_object, date_path, csv_object) + + # 11. CSVアップロード処理を呼び出す + logger.info( + f'I-CTRL-14 [{target_object.object_name}] CSVデータアップロード処理呼び出し') + copy_crm_csvdata(target_object, date_path) + + # 12. 前回取得日時ファイル更新処理を呼びだす + logger.info( + f'I-CTRL-15 [{target_object.object_name}] 前回取得日時ファイル更新処理呼び出し') + updload_last_fetch_datetime(target_object, last_fetch_datetime) + + # 13. オブジェクト処理結果の更新 + process_result[target_object.object_name] = 'success' + + # 14. オブジェクトのアップロードが完了した旨をログに出力する + logger.info(f'I-CTRL-16 [{target_object.object_name}] 処理正常終了') + + except MeDaCaCRMDataFetchException as e: + logger.error(f'{e.error_id} {e}') + logger.exception( + f'I-ERR-03 [{object_info.get("object_name")}] の[{e.func_name}]でエラーが発生しました 次のオブジェクトの処理に移行します') + continue + + except Exception as e: + logger.exception( + f'I-ERR-04 [{object_info.get("object_name")}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e) + continue + + # ④ すべてのオブジェクトの処理が完了したことと、オブジェクト毎の処理結果をログに出力する + logger.info(f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result}]') + + # ⑤ 取得処理実施結果アップロード処理を呼び出す + logger.info('I-CTRL-19 CRM_取得処理実施結果ファイルアップロード処理開始') + updload_result_data(process_result, date_path) + + # ⑥ 最終結果をチェックし、チェック結果をログに出力 + if not all([v == 'success' for v in process_result.values()]): + logger.error('E-CTRL-01 一部のデータ取得に失敗しています。') + else: + logger.info('I-CTRL-18 すべてのデータの取得に成功しました。') + + # ⑦ CRMデータ取得処理終了ログを出力する + logger.info('I-CTRL-20 CRMデータ取得処理を終了します') + + return exit(0) + + except MeDaCaCRMDataFetchException as e: + logger.error(f'{e.error_id} {e}') + logger.exception(f'I-ERR-01 [{e.func_name}]でエラーが発生したため、処理を終了します') + return exit(0) + + except Exception as e: + logger.exception('I-ERR-02 予期せぬエラーが発生したため、処理を終了します', e) + return exit(0) + + +if __name__ == '__main__': + main() diff --git a/ecs/crm-datafetch/src/resbk.py b/ecs/crm-datafetch/src/resbk.py index e69de29b..02b3c53b 100644 --- a/ecs/crm-datafetch/src/resbk.py +++ b/ecs/crm-datafetch/src/resbk.py @@ -0,0 +1,35 @@ +from src.util.logger import Logger +from src.constants import( + RESBK_JP_NAME +) +from src.error.exceptions import( + FileUploadException +) +from src.aws.s3 import BackupBucket + + +logger = Logger().get_logger() + + +def backup_crm_data(object_name, sf_object_jsons, date_path): + # ① CRM電文データバックアップ処理の開始ログを出力する + logger.info(f'I-RESBK-01 [{object_name}] のCRM電文データバックアップ処理を開始します') + + try: + # ② CRMバックアップ保管用バケットに、CRMから取得したJSONの電文データのバックアップを保管する + file_name = f'{date_path}/{object_name}.json' + + backup_bucket = BackupBucket() + backup_bucket.put_response_json(file_name, sf_object_jsons) + + logger.debug(f'D-RESBK-02 [{object_name}] のJSONデータバックアップ 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-RESBK-01', RESBK_JP_NAME, f'[{object_name}] 電文データのバックアップに失敗しました ファイル名：[{object_name}.json] エラー内容:[{e}]') + + # ③ CRM電文データバックアップ処理の終了ログを出力する + logger.info(f'I-RESBK-03 [{object_name}] のCRM電文データバックアップ処理を終了します') + + # ④ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/salesforce/__init__.py b/ecs/crm-datafetch/src/salesforce/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py new file mode 100644 index 00000000..ad466782 --- /dev/null +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -0,0 +1,78 @@ +from tenacity import retry, stop_after_attempt, stop_after_delay +from tenacity.wait import wait_exponential +from simple_salesforce import Salesforce +from src.util.async_retry import TimeoutManager, AsyncRetry +from src.environments import( + CRM_AUTH_DOMAIN, + CRM_USER_NAME, + CRM_USER_PASSWORD, + CRM_USER_SECURITY_TOKEN, + CRM_AUTH_TIMEOUT, + CRM_AUTH_MAX_RETRY_ATTEMPT, + CRM_AUTH_RETRY_INTERVAL, + CRM_AUTH_RETRY_MIN_INTERVAL, + CRM_AUTH_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_TIMEOUT, # CRMのレコード取得処理のタイムアウト秒数 + CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + CRM_FETCH_RECORD_RETRY_MAX_INTERVAL +) + +FETCH_SOQL = """SELECT {column_or_expression} FROM {object_name} + WHERE SystemModStamp > {last_update_datetime_from} + AND SystemModStamp <= {last_update_datetime_to} +""" + + +class SalesfoeceApi(): + def __init__(self) -> None: + self.__sf = Salesforce(username=CRM_USER_NAME, password=CRM_USER_PASSWORD, + security_token=CRM_USER_SECURITY_TOKEN, + domain=CRM_AUTH_DOMAIN + ) + + def sf_query(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + return self.__sf.query(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) + + def sf_query_all_iter(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + return self.__sf.query_all_iter(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) + + +class SalesForceCount(): + def fetch_sf_count(self, object_name, last_update_datetime_from, last_update_datetime_to): + count_soql = FETCH_SOQL.format( + column_or_expression='COUNT(Id)', + object_name=object_name, + last_update_datetime_from=last_update_datetime_from, + last_update_datetime_to=last_update_datetime_to + ) + self.__sf = SalesfoeceApi() + count_res = self.__sf.sf_query( + count_soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) + return count_res.get('records')[0].get('expr0') + + @retry(wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT) | stop_after_delay(CRM_GET_RECORD_COUNT_TIMEOUT)) + def fetch_sf_count_retry(self, object_name, last_update_datetime_from, last_update_datetime_to): + return self.fetch_sf_count(object_name, last_update_datetime_from, last_update_datetime_to) + + +class SalesForceData(): + def fetch_sf_data(self, columns, object_name, last_update_datetime_from, last_update_datetime_to): + soql = FETCH_SOQL.format( + column_or_expression=columns, + object_name=object_name, + last_update_datetime_from=last_update_datetime_from, + last_update_datetime_to=last_update_datetime_to + ) + self.__sf = SalesfoeceApi() + return self.__sf.sf_query_all_iter(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + + @retry(wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT) | stop_after_delay(CRM_FETCH_RECORD_TIMEOUT)) + def fetch_sf_data_retry(self, columns, object_name, last_update_datetime_from, last_update_datetime_to): + return self.fetch_sf_data(columns, object_name, last_update_datetime_from, last_update_datetime_to) diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upd.py index e69de29b..4e3cb90d 100644 --- a/ecs/crm-datafetch/src/upd.py +++ b/ecs/crm-datafetch/src/upd.py @@ -0,0 +1,49 @@ +import json +from src.util.logger import Logger +from src.aws.s3 import ConfigBucket +from src.constants import ( + UPD_JP_NAME +) +from src.error.exceptions import( + FileUploadException +) + + +logger = Logger().get_logger() + + +def updload_last_fetch_datetime(target_object, last_fetch_datetime): + # ① 前回取得日時ファイル更新処理の開始ログを出力する + logger.info( + f'I-UPD-01 [{target_object.object_name}] の前回取得日時ファイルの更新処理を開始します') + + try: + if target_object.is_update_last_fetch_datetime == False: + # ② オブジェクト情報.is_update_last_fetch_datetimeがfalseの場合、以降の処理をスキップする + logger.info( + f'I-UPD-02 [{target_object.object_name}] の前回取得日時ファイルの更新処理をスキップします') + else: + # ③ 前回取得日時ファイル.last_fetch_datetime_fromに取得処理開始年月日時分秒を設定する + # 前回取得日時ファイル.last_fetch_datetime_toに空文字を設定する + last_fetch_datetime_dict = { + 'last_fetch_datetime_from': last_fetch_datetime.last_fetch_datetime_to, + 'last_fetch_datetime_to': '' + } + + config_bucket = ConfigBucket() + config_bucket.put_last_fetch_datetime_file( + target_object.last_fetch_datetime_file_name, json.dumps(last_fetch_datetime_dict)) + + logger.info( + f'D-UPD-03 [{target_object.object_name}] の前回取得日時ファイル更新処理 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-UPD-01', UPD_JP_NAME, f'[{target_object.object_name}] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[{target_object.last_fetch_datetime_file_name}] エラー内容:[{e}]') + + # ④ 前回取得日時ファイル更新処理の終了ログを出力する + logger.info( + f'I-UPD-04 [{target_object.object_name}] の前回取得日時ファイルの更新処理を終了します') + + # ⑤ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/upld.py b/ecs/crm-datafetch/src/upld.py index e69de29b..57e33e25 100644 --- a/ecs/crm-datafetch/src/upld.py +++ b/ecs/crm-datafetch/src/upld.py @@ -0,0 +1,43 @@ +from src.util.logger import Logger +from src.constants import ( + UPLD_JP_NAME +) +from src.environments import( + CRM_BACKUP_BUCKET, + IMPORT_DATA_BUCKET, + CRM_IMPORT_DATA_FOLDER, + CRM_IMPORT_DATA_BACKUP_FOLDER, +) +from src.error.exceptions import( + FileUploadException +) +from src.aws.s3 import S3ResourceNonBucket + + +logger = Logger().get_logger() + + +def copy_crm_csvdata(target_object, date_path): + # ① CSVデータアップロード処理の開始ログを出力する + logger.info( + f'I-UPLD-01 [{target_object.object_name}] のCSVデータアップロード処理を開始します ファイル名：[{target_object.upload_file_name}.csv]') + + try: + # ② CRMバックアップ保管用バケットに保管した変換後のCSVデータをデータ取込バケットにコピーする + s3_resource_non_bucket = S3ResourceNonBucket() + s3_resource_non_bucket.copy(CRM_BACKUP_BUCKET, f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{date_path}/{target_object.upload_file_name}.csv', + IMPORT_DATA_BUCKET, f'{CRM_IMPORT_DATA_FOLDER}/{target_object.upload_file_name}.csv') + + logger.debug( + f'D-UPLD-02 [{target_object.object_name}] のCSVデータアップロード 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-UPLD-01', UPLD_JP_NAME, f'[{target_object.object_name}] CSVデータのアップロードに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') + + # ③ CSVデータアップロード処理の終了ログを出力する + logger.info( + f'I-UPLD-03 [{target_object.object_name}] のCSVデータのアップロード処理を終了します') + + # ④ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/util/__init__.py b/ecs/crm-datafetch/src/util/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py new file mode 100644 index 00000000..a8bfed62 --- /dev/null +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -0,0 +1,64 @@ +import re + + +class DictCheck: + def __init__(self) -> None: + pass + + def check_key_exist(self, object_dict: dict, check_key: str) -> bool: + ''' + 辞書型キー存在チェック + ''' + return True if check_key in object_dict and object_dict[check_key] != '' else False + + def check_value_type(self, object_dict: dict, check_key: str, check_type: type) -> bool: + ''' + 辞書型バリュー型チェック + ''' + return True if isinstance(object_dict[check_key], check_type) else False + + def check_regex(self, regex_str: str, object_dict: dict, check_key: str) -> bool: + ''' + 辞書型バリュー正規表現チェック + ''' + return True if re.fullmatch(regex_str, object_dict[check_key]) else False + + def check_key_exist_and_value_type(self, object_dict: dict, check_key: str, check_type: type) -> None: + ''' + 辞書型キー存在チェック＆バリュー型チェック + ''' + if not self.check_key_exist(object_dict, check_key): + raise Exception(f'「{check_key}」キーは必須です') + + elif not self.check_value_type(object_dict, check_key, check_type): + raise Exception(f'「{check_key}」キーの値は「{check_type}」でなければなりません') + + def check_key_exist_case_value_type(self, object_dict: dict, check_key: str, check_type: type): + ''' + 辞書型キー存在した場合のバリュー型チェック + ''' + if not self.check_key_exist(object_dict, check_key): + pass + + elif not self.check_value_type(object_dict, check_key, check_type): + raise Exception(f'「{check_key}」キーの値は「{check_type}」でなければなりません') + + def check_key_exsit_and_regex(self, object_dict: dict, check_key: str, regex_str: str): + ''' + 辞書型キー存在チェック＆バリュー正規表現チェック + ''' + if not self.check_key_exist(object_dict, check_key): + raise Exception(f'「{check_key}」キーは必須です') + + elif not self.check_regex(regex_str, object_dict, check_key): + raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') + + def check_key_exsit_case_regex(self, object_dict: dict, check_key: str, regex_str: str): + ''' + 辞書型キー存在した場合のバリュー正規表現チェック + ''' + if not self.check_key_exist(object_dict, check_key): + pass + + elif not self.check_regex(regex_str, object_dict, check_key): + raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py new file mode 100644 index 00000000..d9892da8 --- /dev/null +++ b/ecs/crm-datafetch/src/util/logger.py @@ -0,0 +1,29 @@ +import logging +from src.environments import LOG_LEVEL + + +class Logger(): + __logger: logging.Logger + + def __init__(self): + self.__logger = logging.getLogger() + + level = logging.getLevelName(LOG_LEVEL) + if not isinstance(level, int): + level = logging.INFO + self.__logger.setLevel(level) + + if not self.__logger.hasHandlers(): + handler = logging.StreamHandler() + self.__logger.addHandler(handler) + + formatter = logging.Formatter( + '[%(levelname)s]\t%(asctime)s\t%(message)s\n', + '%Y-%m-%d %H:%M:%S' + ) + + for handler in self.__logger.handlers: + handler.setFormatter(formatter) + + def get_logger(self) -> logging.Logger: + return self.__logger diff --git a/ecs/crm-datafetch/src/utils/data_retention.py b/ecs/crm-datafetch/src/utils/data_retention.py deleted file mode 100644 index 89d7d4c3..00000000 --- a/ecs/crm-datafetch/src/utils/data_retention.py +++ /dev/null @@ -1,169 +0,0 @@ -# モジュールロード -import dataclasses -#from fileinput import filename -import os -#from tracemalloc import DomainFilter - -# 変数 -# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する -LOG_LEVEL = os.environ["LOG_LEVEL"] - -CRM_AUTH_TIMEOUT = os.environ["CRM_AUTH_TIMEOUT"] # CRMへの認証処理のタイムアウト秒数 -# CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ["CRM_AUTH_MAX_RETRY_ATTEMPT"] -# CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_INTERVAL = os.environ["CRM_AUTH_RETRY_INTERVAL"] -# CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ["CRM_AUTH_RETRY_MIN_INTERVAL"] -# CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ["CRM_AUTH_RETRY_MAX_INTERVAL"] - -# CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ["CRM_GET_RECORD_COUNT_TIMEOUT"] -# CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ["CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT"] -# CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_INTERVAL"] -# CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL"] -# CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL"] - -# CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ["CRM_FETCH_RECORD_TIMEOUT"] -# CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ["CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT"] -# CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_INTERVAL"] -# CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MIN_INTERVAL"] -# CRMのレコード取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MAX_INTERVAL"] - - -CRM_AUTH_DOMAIN = os.environ["CRM_AUTH_DOMAIN"] # CRMのAPI実行のための認証エンドポイントのドメイン -CRM_USER_NAME = os.environ["CRM_USER_NAME"] # CRMのAPI実行用ユーザ名 -CRM_USER_PASSWORD = os.environ["CRM_USER_PASSWORD"] # CRMのAPI実行用ユーザパスワード -# CRMのAPI実行用ユーザのセキュリティトークン -CRM_USER_SECURITY_TOKEN = os.environ["CRM_USER_SECURITY_TOKEN"] - -# CRMデータ取得用の設定ファイルを格納するバケット名 -CRM_CONFIG_BUCKET = os.environ["CRM_CONFIG_BUCKET"] -CRM_BACKUP_BUCKET = os.environ["CRM_BACKUP_BUCKET"] # CRMのバックアップデータを格納するバケット名 -IMPORT_DATA_BUCKET = os.environ["IMPORT_DATA_BUCKET"] # CRMの取込データを格納するバケット名 - -# TASK_SETTING_FOLDER = os.environ["TASK_SETTING_FOLDER"] # CRM取得処理タスクの設定ファイルを格納するフォルダパス -# TASK_SETTING_FILENAME = os.environ["TASK_SETTING_FILENAME"] # CRM取得処理タスクの設定ファイルのファイル名 - -# CRM取得対象オブジェクトの情報を格納するフォルダパス -OBJECT_INFO_FOLDER = os.environ["OBJECT_INFO_FOLDER"] -# CRM取得対象オブジェクトの情報のファイル名 -OBJECT_INFO_FILENAME = os.environ["OBJECT_INFO_FILENAME"] - -# CRMデータ取得結果を格納するフォルダパス -FETCH_RESULT_FOLDER = os.environ["FETCH_RESULT_FOLDER"] -# CRMデータ取得結果を格納するファイル名 -FETCH_RESULT_FILENAME = os.environ["FETCH_RESULT_FILENAME"] - -# CRMからの最終取得日時ファイルを格納するフォルダパス -LAST_FETCH_DATE_FOLDER = os.environ["LAST_FETCH_DATE_FOLDER"] -# CRMから取得し、取込用に変換したデータを格納するフォルダ -CRM_IMPORT_DATA_FOLDER = os.environ["CRM_IMPORT_DATA_FOLDER"] - -# CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ["LAST_FETCH_DATE_BACKUP_FOLDER"] -# CRMから取得した生データのバックアップを格納するフォルダパス -FETCH_DATA_BACKUP_FOLDER = os.environ["FETCH_DATA_BACKUP_FOLDER"] -# CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ -CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ["CRM_IMPORT_DATA_BACKUP_FOLDER"] - - -# 処理 -@dataclasses.dataclass -class LogLevel: - loglevel: str = LOG_LEVEL - - -@dataclasses.dataclass -class CrmAuthVar: - auth_domain: str = CRM_AUTH_DOMAIN - user_name: str = CRM_USER_NAME - user_password: str = CRM_USER_PASSWORD - security_token: str = CRM_USER_SECURITY_TOKEN - timeout: int = CRM_AUTH_TIMEOUT - max_retry_attempt: int = CRM_AUTH_MAX_RETRY_ATTEMPT - retry_interval: int = CRM_AUTH_RETRY_INTERVAL - retry_min_interval: int = CRM_AUTH_RETRY_MIN_INTERVAL - retry_max_interval: int = CRM_AUTH_RETRY_MAX_INTERVAL - - -@dataclasses.dataclass -class CrmGetRecordCountVar: - timeout: int = CRM_GET_RECORD_COUNT_TIMEOUT - max_retry_attempt: int = CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT - retry_interval: int = CRM_GET_RECORD_COUNT_RETRY_INTERVAL - retry_min_interval: int = CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL - retry_max_interval: int = CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL - - -@dataclasses.dataclass -class CrmFetchRecordVar: - timeout: int = CRM_FETCH_RECORD_TIMEOUT - max_retry_attempt: int = CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT - retry_interval: int = CRM_FETCH_RECORD_RETRY_INTERVAL - retry_min_interval: int = CRM_FETCH_RECORD_RETRY_MIN_INTERVAL - retry_max_interval: int = CRM_FETCH_RECORD_RETRY_MAX_INTERVAL - - -@dataclasses.dataclass -class CrmBucketVar: - config_bucket: str = CRM_CONFIG_BUCKET - backup_bucket: str = CRM_BACKUP_BUCKET - data_bucket: str = IMPORT_DATA_BUCKET - - -@dataclasses.dataclass -class CrmObjectInfoVar: - bucket: str = CRM_CONFIG_BUCKET - folder: str = OBJECT_INFO_FOLDER - file_name: str = OBJECT_INFO_FILENAME - file_name_key: str = f'{folder}/{file_name}' - - -@dataclasses.dataclass -class FetchResultVar: - folder: str = FETCH_RESULT_FOLDER - file_name: str = FETCH_RESULT_FILENAME - file_name_key: str = f'{folder}/{file_name}' - - -@dataclasses.dataclass -class FetchResultVar: - folder: str = FETCH_RESULT_FOLDER - file_name: str = FETCH_RESULT_FILENAME - file_name_key: str = f'{folder}/{file_name}' - - -@dataclasses.dataclass -class LastFetchDateVar: - folder: str = LAST_FETCH_DATE_FOLDER - - -@dataclasses.dataclass -class CrmImportDataVar: - folder: str = CRM_IMPORT_DATA_FOLDER - - -@dataclasses.dataclass -class LastFetchDateBackupVar: - folder: str = LAST_FETCH_DATE_BACKUP_FOLDER - - -@dataclasses.dataclass -class FetchDataBackupVar: - folder: str = FETCH_DATA_BACKUP_FOLDER - - -@dataclasses.dataclass -class CrmImportDataBackupVar: - folder: str = CRM_IMPORT_DATA_BACKUP_FOLDER diff --git a/ecs/crm-datafetch/src/utils/json_parser.py b/ecs/crm-datafetch/src/utils/json_parser.py deleted file mode 100644 index b6d288a9..00000000 --- a/ecs/crm-datafetch/src/utils/json_parser.py +++ /dev/null @@ -1,28 +0,0 @@ -# モジュールロード -import re -import json - - -# 変数 -SYMBOL = ['#', '/'] - - -# 処理 -class JsonExculudeComment: - def __init__(self) -> None: - pass - - def json_exclude_comment(self, json_str) -> dict: - for symbol in SYMBOL: - # コメントアウトシンボルを含む部分を置き換える正規表現 - replace_comment_regex = rf'\s(?!\"){symbol}[\s\S]*?.*' - - json_str = self.json_regex_parse(replace_comment_regex, json_str) - - result = json.loads(json_str) - - return result - - def json_regex_parse(replace_comment_regex, json_str) -> str: - json_without_comment = re.sub(replace_comment_regex, '', json_str) - return json_without_comment diff --git a/ecs/crm-datafetch/src/utils/logger.py b/ecs/crm-datafetch/src/utils/logger.py deleted file mode 100644 index 5ec076fd..00000000 --- a/ecs/crm-datafetch/src/utils/logger.py +++ /dev/null @@ -1,33 +0,0 @@ -import logging -import os - -LOG_LEVEL = { - 'critical': logging.CRITICAL, - 'error': logging.ERROR, - 'warn': logging.WARNING, - 'info': logging.INFO, - 'debug': logging.DEBUG -} - - -class Logger(): - __logger: logging.Logger - - def __init__(self): - # logger設定 - self.__logger = logging.getLogger() - formatter = logging.Formatter( - '[%(levelname)s]\t%(asctime)s\t%(message)s\n', - '%Y-%m-%d %H:%M:%S' - ) - handler = logging.StreamHandler() - self.__logger.addHandler(handler) - for handler in self.__logger.handlers: - handler.setFormatter(formatter) - level = logging.getLevelName(os.environ.get('LOG_LEVEL', 'WARN').upper()) - if not isinstance(level, int): - level = logging.INFO - self.__logger.setLevel(level) - - def get_logger(self) -> logging.Logger: - return self.__logger diff --git a/ecs/crm-datafetch/src/utils/s3_access.py b/ecs/crm-datafetch/src/utils/s3_access.py deleted file mode 100644 index 7ffb697d..00000000 --- a/ecs/crm-datafetch/src/utils/s3_access.py +++ /dev/null @@ -1,48 +0,0 @@ -# モジュールロード -import boto3 -from tenacity import retry, stop_after_attempt -from tenacity.wait import wait_exponential - -# 変数 -s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') - -# 処理 - - -class UploadFileS3: - def __init__(self, bucket_name, local_file_path, file_name_key) -> None: - self.bucket_name: str = bucket_name - self.local_file_path: str = local_file_path - self.file_name_key: str = file_name_key - - def upload_file_to_s3(self) -> None: - bucket = s3_resource.Bucket(self.bucket_name) - bucket.upload_file(self.local_file_path, self.file_name_key) - - -class GetFileS3: - def __init__(self, bucket_name, file_name_key, multiplier=5, min=5, max=50, count=3) -> None: - self.bucket_name: str = bucket_name - self.file_name_key: str = file_name_key - self.multiplier: int = multiplier - self.min: int = min - self.max: int = max - self.count: int = count - - def get_file_from_s3(self) -> str: - bucket = s3_resource.Bucket(self.bucket_name) - response = bucket.Object(self.file_name_key).get() - body = response['Body'].read() - - return body.decode('utf-8') - - def get_file_from_s3_with_retry(self) -> str: - - @retry(wait=wait_exponential(multiplier=self.multiplier, min=self.min, max=self.max), stop=stop_after_attempt(self.count)) - def get_file_from_s3_with_retry_deco(*args, **kwargs) -> str: - return self.get_file_from_s3(self.bucket_name, self.file_name_key) - - response = get_file_from_s3_with_retry_deco(self) - - return response From 3b5b0dc33c1b8351ad6f76b134b6d98da3af4344 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 21 Jul 2022 10:01:24 +0900 Subject: [PATCH 03/21] =?UTF-8?q?feat:=E3=83=AD=E3=82=B0=E5=87=BA=E5=8A=9B?= =?UTF-8?q?=E3=82=84=E3=80=81Salesforce=E3=81=AE=E6=8E=A5=E7=B6=9A?= =?UTF-8?q?=E3=83=AA=E3=83=88=E3=83=A9=E3=82=A4=E3=82=92=E4=B8=BB=E3=81=A8?= =?UTF-8?q?=E3=81=97=E3=81=9F=E6=A9=9F=E8=83=BD=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/launch.json | 19 ++++ ecs/crm-datafetch/src/aws/s3.py | 19 ++-- ecs/crm-datafetch/src/chk.py | 5 +- ecs/crm-datafetch/src/config/__init__.py | 0 ecs/crm-datafetch/src/constants.py | 64 +++++------ ecs/crm-datafetch/src/conv.py | 5 +- ecs/crm-datafetch/src/converter/__init__.py | 0 ecs/crm-datafetch/src/csvbk.py | 8 +- ecs/crm-datafetch/src/date.py | 5 +- ecs/crm-datafetch/src/end.py | 5 +- ecs/crm-datafetch/src/environments.py | 101 ++++++------------ ecs/crm-datafetch/src/error/exceptions.py | 10 -- ecs/crm-datafetch/src/fetch.py | 99 +++++++++++++++-- ecs/crm-datafetch/src/pre.py | 5 +- ecs/crm-datafetch/src/process.py | 26 +++-- ecs/crm-datafetch/src/resbk.py | 5 +- .../src/salesforce/salesforce_api.py | 14 +-- ecs/crm-datafetch/src/upd.py | 5 +- ecs/crm-datafetch/src/upld.py | 5 +- ecs/crm-datafetch/src/util/logger.py | 6 ++ 20 files changed, 214 insertions(+), 192 deletions(-) create mode 100644 .vscode/launch.json create mode 100644 ecs/crm-datafetch/src/config/__init__.py create mode 100644 ecs/crm-datafetch/src/converter/__init__.py diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 00000000..d8f8414f --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,19 @@ +{ + // IntelliSense を使用して利用可能な属性を学べます。 + // 既存の属性の説明をホバーして表示します。 + // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: CRMデータ取得ローカル実行", + "type": "python", + "request": "launch", + // windowsだと\区切りかも + "program": "ecs\\crm-datafetch\\main.py", + "console": "integratedTerminal", + "justMyCode": true, + "envFile": "${workspaceFolder}/.env", + "args": ["local"] + } + ] + } \ No newline at end of file diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index e5235016..229b8666 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -6,6 +6,7 @@ from src.environments import ( IMPORT_DATA_BUCKET, OBJECT_INFO_FOLDER, OBJECT_INFO_FILENAME, + PROCESS_RESULT_FOLDER, CRM_IMPORT_DATA_FOLDER, CRM_IMPORT_DATA_BACKUP_FOLDER, LAST_FETCH_DATE_FOLDER, @@ -49,7 +50,7 @@ class S3ResourceNonBucket: def __init__(self) -> None: self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) - def copy(self, src_bucket, src_key, dest_bucket, dest_key): + def copy(self, src_bucket, src_key, dest_bucket, dest_key) -> None: copy_source = {'Bucket': src_bucket, 'Key': src_key} self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) @@ -62,13 +63,13 @@ class ConfigBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(CRM_CONFIG_BUCKET) - def get_object_info_file(self): + def get_object_info_file(self) -> str: return self.__s3_resource.get_object(f'{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}') - def get_last_fetch_datetime_file(self, file_name): + def get_last_fetch_datetime_file(self, file_name) -> str: return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_name}') - def put_last_fetch_datetime_file(self, file_name, data): + def put_last_fetch_datetime_file(self, file_name, data) -> None: self.__s3_resource.put_object( f'{LAST_FETCH_DATE_FOLDER}/{file_name}', data) return @@ -80,7 +81,7 @@ class DataBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(IMPORT_DATA_BUCKET) - def put_csv(self, file_name, data): + def put_csv(self, file_name, data) -> None: object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_name}' self.__s3_resource.put_object(object_key, data) return @@ -92,16 +93,16 @@ class BackupBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(CRM_BACKUP_BUCKET) - def put_response_json(self, file_name, data): + def put_response_json(self, file_name, data) -> None: object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_name}' self.__s3_resource.put_object(object_key, json.dumps(data)) return - def put_csv_bk(self, file_name, data): + def put_csv_bk(self, file_name, data) -> None: object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_name}' self.__s3_resource.put_object(object_key, data) return - def put_result_json(self, file_name, data): - object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_name}' + def put_result_json(self, file_name, data) -> None: + object_key = f'{PROCESS_RESULT_FOLDER}/{file_name}' self.__s3_resource.put_object(object_key, json.dumps(data)) diff --git a/ecs/crm-datafetch/src/chk.py b/ecs/crm-datafetch/src/chk.py index 1342243e..077f0ee2 100644 --- a/ecs/crm-datafetch/src/chk.py +++ b/ecs/crm-datafetch/src/chk.py @@ -1,4 +1,4 @@ -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import( CHK_JP_NAME ) @@ -8,9 +8,6 @@ from src.error.exceptions import( from src.config.objects import TargetObject -logger = Logger().get_logger() - - def check_object_info(object_info, execute_datetime): # ① オブジェクト情報形式チェック処理開始ログを出力する logger.info('I-CHK-01 オブジェクト情報形式チェック処理を開始します') diff --git a/ecs/crm-datafetch/src/config/__init__.py b/ecs/crm-datafetch/src/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/constants.py b/ecs/crm-datafetch/src/constants.py index fcd6b4a3..a394df16 100644 --- a/ecs/crm-datafetch/src/constants.py +++ b/ecs/crm-datafetch/src/constants.py @@ -1,38 +1,38 @@ # environments(task settings file) -LOG_LEVEL = "LOG_LEVEL" # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する -CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = 'CRM_AUTH_RETRY_MIN_INTERVAL' # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = 'CRM_AUTH_RETRY_MAX_INTERVAL' # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = 'CRM_GET_RECORD_COUNT_TIMEOUT' # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = 'CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT' # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_INTERVAL' # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL' # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL' # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_TIMEOUT = 'CRM_FETCH_RECORD_TIMEOUT' # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = 'CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT' # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_RETRY_INTERVAL = 'CRM_FETCH_RECORD_RETRY_INTERVAL' # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MIN_INTERVAL' # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MAX_INTERVAL' # CRMのレコード取得処理のリトライ時の最大待ち秒数 +LOG_LEVEL = "LOG_LEVEL" # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 +CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = 'CRM_AUTH_RETRY_MIN_INTERVAL' # CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = 'CRM_AUTH_RETRY_MAX_INTERVAL' # CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = 'CRM_GET_RECORD_COUNT_TIMEOUT' # CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = 'CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT' # CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_INTERVAL' # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL' # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = 'CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL' # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_TIMEOUT = 'CRM_FETCH_RECORD_TIMEOUT' # CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = 'CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT' # CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_RETRY_INTERVAL = 'CRM_FETCH_RECORD_RETRY_INTERVAL' # CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MIN_INTERVAL' # CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MAX_INTERVAL' # CRMのレコード取得処理のリトライ時の最大待ち秒数 # environments(ECS Task Enviroment) -CRM_AUTH_DOMAIN = 'CRM_AUTH_DOMAIN' # CRMのAPI実行のための認証エンドポイントのドメイン -CRM_USER_NAME = 'CRM_USER_NAME' # CRMのAPI実行用ユーザ名 -CRM_USER_PASSWORD = 'CRM_USER_PASSWORD' # CRMのAPI実行用ユーザパスワード -CRM_USER_SECURITY_TOKEN = 'CRM_USER_SECURITY_TOKEN' # CRMのAPI実行用ユーザのセキュリティトークン -CRM_CONFIG_BUCKET = 'CRM_CONFIG_BUCKET' # CRMデータ取得用の設定ファイルを格納するバケット名 -CRM_BACKUP_BUCKET = 'CRM_BACKUP_BUCKET' # CRMのバックアップデータを格納するバケット名 -IMPORT_DATA_BUCKET = 'IMPORT_DATA_BUCKET' # CRMの取込データを格納するバケット名 -OBJECT_INFO_FOLDER = 'OBJECT_INFO_FOLDER' # CRM取得対象オブジェクトの情報を格納するフォルダパス -OBJECT_INFO_FILENAME = 'OBJECT_INFO_FILENAME' # CRM取得対象オブジェクトの情報のファイル名 -PROCESS_RESULT_FOLDER = 'PROCESS_RESULT_FOLDER' # CRMデータ取得結果を格納するフォルダパス -PROCESS_RESULT_FILENAME = 'PROCESS_RESULT_FILENAME' # CRMデータ取得結果を格納するファイル名 -LAST_FETCH_DATE_FOLDER = 'LAST_FETCH_DATE_FOLDER' # CRMからの最終取得日時ファイルを格納するフォルダパス -CRM_IMPORT_DATA_FOLDER ='CRM_IMPORT_DATA_FOLDER' # CRMから取得し、取込用に変換したデータを格納するフォルダ -LAST_FETCH_DATE_BACKUP_FOLDER = 'LAST_FETCH_DATE_BACKUP_FOLDER' # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -RESPONSE_JSON_BACKUP_FOLDER = 'RESPONSE_JSON_BACKUP_FOLDER' # CRMから取得した生データのバックアップを格納するフォルダパス -CRM_IMPORT_DATA_BACKUP_FOLDER = 'CRM_IMPORT_DATA_BACKUP_FOLDER' # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ +CRM_AUTH_DOMAIN = 'CRM_AUTH_DOMAIN' # CRMのAPI実行のための認証エンドポイントのドメイン +CRM_USER_NAME = 'CRM_USER_NAME' # CRMのAPI実行用ユーザ名 +CRM_USER_PASSWORD = 'CRM_USER_PASSWORD' # CRMのAPI実行用ユーザパスワード +CRM_USER_SECURITY_TOKEN = 'CRM_USER_SECURITY_TOKEN' # CRMのAPI実行用ユーザのセキュリティトークン +CRM_CONFIG_BUCKET = 'CRM_CONFIG_BUCKET' # CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_BACKUP_BUCKET = 'CRM_BACKUP_BUCKET' # CRMのバックアップデータを格納するバケット名 +IMPORT_DATA_BUCKET = 'IMPORT_DATA_BUCKET' # CRMの取込データを格納するバケット名 +OBJECT_INFO_FOLDER = 'OBJECT_INFO_FOLDER' # CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FILENAME = 'OBJECT_INFO_FILENAME' # CRM取得対象オブジェクトの情報のファイル名 +PROCESS_RESULT_FOLDER = 'PROCESS_RESULT_FOLDER' # CRMデータ取得結果を格納するフォルダパス +PROCESS_RESULT_FILENAME = 'PROCESS_RESULT_FILENAME' # CRMデータ取得結果を格納するファイル名 +LAST_FETCH_DATE_FOLDER = 'LAST_FETCH_DATE_FOLDER' # CRMからの最終取得日時ファイルを格納するフォルダパス +CRM_IMPORT_DATA_FOLDER = 'CRM_IMPORT_DATA_FOLDER' # CRMから取得し、取込用に変換したデータを格納するフォルダ +LAST_FETCH_DATE_BACKUP_FOLDER = 'LAST_FETCH_DATE_BACKUP_FOLDER' # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +RESPONSE_JSON_BACKUP_FOLDER = 'RESPONSE_JSON_BACKUP_FOLDER' # CRMから取得した生データのバックアップを格納するフォルダパス +CRM_IMPORT_DATA_BACKUP_FOLDER = 'CRM_IMPORT_DATA_BACKUP_FOLDER' # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ # 時刻フォーマット # .000ZはUTCを表す。ミリ秒までの考慮は不要なので固定で指定 diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/conv.py index 88b86ae8..eb17c4c3 100644 --- a/ecs/crm-datafetch/src/conv.py +++ b/ecs/crm-datafetch/src/conv.py @@ -1,5 +1,5 @@ from datetime import datetime -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import ( CONV_JP_NAME, ) @@ -9,9 +9,6 @@ from src.error.exceptions import( from src.converter.converter import CSVStringConverter -logger = Logger().get_logger() - - def convert_crm_csvdata(target_object, sf_object_jsons): # ① CSV変換処理の開始ログを出力する logger.info(f'I-CONV-01 [{target_object.object_name}] のCSV変換処理を開始します') diff --git a/ecs/crm-datafetch/src/converter/__init__.py b/ecs/crm-datafetch/src/converter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/crm-datafetch/src/csvbk.py b/ecs/crm-datafetch/src/csvbk.py index db490c63..75c5f780 100644 --- a/ecs/crm-datafetch/src/csvbk.py +++ b/ecs/crm-datafetch/src/csvbk.py @@ -1,4 +1,4 @@ -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.aws.s3 import BackupBucket from src.constants import ( CSVBK_JP_NAME @@ -8,9 +8,6 @@ from src.error.exceptions import( ) -logger = Logger().get_logger() - - def backup_crm_csvdata(target_object, date_path, csv_object): # ① CSVバックアップ処理の開始ログを出力する logger.info( @@ -22,6 +19,9 @@ def backup_crm_csvdata(target_object, date_path, csv_object): backup_bucket.put_csv_bk( f'{date_path}/{target_object.upload_file_name}.csv', csv_object.csv_buffer) + logger.debug( + f'D-CSVBK-02 [{target_object.object_name}] のCSVデータバックアップ 正常終了') + except Exception as e: raise FileUploadException( 'E-CSVBK-01', CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') diff --git a/ecs/crm-datafetch/src/date.py b/ecs/crm-datafetch/src/date.py index 8fff0d94..adcfdd4c 100644 --- a/ecs/crm-datafetch/src/date.py +++ b/ecs/crm-datafetch/src/date.py @@ -1,4 +1,4 @@ -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import( DATE_JP_NAME ) @@ -15,9 +15,6 @@ from src.config.objects import LastFetchDatetime from src.aws.s3 import ConfigBucket -logger = Logger().get_logger() - - def set_datetime_period(target_object, execute_datetime): # ① データ取得期間設定処理の開始ログを出力する logger.info( diff --git a/ecs/crm-datafetch/src/end.py b/ecs/crm-datafetch/src/end.py index e2b274b3..7604b5b3 100644 --- a/ecs/crm-datafetch/src/end.py +++ b/ecs/crm-datafetch/src/end.py @@ -1,4 +1,4 @@ -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import ( END_JP_NAME ) @@ -11,9 +11,6 @@ from src.error.exceptions import( from src.aws.s3 import BackupBucket -logger = Logger().get_logger() - - def updload_result_data(process_result, date_path): # ① 取得処理実施結果アップロード処理のログを出力する logger.info( diff --git a/ecs/crm-datafetch/src/environments.py b/ecs/crm-datafetch/src/environments.py index 8d63cc64..262a2fbd 100644 --- a/ecs/crm-datafetch/src/environments.py +++ b/ecs/crm-datafetch/src/environments.py @@ -1,73 +1,38 @@ import os - -from src.constants import ( - LOG_LEVEL, - LOG_LEVEL_INFO, - CRM_AUTH_TIMEOUT, - CRM_AUTH_MAX_RETRY_ATTEMPT, - CRM_AUTH_RETRY_INTERVAL, - CRM_AUTH_RETRY_MIN_INTERVAL, - CRM_AUTH_RETRY_MAX_INTERVAL, - CRM_GET_RECORD_COUNT_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, - CRM_AUTH_DOMAIN, - CRM_USER_NAME, - CRM_USER_PASSWORD, - CRM_USER_SECURITY_TOKEN, - CRM_CONFIG_BUCKET, - CRM_BACKUP_BUCKET, - IMPORT_DATA_BUCKET, - OBJECT_INFO_FOLDER, - OBJECT_INFO_FILENAME, - PROCESS_RESULT_FOLDER, - PROCESS_RESULT_FILENAME, - LAST_FETCH_DATE_FOLDER, - CRM_IMPORT_DATA_FOLDER, - LAST_FETCH_DATE_BACKUP_FOLDER, - RESPONSE_JSON_BACKUP_FOLDER, - CRM_IMPORT_DATA_BACKUP_FOLDER -) +import src.constants as constants # environments(task settings file) -LOG_LEVEL = os.environ.get(LOG_LEVEL, LOG_LEVEL_INFO) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する -CRM_AUTH_TIMEOUT = os.environ["CRM_AUTH_TIMEOUT"] # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ["CRM_AUTH_MAX_RETRY_ATTEMPT"] # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_RETRY_INTERVAL = os.environ["CRM_AUTH_RETRY_INTERVAL"] # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ["CRM_AUTH_RETRY_MIN_INTERVAL"] # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ["CRM_AUTH_RETRY_MAX_INTERVAL"] # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ["CRM_GET_RECORD_COUNT_TIMEOUT"] # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ["CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT"] # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_INTERVAL"] # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL"] # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ["CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL"] # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ["CRM_FETCH_RECORD_TIMEOUT"] # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ["CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT"] # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_INTERVAL"] # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MIN_INTERVAL"] # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ["CRM_FETCH_RECORD_RETRY_MAX_INTERVAL"] # CRMのレコード取得処理のリトライ時の最大待ち秒数 +LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT] # CRMへの認証処理のタイムアウト秒数 +CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT] # CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL] # CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL] # CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL] # CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT] # CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT] # CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL] # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL] # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL] # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT] # CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT] # CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL] # CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL] # CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL] # CRMのレコード取得処理のリトライ時の最大待ち秒数 # environments(ECS Task Enviroment) -CRM_AUTH_DOMAIN = os.environ["CRM_AUTH_DOMAIN"] # CRMのAPI実行のための認証エンドポイントのドメイン -CRM_USER_NAME = os.environ["CRM_USER_NAME"] # CRMのAPI実行用ユーザ名 -CRM_USER_PASSWORD = os.environ["CRM_USER_PASSWORD"] # CRMのAPI実行用ユーザパスワード -CRM_USER_SECURITY_TOKEN = os.environ["CRM_USER_SECURITY_TOKEN"] # CRMのAPI実行用ユーザのセキュリティトークン -CRM_CONFIG_BUCKET = os.environ["CRM_CONFIG_BUCKET"] # CRMデータ取得用の設定ファイルを格納するバケット名 -CRM_BACKUP_BUCKET = os.environ["CRM_BACKUP_BUCKET"] # CRMのバックアップデータを格納するバケット名 -IMPORT_DATA_BUCKET = os.environ["IMPORT_DATA_BUCKET"] # CRMの取込データを格納するバケット名 -OBJECT_INFO_FOLDER = os.environ["OBJECT_INFO_FOLDER"] # CRM取得対象オブジェクトの情報を格納するフォルダパス -OBJECT_INFO_FILENAME = os.environ["OBJECT_INFO_FILENAME"] # CRM取得対象オブジェクトの情報のファイル名 -PROCESS_RESULT_FOLDER = os.environ["PROCESS_RESULT_FOLDER"] # CRMデータ取得結果を格納するフォルダパス -PROCESS_RESULT_FILENAME = os.environ["PROCESS_RESULT_FILENAME"] # CRMデータ取得結果を格納するファイル名 -LAST_FETCH_DATE_FOLDER = os.environ["LAST_FETCH_DATE_FOLDER"] # CRMからの最終取得日時ファイルを格納するフォルダパス -CRM_IMPORT_DATA_FOLDER = os.environ["CRM_IMPORT_DATA_FOLDER"] # CRMから取得し、取込用に変換したデータを格納するフォルダ -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ["LAST_FETCH_DATE_BACKUP_FOLDER"] # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -RESPONSE_JSON_BACKUP_FOLDER = os.environ["FETCH_DATA_BACKUP_FOLDER"] # CRMから取得した生データのバックアップを格納するフォルダパス -CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ["CRM_IMPORT_DATA_BACKUP_FOLDER"] # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ +CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN] # CRMのAPI実行のための認証エンドポイントのドメイン +CRM_USER_NAME = os.environ[constants.CRM_USER_NAME] # CRMのAPI実行用ユーザ名 +CRM_USER_PASSWORD = os.environ[constants.CRM_USER_PASSWORD] # CRMのAPI実行用ユーザパスワード +CRM_USER_SECURITY_TOKEN = os.environ[constants.CRM_USER_SECURITY_TOKEN] # CRMのAPI実行用ユーザのセキュリティトークン +CRM_CONFIG_BUCKET = os.environ[constants.CRM_CONFIG_BUCKET] # CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET] # CRMのバックアップデータを格納するバケット名 +IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET] # CRMの取込データを格納するバケット名 +OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER] # CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME] # CRM取得対象オブジェクトの情報のファイル名 +PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER] # CRMデータ取得結果を格納するフォルダパス +PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME] # CRMデータ取得結果を格納するファイル名 +LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER] # CRMからの最終取得日時ファイルを格納するフォルダパス +CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER] # CRMから取得し、取込用に変換したデータを格納するフォルダ +LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER] # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER] # CRMから取得した生データのバックアップを格納するフォルダパス +CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER] # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ diff --git a/ecs/crm-datafetch/src/error/exceptions.py b/ecs/crm-datafetch/src/error/exceptions.py index 528c262d..71a174c0 100644 --- a/ecs/crm-datafetch/src/error/exceptions.py +++ b/ecs/crm-datafetch/src/error/exceptions.py @@ -30,16 +30,6 @@ class DataConvertException(MeDaCaCRMDataFetchException): pass -class TimeOutException(MeDaCaCRMDataFetchException): - """タイムアウトが発生した場合の例外""" - pass - - -class RetryExceededException(MeDaCaCRMDataFetchException): - """リトライ処理が発生した場合の例外""" - pass - - class SalesforceAPIException(MeDaCaCRMDataFetchException): """SalseforceのAPI実行失敗が発生した場合の例外""" pass diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch.py index bac313fd..9670767a 100644 --- a/ecs/crm-datafetch/src/fetch.py +++ b/ecs/crm-datafetch/src/fetch.py @@ -1,7 +1,23 @@ -from src.util.logger import Logger +from requests.exceptions import ConnectTimeout, ReadTimeout +from tenacity import retry, stop_after_attempt +from tenacity.wait import wait_exponential +from src.util.logger import logger_instance as logger from src.constants import( FETCH_JP_NAME ) +from src.environments import( + CRM_AUTH_TIMEOUT, + CRM_GET_RECORD_COUNT_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_TIMEOUT, + CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + CRM_FETCH_RECORD_RETRY_MAX_INTERVAL +) from src.error.exceptions import( SalesforceAPIException, DataConvertException @@ -12,9 +28,6 @@ from src.salesforce.salesforce_api import( ) -logger = Logger().get_logger() - - def fetch_crm_data(target_object, last_fetch_datetime): # ① CRMデータ取得処理開始ログを出力する logger.info( @@ -25,31 +38,37 @@ def fetch_crm_data(target_object, last_fetch_datetime): last_fetch_datetime_from = last_fetch_datetime.last_fetch_datetime_from last_fetch_datetime_to = last_fetch_datetime.last_fetch_datetime_to + global count_contime_counter, count_readtime_counter, count_counter, data_contime_counter, data_readtime_counter, data_counter + count_contime_counter = 1 + count_readtime_counter = 1 + count_counter = 1 + data_contime_counter = 1 + data_readtime_counter = 1 + data_counter = 1 + try: # ② 取得対象オブジェクトの取得期間内のレコード件数を取得する logger.info(f'I-FETCH-02 [{object_name}] の件数取得を開始します') - saleforce_count = SalesForceCount() - record_count = saleforce_count.fetch_sf_count_retry( + record_count = fetch_sf_count_retry( object_name, last_fetch_datetime_from, last_fetch_datetime_to) logger.info(f'I-FETCH-03 [{object_name}] の件数：[{record_count}]') except Exception as e: raise SalesforceAPIException( - 'E-FETCH-01', FETCH_JP_NAME, f'[{object_name}] の件数取得に失敗しました [{e}]') + 'E-FETCH-01', FETCH_JP_NAME, f'[{object_name}] の件数取得に失敗しました エラー内容：[{e}]') try: # ③ 取得対象オブジェクトのレコードを取得する logger.info(f'I-FETCH-04 [{object_name}] のレコード取得を開始します') - saleforce_data = SalesForceData() - record_generator = saleforce_data.fetch_sf_data_retry( + record_generator = fetch_sf_data_retry( columns, object_name, last_fetch_datetime_from, last_fetch_datetime_to) except Exception as e: raise SalesforceAPIException( - 'E-FETCH-02', FETCH_JP_NAME, f'[{object_name}] のレコード取得に失敗しました [{e}]') + 'E-FETCH-02', FETCH_JP_NAME, f'[{object_name}] のレコード取得に失敗しました エラー内容：[{e}]') try: # ④ 取得対象オブジェクトをJSONに変換 @@ -62,10 +81,68 @@ def fetch_crm_data(target_object, last_fetch_datetime): except Exception as e: raise DataConvertException( - 'E-FETCH-03', FETCH_JP_NAME, f'[{object_name}] のレコードのJSON変換に失敗しました [{e}]') + 'E-FETCH-03', FETCH_JP_NAME, f'[{object_name}] のレコードのJSON変換に失敗しました エラー内容：[{e}]') # ⑤ CRMデータ取得処理終了ログを出力する logger.info(f'I-FETCH-06 [{object_name}] のレコード取得が成功しました') # ⑥ 次の処理へ移行する return sf_object_jsons + + +@retry(wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) +def fetch_sf_count_retry(object_name, last_update_datetime_from, last_update_datetime_to): + try: + global count_contime_counter, count_readtime_counter, count_counter + + saleforce_count = SalesForceCount() + return saleforce_count.fetch_sf_count(object_name, last_update_datetime_from, last_update_datetime_to) + + except ConnectTimeout as e: + if count_contime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_contime_counter += 1 + logger.warn(f'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') + raise e + + except ReadTimeout as e: + if count_readtime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_readtime_counter += 1 + logger.warn( + f'W-FETCH-02 [{object_name}] の件数取得処理がタイムアウトしたため、リトライします：[{CRM_GET_RECORD_COUNT_TIMEOUT}] エラー内容：[{e}]') + raise e + + except Exception as e: + if count_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_counter += 1 + logger.warn( + f'W-FETCH-03 [{object_name}] の件数取得に失敗したため、リトライします エラー内容：[{e}]') + raise e + + +@retry(wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) +def fetch_sf_data_retry(columns, object_name, last_update_datetime_from, last_update_datetime_to): + try: + global data_contime_counter, data_readtime_counter, data_counter + + saleforce_data = SalesForceData() + return saleforce_data.fetch_sf_data(columns, object_name, last_update_datetime_from, last_update_datetime_to) + + except ConnectTimeout as e: + if data_contime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_contime_counter += 1 + logger.warn(f'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') + raise e + + except ReadTimeout as e: + if data_readtime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_readtime_counter += 1 + logger.warn( + f'W-FETCH-05 [{object_name}] のレコード取得処理がタイムアウトしたため、リトライします：[{CRM_FETCH_RECORD_TIMEOUT}] エラー内容：[{e}]') + raise e + + except Exception as e: + if data_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_counter += 1 + logger.warn( + f'W-FETCH-06 [{object_name}] のレコード取得に失敗したため、リトライします エラー内容：[{e}]') + raise e diff --git a/ecs/crm-datafetch/src/pre.py b/ecs/crm-datafetch/src/pre.py index 50071fe1..c8190f12 100644 --- a/ecs/crm-datafetch/src/pre.py +++ b/ecs/crm-datafetch/src/pre.py @@ -1,5 +1,5 @@ from datetime import datetime -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import( PRE_JP_NAME, YYYYMMDDTHHMMSSTZ @@ -18,9 +18,6 @@ from src.parser.json_parse import JsonParser from src.config.objects import FetchTargetObjects -logger = Logger().get_logger() - - def prepare_get_data(): # ① データ取得準備処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/process.py b/ecs/crm-datafetch/src/process.py index 5ccbd716..09d938af 100644 --- a/ecs/crm-datafetch/src/process.py +++ b/ecs/crm-datafetch/src/process.py @@ -8,13 +8,11 @@ from src.csvbk import backup_crm_csvdata # CS from src.upld import copy_crm_csvdata # CSVアップロード処理 from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 from src.end import updload_result_data # 取得処理実施結果アップロード処理 -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.error.exceptions import( MeDaCaCRMDataFetchException ) -logger = Logger().get_logger() - def main() -> None: try: @@ -108,14 +106,14 @@ def main() -> None: logger.info(f'I-CTRL-16 [{target_object.object_name}] 処理正常終了') except MeDaCaCRMDataFetchException as e: - logger.error(f'{e.error_id} {e}') - logger.exception( - f'I-ERR-03 [{object_info.get("object_name")}] の[{e.func_name}]でエラーが発生しました 次のオブジェクトの処理に移行します') + logger.info(f'{e.error_id} {e}') + logger.info( + f'I-ERR-03 [{object_info.get("object_name")}] の[{e.func_name}]でエラーが発生しました 次のオブジェクトの処理に移行します', exc_info=True) continue except Exception as e: - logger.exception( - f'I-ERR-04 [{object_info.get("object_name")}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e) + logger.info( + f'I-ERR-04 [{object_info.get("object_name")}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e, exc_info=True) continue # ④ すべてのオブジェクトの処理が完了したことと、オブジェクト毎の処理結果をログに出力する @@ -127,22 +125,22 @@ def main() -> None: # ⑥ 最終結果をチェックし、チェック結果をログに出力 if not all([v == 'success' for v in process_result.values()]): - logger.error('E-CTRL-01 一部のデータ取得に失敗しています。') + logger.error('E-CTRL-01 一部のデータ取得に失敗しています。詳細はログをご確認ください。') else: - logger.info('I-CTRL-18 すべてのデータの取得に成功しました。') + logger.info('I-CTRL-20 すべてのデータの取得に成功しました。') # ⑦ CRMデータ取得処理終了ログを出力する - logger.info('I-CTRL-20 CRMデータ取得処理を終了します') + logger.info('I-CTRL-21 CRMデータ取得処理を終了します') return exit(0) except MeDaCaCRMDataFetchException as e: - logger.error(f'{e.error_id} {e}') - logger.exception(f'I-ERR-01 [{e.func_name}]でエラーが発生したため、処理を終了します') + logger.error(f'E-ERR-01 [{e.func_name}]でエラーが発生したため、処理を終了します') + logger.exception(f'{e.error_id} {e}') return exit(0) except Exception as e: - logger.exception('I-ERR-02 予期せぬエラーが発生したため、処理を終了します', e) + logger.exception('E-ERR-02 予期せぬエラーが発生したため、処理を終了します', e) return exit(0) diff --git a/ecs/crm-datafetch/src/resbk.py b/ecs/crm-datafetch/src/resbk.py index 02b3c53b..21d516ef 100644 --- a/ecs/crm-datafetch/src/resbk.py +++ b/ecs/crm-datafetch/src/resbk.py @@ -1,4 +1,4 @@ -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import( RESBK_JP_NAME ) @@ -8,9 +8,6 @@ from src.error.exceptions import( from src.aws.s3 import BackupBucket -logger = Logger().get_logger() - - def backup_crm_data(object_name, sf_object_jsons, date_path): # ① CRM電文データバックアップ処理の開始ログを出力する logger.info(f'I-RESBK-01 [{object_name}] のCRM電文データバックアップ処理を開始します') diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index ad466782..0bce5675 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,7 +1,4 @@ -from tenacity import retry, stop_after_attempt, stop_after_delay -from tenacity.wait import wait_exponential from simple_salesforce import Salesforce -from src.util.async_retry import TimeoutManager, AsyncRetry from src.environments import( CRM_AUTH_DOMAIN, CRM_USER_NAME, @@ -17,13 +14,14 @@ from src.environments import( CRM_GET_RECORD_COUNT_RETRY_INTERVAL, CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, # CRMのレコード取得処理のタイムアウト秒数 + CRM_FETCH_RECORD_TIMEOUT, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, CRM_FETCH_RECORD_RETRY_INTERVAL, CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, CRM_FETCH_RECORD_RETRY_MAX_INTERVAL ) + FETCH_SOQL = """SELECT {column_or_expression} FROM {object_name} WHERE SystemModStamp > {last_update_datetime_from} AND SystemModStamp <= {last_update_datetime_to} @@ -57,10 +55,6 @@ class SalesForceCount(): count_soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) return count_res.get('records')[0].get('expr0') - @retry(wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT) | stop_after_delay(CRM_GET_RECORD_COUNT_TIMEOUT)) - def fetch_sf_count_retry(self, object_name, last_update_datetime_from, last_update_datetime_to): - return self.fetch_sf_count(object_name, last_update_datetime_from, last_update_datetime_to) - class SalesForceData(): def fetch_sf_data(self, columns, object_name, last_update_datetime_from, last_update_datetime_to): @@ -72,7 +66,3 @@ class SalesForceData(): ) self.__sf = SalesfoeceApi() return self.__sf.sf_query_all_iter(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) - - @retry(wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT) | stop_after_delay(CRM_FETCH_RECORD_TIMEOUT)) - def fetch_sf_data_retry(self, columns, object_name, last_update_datetime_from, last_update_datetime_to): - return self.fetch_sf_data(columns, object_name, last_update_datetime_from, last_update_datetime_to) diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upd.py index 4e3cb90d..f8dd4350 100644 --- a/ecs/crm-datafetch/src/upd.py +++ b/ecs/crm-datafetch/src/upd.py @@ -1,5 +1,5 @@ import json -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.aws.s3 import ConfigBucket from src.constants import ( UPD_JP_NAME @@ -9,9 +9,6 @@ from src.error.exceptions import( ) -logger = Logger().get_logger() - - def updload_last_fetch_datetime(target_object, last_fetch_datetime): # ① 前回取得日時ファイル更新処理の開始ログを出力する logger.info( diff --git a/ecs/crm-datafetch/src/upld.py b/ecs/crm-datafetch/src/upld.py index 57e33e25..c3c3140f 100644 --- a/ecs/crm-datafetch/src/upld.py +++ b/ecs/crm-datafetch/src/upld.py @@ -1,4 +1,4 @@ -from src.util.logger import Logger +from src.util.logger import logger_instance as logger from src.constants import ( UPLD_JP_NAME ) @@ -14,9 +14,6 @@ from src.error.exceptions import( from src.aws.s3 import S3ResourceNonBucket -logger = Logger().get_logger() - - def copy_crm_csvdata(target_object, date_path): # ① CSVデータアップロード処理の開始ログを出力する logger.info( diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index d9892da8..ec0baf55 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -1,6 +1,9 @@ import logging from src.environments import LOG_LEVEL +for name in ["boto3", "botocore", "s3transfer", "urllib3"]: + logging.getLogger(name).setLevel(logging.WARNING) + class Logger(): __logger: logging.Logger @@ -27,3 +30,6 @@ class Logger(): def get_logger(self) -> logging.Logger: return self.__logger + + +logger_instance = Logger().get_logger() From 06e8a5769eb955712d83aff6a98d9af5ce9c30e2 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 21 Jul 2022 14:00:25 +0900 Subject: [PATCH 04/21] =?UTF-8?q?refactor:=20launch.json=E3=81=AE=E5=89=8A?= =?UTF-8?q?=E9=99=A4=EF=BC=88=E7=AB=B6=E5=90=88=E3=81=AE=E3=81=9F=E3=82=81?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/launch.json | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json deleted file mode 100644 index d8f8414f..00000000 --- a/.vscode/launch.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - // IntelliSense を使用して利用可能な属性を学べます。 - // 既存の属性の説明をホバーして表示します。 - // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "name": "Python: CRMデータ取得ローカル実行", - "type": "python", - "request": "launch", - // windowsだと\区切りかも - "program": "ecs\\crm-datafetch\\main.py", - "console": "integratedTerminal", - "justMyCode": true, - "envFile": "${workspaceFolder}/.env", - "args": ["local"] - } - ] - } \ No newline at end of file From df6d75780b62e255adbb319a5215eb91799ecbd5 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 19:20:00 +0900 Subject: [PATCH 05/21] =?UTF-8?q?fix(Pipfile,Pipfile.lock):=20Pipfile?= =?UTF-8?q?=E3=81=AB=E3=82=88=E3=82=8B=E4=BE=9D=E5=AD=98=E9=96=A2=E4=BF=82?= =?UTF-8?q?=E3=81=AE=E7=AE=A1=E7=90=86=E3=81=A8=E3=82=B3=E3=83=B3=E3=83=86?= =?UTF-8?q?=E3=83=8A=E3=81=AB=E3=83=AA=E3=83=AA=E3=83=BC=E3=82=B9=E3=81=99?= =?UTF-8?q?=E3=82=8B=E3=81=9F=E3=82=81=E3=81=AElock=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=A4=E3=83=AB=E3=81=8C=E6=9B=B4=E6=96=B0=E3=81=95=E3=82=8C?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=AA=E3=81=8B=E3=81=A3=E3=81=9F=E3=81=AE?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Pipfile | 21 +- ecs/crm-datafetch/Pipfile.lock | 419 +++++++++++++++++++++++++++++++++ 2 files changed, 437 insertions(+), 3 deletions(-) diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index ec9569f7..c9758d2f 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -1,4 +1,19 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + [packages] -boto3 -simple-salesforce -tenacity \ No newline at end of file +boto3 = "*" +simple-salesforce = "*" +tenacity = "*" + +[dev-packages] +autopep8 = "*" +flake8 = "*" + +[requires] +python_version = "3.8" + +[pipenv] +allow_prereleases = true diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index e69de29b..a2cb4458 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -0,0 +1,419 @@ +{ + "_meta": { + "hash": { + "sha256": "ec1d83143aff859500979be73f67196dcfe2298ad3553a7d81ad0605a277d672" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "attrs": { + "hashes": [ + "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", + "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==21.4.0" + }, + "authlib": { + "hashes": [ + "sha256:1286e2d5ef5bfe5a11cc2d0a0d1031f0393f6ce4d61f5121cfe87fa0054e98bd", + "sha256:6e74a4846ac36dfc882b3cc2fbd3d9eb410a627f2f2dc11771276655345223b1" + ], + "version": "==1.0.1" + }, + "boto3": { + "hashes": [ + "sha256:5c775dcb12ca5d6be3f5aa3c49d77783faa64eb30fd3f4af93ff116bb42f9ffb", + "sha256:5d9bcc355cf6edd7f3849fedac4252e12a0aa2b436cdbc0d4371b16a0f852a30" + ], + "index": "pypi", + "version": "==1.24.34" + }, + "botocore": { + "hashes": [ + "sha256:0d824a5315f5f5c3bea53c14107a69695ef43190edf647f1281bac8f172ca77c", + "sha256:9c695d47f1f1212f3e306e51f7bacdf67e58055194ddcf7d8296660b124cf135" + ], + "markers": "python_version >= '3.7'", + "version": "==1.27.34" + }, + "cached-property": { + "hashes": [ + "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", + "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" + ], + "version": "==1.5.2" + }, + "certifi": { + "hashes": [ + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==2022.6.15" + }, + "cffi": { + "hashes": [ + "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", + "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", + "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", + "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", + "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", + "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", + "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", + "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", + "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", + "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", + "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", + "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", + "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", + "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", + "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", + "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", + "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", + "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", + "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", + "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", + "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", + "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", + "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", + "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", + "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", + "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", + "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", + "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", + "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", + "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", + "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", + "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", + "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", + "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", + "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", + "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", + "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", + "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", + "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", + "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", + "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", + "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", + "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", + "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", + "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", + "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", + "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", + "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", + "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", + "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", + "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", + "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", + "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", + "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", + "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", + "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", + "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", + "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", + "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", + "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", + "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", + "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", + "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", + "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" + ], + "version": "==1.15.1" + }, + "charset-normalizer": { + "hashes": [ + "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", + "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==2.1.0" + }, + "cryptography": { + "hashes": [ + "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", + "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596", + "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", + "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", + "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab", + "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", + "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", + "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b", + "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441", + "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa", + "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d", + "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", + "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a", + "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", + "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", + "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", + "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282", + "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67", + "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", + "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", + "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", + "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==37.0.4" + }, + "idna": { + "hashes": [ + "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + ], + "markers": "python_version >= '3.5'", + "version": "==3.3" + }, + "isodate": { + "hashes": [ + "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96", + "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9" + ], + "version": "==0.6.1" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "lxml": { + "hashes": [ + "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318", + "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c", + "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b", + "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000", + "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73", + "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d", + "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb", + "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8", + "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2", + "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345", + "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94", + "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e", + "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b", + "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc", + "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a", + "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9", + "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc", + "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387", + "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb", + "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7", + "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4", + "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97", + "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67", + "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627", + "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7", + "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd", + "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3", + "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7", + "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130", + "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b", + "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036", + "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785", + "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca", + "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91", + "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc", + "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536", + "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391", + "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3", + "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d", + "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21", + "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3", + "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d", + "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29", + "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715", + "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed", + "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25", + "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c", + "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785", + "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837", + "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4", + "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b", + "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2", + "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067", + "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448", + "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d", + "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2", + "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc", + "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c", + "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5", + "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84", + "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8", + "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf", + "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7", + "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e", + "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb", + "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b", + "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3", + "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad", + "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8", + "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.9.1" + }, + "platformdirs": { + "hashes": [ + "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788", + "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19" + ], + "markers": "python_version >= '3.7'", + "version": "==2.5.2" + }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.8.2" + }, + "pytz": { + "hashes": [ + "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", + "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" + ], + "version": "==2022.1" + }, + "requests": { + "hashes": [ + "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" + ], + "markers": "python_version >= '3.7' and python_version < '4'", + "version": "==2.28.1" + }, + "requests-file": { + "hashes": [ + "sha256:07d74208d3389d01c38ab89ef403af0cfec63957d53a0081d8eca738d0247d8e", + "sha256:dfe5dae75c12481f68ba353183c53a65e6044c923e64c24b2209f6c7570ca953" + ], + "version": "==1.5.1" + }, + "requests-toolbelt": { + "hashes": [ + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" + ], + "version": "==0.9.1" + }, + "s3transfer": { + "hashes": [ + "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", + "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" + ], + "markers": "python_version >= '3.7'", + "version": "==0.6.0" + }, + "simple-salesforce": { + "hashes": [ + "sha256:15d6943e52252c9cc28e1779803354f2a36c88b72056499e07eb06cd652f149c", + "sha256:7931038081c445e9459ddc014aaf7f540b1131a31596956cb5d7c0e7b7e0c4cb" + ], + "index": "pypi", + "version": "==1.12.1" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.16.0" + }, + "tenacity": { + "hashes": [ + "sha256:43242a20e3e73291a28bcbcacfd6e000b02d3857a9a9fff56b297a27afdc932f", + "sha256:f78f4ea81b0fabc06728c11dc2a8c01277bfc5181b321a4770471902e3eb844a" + ], + "index": "pypi", + "version": "==8.0.1" + }, + "urllib3": { + "hashes": [ + "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", + "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.10" + }, + "zeep": { + "hashes": [ + "sha256:5867f2eadd6b028d9751f4155af590d3aaf9280e3a0ed5e15a53343921c956e5", + "sha256:81c491092b71f5b276de8c63dfd452be3f322622c48a54f3a497cf913bdfb2f4" + ], + "markers": "python_full_version >= '3.6.0'", + "version": "==4.1.0" + } + }, + "develop": { + "autopep8": { + "hashes": [ + "sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979", + "sha256:ed77137193bbac52d029a52c59bec1b0629b5a186c495f1eb21b126ac466083f" + ], + "index": "pypi", + "version": "==1.6.0" + }, + "flake8": { + "hashes": [ + "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", + "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" + ], + "index": "pypi", + "version": "==4.0.1" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "pycodestyle": { + "hashes": [ + "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", + "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.8.0" + }, + "pyflakes": { + "hashes": [ + "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", + "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "version": "==0.10.2" + } + } +} From 5e88f3b519c5d89ff6d84e3807b1a224351818f6 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 21 Jul 2022 19:32:33 +0900 Subject: [PATCH 06/21] =?UTF-8?q?feat:=20Dokcer=E9=96=A2=E9=80=A3=E3=81=AE?= =?UTF-8?q?=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Dockerfile | 12 +- ecs/crm-datafetch/Pipfile | 16 +- ecs/crm-datafetch/Pipfile.lock | 371 +++++++++++++++++++++++++++++++++ 3 files changed, 390 insertions(+), 9 deletions(-) diff --git a/ecs/crm-datafetch/Dockerfile b/ecs/crm-datafetch/Dockerfile index 88e588e1..56087bf0 100644 --- a/ecs/crm-datafetch/Dockerfile +++ b/ecs/crm-datafetch/Dockerfile @@ -4,16 +4,16 @@ ENV TZ="Asia/Tokyo" WORKDIR /usr/src/app COPY Pipfile Pipfile.lock ./ -RUN pip install --no-cache-dir -r requirements.txt && \ - cp /usr/share/zoneinfo/Asia/Tokyo /etc/localtime - -RUN \ +RUN \ apt update -y && \ # パッケージのセキュリティアップデートのみを適用するコマンド apt install -y unattended-upgrades && \ - unattended-upgrades + unattended-upgrades && \ + pip install pipenv --no-cache-dir && \ + pipenv install --system --deploy && \ + pip uninstall -y pipenv virtualenv-clone virtualenv COPY main.py ./ -COPY crm-datafetch /usr/src +COPY src ./ CMD [ "python", "./main.py" ] diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index ec9569f7..3c0e1a61 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -1,4 +1,14 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + [packages] -boto3 -simple-salesforce -tenacity \ No newline at end of file +boto3 = "*" +simple-salesforce = "*" +tenacity = "*" + +[dev-packages] + +[requires] +python_version = "3.8" diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index e69de29b..b66b415b 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -0,0 +1,371 @@ +{ + "_meta": { + "hash": { + "sha256": "84e1c5fdd458ebb8022dd649f03e59828d8658b62a572f524a60850e2a1d626c" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "attrs": { + "hashes": [ + "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", + "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==21.4.0" + }, + "authlib": { + "hashes": [ + "sha256:1286e2d5ef5bfe5a11cc2d0a0d1031f0393f6ce4d61f5121cfe87fa0054e98bd", + "sha256:6e74a4846ac36dfc882b3cc2fbd3d9eb410a627f2f2dc11771276655345223b1" + ], + "version": "==1.0.1" + }, + "boto3": { + "hashes": [ + "sha256:5c775dcb12ca5d6be3f5aa3c49d77783faa64eb30fd3f4af93ff116bb42f9ffb", + "sha256:5d9bcc355cf6edd7f3849fedac4252e12a0aa2b436cdbc0d4371b16a0f852a30" + ], + "index": "pypi", + "version": "==1.24.34" + }, + "botocore": { + "hashes": [ + "sha256:0d824a5315f5f5c3bea53c14107a69695ef43190edf647f1281bac8f172ca77c", + "sha256:9c695d47f1f1212f3e306e51f7bacdf67e58055194ddcf7d8296660b124cf135" + ], + "markers": "python_version >= '3.7'", + "version": "==1.27.34" + }, + "cached-property": { + "hashes": [ + "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", + "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" + ], + "version": "==1.5.2" + }, + "certifi": { + "hashes": [ + "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", + "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" + ], + "markers": "python_version >= '3.6'", + "version": "==2022.6.15" + }, + "cffi": { + "hashes": [ + "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", + "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", + "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", + "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", + "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", + "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", + "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", + "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", + "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", + "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", + "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", + "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", + "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", + "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", + "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", + "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", + "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", + "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", + "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", + "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", + "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", + "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", + "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", + "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", + "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", + "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", + "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", + "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", + "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", + "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", + "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", + "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", + "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", + "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", + "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", + "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", + "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", + "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", + "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", + "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", + "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", + "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", + "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", + "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", + "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", + "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", + "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", + "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", + "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", + "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", + "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", + "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", + "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", + "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", + "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", + "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", + "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", + "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", + "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", + "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", + "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", + "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", + "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", + "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" + ], + "version": "==1.15.1" + }, + "charset-normalizer": { + "hashes": [ + "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", + "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" + ], + "markers": "python_version >= '3.6'", + "version": "==2.1.0" + }, + "cryptography": { + "hashes": [ + "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", + "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596", + "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", + "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", + "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab", + "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", + "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", + "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b", + "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441", + "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa", + "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d", + "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", + "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a", + "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", + "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", + "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", + "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282", + "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67", + "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", + "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", + "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", + "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" + ], + "markers": "python_version >= '3.6'", + "version": "==37.0.4" + }, + "idna": { + "hashes": [ + "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" + ], + "markers": "python_version >= '3.5'", + "version": "==3.3" + }, + "isodate": { + "hashes": [ + "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96", + "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9" + ], + "version": "==0.6.1" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "lxml": { + "hashes": [ + "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318", + "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c", + "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b", + "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000", + "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73", + "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d", + "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb", + "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8", + "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2", + "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345", + "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94", + "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e", + "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b", + "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc", + "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a", + "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9", + "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc", + "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387", + "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb", + "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7", + "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4", + "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97", + "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67", + "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627", + "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7", + "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd", + "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3", + "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7", + "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130", + "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b", + "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036", + "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785", + "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca", + "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91", + "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc", + "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536", + "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391", + "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3", + "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d", + "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21", + "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3", + "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d", + "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29", + "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715", + "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed", + "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25", + "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c", + "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785", + "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837", + "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4", + "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b", + "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2", + "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067", + "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448", + "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d", + "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2", + "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc", + "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c", + "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5", + "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84", + "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8", + "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf", + "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7", + "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e", + "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb", + "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b", + "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3", + "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad", + "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8", + "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==4.9.1" + }, + "platformdirs": { + "hashes": [ + "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788", + "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19" + ], + "markers": "python_version >= '3.7'", + "version": "==2.5.2" + }, + "pycparser": { + "hashes": [ + "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", + "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" + ], + "version": "==2.21" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "pytz": { + "hashes": [ + "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", + "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" + ], + "version": "==2022.1" + }, + "requests": { + "hashes": [ + "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", + "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" + ], + "markers": "python_version >= '3.7' and python_version < '4'", + "version": "==2.28.1" + }, + "requests-file": { + "hashes": [ + "sha256:07d74208d3389d01c38ab89ef403af0cfec63957d53a0081d8eca738d0247d8e", + "sha256:dfe5dae75c12481f68ba353183c53a65e6044c923e64c24b2209f6c7570ca953" + ], + "version": "==1.5.1" + }, + "requests-toolbelt": { + "hashes": [ + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" + ], + "version": "==0.9.1" + }, + "s3transfer": { + "hashes": [ + "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", + "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" + ], + "markers": "python_version >= '3.7'", + "version": "==0.6.0" + }, + "simple-salesforce": { + "hashes": [ + "sha256:15d6943e52252c9cc28e1779803354f2a36c88b72056499e07eb06cd652f149c", + "sha256:7931038081c445e9459ddc014aaf7f540b1131a31596956cb5d7c0e7b7e0c4cb" + ], + "index": "pypi", + "version": "==1.12.1" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "tenacity": { + "hashes": [ + "sha256:43242a20e3e73291a28bcbcacfd6e000b02d3857a9a9fff56b297a27afdc932f", + "sha256:f78f4ea81b0fabc06728c11dc2a8c01277bfc5181b321a4770471902e3eb844a" + ], + "index": "pypi", + "version": "==8.0.1" + }, + "urllib3": { + "hashes": [ + "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", + "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", + "version": "==1.26.10" + }, + "zeep": { + "hashes": [ + "sha256:5867f2eadd6b028d9751f4155af590d3aaf9280e3a0ed5e15a53343921c956e5", + "sha256:81c491092b71f5b276de8c63dfd452be3f322622c48a54f3a497cf913bdfb2f4" + ], + "markers": "python_version >= '3.6'", + "version": "==4.1.0" + } + }, + "develop": {} +} From 91e70710494f26bd6ef14c01b1fb82e7a38a7fa9 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 19:41:28 +0900 Subject: [PATCH 07/21] =?UTF-8?q?Revert=20"fix(Pipfile,Pipfile.lock):=20Pi?= =?UTF-8?q?pfile=E3=81=AB=E3=82=88=E3=82=8B=E4=BE=9D=E5=AD=98=E9=96=A2?= =?UTF-8?q?=E4=BF=82=E3=81=AE=E7=AE=A1=E7=90=86=E3=81=A8=E3=82=B3=E3=83=B3?= =?UTF-8?q?=E3=83=86=E3=83=8A=E3=81=AB=E3=83=AA=E3=83=AA=E3=83=BC=E3=82=B9?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=81=9F=E3=82=81=E3=81=AElock=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=8C=E6=9B=B4=E6=96=B0=E3=81=95?= =?UTF-8?q?=E3=82=8C=E3=81=A6=E3=81=84=E3=81=AA=E3=81=8B=E3=81=A3=E3=81=9F?= =?UTF-8?q?=E3=81=AE=E3=82=92=E4=BF=AE=E6=AD=A3"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit df6d75780b62e255adbb319a5215eb91799ecbd5. --- ecs/crm-datafetch/Pipfile | 21 +- ecs/crm-datafetch/Pipfile.lock | 419 --------------------------------- 2 files changed, 3 insertions(+), 437 deletions(-) diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index c9758d2f..ec9569f7 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -1,19 +1,4 @@ -[[source]] -url = "https://pypi.org/simple" -verify_ssl = true -name = "pypi" - [packages] -boto3 = "*" -simple-salesforce = "*" -tenacity = "*" - -[dev-packages] -autopep8 = "*" -flake8 = "*" - -[requires] -python_version = "3.8" - -[pipenv] -allow_prereleases = true +boto3 +simple-salesforce +tenacity \ No newline at end of file diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index a2cb4458..e69de29b 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -1,419 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "ec1d83143aff859500979be73f67196dcfe2298ad3553a7d81ad0605a277d672" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.8" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "attrs": { - "hashes": [ - "sha256:2d27e3784d7a565d36ab851fe94887c5eccd6a463168875832a1be79c82828b4", - "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==21.4.0" - }, - "authlib": { - "hashes": [ - "sha256:1286e2d5ef5bfe5a11cc2d0a0d1031f0393f6ce4d61f5121cfe87fa0054e98bd", - "sha256:6e74a4846ac36dfc882b3cc2fbd3d9eb410a627f2f2dc11771276655345223b1" - ], - "version": "==1.0.1" - }, - "boto3": { - "hashes": [ - "sha256:5c775dcb12ca5d6be3f5aa3c49d77783faa64eb30fd3f4af93ff116bb42f9ffb", - "sha256:5d9bcc355cf6edd7f3849fedac4252e12a0aa2b436cdbc0d4371b16a0f852a30" - ], - "index": "pypi", - "version": "==1.24.34" - }, - "botocore": { - "hashes": [ - "sha256:0d824a5315f5f5c3bea53c14107a69695ef43190edf647f1281bac8f172ca77c", - "sha256:9c695d47f1f1212f3e306e51f7bacdf67e58055194ddcf7d8296660b124cf135" - ], - "markers": "python_version >= '3.7'", - "version": "==1.27.34" - }, - "cached-property": { - "hashes": [ - "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", - "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" - ], - "version": "==1.5.2" - }, - "certifi": { - "hashes": [ - "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d", - "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412" - ], - "markers": "python_full_version >= '3.6.0'", - "version": "==2022.6.15" - }, - "cffi": { - "hashes": [ - "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5", - "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef", - "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104", - "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426", - "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405", - "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375", - "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a", - "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e", - "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc", - "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf", - "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185", - "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497", - "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3", - "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35", - "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c", - "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83", - "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21", - "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca", - "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984", - "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac", - "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd", - "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee", - "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a", - "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2", - "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192", - "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7", - "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585", - "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f", - "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e", - "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27", - "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b", - "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e", - "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e", - "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d", - "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c", - "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415", - "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82", - "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02", - "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314", - "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325", - "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c", - "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3", - "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914", - "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045", - "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d", - "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9", - "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5", - "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2", - "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c", - "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3", - "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2", - "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8", - "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d", - "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d", - "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9", - "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162", - "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76", - "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4", - "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e", - "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9", - "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6", - "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b", - "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01", - "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0" - ], - "version": "==1.15.1" - }, - "charset-normalizer": { - "hashes": [ - "sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5", - "sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413" - ], - "markers": "python_full_version >= '3.6.0'", - "version": "==2.1.0" - }, - "cryptography": { - "hashes": [ - "sha256:190f82f3e87033821828f60787cfa42bff98404483577b591429ed99bed39d59", - "sha256:2be53f9f5505673eeda5f2736bea736c40f051a739bfae2f92d18aed1eb54596", - "sha256:30788e070800fec9bbcf9faa71ea6d8068f5136f60029759fd8c3efec3c9dcb3", - "sha256:3d41b965b3380f10e4611dbae366f6dc3cefc7c9ac4e8842a806b9672ae9add5", - "sha256:4c590ec31550a724ef893c50f9a97a0c14e9c851c85621c5650d699a7b88f7ab", - "sha256:549153378611c0cca1042f20fd9c5030d37a72f634c9326e225c9f666d472884", - "sha256:63f9c17c0e2474ccbebc9302ce2f07b55b3b3fcb211ded18a42d5764f5c10a82", - "sha256:6bc95ed67b6741b2607298f9ea4932ff157e570ef456ef7ff0ef4884a134cc4b", - "sha256:7099a8d55cd49b737ffc99c17de504f2257e3787e02abe6d1a6d136574873441", - "sha256:75976c217f10d48a8b5a8de3d70c454c249e4b91851f6838a4e48b8f41eb71aa", - "sha256:7bc997818309f56c0038a33b8da5c0bfbb3f1f067f315f9abd6fc07ad359398d", - "sha256:80f49023dd13ba35f7c34072fa17f604d2f19bf0989f292cedf7ab5770b87a0b", - "sha256:91ce48d35f4e3d3f1d83e29ef4a9267246e6a3be51864a5b7d2247d5086fa99a", - "sha256:a958c52505c8adf0d3822703078580d2c0456dd1d27fabfb6f76fe63d2971cd6", - "sha256:b62439d7cd1222f3da897e9a9fe53bbf5c104fff4d60893ad1355d4c14a24157", - "sha256:b7f8dd0d4c1f21759695c05a5ec8536c12f31611541f8904083f3dc582604280", - "sha256:d204833f3c8a33bbe11eda63a54b1aad7aa7456ed769a982f21ec599ba5fa282", - "sha256:e007f052ed10cc316df59bc90fbb7ff7950d7e2919c9757fd42a2b8ecf8a5f67", - "sha256:f2dcb0b3b63afb6df7fd94ec6fbddac81b5492513f7b0436210d390c14d46ee8", - "sha256:f721d1885ecae9078c3f6bbe8a88bc0786b6e749bf32ccec1ef2b18929a05046", - "sha256:f7a6de3e98771e183645181b3627e2563dcde3ce94a9e42a3f427d2255190327", - "sha256:f8c0a6e9e1dd3eb0414ba320f85da6b0dcbd543126e30fcc546e7372a7fbf3b9" - ], - "markers": "python_full_version >= '3.6.0'", - "version": "==37.0.4" - }, - "idna": { - "hashes": [ - "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", - "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d" - ], - "markers": "python_version >= '3.5'", - "version": "==3.3" - }, - "isodate": { - "hashes": [ - "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96", - "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9" - ], - "version": "==0.6.1" - }, - "jmespath": { - "hashes": [ - "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", - "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" - ], - "markers": "python_version >= '3.7'", - "version": "==1.0.1" - }, - "lxml": { - "hashes": [ - "sha256:04da965dfebb5dac2619cb90fcf93efdb35b3c6994fea58a157a834f2f94b318", - "sha256:0538747a9d7827ce3e16a8fdd201a99e661c7dee3c96c885d8ecba3c35d1032c", - "sha256:0645e934e940107e2fdbe7c5b6fb8ec6232444260752598bc4d09511bd056c0b", - "sha256:079b68f197c796e42aa80b1f739f058dcee796dc725cc9a1be0cdb08fc45b000", - "sha256:0f3f0059891d3254c7b5fb935330d6db38d6519ecd238ca4fce93c234b4a0f73", - "sha256:10d2017f9150248563bb579cd0d07c61c58da85c922b780060dcc9a3aa9f432d", - "sha256:1355755b62c28950f9ce123c7a41460ed9743c699905cbe664a5bcc5c9c7c7fb", - "sha256:13c90064b224e10c14dcdf8086688d3f0e612db53766e7478d7754703295c7c8", - "sha256:1423631e3d51008871299525b541413c9b6c6423593e89f9c4cfbe8460afc0a2", - "sha256:1436cf0063bba7888e43f1ba8d58824f085410ea2025befe81150aceb123e345", - "sha256:1a7c59c6ffd6ef5db362b798f350e24ab2cfa5700d53ac6681918f314a4d3b94", - "sha256:1e1cf47774373777936c5aabad489fef7b1c087dcd1f426b621fda9dcc12994e", - "sha256:206a51077773c6c5d2ce1991327cda719063a47adc02bd703c56a662cdb6c58b", - "sha256:21fb3d24ab430fc538a96e9fbb9b150029914805d551deeac7d7822f64631dfc", - "sha256:27e590352c76156f50f538dbcebd1925317a0f70540f7dc8c97d2931c595783a", - "sha256:287605bede6bd36e930577c5925fcea17cb30453d96a7b4c63c14a257118dbb9", - "sha256:2aaf6a0a6465d39b5ca69688fce82d20088c1838534982996ec46633dc7ad6cc", - "sha256:32a73c53783becdb7eaf75a2a1525ea8e49379fb7248c3eeefb9412123536387", - "sha256:41fb58868b816c202e8881fd0f179a4644ce6e7cbbb248ef0283a34b73ec73bb", - "sha256:4780677767dd52b99f0af1f123bc2c22873d30b474aa0e2fc3fe5e02217687c7", - "sha256:4878e667ebabe9b65e785ac8da4d48886fe81193a84bbe49f12acff8f7a383a4", - "sha256:487c8e61d7acc50b8be82bda8c8d21d20e133c3cbf41bd8ad7eb1aaeb3f07c97", - "sha256:4beea0f31491bc086991b97517b9683e5cfb369205dac0148ef685ac12a20a67", - "sha256:4cfbe42c686f33944e12f45a27d25a492cc0e43e1dc1da5d6a87cbcaf2e95627", - "sha256:4d5bae0a37af799207140652a700f21a85946f107a199bcb06720b13a4f1f0b7", - "sha256:4e285b5f2bf321fc0857b491b5028c5f276ec0c873b985d58d7748ece1d770dd", - "sha256:57e4d637258703d14171b54203fd6822fda218c6c2658a7d30816b10995f29f3", - "sha256:5974895115737a74a00b321e339b9c3f45c20275d226398ae79ac008d908bff7", - "sha256:5ef87fca280fb15342726bd5f980f6faf8b84a5287fcc2d4962ea8af88b35130", - "sha256:603a464c2e67d8a546ddaa206d98e3246e5db05594b97db844c2f0a1af37cf5b", - "sha256:6653071f4f9bac46fbc30f3c7838b0e9063ee335908c5d61fb7a4a86c8fd2036", - "sha256:6ca2264f341dd81e41f3fffecec6e446aa2121e0b8d026fb5130e02de1402785", - "sha256:6d279033bf614953c3fc4a0aa9ac33a21e8044ca72d4fa8b9273fe75359d5cca", - "sha256:6d949f53ad4fc7cf02c44d6678e7ff05ec5f5552b235b9e136bd52e9bf730b91", - "sha256:6daa662aba22ef3258934105be2dd9afa5bb45748f4f702a3b39a5bf53a1f4dc", - "sha256:6eafc048ea3f1b3c136c71a86db393be36b5b3d9c87b1c25204e7d397cee9536", - "sha256:830c88747dce8a3e7525defa68afd742b4580df6aa2fdd6f0855481e3994d391", - "sha256:86e92728ef3fc842c50a5cb1d5ba2bc66db7da08a7af53fb3da79e202d1b2cd3", - "sha256:8caf4d16b31961e964c62194ea3e26a0e9561cdf72eecb1781458b67ec83423d", - "sha256:8d1a92d8e90b286d491e5626af53afef2ba04da33e82e30744795c71880eaa21", - "sha256:8f0a4d179c9a941eb80c3a63cdb495e539e064f8054230844dcf2fcb812b71d3", - "sha256:9232b09f5efee6a495a99ae6824881940d6447debe272ea400c02e3b68aad85d", - "sha256:927a9dd016d6033bc12e0bf5dee1dde140235fc8d0d51099353c76081c03dc29", - "sha256:93e414e3206779ef41e5ff2448067213febf260ba747fc65389a3ddaa3fb8715", - "sha256:98cafc618614d72b02185ac583c6f7796202062c41d2eeecdf07820bad3295ed", - "sha256:9c3a88d20e4fe4a2a4a84bf439a5ac9c9aba400b85244c63a1ab7088f85d9d25", - "sha256:9f36de4cd0c262dd9927886cc2305aa3f2210db437aa4fed3fb4940b8bf4592c", - "sha256:a60f90bba4c37962cbf210f0188ecca87daafdf60271f4c6948606e4dabf8785", - "sha256:a614e4afed58c14254e67862456d212c4dcceebab2eaa44d627c2ca04bf86837", - "sha256:ae06c1e4bc60ee076292e582a7512f304abdf6c70db59b56745cca1684f875a4", - "sha256:b122a188cd292c4d2fcd78d04f863b789ef43aa129b233d7c9004de08693728b", - "sha256:b570da8cd0012f4af9fa76a5635cd31f707473e65a5a335b186069d5c7121ff2", - "sha256:bcaa1c495ce623966d9fc8a187da80082334236a2a1c7e141763ffaf7a405067", - "sha256:bd34f6d1810d9354dc7e35158aa6cc33456be7706df4420819af6ed966e85448", - "sha256:be9eb06489bc975c38706902cbc6888f39e946b81383abc2838d186f0e8b6a9d", - "sha256:c4b2e0559b68455c085fb0f6178e9752c4be3bba104d6e881eb5573b399d1eb2", - "sha256:c62e8dd9754b7debda0c5ba59d34509c4688f853588d75b53c3791983faa96fc", - "sha256:c852b1530083a620cb0de5f3cd6826f19862bafeaf77586f1aef326e49d95f0c", - "sha256:d9fc0bf3ff86c17348dfc5d322f627d78273eba545db865c3cd14b3f19e57fa5", - "sha256:dad7b164905d3e534883281c050180afcf1e230c3d4a54e8038aa5cfcf312b84", - "sha256:e5f66bdf0976ec667fc4594d2812a00b07ed14d1b44259d19a41ae3fff99f2b8", - "sha256:e8f0c9d65da595cfe91713bc1222af9ecabd37971762cb830dea2fc3b3bb2acf", - "sha256:edffbe3c510d8f4bf8640e02ca019e48a9b72357318383ca60e3330c23aaffc7", - "sha256:eea5d6443b093e1545ad0210e6cf27f920482bfcf5c77cdc8596aec73523bb7e", - "sha256:ef72013e20dd5ba86a8ae1aed7f56f31d3374189aa8b433e7b12ad182c0d2dfb", - "sha256:f05251bbc2145349b8d0b77c0d4e5f3b228418807b1ee27cefb11f69ed3d233b", - "sha256:f1be258c4d3dc609e654a1dc59d37b17d7fef05df912c01fc2e15eb43a9735f3", - "sha256:f9ced82717c7ec65a67667bb05865ffe38af0e835cdd78728f1209c8fffe0cad", - "sha256:fe17d10b97fdf58155f858606bddb4e037b805a60ae023c009f760d8361a4eb8", - "sha256:fe749b052bb7233fe5d072fcb549221a8cb1a16725c47c37e42b0b9cb3ff2c3f" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==4.9.1" - }, - "platformdirs": { - "hashes": [ - "sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788", - "sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19" - ], - "markers": "python_version >= '3.7'", - "version": "==2.5.2" - }, - "pycparser": { - "hashes": [ - "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9", - "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206" - ], - "version": "==2.21" - }, - "python-dateutil": { - "hashes": [ - "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", - "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", - "version": "==2.8.2" - }, - "pytz": { - "hashes": [ - "sha256:1e760e2fe6a8163bc0b3d9a19c4f84342afa0a2affebfaa84b01b978a02ecaa7", - "sha256:e68985985296d9a66a881eb3193b0906246245294a881e7c8afe623866ac6a5c" - ], - "version": "==2022.1" - }, - "requests": { - "hashes": [ - "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", - "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" - ], - "markers": "python_version >= '3.7' and python_version < '4'", - "version": "==2.28.1" - }, - "requests-file": { - "hashes": [ - "sha256:07d74208d3389d01c38ab89ef403af0cfec63957d53a0081d8eca738d0247d8e", - "sha256:dfe5dae75c12481f68ba353183c53a65e6044c923e64c24b2209f6c7570ca953" - ], - "version": "==1.5.1" - }, - "requests-toolbelt": { - "hashes": [ - "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", - "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" - ], - "version": "==0.9.1" - }, - "s3transfer": { - "hashes": [ - "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", - "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" - ], - "markers": "python_version >= '3.7'", - "version": "==0.6.0" - }, - "simple-salesforce": { - "hashes": [ - "sha256:15d6943e52252c9cc28e1779803354f2a36c88b72056499e07eb06cd652f149c", - "sha256:7931038081c445e9459ddc014aaf7f540b1131a31596956cb5d7c0e7b7e0c4cb" - ], - "index": "pypi", - "version": "==1.12.1" - }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", - "version": "==1.16.0" - }, - "tenacity": { - "hashes": [ - "sha256:43242a20e3e73291a28bcbcacfd6e000b02d3857a9a9fff56b297a27afdc932f", - "sha256:f78f4ea81b0fabc06728c11dc2a8c01277bfc5181b321a4770471902e3eb844a" - ], - "index": "pypi", - "version": "==8.0.1" - }, - "urllib3": { - "hashes": [ - "sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec", - "sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'", - "version": "==1.26.10" - }, - "zeep": { - "hashes": [ - "sha256:5867f2eadd6b028d9751f4155af590d3aaf9280e3a0ed5e15a53343921c956e5", - "sha256:81c491092b71f5b276de8c63dfd452be3f322622c48a54f3a497cf913bdfb2f4" - ], - "markers": "python_full_version >= '3.6.0'", - "version": "==4.1.0" - } - }, - "develop": { - "autopep8": { - "hashes": [ - "sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979", - "sha256:ed77137193bbac52d029a52c59bec1b0629b5a186c495f1eb21b126ac466083f" - ], - "index": "pypi", - "version": "==1.6.0" - }, - "flake8": { - "hashes": [ - "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", - "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" - ], - "index": "pypi", - "version": "==4.0.1" - }, - "mccabe": { - "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" - ], - "version": "==0.6.1" - }, - "pycodestyle": { - "hashes": [ - "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", - "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==2.8.0" - }, - "pyflakes": { - "hashes": [ - "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", - "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.4.0" - }, - "toml": { - "hashes": [ - "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", - "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" - ], - "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", - "version": "==0.10.2" - } - } -} From c6f15e22b9b4d01b34684171b17056ae47efde29 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 19:46:49 +0900 Subject: [PATCH 08/21] =?UTF-8?q?feat(Pipfile,Pipefile.lock):=20=E9=96=8B?= =?UTF-8?q?=E7=99=BA=E7=94=A8=E3=83=91=E3=83=83=E3=82=B1=E3=83=BC=E3=82=B8?= =?UTF-8?q?=E3=81=A8=E3=81=97=E3=81=A6=E3=80=81=E3=83=AA=E3=83=B3=E3=82=BF?= =?UTF-8?q?=E3=83=BC(flake8)=E3=81=A8=E3=83=95=E3=82=A9=E3=83=BC=E3=83=9E?= =?UTF-8?q?=E3=83=83=E3=82=BF(autopep8)=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/Pipfile | 2 ++ ecs/crm-datafetch/Pipfile.lock | 52 ++++++++++++++++++++++++++++++++-- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/ecs/crm-datafetch/Pipfile b/ecs/crm-datafetch/Pipfile index 3c0e1a61..33501d98 100644 --- a/ecs/crm-datafetch/Pipfile +++ b/ecs/crm-datafetch/Pipfile @@ -9,6 +9,8 @@ simple-salesforce = "*" tenacity = "*" [dev-packages] +autopep8 = "*" +flake8 = "*" [requires] python_version = "3.8" diff --git a/ecs/crm-datafetch/Pipfile.lock b/ecs/crm-datafetch/Pipfile.lock index b66b415b..c8c60540 100644 --- a/ecs/crm-datafetch/Pipfile.lock +++ b/ecs/crm-datafetch/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "84e1c5fdd458ebb8022dd649f03e59828d8658b62a572f524a60850e2a1d626c" + "sha256": "ec1d83143aff859500979be73f67196dcfe2298ad3553a7d81ad0605a277d672" }, "pipfile-spec": 6, "requires": { @@ -367,5 +367,53 @@ "version": "==4.1.0" } }, - "develop": {} + "develop": { + "autopep8": { + "hashes": [ + "sha256:44f0932855039d2c15c4510d6df665e4730f2b8582704fa48f9c55bd3e17d979", + "sha256:ed77137193bbac52d029a52c59bec1b0629b5a186c495f1eb21b126ac466083f" + ], + "index": "pypi", + "version": "==1.6.0" + }, + "flake8": { + "hashes": [ + "sha256:479b1304f72536a55948cb40a32dce8bb0ffe3501e26eaf292c7e60eb5e0428d", + "sha256:806e034dda44114815e23c16ef92f95c91e4c71100ff52813adf7132a6ad870d" + ], + "index": "pypi", + "version": "==4.0.1" + }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, + "pycodestyle": { + "hashes": [ + "sha256:720f8b39dde8b293825e7ff02c475f3077124006db4f440dcbc9a20b76548a20", + "sha256:eddd5847ef438ea1c7870ca7eb78a9d47ce0cdb4851a5523949f2601d0cbbe7f" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", + "version": "==2.8.0" + }, + "pyflakes": { + "hashes": [ + "sha256:05a85c2872edf37a4ed30b0cce2f6093e1d0581f8c19d7393122da7e25b2b24c", + "sha256:3bb3a3f256f4b7968c9c788781e4ff07dce46bdf12339dcda61053375426ee2e" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.10.2" + } + } } From a53e60cbf19934f4a4259452b51e895dd150dbf1 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 19:48:59 +0900 Subject: [PATCH 09/21] =?UTF-8?q?docs:=20=E3=83=AD=E3=83=BC=E3=82=AB?= =?UTF-8?q?=E3=83=AB=E9=96=8B=E7=99=BA=E7=92=B0=E5=A2=83=E3=81=AE=E6=A7=8B?= =?UTF-8?q?=E7=AF=89=E3=82=84=E5=AE=9F=E8=A1=8C=E6=96=B9=E6=B3=95=E3=81=AB?= =?UTF-8?q?=E3=81=A4=E3=81=84=E3=81=A6=E3=80=81README=E3=81=AB=E8=BF=BD?= =?UTF-8?q?=E8=A8=98=E3=81=97=E3=80=81=E5=91=A8=E8=BE=BA=E3=81=AE=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=82=E4=B8=80=E7=B7=92=E3=81=AB?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 5 +- .../.vscode/recommend_settings.json | 20 ++++++ ecs/crm-datafetch/README.md | 69 +++++++++++++++++++ 3 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 ecs/crm-datafetch/.vscode/recommend_settings.json create mode 100644 ecs/crm-datafetch/README.md diff --git a/.gitignore b/.gitignore index e4f6d9fb..6733371f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ lambda/mbj-newdwh2021-staging-NoticeToSlack/package-lock.json lambda/mbj-newdwh2021-staging-NoticeToSlack/node_modules/* lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json -lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* \ No newline at end of file +lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/* + +**/.vscode/settings.json +.env \ No newline at end of file diff --git a/ecs/crm-datafetch/.vscode/recommend_settings.json b/ecs/crm-datafetch/.vscode/recommend_settings.json new file mode 100644 index 00000000..a29de1db --- /dev/null +++ b/ecs/crm-datafetch/.vscode/recommend_settings.json @@ -0,0 +1,20 @@ +{ + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": ["--max-line-length=120"], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Args": [ + "--max-line-length", + "120" + ] + } + \ No newline at end of file diff --git a/ecs/crm-datafetch/README.md b/ecs/crm-datafetch/README.md new file mode 100644 index 00000000..5a23e966 --- /dev/null +++ b/ecs/crm-datafetch/README.md @@ -0,0 +1,69 @@ +# CRMデータ連携 データ取得処理 ECSタスク + +## 前提事項 + +### ツールのバージョン + +- Python 3.8.x +- PipEnv(Pythonの依存関係管理用モジュール) + +### 開発環境 + +- Visual Studio Code + +## 開発環境構築 + +※下記の操作は基本的にVSCode上で行います。 + +- [ファイル]-[フォルダーを開く]から、当フォルダを選択して開く + +- [Wiki | Pythonの環境構築](https://nds-tyo.backlog.com/alias/wiki/1874930)にて、pyenvの導入まで完了させる + - **pyenvの導入はマストではないが、Pythonのバージョンが前提のバージョンと同一であることを確認して開発を進めてください** + - **確認しながら開発するのは煩わしいため、導入を強く推奨します。** + +- ローカルのPythonでPipEnvをインストールする + + ```sh + pip install pipenv + ``` + +- pipenvの仮想環境と依存パッケージをインストール。このとき、初回実行にはpythonの仮想環境のパスがターミナルに表示されるため、控えておく + + ```sh + # 開発用パッケージも含めてインストール + pipenv install --dev + ``` + +- VSCodeのコマンドパレットを[表示]-[コマンドパレット]から開き、`Python: Select interpreter`を選択して実行する +- Pythonの実行環境を聞かれるため、先に控えたパスと一致するものを選択する + - 出てこない場合、一度VSCodeを閉じて再度開き直す + +- 当フォルダ直下の`.vscode`フォルダ内にある`recommend_settings.json`をコピーし、同フォルダ内に`settings.json`を作成する + +## ローカルでの起動方法 + +- 当フォルダ直下の`.vscode`フォルダ内に`launch.json`を作成する +- 以下のJSONを入力して保存する + + ```json + { + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + // エントリーポイントのファイルに変更すること + "program": "<エントリーポイントになるファイル>", + "console": "integratedTerminal", + "justMyCode": true, + // 環境変数が必要な場合に読み込む環境変数ファイル + "envFile": "${workspaceFolder}/.env", + } + ] + } + ``` + +- 環境変数が必要な場合、直接設定するか、上記JSONの`"envFile"`に設定されたパスに`.env`ファイルを作成し、環境変数を入力する +- キーボードの「F5」キーを押して起動する + - デバッグモードで実行されるため、適当なところにブレークポイントを置いてデバッグすることができる From 24149be0b1b00071ac8abbed876dc563ff887247 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 19:55:37 +0900 Subject: [PATCH 10/21] =?UTF-8?q?style:=20=E3=83=95=E3=82=A9=E3=83=BC?= =?UTF-8?q?=E3=83=9E=E3=83=83=E3=82=BF=E3=82=92=E9=81=A9=E7=94=A8=E3=80=82?= =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=81=AE=E6=9B=B8=E3=81=8D?= =?UTF-8?q?=E3=81=A3=E3=81=B7=E3=82=8A=E3=82=92=E3=82=92=E8=80=83=E6=85=AE?= =?UTF-8?q?=E3=81=97=E3=81=A61=E8=A1=8C150=E6=96=87=E5=AD=97=E3=81=BE?= =?UTF-8?q?=E3=81=A7=E3=81=AF=E8=A8=B1=E5=AE=B9=E3=81=99=E3=82=8B=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.vscode/recommend_settings.json | 34 +++---- ecs/crm-datafetch/src/aws/s3.py | 27 ++---- ecs/crm-datafetch/src/chk.py | 10 +- ecs/crm-datafetch/src/config/objects.py | 4 +- ecs/crm-datafetch/src/constants.py | 5 +- ecs/crm-datafetch/src/conv.py | 11 +-- ecs/crm-datafetch/src/converter/converter.py | 13 +-- ecs/crm-datafetch/src/csvbk.py | 13 +-- ecs/crm-datafetch/src/date.py | 20 ++-- ecs/crm-datafetch/src/end.py | 14 +-- ecs/crm-datafetch/src/environments.py | 97 +++++++++++++------ ecs/crm-datafetch/src/fetch.py | 39 +++----- ecs/crm-datafetch/src/parser/json_parse.py | 4 +- ecs/crm-datafetch/src/pre.py | 22 ++--- ecs/crm-datafetch/src/process.py | 24 +++-- ecs/crm-datafetch/src/resbk.py | 10 +- .../src/salesforce/salesforce_api.py | 37 +++---- ecs/crm-datafetch/src/upd.py | 11 +-- ecs/crm-datafetch/src/upld.py | 18 +--- ecs/crm-datafetch/src/util/logger.py | 1 + 20 files changed, 184 insertions(+), 230 deletions(-) diff --git a/ecs/crm-datafetch/.vscode/recommend_settings.json b/ecs/crm-datafetch/.vscode/recommend_settings.json index a29de1db..72dacf55 100644 --- a/ecs/crm-datafetch/.vscode/recommend_settings.json +++ b/ecs/crm-datafetch/.vscode/recommend_settings.json @@ -1,20 +1,16 @@ { - "[python]": { - "editor.defaultFormatter": null, - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.organizeImports": true - } - }, - "python.linting.lintOnSave": true, - "python.linting.enabled": true, - "python.linting.pylintEnabled": false, - "python.linting.flake8Enabled": true, - "python.linting.flake8Args": ["--max-line-length=120"], - "python.formatting.provider": "autopep8", - "python.formatting.autopep8Args": [ - "--max-line-length", - "120" - ] - } - \ No newline at end of file + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": ["--max-line-length=150"], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Args": ["--max-line-length", "150"] +} diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index 229b8666..e8691e16 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -1,22 +1,13 @@ -import boto3 import json -from src.environments import ( - CRM_CONFIG_BUCKET, - CRM_BACKUP_BUCKET, - IMPORT_DATA_BUCKET, - OBJECT_INFO_FOLDER, - OBJECT_INFO_FILENAME, - PROCESS_RESULT_FOLDER, - CRM_IMPORT_DATA_FOLDER, - CRM_IMPORT_DATA_BACKUP_FOLDER, - LAST_FETCH_DATE_FOLDER, - RESPONSE_JSON_BACKUP_FOLDER -) -from src.constants import ( - AWS_RESOURCE_S3, - AWS_CLINET_S3, - S3_RESPONSE_BODY -) + +import boto3 +from src.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY +from src.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, + CRM_IMPORT_DATA_BACKUP_FOLDER, + CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET, + LAST_FETCH_DATE_FOLDER, OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER, PROCESS_RESULT_FOLDER, + RESPONSE_JSON_BACKUP_FOLDER) class S3Clinet: diff --git a/ecs/crm-datafetch/src/chk.py b/ecs/crm-datafetch/src/chk.py index 077f0ee2..b842d19b 100644 --- a/ecs/crm-datafetch/src/chk.py +++ b/ecs/crm-datafetch/src/chk.py @@ -1,11 +1,7 @@ -from src.util.logger import logger_instance as logger -from src.constants import( - CHK_JP_NAME -) -from src.error.exceptions import( - InvalidConfigException -) from src.config.objects import TargetObject +from src.constants import CHK_JP_NAME +from src.error.exceptions import InvalidConfigException +from src.util.logger import logger_instance as logger def check_object_info(object_info, execute_datetime): diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index edce5b33..8d9f6f85 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,6 +1,4 @@ -from src.constants import ( - DATE_PATTERN_YYYYMMDDTHHMMSSTZ -) +from src.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ from src.util.dict_checker import DictCheck diff --git a/ecs/crm-datafetch/src/constants.py b/ecs/crm-datafetch/src/constants.py index a394df16..fde2ec65 100644 --- a/ecs/crm-datafetch/src/constants.py +++ b/ecs/crm-datafetch/src/constants.py @@ -1,5 +1,6 @@ # environments(task settings file) -LOG_LEVEL = "LOG_LEVEL" # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +LOG_LEVEL = "LOG_LEVEL" CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 @@ -70,4 +71,4 @@ END_JP_NAME = '取得処理実施結果アップロード処理' # CSVチェック CSV_TRUE_VALUE = '1' -CSV_FALSE_VALUE = '0' \ No newline at end of file +CSV_FALSE_VALUE = '0' diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/conv.py index eb17c4c3..cfcc6a8a 100644 --- a/ecs/crm-datafetch/src/conv.py +++ b/ecs/crm-datafetch/src/conv.py @@ -1,12 +1,9 @@ from datetime import datetime -from src.util.logger import logger_instance as logger -from src.constants import ( - CONV_JP_NAME, -) -from src.error.exceptions import( - DataConvertException -) + +from src.constants import CONV_JP_NAME from src.converter.converter import CSVStringConverter +from src.error.exceptions import DataConvertException +from src.util.logger import logger_instance as logger def convert_crm_csvdata(target_object, sf_object_jsons): diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 1b785c35..18fc2923 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -1,13 +1,10 @@ -import re -import io import csv +import io +import re from datetime import datetime -from src.constants import( - CSV_TRUE_VALUE, - CSV_FALSE_VALUE, - CRM_DATETIME_FORMAT, - YYYYMMDDHHMMSS -) + +from src.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, + CSV_TRUE_VALUE, YYYYMMDDHHMMSS) class CSVStringConverter: diff --git a/ecs/crm-datafetch/src/csvbk.py b/ecs/crm-datafetch/src/csvbk.py index 75c5f780..375b273e 100644 --- a/ecs/crm-datafetch/src/csvbk.py +++ b/ecs/crm-datafetch/src/csvbk.py @@ -1,11 +1,7 @@ -from src.util.logger import logger_instance as logger from src.aws.s3 import BackupBucket -from src.constants import ( - CSVBK_JP_NAME -) -from src.error.exceptions import( - FileUploadException -) +from src.constants import CSVBK_JP_NAME +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def backup_crm_csvdata(target_object, date_path, csv_object): @@ -24,7 +20,8 @@ def backup_crm_csvdata(target_object, date_path, csv_object): except Exception as e: raise FileUploadException( - 'E-CSVBK-01', CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') + 'E-CSVBK-01', + CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') # ③ CSVバックアップ処理の終了ログを出力する logger.info( diff --git a/ecs/crm-datafetch/src/date.py b/ecs/crm-datafetch/src/date.py index adcfdd4c..2896c366 100644 --- a/ecs/crm-datafetch/src/date.py +++ b/ecs/crm-datafetch/src/date.py @@ -1,18 +1,10 @@ -from src.util.logger import logger_instance as logger -from src.constants import( - DATE_JP_NAME -) -from src.environments import ( - CRM_CONFIG_BUCKET, - LAST_FETCH_DATE_FOLDER -) -from src.error.exceptions import( - FileNotFoundException, - InvalidConfigException -) -from src.parser.json_parse import JsonParser -from src.config.objects import LastFetchDatetime from src.aws.s3 import ConfigBucket +from src.config.objects import LastFetchDatetime +from src.constants import DATE_JP_NAME +from src.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.error.exceptions import FileNotFoundException, InvalidConfigException +from src.parser.json_parse import JsonParser +from src.util.logger import logger_instance as logger def set_datetime_period(target_object, execute_datetime): diff --git a/ecs/crm-datafetch/src/end.py b/ecs/crm-datafetch/src/end.py index 7604b5b3..471effc2 100644 --- a/ecs/crm-datafetch/src/end.py +++ b/ecs/crm-datafetch/src/end.py @@ -1,14 +1,8 @@ -from src.util.logger import logger_instance as logger -from src.constants import ( - END_JP_NAME -) -from src.environments import( - PROCESS_RESULT_FILENAME -) -from src.error.exceptions import( - FileUploadException -) from src.aws.s3 import BackupBucket +from src.constants import END_JP_NAME +from src.environments import PROCESS_RESULT_FILENAME +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def updload_result_data(process_result, date_path): diff --git a/ecs/crm-datafetch/src/environments.py b/ecs/crm-datafetch/src/environments.py index 262a2fbd..4a09b2a2 100644 --- a/ecs/crm-datafetch/src/environments.py +++ b/ecs/crm-datafetch/src/environments.py @@ -1,38 +1,71 @@ import os + import src.constants as constants # environments(task settings file) -LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する -CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT] # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT] # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL] # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL] # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL] # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT] # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT] # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL] # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL] # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL] # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT] # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT] # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL] # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL] # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL] # CRMのレコード取得処理のリトライ時の最大待ち秒数 +# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する +LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) +# CRMへの認証処理のタイムアウト秒数 +CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT] +# CRMへの認証処理の最大リトライ試行回数 +CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT] +# CRMへの認証処理のリトライ時の初回待ち秒数 +CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL] +# CRMへの認証処理のリトライ時の最小待ち秒数 +CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL] +# CRMへの認証処理のリトライ時の最大待ち秒数 +CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL] +# CRMのレコード件数取得処理のタイムアウト秒数 +CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT] +# CRMのレコード件数取得処理の最大リトライ試行回数 +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT] +# CRMのレコード件数取得処理のリトライ時の初回待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL] +# CRMのレコード件数取得処理のリトライ時の最小待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL] +# CRMのレコード件数取得処理のリトライ時の最大待ち秒数 +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL] +# CRMのレコード取得処理のタイムアウト秒数 +CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT] +# CRMのレコード取得処理の最大リトライ試行回数 +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT] +# CRMのレコード取得処理のリトライ時の初回待ち秒数 +CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL] +# CRMのレコード取得処理のリトライ時の最小待ち秒数 +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL] +# CRMのレコード取得処理のリトライ時の最大待ち秒数 +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL] # environments(ECS Task Enviroment) -CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN] # CRMのAPI実行のための認証エンドポイントのドメイン -CRM_USER_NAME = os.environ[constants.CRM_USER_NAME] # CRMのAPI実行用ユーザ名 -CRM_USER_PASSWORD = os.environ[constants.CRM_USER_PASSWORD] # CRMのAPI実行用ユーザパスワード -CRM_USER_SECURITY_TOKEN = os.environ[constants.CRM_USER_SECURITY_TOKEN] # CRMのAPI実行用ユーザのセキュリティトークン -CRM_CONFIG_BUCKET = os.environ[constants.CRM_CONFIG_BUCKET] # CRMデータ取得用の設定ファイルを格納するバケット名 -CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET] # CRMのバックアップデータを格納するバケット名 -IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET] # CRMの取込データを格納するバケット名 -OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER] # CRM取得対象オブジェクトの情報を格納するフォルダパス -OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME] # CRM取得対象オブジェクトの情報のファイル名 -PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER] # CRMデータ取得結果を格納するフォルダパス -PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME] # CRMデータ取得結果を格納するファイル名 -LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER] # CRMからの最終取得日時ファイルを格納するフォルダパス -CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER] # CRMから取得し、取込用に変換したデータを格納するフォルダ -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER] # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER] # CRMから取得した生データのバックアップを格納するフォルダパス -CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER] # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ +# CRMのAPI実行のための認証エンドポイントのドメイン +CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN] +# CRMのAPI実行用ユーザ名 +CRM_USER_NAME = os.environ[constants.CRM_USER_NAME] +# CRMのAPI実行用ユーザパスワード +CRM_USER_PASSWORD = os.environ[constants.CRM_USER_PASSWORD] +# CRMのAPI実行用ユーザのセキュリティトークン +CRM_USER_SECURITY_TOKEN = os.environ[constants.CRM_USER_SECURITY_TOKEN] +# CRMデータ取得用の設定ファイルを格納するバケット名 +CRM_CONFIG_BUCKET = os.environ[constants.CRM_CONFIG_BUCKET] +# CRMのバックアップデータを格納するバケット名 +CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET] +# CRMの取込データを格納するバケット名 +IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET] +# CRM取得対象オブジェクトの情報を格納するフォルダパス +OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER] +# CRM取得対象オブジェクトの情報のファイル名 +OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME] +# CRMデータ取得結果を格納するフォルダパス +PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER] +# CRMデータ取得結果を格納するファイル名 +PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME] +# CRMからの最終取得日時ファイルを格納するフォルダパス +LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER] +# CRMから取得し、取込用に変換したデータを格納するフォルダ +CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER] +# CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス +LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER] +# CRMから取得した生データのバックアップを格納するフォルダパス +RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER] +# CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ +CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER] diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch.py index 9670767a..893aaadc 100644 --- a/ecs/crm-datafetch/src/fetch.py +++ b/ecs/crm-datafetch/src/fetch.py @@ -1,31 +1,22 @@ from requests.exceptions import ConnectTimeout, ReadTimeout from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential + +from src.constants import FETCH_JP_NAME +from src.environments import (CRM_AUTH_TIMEOUT, + CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, + CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + CRM_FETCH_RECORD_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, + CRM_GET_RECORD_COUNT_TIMEOUT) +from src.error.exceptions import DataConvertException, SalesforceAPIException +from src.salesforce.salesforce_api import SalesForceCount, SalesForceData from src.util.logger import logger_instance as logger -from src.constants import( - FETCH_JP_NAME -) -from src.environments import( - CRM_AUTH_TIMEOUT, - CRM_GET_RECORD_COUNT_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL -) -from src.error.exceptions import( - SalesforceAPIException, - DataConvertException -) -from src.salesforce.salesforce_api import( - SalesForceCount, - SalesForceData -) def fetch_crm_data(target_object, last_fetch_datetime): diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parse.py index 4273125d..f9cda105 100644 --- a/ecs/crm-datafetch/src/parser/json_parse.py +++ b/ecs/crm-datafetch/src/parser/json_parse.py @@ -1,7 +1,9 @@ -import re import json +import re + from src.constants import EXCLUDE_SYMBOL + class JsonParser(): def __init__(self, json_str) -> None: self.__json_str = json_str diff --git a/ecs/crm-datafetch/src/pre.py b/ecs/crm-datafetch/src/pre.py index c8190f12..adb6f68e 100644 --- a/ecs/crm-datafetch/src/pre.py +++ b/ecs/crm-datafetch/src/pre.py @@ -1,21 +1,13 @@ from datetime import datetime -from src.util.logger import logger_instance as logger -from src.constants import( - PRE_JP_NAME, - YYYYMMDDTHHMMSSTZ -) -from src.environments import( - CRM_CONFIG_BUCKET, - OBJECT_INFO_FOLDER, - OBJECT_INFO_FILENAME -) -from src.error.exceptions import( - FileNotFoundException, - InvalidConfigException -) + from src.aws.s3 import ConfigBucket -from src.parser.json_parse import JsonParser from src.config.objects import FetchTargetObjects +from src.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ +from src.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER) +from src.error.exceptions import FileNotFoundException, InvalidConfigException +from src.parser.json_parse import JsonParser +from src.util.logger import logger_instance as logger def prepare_get_data(): diff --git a/ecs/crm-datafetch/src/process.py b/ecs/crm-datafetch/src/process.py index 09d938af..add1e98d 100644 --- a/ecs/crm-datafetch/src/process.py +++ b/ecs/crm-datafetch/src/process.py @@ -1,17 +1,15 @@ -from src.pre import prepare_get_data # データ取得準備処理 -from src.chk import check_object_info # オブジェクト情報形式チェック処理 -from src.date import set_datetime_period # データ取得期間設定処理 -from src.fetch import fetch_crm_data # CRMデータ取得処理 -from src.resbk import backup_crm_data # CRM電文データバックアップ処理 -from src.conv import convert_crm_csvdata # CSV変換処理 -from src.csvbk import backup_crm_csvdata # CSVバックアップ処理 -from src.upld import copy_crm_csvdata # CSVアップロード処理 -from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 -from src.end import updload_result_data # 取得処理実施結果アップロード処理 +from src.chk import check_object_info # オブジェクト情報形式チェック処理 +from src.conv import convert_crm_csvdata # CSV変換処理 +from src.csvbk import backup_crm_csvdata # CSVバックアップ処理 +from src.date import set_datetime_period # データ取得期間設定処理 +from src.end import updload_result_data # 取得処理実施結果アップロード処理 +from src.error.exceptions import MeDaCaCRMDataFetchException +from src.fetch import fetch_crm_data # CRMデータ取得処理 +from src.pre import prepare_get_data # データ取得準備処理 +from src.resbk import backup_crm_data # CRM電文データバックアップ処理 +from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 +from src.upld import copy_crm_csvdata # CSVアップロード処理 from src.util.logger import logger_instance as logger -from src.error.exceptions import( - MeDaCaCRMDataFetchException -) def main() -> None: diff --git a/ecs/crm-datafetch/src/resbk.py b/ecs/crm-datafetch/src/resbk.py index 21d516ef..61bfe306 100644 --- a/ecs/crm-datafetch/src/resbk.py +++ b/ecs/crm-datafetch/src/resbk.py @@ -1,11 +1,7 @@ -from src.util.logger import logger_instance as logger -from src.constants import( - RESBK_JP_NAME -) -from src.error.exceptions import( - FileUploadException -) from src.aws.s3 import BackupBucket +from src.constants import RESBK_JP_NAME +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def backup_crm_data(object_name, sf_object_jsons, date_path): diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index 0bce5675..ac6151a9 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,26 +1,19 @@ from simple_salesforce import Salesforce -from src.environments import( - CRM_AUTH_DOMAIN, - CRM_USER_NAME, - CRM_USER_PASSWORD, - CRM_USER_SECURITY_TOKEN, - CRM_AUTH_TIMEOUT, - CRM_AUTH_MAX_RETRY_ATTEMPT, - CRM_AUTH_RETRY_INTERVAL, - CRM_AUTH_RETRY_MIN_INTERVAL, - CRM_AUTH_RETRY_MAX_INTERVAL, - CRM_GET_RECORD_COUNT_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL -) - +from src.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT, + CRM_AUTH_RETRY_INTERVAL, + CRM_AUTH_RETRY_MAX_INTERVAL, + CRM_AUTH_RETRY_MIN_INTERVAL, CRM_AUTH_TIMEOUT, + CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, + CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + CRM_FETCH_RECORD_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, + CRM_GET_RECORD_COUNT_TIMEOUT, CRM_USER_NAME, + CRM_USER_PASSWORD, CRM_USER_SECURITY_TOKEN) FETCH_SOQL = """SELECT {column_or_expression} FROM {object_name} WHERE SystemModStamp > {last_update_datetime_from} diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upd.py index f8dd4350..d69bddcc 100644 --- a/ecs/crm-datafetch/src/upd.py +++ b/ecs/crm-datafetch/src/upd.py @@ -1,12 +1,9 @@ import json -from src.util.logger import logger_instance as logger + from src.aws.s3 import ConfigBucket -from src.constants import ( - UPD_JP_NAME -) -from src.error.exceptions import( - FileUploadException -) +from src.constants import UPD_JP_NAME +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def updload_last_fetch_datetime(target_object, last_fetch_datetime): diff --git a/ecs/crm-datafetch/src/upld.py b/ecs/crm-datafetch/src/upld.py index c3c3140f..2384ae57 100644 --- a/ecs/crm-datafetch/src/upld.py +++ b/ecs/crm-datafetch/src/upld.py @@ -1,17 +1,9 @@ -from src.util.logger import logger_instance as logger -from src.constants import ( - UPLD_JP_NAME -) -from src.environments import( - CRM_BACKUP_BUCKET, - IMPORT_DATA_BUCKET, - CRM_IMPORT_DATA_FOLDER, - CRM_IMPORT_DATA_BACKUP_FOLDER, -) -from src.error.exceptions import( - FileUploadException -) from src.aws.s3 import S3ResourceNonBucket +from src.constants import UPLD_JP_NAME +from src.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, + CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) +from src.error.exceptions import FileUploadException +from src.util.logger import logger_instance as logger def copy_crm_csvdata(target_object, date_path): diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index ec0baf55..cb139c98 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -1,4 +1,5 @@ import logging + from src.environments import LOG_LEVEL for name in ["boto3", "botocore", "s3transfer", "urllib3"]: From e532b395df1f4f0c3e4435fea3f538decad37afb Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 21 Jul 2022 20:00:35 +0900 Subject: [PATCH 11/21] =?UTF-8?q?style:=20Flake8=E3=81=AB=E3=82=88?= =?UTF-8?q?=E3=82=8B=E3=83=AA=E3=83=B3=E3=83=88=E4=B8=8D=E6=AD=A3=E3=81=AE?= =?UTF-8?q?=E8=A7=A3=E6=B6=88=E3=80=82=E3=83=95=E3=82=A9=E3=83=BC=E3=83=9E?= =?UTF-8?q?=E3=83=83=E3=83=88=E6=96=87=E5=AD=97=E5=88=97=E3=81=AF=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E3=81=8C=E5=9F=8B=E3=82=81=E8=BE=BC=E3=81=BE=E3=82=8C?= =?UTF-8?q?=E3=81=A6=E3=81=84=E3=81=AA=E3=81=8F=E3=81=A6=E3=82=82=E3=82=A8?= =?UTF-8?q?=E3=83=A9=E3=83=BC=E3=81=AB=E3=81=97=E3=81=AA=E3=81=84=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=80=81=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF=E3=82=92?= =?UTF-8?q?=E7=84=A1=E8=A6=96=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.vscode/recommend_settings.json | 2 +- ecs/crm-datafetch/src/conv.py | 2 -- ecs/crm-datafetch/src/fetch.py | 10 ++++++++-- ecs/crm-datafetch/src/process.py | 2 +- ecs/crm-datafetch/src/upd.py | 6 ++++-- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/ecs/crm-datafetch/.vscode/recommend_settings.json b/ecs/crm-datafetch/.vscode/recommend_settings.json index 72dacf55..66a8f2b5 100644 --- a/ecs/crm-datafetch/.vscode/recommend_settings.json +++ b/ecs/crm-datafetch/.vscode/recommend_settings.json @@ -10,7 +10,7 @@ "python.linting.enabled": true, "python.linting.pylintEnabled": false, "python.linting.flake8Enabled": true, - "python.linting.flake8Args": ["--max-line-length=150"], + "python.linting.flake8Args": ["--max-line-length=150", "--ignore=F541"], "python.formatting.provider": "autopep8", "python.formatting.autopep8Args": ["--max-line-length", "150"] } diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/conv.py index cfcc6a8a..2a4d3be8 100644 --- a/ecs/crm-datafetch/src/conv.py +++ b/ecs/crm-datafetch/src/conv.py @@ -1,5 +1,3 @@ -from datetime import datetime - from src.constants import CONV_JP_NAME from src.converter.converter import CSVStringConverter from src.error.exceptions import DataConvertException diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch.py index 893aaadc..baa0219f 100644 --- a/ecs/crm-datafetch/src/fetch.py +++ b/ecs/crm-datafetch/src/fetch.py @@ -81,7 +81,10 @@ def fetch_crm_data(target_object, last_fetch_datetime): return sf_object_jsons -@retry(wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) +@retry( + wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), + stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) def fetch_sf_count_retry(object_name, last_update_datetime_from, last_update_datetime_to): try: global count_contime_counter, count_readtime_counter, count_counter @@ -110,7 +113,10 @@ def fetch_sf_count_retry(object_name, last_update_datetime_from, last_update_dat raise e -@retry(wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) +@retry( + wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, + min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), + stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) def fetch_sf_data_retry(columns, object_name, last_update_datetime_from, last_update_datetime_to): try: global data_contime_counter, data_readtime_counter, data_counter diff --git a/ecs/crm-datafetch/src/process.py b/ecs/crm-datafetch/src/process.py index add1e98d..e914a4d4 100644 --- a/ecs/crm-datafetch/src/process.py +++ b/ecs/crm-datafetch/src/process.py @@ -80,7 +80,7 @@ def main() -> None: # 9. CSV変換処理を呼び出す logger.info( f'I-CTRL-12 [{target_object.object_name}] CSV変換処理呼び出し') - csv_object = convert_crm_csvdata(target_object, sf_object_jsons) + csv_object = convert_crm_csvdata(target_object, sf_object_jsons) # 10. CSVバックアップ処理を呼び出す logger.info( diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upd.py index d69bddcc..d6a8ef03 100644 --- a/ecs/crm-datafetch/src/upd.py +++ b/ecs/crm-datafetch/src/upd.py @@ -12,7 +12,7 @@ def updload_last_fetch_datetime(target_object, last_fetch_datetime): f'I-UPD-01 [{target_object.object_name}] の前回取得日時ファイルの更新処理を開始します') try: - if target_object.is_update_last_fetch_datetime == False: + if target_object.is_update_last_fetch_datetime is False: # ② オブジェクト情報.is_update_last_fetch_datetimeがfalseの場合、以降の処理をスキップする logger.info( f'I-UPD-02 [{target_object.object_name}] の前回取得日時ファイルの更新処理をスキップします') @@ -33,7 +33,9 @@ def updload_last_fetch_datetime(target_object, last_fetch_datetime): except Exception as e: raise FileUploadException( - 'E-UPD-01', UPD_JP_NAME, f'[{target_object.object_name}] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[{target_object.last_fetch_datetime_file_name}] エラー内容:[{e}]') + 'E-UPD-01', + UPD_JP_NAME, + f'[{target_object.object_name}] 前回処理日時ファイルのアップロードに失敗しました ファイル名：[{target_object.last_fetch_datetime_file_name}] エラー内容:[{e}]') # ④ 前回取得日時ファイル更新処理の終了ログを出力する logger.info( From 440e6c8fd094f0800d3d8d9b7555361ae45fdf28 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 26 Jul 2022 09:54:16 +0900 Subject: [PATCH 12/21] =?UTF-8?q?fix:=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E5=90=8D=E5=A4=89=E6=9B=B4=E3=80=81=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E3=81=AE=E3=83=91=E3=82=B9=E3=82=92=E5=A4=89=E6=9B=B4?= =?UTF-8?q?=E3=81=97=E3=81=9F=E3=82=82=E3=81=AE=E3=82=92=E4=B8=80=E5=BA=A6?= =?UTF-8?q?=E3=82=B3=E3=83=9F=E3=83=83=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/.vscode/launch.json | 16 ++++++++++++++ ecs/crm-datafetch/main.py | 4 ++-- ecs/crm-datafetch/src/aws/s3.py | 4 ++-- ...csvbk.py => backup_crm_csvdata_process.py} | 2 +- .../{resbk.py => backup_crm_data_process.py} | 2 +- .../{chk.py => check_object_info_process.py} | 2 +- ecs/crm-datafetch/src/config/objects.py | 2 +- .../src/{process.py => controller.py} | 22 +++++++++---------- ...conv.py => convert_crm_csvdata_process.py} | 2 +- ecs/crm-datafetch/src/converter/converter.py | 2 +- .../{upld.py => copy_crm_csvdata_process.py} | 4 ++-- .../{fetch.py => fetch_crm_data_process.py} | 4 ++-- ecs/crm-datafetch/src/parser/json_parse.py | 2 +- .../{pre.py => prepare_get_data_process.py} | 4 ++-- .../src/salesforce/salesforce_api.py | 2 +- ...date.py => set_datetime_period_process.py} | 4 ++-- .../src/{ => system_var}/constants.py | 0 .../src/{ => system_var}/environments.py | 2 +- ...{end.py => updload_result_data_process.py} | 4 ++-- ... => upload_last_fetch_datetime_process.py} | 2 +- ecs/crm-datafetch/src/util/logger.py | 2 +- 21 files changed, 52 insertions(+), 36 deletions(-) create mode 100644 ecs/crm-datafetch/.vscode/launch.json rename ecs/crm-datafetch/src/{csvbk.py => backup_crm_csvdata_process.py} (96%) rename ecs/crm-datafetch/src/{resbk.py => backup_crm_data_process.py} (96%) rename ecs/crm-datafetch/src/{chk.py => check_object_info_process.py} (95%) rename ecs/crm-datafetch/src/{process.py => controller.py} (87%) rename ecs/crm-datafetch/src/{conv.py => convert_crm_csvdata_process.py} (95%) rename ecs/crm-datafetch/src/{upld.py => copy_crm_csvdata_process.py} (92%) rename ecs/crm-datafetch/src/{fetch.py => fetch_crm_data_process.py} (98%) rename ecs/crm-datafetch/src/{pre.py => prepare_get_data_process.py} (95%) rename ecs/crm-datafetch/src/{date.py => set_datetime_period_process.py} (95%) rename ecs/crm-datafetch/src/{ => system_var}/constants.py (100%) rename ecs/crm-datafetch/src/{ => system_var}/environments.py (99%) rename ecs/crm-datafetch/src/{end.py => updload_result_data_process.py} (91%) rename ecs/crm-datafetch/src/{upd.py => upload_last_fetch_datetime_process.py} (97%) diff --git a/ecs/crm-datafetch/.vscode/launch.json b/ecs/crm-datafetch/.vscode/launch.json new file mode 100644 index 00000000..2e39a541 --- /dev/null +++ b/ecs/crm-datafetch/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Current File", + "type": "python", + "request": "launch", + // エントリーポイントのファイルに変更すること + "program": "main.py", + "console": "integratedTerminal", + "justMyCode": true, + // 環境変数が必要な場合に読み込む環境変数ファイル + "envFile": "${workspaceFolder}/.env", + } + ] +} \ No newline at end of file diff --git a/ecs/crm-datafetch/main.py b/ecs/crm-datafetch/main.py index b0edf545..6e197d30 100644 --- a/ecs/crm-datafetch/main.py +++ b/ecs/crm-datafetch/main.py @@ -1,4 +1,4 @@ -import src.process as process +import src.controller as controller if __name__ == '__main__': - process.main() + controller.main() diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index e8691e16..dec0ec62 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -1,8 +1,8 @@ import json import boto3 -from src.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY -from src.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, +from src.system_var.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY +from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET, LAST_FETCH_DATE_FOLDER, OBJECT_INFO_FILENAME, diff --git a/ecs/crm-datafetch/src/csvbk.py b/ecs/crm-datafetch/src/backup_crm_csvdata_process.py similarity index 96% rename from ecs/crm-datafetch/src/csvbk.py rename to ecs/crm-datafetch/src/backup_crm_csvdata_process.py index 375b273e..1960e7b0 100644 --- a/ecs/crm-datafetch/src/csvbk.py +++ b/ecs/crm-datafetch/src/backup_crm_csvdata_process.py @@ -1,5 +1,5 @@ from src.aws.s3 import BackupBucket -from src.constants import CSVBK_JP_NAME +from src.system_var.constants import CSVBK_JP_NAME from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/resbk.py b/ecs/crm-datafetch/src/backup_crm_data_process.py similarity index 96% rename from ecs/crm-datafetch/src/resbk.py rename to ecs/crm-datafetch/src/backup_crm_data_process.py index 61bfe306..130da769 100644 --- a/ecs/crm-datafetch/src/resbk.py +++ b/ecs/crm-datafetch/src/backup_crm_data_process.py @@ -1,5 +1,5 @@ from src.aws.s3 import BackupBucket -from src.constants import RESBK_JP_NAME +from src.system_var.constants import RESBK_JP_NAME from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/chk.py b/ecs/crm-datafetch/src/check_object_info_process.py similarity index 95% rename from ecs/crm-datafetch/src/chk.py rename to ecs/crm-datafetch/src/check_object_info_process.py index b842d19b..91551d21 100644 --- a/ecs/crm-datafetch/src/chk.py +++ b/ecs/crm-datafetch/src/check_object_info_process.py @@ -1,5 +1,5 @@ from src.config.objects import TargetObject -from src.constants import CHK_JP_NAME +from src.system_var.constants import CHK_JP_NAME from src.error.exceptions import InvalidConfigException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 8d9f6f85..c00274bf 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,4 +1,4 @@ -from src.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ +from src.system_var.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ from src.util.dict_checker import DictCheck diff --git a/ecs/crm-datafetch/src/process.py b/ecs/crm-datafetch/src/controller.py similarity index 87% rename from ecs/crm-datafetch/src/process.py rename to ecs/crm-datafetch/src/controller.py index e914a4d4..ebd3b3cd 100644 --- a/ecs/crm-datafetch/src/process.py +++ b/ecs/crm-datafetch/src/controller.py @@ -1,14 +1,14 @@ -from src.chk import check_object_info # オブジェクト情報形式チェック処理 -from src.conv import convert_crm_csvdata # CSV変換処理 -from src.csvbk import backup_crm_csvdata # CSVバックアップ処理 -from src.date import set_datetime_period # データ取得期間設定処理 -from src.end import updload_result_data # 取得処理実施結果アップロード処理 +from src.check_object_info_process import check_object_info # オブジェクト情報形式チェック処理 +from src.convert_crm_csvdata_process import convert_crm_csvdata # CSV変換処理 +from src.backup_crm_csvdata_process import backup_crm_csvdata # CSVバックアップ処理 +from src.set_datetime_period_process import set_datetime_period # データ取得期間設定処理 +from src.updload_result_data_process import updload_result_data # 取得処理実施結果アップロード処理 from src.error.exceptions import MeDaCaCRMDataFetchException -from src.fetch import fetch_crm_data # CRMデータ取得処理 -from src.pre import prepare_get_data # データ取得準備処理 -from src.resbk import backup_crm_data # CRM電文データバックアップ処理 -from src.upd import updload_last_fetch_datetime # 前回取得日時ファイル更新 -from src.upld import copy_crm_csvdata # CSVアップロード処理 +from src.fetch_crm_data_process import fetch_crm_data # CRMデータ取得処理 +from src.prepare_get_data_process import prepare_get_data # データ取得準備処理 +from src.backup_crm_data_process import backup_crm_data # CRM電文データバックアップ処理 +from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime # 前回取得日時ファイル更新 +from src.copy_crm_csvdata_process import copy_crm_csvdata # CSVアップロード処理 from src.util.logger import logger_instance as logger @@ -95,7 +95,7 @@ def main() -> None: # 12. 前回取得日時ファイル更新処理を呼びだす logger.info( f'I-CTRL-15 [{target_object.object_name}] 前回取得日時ファイル更新処理呼び出し') - updload_last_fetch_datetime(target_object, last_fetch_datetime) + upload_last_fetch_datetime(target_object, last_fetch_datetime) # 13. オブジェクト処理結果の更新 process_result[target_object.object_name] = 'success' diff --git a/ecs/crm-datafetch/src/conv.py b/ecs/crm-datafetch/src/convert_crm_csvdata_process.py similarity index 95% rename from ecs/crm-datafetch/src/conv.py rename to ecs/crm-datafetch/src/convert_crm_csvdata_process.py index 2a4d3be8..4ef6dd11 100644 --- a/ecs/crm-datafetch/src/conv.py +++ b/ecs/crm-datafetch/src/convert_crm_csvdata_process.py @@ -1,4 +1,4 @@ -from src.constants import CONV_JP_NAME +from src.system_var.constants import CONV_JP_NAME from src.converter.converter import CSVStringConverter from src.error.exceptions import DataConvertException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 18fc2923..7b4adb1e 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -3,7 +3,7 @@ import io import re from datetime import datetime -from src.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, +from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, CSV_TRUE_VALUE, YYYYMMDDHHMMSS) diff --git a/ecs/crm-datafetch/src/upld.py b/ecs/crm-datafetch/src/copy_crm_csvdata_process.py similarity index 92% rename from ecs/crm-datafetch/src/upld.py rename to ecs/crm-datafetch/src/copy_crm_csvdata_process.py index 2384ae57..ecebe9ea 100644 --- a/ecs/crm-datafetch/src/upld.py +++ b/ecs/crm-datafetch/src/copy_crm_csvdata_process.py @@ -1,6 +1,6 @@ from src.aws.s3 import S3ResourceNonBucket -from src.constants import UPLD_JP_NAME -from src.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, +from src.system_var.constants import UPLD_JP_NAME +from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/fetch.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py similarity index 98% rename from ecs/crm-datafetch/src/fetch.py rename to ecs/crm-datafetch/src/fetch_crm_data_process.py index baa0219f..eac5478b 100644 --- a/ecs/crm-datafetch/src/fetch.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -2,8 +2,8 @@ from requests.exceptions import ConnectTimeout, ReadTimeout from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential -from src.constants import FETCH_JP_NAME -from src.environments import (CRM_AUTH_TIMEOUT, +from src.system_var.constants import FETCH_JP_NAME +from src.system_var.environments import (CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, CRM_FETCH_RECORD_RETRY_INTERVAL, CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parse.py index f9cda105..5e7249b9 100644 --- a/ecs/crm-datafetch/src/parser/json_parse.py +++ b/ecs/crm-datafetch/src/parser/json_parse.py @@ -1,7 +1,7 @@ import json import re -from src.constants import EXCLUDE_SYMBOL +from src.system_var.constants import EXCLUDE_SYMBOL class JsonParser(): diff --git a/ecs/crm-datafetch/src/pre.py b/ecs/crm-datafetch/src/prepare_get_data_process.py similarity index 95% rename from ecs/crm-datafetch/src/pre.py rename to ecs/crm-datafetch/src/prepare_get_data_process.py index adb6f68e..e96de09c 100644 --- a/ecs/crm-datafetch/src/pre.py +++ b/ecs/crm-datafetch/src/prepare_get_data_process.py @@ -2,8 +2,8 @@ from datetime import datetime from src.aws.s3 import ConfigBucket from src.config.objects import FetchTargetObjects -from src.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ -from src.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, +from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ +from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, OBJECT_INFO_FOLDER) from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index ac6151a9..a6b9c458 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,5 +1,5 @@ from simple_salesforce import Salesforce -from src.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT, +from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT, CRM_AUTH_RETRY_INTERVAL, CRM_AUTH_RETRY_MAX_INTERVAL, CRM_AUTH_RETRY_MIN_INTERVAL, CRM_AUTH_TIMEOUT, diff --git a/ecs/crm-datafetch/src/date.py b/ecs/crm-datafetch/src/set_datetime_period_process.py similarity index 95% rename from ecs/crm-datafetch/src/date.py rename to ecs/crm-datafetch/src/set_datetime_period_process.py index 2896c366..ea281fba 100644 --- a/ecs/crm-datafetch/src/date.py +++ b/ecs/crm-datafetch/src/set_datetime_period_process.py @@ -1,7 +1,7 @@ from src.aws.s3 import ConfigBucket from src.config.objects import LastFetchDatetime -from src.constants import DATE_JP_NAME -from src.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.system_var.constants import DATE_JP_NAME +from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/constants.py b/ecs/crm-datafetch/src/system_var/constants.py similarity index 100% rename from ecs/crm-datafetch/src/constants.py rename to ecs/crm-datafetch/src/system_var/constants.py diff --git a/ecs/crm-datafetch/src/environments.py b/ecs/crm-datafetch/src/system_var/environments.py similarity index 99% rename from ecs/crm-datafetch/src/environments.py rename to ecs/crm-datafetch/src/system_var/environments.py index 4a09b2a2..dfa87ae3 100644 --- a/ecs/crm-datafetch/src/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -1,6 +1,6 @@ import os -import src.constants as constants +import src.system_var.constants as constants # environments(task settings file) # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する diff --git a/ecs/crm-datafetch/src/end.py b/ecs/crm-datafetch/src/updload_result_data_process.py similarity index 91% rename from ecs/crm-datafetch/src/end.py rename to ecs/crm-datafetch/src/updload_result_data_process.py index 471effc2..e70f4679 100644 --- a/ecs/crm-datafetch/src/end.py +++ b/ecs/crm-datafetch/src/updload_result_data_process.py @@ -1,6 +1,6 @@ from src.aws.s3 import BackupBucket -from src.constants import END_JP_NAME -from src.environments import PROCESS_RESULT_FILENAME +from src.system_var.constants import END_JP_NAME +from src.system_var.environments import PROCESS_RESULT_FILENAME from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/upd.py b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py similarity index 97% rename from ecs/crm-datafetch/src/upd.py rename to ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py index d6a8ef03..f606c9ad 100644 --- a/ecs/crm-datafetch/src/upd.py +++ b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py @@ -1,7 +1,7 @@ import json from src.aws.s3 import ConfigBucket -from src.constants import UPD_JP_NAME +from src.system_var.constants import UPD_JP_NAME from src.error.exceptions import FileUploadException from src.util.logger import logger_instance as logger diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index cb139c98..689d9c36 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -1,6 +1,6 @@ import logging -from src.environments import LOG_LEVEL +from src.system_var.environments import LOG_LEVEL for name in ["boto3", "botocore", "s3transfer", "urllib3"]: logging.getLogger(name).setLevel(logging.WARNING) From a41a40004b4f9ed97304b5d77537f0994be52da7 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Wed, 27 Jul 2022 23:28:28 +0900 Subject: [PATCH 13/21] =?UTF-8?q?fix:=20=E3=82=BD=E3=83=BC=E3=82=B9?= =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=89=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E3=81=AE=E6=8C=87=E6=91=98=E5=8F=8D=E6=98=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/main.py | 5 +- ecs/crm-datafetch/src/aws/s3.py | 76 +- .../src/backup_crm_csv_data_process.py | 58 + .../src/backup_crm_csvdata_process.py | 31 - .../src/backup_crm_data_process.py | 31 +- .../src/check_object_info_process.py | 26 +- ecs/crm-datafetch/src/config/objects.py | 243 +- ecs/crm-datafetch/src/controller.py | 257 +- .../src/convert_crm_csv_data_process.py | 51 + .../src/convert_crm_csvdata_process.py | 25 - .../src/converter/convert_factory.py | 55 + ecs/crm-datafetch/src/converter/converter.py | 99 +- .../src/copy_crm_csv_data_process.py | 57 + .../src/copy_crm_csvdata_process.py | 32 - ecs/crm-datafetch/src/error/exceptions.py | 2 +- .../src/fetch_crm_data_process.py | 64 +- ...ocess.py => prepare_data_fetch_process.py} | 46 +- .../src/salesforce/salesforce_api.py | 53 +- .../src/salesforce/soql_builder.py | 34 + .../src/set_datetime_period_process.py | 41 +- ecs/crm-datafetch/src/system_var/constants.py | 33 +- .../src/system_var/environments.py | 48 +- .../src/upload_last_fetch_datetime_process.py | 50 +- ...ocess.py => upload_result_data_process.py} | 27 +- ecs/crm-datafetch/src/util/dict_checker.py | 75 +- .../src/util/execute_datetime.py | 17 + ecs/crm-datafetch/src/util/logger.py | 1 + .../crm/object_info/crm_object_list_all.json | 47 + .../crm/object_info/crm_object_list_diff.json | 3043 ++++++++++++++++- 29 files changed, 3947 insertions(+), 680 deletions(-) create mode 100644 ecs/crm-datafetch/src/backup_crm_csv_data_process.py delete mode 100644 ecs/crm-datafetch/src/backup_crm_csvdata_process.py create mode 100644 ecs/crm-datafetch/src/convert_crm_csv_data_process.py delete mode 100644 ecs/crm-datafetch/src/convert_crm_csvdata_process.py create mode 100644 ecs/crm-datafetch/src/converter/convert_factory.py create mode 100644 ecs/crm-datafetch/src/copy_crm_csv_data_process.py delete mode 100644 ecs/crm-datafetch/src/copy_crm_csvdata_process.py rename ecs/crm-datafetch/src/{prepare_get_data_process.py => prepare_data_fetch_process.py} (76%) create mode 100644 ecs/crm-datafetch/src/salesforce/soql_builder.py rename ecs/crm-datafetch/src/{updload_result_data_process.py => upload_result_data_process.py} (64%) create mode 100644 ecs/crm-datafetch/src/util/execute_datetime.py create mode 100644 s3/config/crm/object_info/crm_object_list_all.json diff --git a/ecs/crm-datafetch/main.py b/ecs/crm-datafetch/main.py index 6e197d30..dcae9825 100644 --- a/ecs/crm-datafetch/main.py +++ b/ecs/crm-datafetch/main.py @@ -1,4 +1,5 @@ -import src.controller as controller +from src.controller import controller +"""CRMデータ取得処理のエントリーポイント""" if __name__ == '__main__': - controller.main() + controller() diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index dec0ec62..e1d0dc70 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -1,7 +1,7 @@ import json import boto3 -from src.system_var.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY +from src.system_var.constants import AWS_RESOURCE_S3, S3_RESPONSE_BODY, S3_CHAR_CODE from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET, @@ -10,16 +10,6 @@ from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, RESPONSE_JSON_BACKUP_FOLDER) -class S3Clinet: - def __init__(self) -> None: - self.__s3_client = boto3.client(AWS_CLINET_S3) - - def copy_object(self, src_bucket, src_key, dest_bucket, dest_key): - self.__s3_client.copy( - {"Bucket": src_bucket, "key": src_key}, dest_bucket, dest_key) - return - - class S3Resource: def __init__(self, bucket_name: str) -> None: self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) @@ -28,41 +18,47 @@ class S3Resource: def get_object(self, object_key: str) -> str: response = self.__s3_bucket.Object(object_key).get() body = response[S3_RESPONSE_BODY].read() + return body.decode(S3_CHAR_CODE) - return body.decode('utf-8') - - def put_object(self, object_key: str, data) -> str: + def put_object(self, object_key: str, data: str) -> None: s3_object = self.__s3_bucket.Object(object_key) - s3_object.put(Body=data.encode('utf-8'), ContentEncoding='utf-8') + s3_object.put(Body=data.encode(S3_CHAR_CODE), ContentEncoding=S3_CHAR_CODE) return - -class S3ResourceNonBucket: - def __init__(self) -> None: - self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) - - def copy(self, src_bucket, src_key, dest_bucket, dest_key) -> None: + def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: copy_source = {'Bucket': src_bucket, 'Key': src_key} self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) - return +#class S3ResourceNonBucket: +# def __init__(self) -> None: +# self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) +# +# def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: +# copy_source = {'Bucket': src_bucket, 'Key': src_key} +# self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) +# return + + class ConfigBucket: __s3_resource: S3Resource = None def __init__(self) -> None: self.__s3_resource = S3Resource(CRM_CONFIG_BUCKET) + def __str__(self) -> str: + return CRM_CONFIG_BUCKET + def get_object_info_file(self) -> str: return self.__s3_resource.get_object(f'{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}') - def get_last_fetch_datetime_file(self, file_name) -> str: - return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_name}') + def get_last_fetch_datetime_file(self, file_path: str) -> str: + return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_path}') - def put_last_fetch_datetime_file(self, file_name, data) -> None: + def put_last_fetch_datetime_file(self, file_path: str, data: str) -> None: self.__s3_resource.put_object( - f'{LAST_FETCH_DATE_FOLDER}/{file_name}', data) + f'{LAST_FETCH_DATE_FOLDER}/{file_path}', data) return @@ -72,11 +68,19 @@ class DataBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(IMPORT_DATA_BUCKET) - def put_csv(self, file_name, data) -> None: - object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_name}' + def __str__(self) -> str: + return IMPORT_DATA_BUCKET + + def put_csv(self, file_path: str, data: str) -> None: + object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, data) return + def put_csv_from(self, src_bucket: str, src_key: str): + self.__s3_resource.copy(src_bucket, src_key, str(self), CRM_IMPORT_DATA_FOLDER) + return + + class BackupBucket: __s3_resource: S3Resource = None @@ -84,16 +88,20 @@ class BackupBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(CRM_BACKUP_BUCKET) - def put_response_json(self, file_name, data) -> None: - object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_name}' + def __str__(self) -> str: + return CRM_BACKUP_BUCKET + + def put_response_json(self, file_path: str, data: dict) -> None: + object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, json.dumps(data)) return - def put_csv_bk(self, file_name, data) -> None: - object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_name}' + def put_csv(self, file_path: str, data: str) -> None: + object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, data) return - def put_result_json(self, file_name, data) -> None: - object_key = f'{PROCESS_RESULT_FOLDER}/{file_name}' + def put_result_json(self, file_path: str, data: dict) -> None: + object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, json.dumps(data)) + return \ No newline at end of file diff --git a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py new file mode 100644 index 00000000..f14490cf --- /dev/null +++ b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py @@ -0,0 +1,58 @@ +from src.aws.s3 import BackupBucket +from src.config.objects import TargetObject +from src.system_var.constants import CSVBK_JP_NAME +from src.error.exceptions import FileUploadException +from src.util.execute_datetime import ExecuteDateTime +from src.util.logger import logger_instance as logger + + +def backup_crm_csv_data_process(target_object: TargetObject, exetute_datetime: ExecuteDateTime, csv_string: str): + """ + CSVバックアップ処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + csv_string : str + csvデータ + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + + # ① CSVバックアップ処理の開始ログを出力する + target_object_name = target_object.object_name + upload_file_name = target_object.upload_file_name + + logger.info( + f'I-CSVBK-01 [{target_object_name}] のCSVデータのバックアップ処理を開始します ファイル名：[{upload_file_name}.csv]') + + try: + # ② CRMバックアップ保管用バケットに、変換後のCSVデータのバックアップを保管する + backup_bucket = BackupBucket() + backup_bucket.put_csv( + f'{exetute_datetime.to_path()}/{upload_file_name}.csv', csv_string) + + logger.debug( + f'D-CSVBK-02 [{target_object_name}] のCSVデータバックアップ 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-CSVBK-01', + CSVBK_JP_NAME, f'[{target_object_name}] CSVデータのバックアップに失敗しました ファイル名：[{upload_file_name}.csv] エラー内容:[{e}]') + + # ③ CSVバックアップ処理の終了ログを出力する + logger.info( + f'I-CSVBK-03 [{target_object_name}] のCSVデータのバックアップ処理を終了します') + + # ④ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/backup_crm_csvdata_process.py b/ecs/crm-datafetch/src/backup_crm_csvdata_process.py deleted file mode 100644 index 1960e7b0..00000000 --- a/ecs/crm-datafetch/src/backup_crm_csvdata_process.py +++ /dev/null @@ -1,31 +0,0 @@ -from src.aws.s3 import BackupBucket -from src.system_var.constants import CSVBK_JP_NAME -from src.error.exceptions import FileUploadException -from src.util.logger import logger_instance as logger - - -def backup_crm_csvdata(target_object, date_path, csv_object): - # ① CSVバックアップ処理の開始ログを出力する - logger.info( - f'I-CSVBK-01 [{target_object.object_name}] のCSVデータのバックアップ処理を開始します ファイル名：[{target_object.upload_file_name}.csv]') - - try: - # ② CRMバックアップ保管用バケットに、変換後のCSVデータのバックアップを保管する - backup_bucket = BackupBucket() - backup_bucket.put_csv_bk( - f'{date_path}/{target_object.upload_file_name}.csv', csv_object.csv_buffer) - - logger.debug( - f'D-CSVBK-02 [{target_object.object_name}] のCSVデータバックアップ 正常終了') - - except Exception as e: - raise FileUploadException( - 'E-CSVBK-01', - CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') - - # ③ CSVバックアップ処理の終了ログを出力する - logger.info( - f'I-CSVBK-03 [{target_object.object_name}] のCSVデータのバックアップ処理を終了します') - - # ④ 次の処理へ移行する - return diff --git a/ecs/crm-datafetch/src/backup_crm_data_process.py b/ecs/crm-datafetch/src/backup_crm_data_process.py index 130da769..a37eb583 100644 --- a/ecs/crm-datafetch/src/backup_crm_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_data_process.py @@ -1,19 +1,42 @@ from src.aws.s3 import BackupBucket -from src.system_var.constants import RESBK_JP_NAME from src.error.exceptions import FileUploadException +from src.system_var.constants import RESBK_JP_NAME +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def backup_crm_data(object_name, sf_object_jsons, date_path): +def backup_crm_data_process(object_name: str, sf_object_dict: dict, execute_datetime: ExecuteDateTime): + """ + CRM電文データバックアップ処理 + + Parameters + ---------- + object_name : str + 取得対象オブジェクト情報インスタンス + sf_object_dict : dict + Salesforceオブジェクトデータ + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + # ① CRM電文データバックアップ処理の開始ログを出力する logger.info(f'I-RESBK-01 [{object_name}] のCRM電文データバックアップ処理を開始します') try: # ② CRMバックアップ保管用バケットに、CRMから取得したJSONの電文データのバックアップを保管する - file_name = f'{date_path}/{object_name}.json' + file_name = f'{execute_datetime.to_path()}/{object_name}.json' backup_bucket = BackupBucket() - backup_bucket.put_response_json(file_name, sf_object_jsons) + backup_bucket.put_response_json(file_name, sf_object_dict) logger.debug(f'D-RESBK-02 [{object_name}] のJSONデータバックアップ 正常終了') diff --git a/ecs/crm-datafetch/src/check_object_info_process.py b/ecs/crm-datafetch/src/check_object_info_process.py index 91551d21..ed2a34da 100644 --- a/ecs/crm-datafetch/src/check_object_info_process.py +++ b/ecs/crm-datafetch/src/check_object_info_process.py @@ -1,10 +1,32 @@ from src.config.objects import TargetObject -from src.system_var.constants import CHK_JP_NAME from src.error.exceptions import InvalidConfigException +from src.util.execute_datetime import ExecuteDateTime +from src.system_var.constants import CHK_JP_NAME from src.util.logger import logger_instance as logger -def check_object_info(object_info, execute_datetime): +def check_object_info_process(object_info: dict, execute_datetime: ExecuteDateTime): + """ + オブジェクト情報形式チェック処理 + + Parameters + ---------- + object_info : dict + 取得対象オブジェクト情報 + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + + Returns + ------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + + Raises + ------ + InvalidConfigException + オブジェクト情報定義が不正だった場合 + """ + # ① オブジェクト情報形式チェック処理開始ログを出力する logger.info('I-CHK-01 オブジェクト情報形式チェック処理を開始します') diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index c00274bf..464f7b0e 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,193 +1,138 @@ -from src.system_var.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ -from src.util.dict_checker import DictCheck +from src.system_var.constants import ( DATE_PATTERN_YYYYMMDDTHHMMSSTZ, + OBJECTS_KEY, + OBJECTS_TYPE, + OBJECT_NAME_KEY, + OBJECT_NAME_TYPE, + COLUMNS_KEY, + COLUMNS_TYPE, + IS_SKIP_KEY, + IS_SKIP_TYPE, + IS_UPDATE_LAST_FETCH_DATETIME_KEY, + IS_UPDATE_LAST_FETCH_DATETIME_TYPE, + LAST_FETCH_DATETIME_FILE_NAME_KEY, + LAST_FETCH_DATETIME_FILE_NAME_TYPE, + UPLOAD_FILE_NAME_KEY, + UPLOAD_FILE_NAME_TYPE, + DATETIME_COLUMN_KEY, + DATETIME_COLUMN_TYPE, + LAST_FETCH_DATETIME_FROM_KEY, + LAST_FETCH_DATETIME_TO_KEY, + DATETIME_COLUMN_DEFAULT_VALUE +) + +from src.util.dict_checker import DictChecker class FetchTargetObjects(): def __init__(self, object_info_file_dict) -> None: - self.__dict_check = DictCheck() self.__objects = object_info_file_dict - self.__key = 'objects' - self.check_key_objects() + self.__dict_checker = DictChecker(self.__objects) + self.validate() self.__i = 0 def __iter__(self): return self def __next__(self): - if self.__i == len(self.__objects[self.__key]): + if self.__i == len(self.__objects[OBJECTS_KEY]): raise StopIteration() - value = self.__objects['objects'][self.__i] + value = self.__objects[OBJECTS_KEY][self.__i] self.__i += 1 return value - def check_key_objects(self) -> None: - __check_key = self.__key - __check_type = list - self.__dict_check.check_key_exist_and_value_type( - self.__objects, __check_key, __check_type) + def validate(self) -> None: + self.__dict_checker.assert_key_exist(OBJECTS_KEY) + self.__dict_checker.assert_data_type(OBJECTS_KEY, OBJECTS_TYPE) class TargetObject(): def __init__(self, object_info, execute_datetime) -> None: - self.__dict_check = DictCheck() + self.__dict_checker = DictChecker(object_info) self.__object_info = object_info - self.execute_datetime = execute_datetime - self.check_key_object_name() - self.check_key_columns() - self.check_key_is_skip() - self.check_key_is_update_last_fetch_datetime() - self.check_key_last_fetch_datetime_file_name() - self.check_key_upload_file_name() - self.object_name = self.__object_info['object_name'] - self.columns = self.__object_info['columns'] - self.is_skip = self.set_is_skip() - self.is_update_last_fetch_datetime = self.set_is_update_last_fetch_datetime() - self.last_fetch_datetime_file_name = self.set_fetch_datetime_file_name() - self.upload_file_name = self.set_upload_file_name() + self.__execute_datetime = execute_datetime + self.__validate() - def check_key_object_name(self) -> None: - ''' - オブジェクト名チェック - ''' - __check_key = 'object_name' - __check_type = str - self.__dict_check.check_key_exist_and_value_type( - self.__object_info, __check_key, __check_type) + def __validate(self) -> None: + self.__validate_required_properties() + self.__validate_optional_properties() return - def check_key_columns(self) -> None: - ''' - カラム情報チェック - ''' - __key = 'columns' - __type = list - self.__dict_check.check_key_exist_and_value_type( - self.__object_info, __key, __type) + def __validate_required_properties(self) -> None: + self.__dict_checker.assert_key_exist(OBJECT_NAME_KEY) + self.__dict_checker.assert_data_type(OBJECT_NAME_KEY, OBJECT_NAME_TYPE) + self.__dict_checker.assert_key_exist(COLUMNS_KEY) + self.__dict_checker.assert_data_type(COLUMNS_KEY, COLUMNS_TYPE) return - def check_key_is_skip(self,) -> None: - ''' - スキップフラグ型チェック - ''' - __check_key = 'is_skip' - __check_type = bool - self.__dict_check.check_key_exist_case_value_type( - self.__object_info, __check_key, __check_type) + def __validate_optional_properties(self) -> None: + if self.__dict_checker.check_key_exist(IS_SKIP_KEY): + self.__dict_checker.assert_data_type(IS_SKIP_KEY, IS_SKIP_TYPE) + + if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY): + self.__dict_checker.assert_data_type(IS_UPDATE_LAST_FETCH_DATETIME_KEY, IS_UPDATE_LAST_FETCH_DATETIME_TYPE) + + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY): + self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_FILE_NAME_KEY, LAST_FETCH_DATETIME_FILE_NAME_TYPE) + + if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): + self.__dict_checker.assert_data_type(UPLOAD_FILE_NAME_KEY, UPLOAD_FILE_NAME_TYPE) + + if self.__dict_checker.check_key_exist(DATETIME_COLUMN_KEY): + self.__dict_checker.assert_data_type(DATETIME_COLUMN_KEY, DATETIME_COLUMN_TYPE) return - def check_key_is_update_last_fetch_datetime(self) -> None: - ''' - 前回取得日時ファイル更新フラグチェック - ''' - __check_key = 'is_update_last_fetch_datetime' - __check_type = bool - self.__dict_check.check_key_exist_case_value_type( - self.__object_info, __check_key, __check_type) + @property + def object_name(self) -> str: + return self.__object_info[OBJECT_NAME_KEY] - return + @property + def columns(self) -> list: + return self.__object_info[COLUMNS_KEY] - def check_key_last_fetch_datetime_file_name(self) -> None: - ''' - 前回取得日時ファイル名型チェック - ''' - __check_key = 'last_fetch_datetime_file_name' - __check_type = str - self.__dict_check.check_key_exist_case_value_type( - self.__object_info, __check_key, __check_type) + @property + def is_skip(self) -> bool: + return self.__object_info[IS_SKIP_KEY] if self.__dict_checker.check_key_exist(IS_SKIP_KEY) else False - return + @property + def is_update_last_fetch_datetime(self) -> bool: + return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY) else False - def check_key_upload_file_name(self) -> None: - ''' - アップロードファイル名称型チェック - ''' - __check_key = 'upload_file_name' - __check_type = str - self.__dict_check.check_key_exist_case_value_type( - self.__object_info, __check_key, __check_type) + @property + def last_fetch_datetime_file_name(self) -> str: + return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY] if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}.json' - return - def set_is_skip(self) -> bool: - ''' - スキップフラグ設定 - ''' - __check_key = 'is_skip' - if self.__dict_check.check_key_exist(self.__object_info, __check_key): - return self.__object_info[__check_key] - else: - return False + @property + def upload_file_name(self) -> str: + return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' - def set_is_update_last_fetch_datetime(self) -> bool: - ''' - 前回取得日時ファイル更新フラグ設定 - ''' - __check_key = 'is_update_last_fetch_datetime' - if self.__dict_check.check_key_exist(self.__object_info, __check_key): - return self.__object_info[__check_key] - else: - return False - - def set_fetch_datetime_file_name(self) -> str: - ''' - 前回取得日時ファイル名設定 - ''' - __check_key = 'last_fetch_datetime_file_name' - if self.__dict_check.check_key_exist(self.__object_info, __check_key): - return self.__object_info[__check_key] - else: - return self.__object_info['object_name'] + '.json' - - def set_upload_file_name(self) -> str: - ''' - アップロードファイル名称設定 - ''' - __check_key = 'upload_file_name' - if self.__dict_check.check_key_exist(self.__object_info, __check_key): - return self.__object_info[__check_key].format(execute_datetime=self.execute_datetime) - else: - return 'CRM_' + self.__object_info['object_name'] + '_' + self.execute_datetime + @property + def datetime_column(self) -> str: + return self.__object_info[DATETIME_COLUMN_KEY] if self.__dict_checker.check_key_exist(DATETIME_COLUMN_KEY) else DATETIME_COLUMN_DEFAULT_VALUE class LastFetchDatetime(): - def __init__(self, last_fetch_datetime_file_name, last_fetch_datetime_file_dict, execute_datetime) -> None: - self.__dict_check = DictCheck() - self.execute_datetime = execute_datetime + def __init__(self, last_fetch_datetime_file_dict, execute_datetime) -> None: + self.__dict_checker = DictChecker(last_fetch_datetime_file_dict) + self.__execute_datetime = execute_datetime self.__last_fetch_datetime_file_dict = last_fetch_datetime_file_dict - self.last_fetch_datetime_file_name = last_fetch_datetime_file_name - self.check_key_last_fetch_datetime_from - self.check_key_last_fetch_datetime_to - self.last_fetch_datetime_from = self.__last_fetch_datetime_file_dict[ - 'last_fetch_datetime_from'] - self.last_fetch_datetime_to = self.set_last_fetch_datetime_to() + self.__validate() - def check_key_last_fetch_datetime_from(self) -> None: - ''' - データ取得開始日時チェック - ''' - __check_key = 'last_fetch_datetime_from' - __regex_str = DATE_PATTERN_YYYYMMDDTHHMMSSTZ - self.__dict_check.check_key_exsit_and_regex( - self.__last_fetch_datetime_file_dict, __check_key, __regex_str) + def __validate(self) -> None: + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FROM_KEY): + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) - def check_key_last_fetch_datetime_to(self) -> None: - ''' - データ取得終了日時チェック - ''' - __check_key = 'last_fetch_datetime_to' - __regex_str = DATE_PATTERN_YYYYMMDDTHHMMSSTZ - self.__dict_check.check_key_exsit_case_regex( - self.__last_fetch_datetime_file_dict, __check_key, __regex_str) + return - def set_last_fetch_datetime_to(self) -> str: - ''' - データ取得終了日時設定 - ''' - __check_key = 'last_fetch_datetime_to' - if self.__dict_check.check_key_exist(self.__last_fetch_datetime_file_dict, __check_key): - return self.__last_fetch_datetime_file_dict[__check_key].format(execute_datetime=self.execute_datetime) - else: - return self.execute_datetime + @property + def last_fetch_datetime_from(self) -> str: + return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_FROM_KEY] + + @property + def last_fetch_datetime_to(self) -> str: + return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FROM_KEY) else self.__execute_datetime diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index ebd3b3cd..acbd2383 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -1,125 +1,45 @@ -from src.check_object_info_process import check_object_info # オブジェクト情報形式チェック処理 -from src.convert_crm_csvdata_process import convert_crm_csvdata # CSV変換処理 -from src.backup_crm_csvdata_process import backup_crm_csvdata # CSVバックアップ処理 -from src.set_datetime_period_process import set_datetime_period # データ取得期間設定処理 -from src.updload_result_data_process import updload_result_data # 取得処理実施結果アップロード処理 +import gc + +from src.backup_crm_csv_data_process import backup_crm_csv_data_process +from src.backup_crm_data_process import backup_crm_data_process +from src.check_object_info_process import check_object_info_process +from src.config.objects import FetchTargetObjects +from src.convert_crm_csv_data_process import convert_crm_csv_data_process +from src.copy_crm_csv_data_process import copy_crm_csv_data_process from src.error.exceptions import MeDaCaCRMDataFetchException -from src.fetch_crm_data_process import fetch_crm_data # CRMデータ取得処理 -from src.prepare_get_data_process import prepare_get_data # データ取得準備処理 -from src.backup_crm_data_process import backup_crm_data # CRM電文データバックアップ処理 -from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime # 前回取得日時ファイル更新 -from src.copy_crm_csvdata_process import copy_crm_csvdata # CSVアップロード処理 +from src.util.execute_datetime import ExecuteDateTime +from src.fetch_crm_data_process import fetch_crm_data_process +from src.prepare_data_fetch_process import prepare_data_fetch_process +from src.set_datetime_period_process import set_datetime_period_process +from src.system_var.constants import OBJECT_NAME_KEY +from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime_process +from src.upload_result_data_process import upload_result_data_process from src.util.logger import logger_instance as logger -def main() -> None: +def controller() -> None: + """コントロール処理""" + try: # ① CRMデータ取得処理開始ログを出力する logger.info('I-CTRL-01 CRMデータ取得処理を開始します') - fetch_target_objects = None # オブジェクト情報ファイル用オブジェクト - execute_datetime = None # 実行日次文字列 - date_path = None # 実行日次のパス文字列 - process_result = None # オブジェクトごとの実行結果JSON - # ② データ取得準備処理を呼び出す logger.info('I-CTRL-02 データ取得準備処理呼び出し') - fetch_target_objects, execute_datetime, date_path, process_result = prepare_get_data() + fetch_target_objects, execute_datetime, process_result = prepare_data_fetch_process() # ③ object_infoのobjectsキーの値の件数分ループする logger.info('I-CTRL-03 取得対象オブジェクトのループ処理開始') - for object_info in fetch_target_objects: - try: - # 1. オブジェクト処理結果の初期化 - target_object = None # オブジェクトごとの情報格納用オブジェクト - last_fetch_datetime = None # オブジェクトごとの取得日付格納用オブジェクト - sf_object_jsons = None # オブジェクトごとのSalesforce取得変数JSON - csv_object = None # CSVオブジェクト - process_result[object_info.get('object_name')] = 'fail' # オブジェクト処理結果 - - logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：{object_info}') - - # 2. オブジェクト情報形式チェック処理を呼び出す - logger.info('I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') - - target_object = check_object_info(object_info, execute_datetime) - - # 3. 処理対象のオブジェクト名をログ出力する - logger.info( - f'I-CTRL-06 [{target_object.object_name}]のデータ取得を開始します') - - # 4. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する - if target_object.is_skip is True: - logger.info( - f'I-CTRL-07 [{target_object.object_name}]のデータ取得処理をスキップします') - continue - - # 5. データ取得期間設定処理を呼び出す - logger.info( - f'I-CTRL-08 [{target_object.object_name}]のデータ取得期間設定処理呼び出し') - - last_fetch_datetime = set_datetime_period(target_object, execute_datetime) - - # 6. CRMデータ取得処理を呼び出す - logger.info( - f'I-CTRL-09 [{target_object.object_name}]のデータ取得処理呼び出し') - - sf_object_jsons = fetch_crm_data(target_object, last_fetch_datetime) - - # 7. 出力ファイル名をログ出力する - logger.info( - f'I-CTRL-10 [{target_object.object_name}] の出力ファイル名は [{target_object.upload_file_name}]となります') - - # 8. CRM電文データバックアップ処理を呼び出す - logger.info( - f'I-CTRL-11 [{target_object.object_name}] CRM電文データバックアップ処理呼び出し') - backup_crm_data(target_object.object_name, sf_object_jsons, date_path) - - # 9. CSV変換処理を呼び出す - logger.info( - f'I-CTRL-12 [{target_object.object_name}] CSV変換処理呼び出し') - csv_object = convert_crm_csvdata(target_object, sf_object_jsons) - - # 10. CSVバックアップ処理を呼び出す - logger.info( - f'I-CTRL-13 [{target_object.object_name}] CSVデータバックアップ処理呼び出し') - backup_crm_csvdata(target_object, date_path, csv_object) - - # 11. CSVアップロード処理を呼び出す - logger.info( - f'I-CTRL-14 [{target_object.object_name}] CSVデータアップロード処理呼び出し') - copy_crm_csvdata(target_object, date_path) - - # 12. 前回取得日時ファイル更新処理を呼びだす - logger.info( - f'I-CTRL-15 [{target_object.object_name}] 前回取得日時ファイル更新処理呼び出し') - upload_last_fetch_datetime(target_object, last_fetch_datetime) - - # 13. オブジェクト処理結果の更新 - process_result[target_object.object_name] = 'success' - - # 14. オブジェクトのアップロードが完了した旨をログに出力する - logger.info(f'I-CTRL-16 [{target_object.object_name}] 処理正常終了') - - except MeDaCaCRMDataFetchException as e: - logger.info(f'{e.error_id} {e}') - logger.info( - f'I-ERR-03 [{object_info.get("object_name")}] の[{e.func_name}]でエラーが発生しました 次のオブジェクトの処理に移行します', exc_info=True) - continue - - except Exception as e: - logger.info( - f'I-ERR-04 [{object_info.get("object_name")}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e, exc_info=True) - continue + process_result = fetch_crm_data(fetch_target_objects, execute_datetime, process_result) # ④ すべてのオブジェクトの処理が完了したことと、オブジェクト毎の処理結果をログに出力する logger.info(f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result}]') # ⑤ 取得処理実施結果アップロード処理を呼び出す logger.info('I-CTRL-19 CRM_取得処理実施結果ファイルアップロード処理開始') - updload_result_data(process_result, date_path) + upload_result_data_process(process_result, execute_datetime) # ⑥ 最終結果をチェックし、チェック結果をログに出力 if not all([v == 'success' for v in process_result.values()]): @@ -142,5 +62,136 @@ def main() -> None: return exit(0) -if __name__ == '__main__': - main() +def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: ExecuteDateTime, process_result: dict): + """ + 取得対象オブジェクト情報をループし、１オブジェクトごとのデータを取得する + + Parameters + ---------- + fetch_target_objects : FetchTargetObjects + CRMオブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + process_result : dict + 取得処理実行結果辞書オブジェクト + + Returns + ------- + process_result : dict + 取得処理実行結果辞書オブジェクト + + """ + for object_info in fetch_target_objects: + try: + process_result[object_info.get(OBJECT_NAME_KEY)] = 'fail' + + fetch_crm_data_per_object(object_info, execute_datetime, process_result) + + process_result[object_info.get(OBJECT_NAME_KEY)] = 'success' + + except MeDaCaCRMDataFetchException as e: + logger.info(f'{e.error_id} {e}') + logger.info( + f'I-ERR-03 [{object_info.get(OBJECT_NAME_KEY)}] の[{e.func_name}]でエラーが発生しました 次のオブジェクトの処理に移行します', exc_info=True) + continue + + except Exception as e: + logger.info( + f'I-ERR-04 [{object_info.get(OBJECT_NAME_KEY)}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e, exc_info=True) + continue + + return process_result + +def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTime) -> None: + """ + オブジェクトごとにCRMのデータを取得し、取込フォルダにアップロードする + + Parameters + ---------- + object_info : dict + 取得対象オブジェクト情報 + execute_datetime : object + 実行日次取得インスタンス + + Returns + ------- + なし + + Raises + ------ + FileNotFoundException + S3上のファイルが存在しない場合 + InvalidConfigException + オブジェクト情報定義が不正だった場合 + + """ + + # 1. オブジェクト処理結果の初期化 + logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：{object_info}') + + # 2. オブジェクト情報形式チェック処理を呼び出す + logger.info('I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') + + target_object = check_object_info_process(object_info, execute_datetime) + target_object_name = target_object.object_name + + # 3. 処理対象のオブジェクト名をログ出力する + logger.info( + f'I-CTRL-06 [{target_object_name}]のデータ取得を開始します') + + # 4. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する + if target_object.is_skip is True: + logger.info( + f'I-CTRL-07 [{target_object_name}]のデータ取得処理をスキップします') + return + + # 5. データ取得期間設定処理を呼び出す + logger.info( + f'I-CTRL-08 [{target_object_name}]のデータ取得期間設定処理呼び出し') + + last_fetch_datetime = set_datetime_period_process(target_object, execute_datetime) + + # 6. CRMデータ取得処理を呼び出す + logger.info( + f'I-CTRL-09 [{target_object_name}]のデータ取得処理呼び出し') + + crm_data_response = fetch_crm_data_process(target_object, last_fetch_datetime) + + # 7. 出力ファイル名をログ出力する + logger.info( + f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{target_object.upload_file_name}]となります') + + # 8. CRM電文データバックアップ処理を呼び出す + logger.info( + f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し') + backup_crm_data_process(target_object_name, crm_data_response, execute_datetime) + + # 9. CSV変換処理を呼び出す + logger.info( + f'I-CTRL-12 [{target_object.object_name}] CSV変換処理呼び出し') + csv_string = convert_crm_csv_data_process(target_object, crm_data_response) + + # 10. CSVバックアップ処理を呼び出す + logger.info( + f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し') + backup_crm_csv_data_process(target_object, execute_datetime, csv_string) + + # 11. CSVアップロード処理を呼び出す + logger.info( + f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し') + copy_crm_csv_data_process(target_object, execute_datetime) + + # 12. メモリ解放 + del crm_data_response + del csv_string + gc.collect() + + # 13. 前回取得日時ファイル更新処理を呼びだす + logger.info( + f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し') + upload_last_fetch_datetime_process(target_object, last_fetch_datetime) + + # 14. オブジェクトのアップロードが完了した旨をログに出力する + logger.info(f'I-CTRL-16 [{target_object_name}] 処理正常終了') + + return \ No newline at end of file diff --git a/ecs/crm-datafetch/src/convert_crm_csv_data_process.py b/ecs/crm-datafetch/src/convert_crm_csv_data_process.py new file mode 100644 index 00000000..1ed05c3a --- /dev/null +++ b/ecs/crm-datafetch/src/convert_crm_csv_data_process.py @@ -0,0 +1,51 @@ +from src.config.objects import TargetObject +from src.converter.converter import CSVStringConverter +from src.error.exceptions import DataConvertException +from src.system_var.constants import CONV_JP_NAME +from src.util.logger import logger_instance as logger + + +def convert_crm_csv_data_process(target_object: TargetObject, crm_data_response: dict): + """ + CSV変換処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + crm_data_response : dict + Salesforceオブジェクトデータ + + Returns + ------- + csv_string : str + csvデータ + + Raises + ------ + DataConvertException + データ変換が失敗した場合 + """ + + # ① CSV変換処理の開始ログを出力する + target_object_name = target_object.object_name + + logger.info(f'I-CONV-01 [{target_object_name}] のCSV変換処理を開始します') + + try: + # ② CSV変換 + CSVStringConverter(target_object, crm_data_response) + csv_string = CSVStringConverter.convert() + + + logger.debug(f'D-CONV-02 [{target_object_name}] のCSV変換処理 正常終了') + + except Exception as e: + raise DataConvertException( + 'E-CONV-01', CONV_JP_NAME, f'[{target_object_name}] CSV変換に失敗しました エラー内容:[{e}]') + + # ③ CSV変換処理の終了ログを出力する + logger.info(f'I-CONV-03 [{target_object_name}] のCSV変換処理を終了します') + + # ④ 次の処理へ移行する + return csv_string diff --git a/ecs/crm-datafetch/src/convert_crm_csvdata_process.py b/ecs/crm-datafetch/src/convert_crm_csvdata_process.py deleted file mode 100644 index 4ef6dd11..00000000 --- a/ecs/crm-datafetch/src/convert_crm_csvdata_process.py +++ /dev/null @@ -1,25 +0,0 @@ -from src.system_var.constants import CONV_JP_NAME -from src.converter.converter import CSVStringConverter -from src.error.exceptions import DataConvertException -from src.util.logger import logger_instance as logger - - -def convert_crm_csvdata(target_object, sf_object_jsons): - # ① CSV変換処理の開始ログを出力する - logger.info(f'I-CONV-01 [{target_object.object_name}] のCSV変換処理を開始します') - - try: - # ② CSV変換 - csv_object = CSVStringConverter(target_object, sf_object_jsons) - - logger.debug(f'D-CONV-02 [{target_object.object_name}] のCSV変換処理 正常終了') - - except Exception as e: - raise DataConvertException( - 'E-CONV-01', CONV_JP_NAME, f'[{target_object.object_name}] CSV変換に失敗しました エラー内容:[{e}]') - - # ③ CSV変換処理の終了ログを出力する - logger.info(f'I-CONV-03 [{target_object.object_name}] のCSV変換処理を終了します') - - # ④ 次の処理へ移行する - return csv_object diff --git a/ecs/crm-datafetch/src/converter/convert_factory.py b/ecs/crm-datafetch/src/converter/convert_factory.py new file mode 100644 index 00000000..48673f7e --- /dev/null +++ b/ecs/crm-datafetch/src/converter/convert_factory.py @@ -0,0 +1,55 @@ +import re +from datetime import datetime + +from src.config.objects import TargetObject +from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, + CSV_TRUE_VALUE, YYYYMMDDHHMMSS, DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC) + + +class ConvertStrategyFactory: + def __init__(self) -> None: + self.__none_value_convert_strategy = NoneValueConvertStrategy() + self.__float_convert_strategy = FloatConvertStrategy() + self.__boolean_convert_strategy = BooleanConvertStrategy() + self.__datetime_convert_strategy = DatatimeConvertStrategy() + + def create(self, value): + + converted_value = value + + if value is None: + converted_value = self.__none_value_convert_strategy.convert_value() + + # 指数表記で取得できるパターン。指数表記を整数表記に変換する。 + elif type(value) == float: + converted_value = self.__float_convert_strategy.convert_value(value) + + # SQLの真偽値に対応するために変換する + elif type(value) == bool: + converted_value = self.__boolean_convert_strategy.convert_value(value) + + elif type(value) == str and re.fullmatch(DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, value): + converted_value = self.__datetime_convert_strategy.convert_value(value) + + return converted_value + + +class NoneValueConvertStrategy: + def convert_value(self) -> str: + return '' + + +class BooleanConvertStrategy: + def convert_value(self, convert_value: str) -> bool: + return CSV_TRUE_VALUE if convert_value is True else CSV_FALSE_VALUE + + +class DatatimeConvertStrategy: + def convert_value(self, convert_value: str) -> str: + return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).strftime(YYYYMMDDHHMMSS) + + +class FloatConvertStrategy: + def convert_value(self, convert_value: str) -> int: + return int(convert_value) + diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 7b4adb1e..bf7faaee 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -1,33 +1,35 @@ import csv import io -import re -from datetime import datetime -from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, - CSV_TRUE_VALUE, YYYYMMDDHHMMSS) +from src.config.objects import TargetObject +from src.converter.convert_factory import ConvertStrategyFactory class CSVStringConverter: - def __init__(self, target_object, sf_object_jsons) -> None: + def __init__(self, target_object: TargetObject, sf_object_jsons: dict) -> None: self.__target_object = target_object self.__sf_object_jsons = sf_object_jsons - self.__extracted_sf_object_jsons = self.extract_sf_object_jsons() - self.csv_data = self.convert_to_csv() - self.csv_buffer = self.write_csv() + self.__convert_strategy_factory = ConvertStrategyFactory() - def extract_sf_object_jsons(self) -> list: + def convert(self) -> str: + extracted_sf_object_jsons = self.__extract_sf_object_jsons() + csv_data = self.__convert_to_csv(extracted_sf_object_jsons) + csv_string = self.__write_csv_string(csv_data) + return csv_string + + def __extract_sf_object_jsons(self) -> list: try: extracted_sf_object_jsons = [] for sf_object_json in self.__sf_object_jsons: extracted_sf_object_jsons.append( - self.extract_necessary_props_from(sf_object_json)) + self.__extract_necessary_props_from(sf_object_json)) return extracted_sf_object_jsons except Exception as e: - raise Exception('必要なjsonのデータ抽出に失敗しました') + raise Exception('必要なjsonのデータ抽出に失敗しました', e) - def extract_necessary_props_from(self, sf_object_json) -> dict: + def __extract_necessary_props_from(self, sf_object_json) -> dict: try: clone_sf_object = {**sf_object_json} @@ -39,19 +41,18 @@ class CSVStringConverter: return uppercase_key_sf_object except Exception as e: - raise Exception('必要なjsonのデータ成形に失敗しました') + raise Exception('必要なjsonのデータ成形に失敗しました', e) - def convert_to_csv(self) -> list: + def __convert_to_csv(self, extracted_sf_object_jsons) -> list: try: columns = self.__target_object.columns csv_data = [] - for i, json_object in enumerate(self.__extracted_sf_object_jsons, 1): + for i, json_object in enumerate(extracted_sf_object_jsons, 1): csv_row = [] for column in columns: v = json_object[column.upper()] - converted_value = CSVStringConverterFactory( - v).value_convert() + converted_value = self.__convert_strategy_factory.create(v) csv_row.append(converted_value) @@ -62,74 +63,16 @@ class CSVStringConverter: raise Exception( f'CSV変換に失敗しました カラム名：[{column}] 行番号: [{i}] エラー内容:[{e}]') - def write_csv(self) -> str: + def __write_csv_string(self, csv_data) -> str: try: with io.StringIO(newline='') as string_stream: writer = csv.writer(string_stream, delimiter=',', lineterminator='\r\n', doublequote=True, quotechar='"', quoting=csv.QUOTE_ALL, strict=True) writer.writerow(self.__target_object.columns) - writer.writerows(self.csv_data) + writer.writerows(csv_data) csv_value = string_stream.getvalue() return csv_value except Exception as e: - raise Exception('csvデータの取得に失敗しました') - - -class NoneValueConverter: - def __init__(self, convert_value) -> None: - self.__convert_value = convert_value - self.value = self.convert_value() - - def convert_value(self) -> str: - return '' - - -class BooleanConverter: - def __init__(self, convert_value) -> None: - self.__convert_value = convert_value - self.value = self.convert_value() - - def convert_value(self) -> bool: - return CSV_TRUE_VALUE if self.__convert_value is True else CSV_FALSE_VALUE - - -class DatatimeConverter: - def __init__(self, convert_value) -> None: - self.__convert_value = convert_value - self.value = self.convert_value() - - def convert_value(self) -> str: - return datetime.strptime(self.__convert_value, CRM_DATETIME_FORMAT).strftime(YYYYMMDDHHMMSS) - - -class FloatConverter: - def __init__(self, convert_value) -> None: - self.__convert_value = convert_value - self.value = self.convert_value() - - def convert_value(self) -> int: - return int(self.__convert_value) - - -class CSVStringConverterFactory: - def __init__(self, v) -> None: - self.__value = v - - def value_convert(self): - - converted_value = self.__value - - if self.__value is None: - converted_value = NoneValueConverter(self.__value).value - # 指数表記で取得できるパターン。指数表記を整数表記に変換する。 - elif type(self.__value) == float: - converted_value = FloatConverter(self.__value).value - # SQLの真偽値に対応するために変換する - elif type(self.__value) == bool: - converted_value = BooleanConverter(self.__value).value - elif type(self.__value) == str and re.fullmatch(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.000\+0000', self.__value): - converted_value = DatatimeConverter(self.__value).value - - return converted_value + raise Exception('csvデータの取得に失敗しました', e) diff --git a/ecs/crm-datafetch/src/copy_crm_csv_data_process.py b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py new file mode 100644 index 00000000..57a3659f --- /dev/null +++ b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py @@ -0,0 +1,57 @@ +from src.aws.s3 import BackupBucket, DataBucket +from src.config.objects import TargetObject +from src.error.exceptions import FileUploadException +from src.system_var.constants import UPLD_JP_NAME +from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, + CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) +from src.util.execute_datetime import ExecuteDateTime +from src.util.logger import logger_instance as logger + + +def copy_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime): + """ + CSVアップロード処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + + # ① CSVデータアップロード処理の開始ログを出力する + target_object_name = target_object.object_name + upload_file_name = target_object.upload_file_name + + logger.info( + f'I-UPLD-01 [{target_object_name}] のCSVデータアップロード処理を開始します ファイル名：[{upload_file_name}.csv]') + + try: + # ② CRMバックアップ保管用バケットに保管した変換後のCSVデータをデータ取込バケットにコピーする + data_bucket = DataBucket() + backup_bucket = BackupBucket() + data_bucket.put_csv_from(str(backup_bucket), f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{execute_datetime.to_path()}/{upload_file_name}.csv') + + logger.debug( + f'D-UPLD-02 [{target_object_name}] のCSVデータアップロード 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-UPLD-01', UPLD_JP_NAME, f'[{target_object_name}] CSVデータのアップロードに失敗しました ファイル名：[{upload_file_name}.csv] エラー内容:[{e}]') + + # ③ CSVデータアップロード処理の終了ログを出力する + logger.info( + f'I-UPLD-03 [{target_object_name}] のCSVデータのアップロード処理を終了します') + + # ④ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/copy_crm_csvdata_process.py b/ecs/crm-datafetch/src/copy_crm_csvdata_process.py deleted file mode 100644 index ecebe9ea..00000000 --- a/ecs/crm-datafetch/src/copy_crm_csvdata_process.py +++ /dev/null @@ -1,32 +0,0 @@ -from src.aws.s3 import S3ResourceNonBucket -from src.system_var.constants import UPLD_JP_NAME -from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, - CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) -from src.error.exceptions import FileUploadException -from src.util.logger import logger_instance as logger - - -def copy_crm_csvdata(target_object, date_path): - # ① CSVデータアップロード処理の開始ログを出力する - logger.info( - f'I-UPLD-01 [{target_object.object_name}] のCSVデータアップロード処理を開始します ファイル名：[{target_object.upload_file_name}.csv]') - - try: - # ② CRMバックアップ保管用バケットに保管した変換後のCSVデータをデータ取込バケットにコピーする - s3_resource_non_bucket = S3ResourceNonBucket() - s3_resource_non_bucket.copy(CRM_BACKUP_BUCKET, f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{date_path}/{target_object.upload_file_name}.csv', - IMPORT_DATA_BUCKET, f'{CRM_IMPORT_DATA_FOLDER}/{target_object.upload_file_name}.csv') - - logger.debug( - f'D-UPLD-02 [{target_object.object_name}] のCSVデータアップロード 正常終了') - - except Exception as e: - raise FileUploadException( - 'E-UPLD-01', UPLD_JP_NAME, f'[{target_object.object_name}] CSVデータのアップロードに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') - - # ③ CSVデータアップロード処理の終了ログを出力する - logger.info( - f'I-UPLD-03 [{target_object.object_name}] のCSVデータのアップロード処理を終了します') - - # ④ 次の処理へ移行する - return diff --git a/ecs/crm-datafetch/src/error/exceptions.py b/ecs/crm-datafetch/src/error/exceptions.py index 71a174c0..f231804b 100644 --- a/ecs/crm-datafetch/src/error/exceptions.py +++ b/ecs/crm-datafetch/src/error/exceptions.py @@ -4,7 +4,7 @@ from abc import ABCMeta class MeDaCaCRMDataFetchException(Exception, metaclass=ABCMeta): """MeDaCaシステム固有のカスタムエラークラス""" - def __init__(self, error_id: str, func_name, message) -> None: + def __init__(self, error_id: str, func_name: str, message: str) -> None: super().__init__(message) self.func_name = func_name self.error_id = error_id diff --git a/ecs/crm-datafetch/src/fetch_crm_data_process.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py index eac5478b..3a89da1e 100644 --- a/ecs/crm-datafetch/src/fetch_crm_data_process.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -2,6 +2,10 @@ from requests.exceptions import ConnectTimeout, ReadTimeout from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential +from src.config.objects import TargetObject, LastFetchDatetime +from src.error.exceptions import DataConvertException, SalesforceAPIException +from src.salesforce.salesforce_api import SalesforceApiClient +from src.salesforce.soql_builder import SOQLBuilder from src.system_var.constants import FETCH_JP_NAME from src.system_var.environments import (CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, @@ -14,20 +18,38 @@ from src.system_var.environments import (CRM_AUTH_TIMEOUT, CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_TIMEOUT) -from src.error.exceptions import DataConvertException, SalesforceAPIException -from src.salesforce.salesforce_api import SalesForceCount, SalesForceData from src.util.logger import logger_instance as logger -def fetch_crm_data(target_object, last_fetch_datetime): +def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): + """ + CRMデータ取得処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + last_fetch_datetime : LastFetchDatetime + 取得対象オブジェクト情報インスタンス + + Returns + ------- + sf_object_dict : dict + Salesforceオブジェクトデータ + + Raises + ------ + SalesforceAPIException + SalseforceのAPI実行失敗が発生した場合 + DataConvertException + データ変換が失敗した場合 + """ + # ① CRMデータ取得処理開始ログを出力する logger.info( f'I-FETCH-01 [{target_object.object_name}] のCRMからのデータ取得処理を開始します') object_name = target_object.object_name - columns = ','.join(target_object.columns) - last_fetch_datetime_from = last_fetch_datetime.last_fetch_datetime_from - last_fetch_datetime_to = last_fetch_datetime.last_fetch_datetime_to global count_contime_counter, count_readtime_counter, count_counter, data_contime_counter, data_readtime_counter, data_counter count_contime_counter = 1 @@ -41,8 +63,10 @@ def fetch_crm_data(target_object, last_fetch_datetime): # ② 取得対象オブジェクトの取得期間内のレコード件数を取得する logger.info(f'I-FETCH-02 [{object_name}] の件数取得を開始します') - record_count = fetch_sf_count_retry( - object_name, last_fetch_datetime_from, last_fetch_datetime_to) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + count_soql = soql_builder.create_count_soql() + + record_count = fetch_record_count_retry(count_soql, object_name) logger.info(f'I-FETCH-03 [{object_name}] の件数：[{record_count}]') @@ -54,8 +78,9 @@ def fetch_crm_data(target_object, last_fetch_datetime): # ③ 取得対象オブジェクトのレコードを取得する logger.info(f'I-FETCH-04 [{object_name}] のレコード取得を開始します') - record_generator = fetch_sf_data_retry( - columns, object_name, last_fetch_datetime_from, last_fetch_datetime_to) + fetch_soql = soql_builder.create_fetch_soql() + + record_all = fetch_sf_data_retry(fetch_soql, object_name) except Exception as e: raise SalesforceAPIException( @@ -65,10 +90,7 @@ def fetch_crm_data(target_object, last_fetch_datetime): # ④ 取得対象オブジェクトをJSONに変換 logger.info(f'I-FETCH-05 [{object_name}] のレコードをJSONに変換します') - sf_object_jsons = [] - - for record in record_generator: - sf_object_jsons.append(record) + crm_data_response = [record for record in record_all] except Exception as e: raise DataConvertException( @@ -78,19 +100,19 @@ def fetch_crm_data(target_object, last_fetch_datetime): logger.info(f'I-FETCH-06 [{object_name}] のレコード取得が成功しました') # ⑥ 次の処理へ移行する - return sf_object_jsons + return crm_data_response @retry( wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) -def fetch_sf_count_retry(object_name, last_update_datetime_from, last_update_datetime_to): +def fetch_record_count_retry(soql: str, object_name: str): try: global count_contime_counter, count_readtime_counter, count_counter - saleforce_count = SalesForceCount() - return saleforce_count.fetch_sf_count(object_name, last_update_datetime_from, last_update_datetime_to) + salesforce_api_client = SalesforceApiClient() + return salesforce_api_client.fetch_sf_count(soql) except ConnectTimeout as e: if count_contime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: @@ -117,12 +139,12 @@ def fetch_sf_count_retry(object_name, last_update_datetime_from, last_update_dat wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) -def fetch_sf_data_retry(columns, object_name, last_update_datetime_from, last_update_datetime_to): +def fetch_sf_data_retry(soql: str, object_name: str): try: global data_contime_counter, data_readtime_counter, data_counter - saleforce_data = SalesForceData() - return saleforce_data.fetch_sf_data(columns, object_name, last_update_datetime_from, last_update_datetime_to) + salesforce_api_client = SalesforceApiClient() + return salesforce_api_client.fetch_sf_data(soql) except ConnectTimeout as e: if data_contime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: diff --git a/ecs/crm-datafetch/src/prepare_get_data_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py similarity index 76% rename from ecs/crm-datafetch/src/prepare_get_data_process.py rename to ecs/crm-datafetch/src/prepare_data_fetch_process.py index e96de09c..409651ad 100644 --- a/ecs/crm-datafetch/src/prepare_get_data_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -1,24 +1,46 @@ -from datetime import datetime - from src.aws.s3 import ConfigBucket from src.config.objects import FetchTargetObjects -from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ -from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, - OBJECT_INFO_FOLDER) from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser +from src.system_var.constants import PRE_JP_NAME +from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER) +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def prepare_get_data(): +def prepare_data_fetch_process(): + """ + データ取得準備処理 + + Parameters + ---------- + なし + + Returns + ------- + fetch_target_objects : FetchTargetObjects + CRMオブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + process_result : dict + 取得処理実行結果辞書オブジェクト + + Raises + ------ + FileNotFoundException + S3上のファイルが存在しない場合 + InvalidConfigException + オブジェクト情報定義が不正だった場合 + + """ # ① データ取得準備処理の開始ログを出力する logger.info('I-PRE-01 データ取得準備処理を開始します') # ② 取得処理開始年月日時分秒を控える - execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) - date_path = execute_datetime.rstrip('000Z').translate( - str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) + + execute_datetime = ExecuteDateTime() logger.info(f'I-PRE-02 データ取得処理開始日時：{execute_datetime}') @@ -29,7 +51,7 @@ def prepare_get_data(): f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') config_bucket = ConfigBucket() - object_info_file_json = config_bucket.get_object_info_file() + object_info_file_str = config_bucket.get_object_info_file() logger.debug('D-PRE-04 CRM_取得オブジェクト情報ファイルの取得成功しました') @@ -41,7 +63,7 @@ def prepare_get_data(): # ④ CRM_取得オブジェクト情報ファイルをパースし、メモリ上に展開する logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします') - json_parser = JsonParser(object_info_file_json) + json_parser = JsonParser(object_info_file_str) object_info_file_dict = json_parser.json_parser() logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました') @@ -69,4 +91,4 @@ def prepare_get_data(): logger.info('I-PRE-09 データ取得準備処理を終了します') # ⑧ 次の処理へ移行する - return(fetch_target_objects, execute_datetime, date_path, process_result) + return(fetch_target_objects, execute_datetime, process_result) diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index a6b9c458..0c073b9a 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,61 +1,28 @@ from simple_salesforce import Salesforce -from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT, - CRM_AUTH_RETRY_INTERVAL, - CRM_AUTH_RETRY_MAX_INTERVAL, - CRM_AUTH_RETRY_MIN_INTERVAL, CRM_AUTH_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + +from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_TIMEOUT, CRM_USER_NAME, CRM_USER_PASSWORD, CRM_USER_SECURITY_TOKEN) -FETCH_SOQL = """SELECT {column_or_expression} FROM {object_name} - WHERE SystemModStamp > {last_update_datetime_from} - AND SystemModStamp <= {last_update_datetime_to} -""" -class SalesfoeceApi(): +class SalesforceApiClient(): def __init__(self) -> None: self.__sf = Salesforce(username=CRM_USER_NAME, password=CRM_USER_PASSWORD, security_token=CRM_USER_SECURITY_TOKEN, domain=CRM_AUTH_DOMAIN ) - def sf_query(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + def query(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): return self.__sf.query(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) - def sf_query_all_iter(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): - return self.__sf.query_all_iter(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) + def query_all(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + return self.__sf.query_all(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) - -class SalesForceCount(): - def fetch_sf_count(self, object_name, last_update_datetime_from, last_update_datetime_to): - count_soql = FETCH_SOQL.format( - column_or_expression='COUNT(Id)', - object_name=object_name, - last_update_datetime_from=last_update_datetime_from, - last_update_datetime_to=last_update_datetime_to - ) - self.__sf = SalesfoeceApi() - count_res = self.__sf.sf_query( - count_soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) + def fetch_sf_count(self, soql: str): + count_res = self.query(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) return count_res.get('records')[0].get('expr0') - -class SalesForceData(): - def fetch_sf_data(self, columns, object_name, last_update_datetime_from, last_update_datetime_to): - soql = FETCH_SOQL.format( - column_or_expression=columns, - object_name=object_name, - last_update_datetime_from=last_update_datetime_from, - last_update_datetime_to=last_update_datetime_to - ) - self.__sf = SalesfoeceApi() - return self.__sf.sf_query_all_iter(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + def fetch_sf_data(self, soql: str): + return self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) diff --git a/ecs/crm-datafetch/src/salesforce/soql_builder.py b/ecs/crm-datafetch/src/salesforce/soql_builder.py new file mode 100644 index 00000000..a84744a8 --- /dev/null +++ b/ecs/crm-datafetch/src/salesforce/soql_builder.py @@ -0,0 +1,34 @@ +from src.config.objects import TargetObject, LastFetchDatetime + + +class SOQLBuilder: + def __init__(self, target_object: TargetObject, last_fetch_datetime: LastFetchDatetime) -> None: + self.__SELECT_SOQL = """SELECT {column_or_expression} FROM {object_name} + WHERE {datetime_column} > {last_fetch_datetime_from} + AND {datetime_column} <= {last_fetch_datetime_to} + """ + self.__target_object = target_object + self.__last_fetch_datetime = last_fetch_datetime + + def create_count_soql(self): + count_soql = self.__SELECT_SOQL.format( + column_or_expression='COUNT(Id)', + object_name=self.__target_object.object_name, + last_fetch_datetime_from=self.__last_fetch_datetime.last_fetch_datetime_from, + last_fetch_datetime_to=self.__last_fetch_datetime.last_fetch_datetime_to, + datetime_column=self.__target_object.datetime_column + ) + + return count_soql + + def create_fetch_soql(self): + columns = ','.join(self.__target_object.columns) + fetch_soql = self.__SELECT_SOQL.format( + column_or_expression=columns, + object_name=self.__target_object.object_name, + last_fetch_datetime_from=self.__last_fetch_datetime.last_fetch_datetime_from, + last_fetch_datetime_to=self.__last_fetch_datetime.last_fetch_datetime_to, + datetime_column=self.__target_object.datetime_column + ) + + return fetch_soql diff --git a/ecs/crm-datafetch/src/set_datetime_period_process.py b/ecs/crm-datafetch/src/set_datetime_period_process.py index ea281fba..9795ab2e 100644 --- a/ecs/crm-datafetch/src/set_datetime_period_process.py +++ b/ecs/crm-datafetch/src/set_datetime_period_process.py @@ -1,13 +1,39 @@ from src.aws.s3 import ConfigBucket -from src.config.objects import LastFetchDatetime -from src.system_var.constants import DATE_JP_NAME -from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.config.objects import TargetObject, LastFetchDatetime from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser +from src.system_var.constants import DATE_JP_NAME +from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def set_datetime_period(target_object, execute_datetime): + + +def set_datetime_period_process(target_object: TargetObject, execute_datetime: ExecuteDateTime): + """ + データ取得期間設定処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + + Returns + ------- + last_fetch_datetime : LastFetchDatetime + 取得対象オブジェクト情報インスタンス + + Raises + ------ + FileNotFoundException + S3上のファイルが存在しない場合 + InvalidConfigException + オブジェクト情報定義が不正だった場合 + """ + # ① データ取得期間設定処理の開始ログを出力する logger.info( f'I-DATE-01 [{target_object.object_name}] のデータ取得期間設定処理を開始します') @@ -18,7 +44,7 @@ def set_datetime_period(target_object, execute_datetime): f'I-DATE-02 前回取得日時ファイルの取得開始します ファイルパス：[s3://{CRM_CONFIG_BUCKET}/{LAST_FETCH_DATE_FOLDER}/{target_object.last_fetch_datetime_file_name}]') s3_config_bucket = ConfigBucket() - last_fetch_datetime_file_json = s3_config_bucket.get_last_fetch_datetime_file( + last_fetch_datetime_file_str = s3_config_bucket.get_last_fetch_datetime_file( target_object.last_fetch_datetime_file_name) logger.info(f'I-DATE-03 前回取得日時ファイルの取得成功しました') @@ -32,11 +58,10 @@ def set_datetime_period(target_object, execute_datetime): # ④ データの取得期間を設定する logger.debug(f'D-DATE-04 前回取得日時ファイルの形式チェックを開始します') - json_parser = JsonParser(last_fetch_datetime_file_json) + json_parser = JsonParser(last_fetch_datetime_file_str) last_fetch_datetime_file_dict = json_parser.json_parser() - last_fetch_datetime = LastFetchDatetime(target_object.last_fetch_datetime_file_name, - last_fetch_datetime_file_dict, execute_datetime) + last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_file_dict, execute_datetime) logger.debug(f'D-DATE-05 前回取得日時ファイルの形式チェック 正常終了') logger.info( diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index fde2ec65..b84d8f1d 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -1,6 +1,5 @@ # environments(task settings file) -# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する -LOG_LEVEL = "LOG_LEVEL" +LOG_LEVEL = 'LOG_LEVEL' # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 @@ -17,7 +16,7 @@ CRM_FETCH_RECORD_RETRY_INTERVAL = 'CRM_FETCH_RECORD_RETRY_INTERVAL' CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MIN_INTERVAL' # CRMのレコード取得処理のリトライ時の最小待ち秒数 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MAX_INTERVAL' # CRMのレコード取得処理のリトライ時の最大待ち秒数 -# environments(ECS Task Enviroment) +# environments(ECS Task Environment) CRM_AUTH_DOMAIN = 'CRM_AUTH_DOMAIN' # CRMのAPI実行のための認証エンドポイントのドメイン CRM_USER_NAME = 'CRM_USER_NAME' # CRMのAPI実行用ユーザ名 CRM_USER_PASSWORD = 'CRM_USER_PASSWORD' # CRMのAPI実行用ユーザパスワード @@ -40,16 +39,17 @@ CRM_IMPORT_DATA_BACKUP_FOLDER = 'CRM_IMPORT_DATA_BACKUP_FOLDER' YYYYMMDDTHHMMSSTZ = '%Y-%m-%dT%H:%M:%S.000Z' CRM_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.000%z' YYYYMMDDHHMMSS = '%Y-%m-%d %H:%M:%S' +MILLISEC_FORMAT = '000Z' # aws AWS_RESOURCE_S3 = 's3' -AWS_CLINET_S3 = 's3' S3_RESPONSE_BODY = 'Body' - +S3_CHAR_CODE = 'utf-8' # 正規表現チェック EXCLUDE_SYMBOL = ['#', '/'] -DATE_PATTERN_YYYYMMDDTHHMMSSTZ = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9].000Z' +DATE_PATTERN_YYYYMMDDTHHMMSSTZ = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' +DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.000\+0000' # logger LOG_FORMAT = '[%(levelname)s]\t%(asctime)s\t%(message)s\n' @@ -72,3 +72,24 @@ END_JP_NAME = '取得処理実施結果アップロード処理' # CSVチェック CSV_TRUE_VALUE = '1' CSV_FALSE_VALUE = '0' + +# オブジェクト変数 +OBJECTS_KEY = 'objects' +OBJECTS_TYPE = list +OBJECT_NAME_KEY = 'object_name' +OBJECT_NAME_TYPE = str +COLUMNS_KEY = 'columns' +COLUMNS_TYPE = list +IS_SKIP_KEY = 'is_skip' +IS_SKIP_TYPE = bool +IS_UPDATE_LAST_FETCH_DATETIME_KEY = 'is_update_last_fetch_datetime' +IS_UPDATE_LAST_FETCH_DATETIME_TYPE = bool +LAST_FETCH_DATETIME_FILE_NAME_KEY = 'last_fetch_datetime_file_name' +LAST_FETCH_DATETIME_FILE_NAME_TYPE = str +UPLOAD_FILE_NAME_KEY = 'upload_file_name' +UPLOAD_FILE_NAME_TYPE = str +DATETIME_COLUMN_KEY = 'datetime_column' +DATETIME_COLUMN_TYPE = str +DATETIME_COLUMN_DEFAULT_VALUE = 'SystemModstamp' +LAST_FETCH_DATETIME_TO_KEY = 'last_fetch_datetime_to' +LAST_FETCH_DATETIME_FROM_KEY = 'last_fetch_datetime_from' diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index dfa87ae3..5ff41929 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -6,37 +6,37 @@ import src.system_var.constants as constants # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT] +CRM_AUTH_TIMEOUT = os.environ.get(constants.CRM_AUTH_TIMEOUT, 100) # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT] +CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3) # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL] +CRM_AUTH_RETRY_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5) # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL] +CRM_AUTH_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5) # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL] +CRM_AUTH_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50) # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT] +CRM_GET_RECORD_COUNT_TIMEOUT = os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300) # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT] +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3) # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL] +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5) # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL] +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, 5) # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL] +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50) # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT] +CRM_FETCH_RECORD_TIMEOUT = os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300) # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT] +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3) # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL] +CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5) # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL] +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5) # CRMのレコード取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL] +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50) -# environments(ECS Task Enviroment) +# environments(ECS Task Environment) # CRMのAPI実行のための認証エンドポイントのドメイン CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN] # CRMのAPI実行用ユーザ名 @@ -52,20 +52,20 @@ CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET] # CRMの取込データを格納するバケット名 IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET] # CRM取得対象オブジェクトの情報を格納するフォルダパス -OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER] +OBJECT_INFO_FOLDER = os.environ.get(constants.OBJECT_INFO_FOLDER, 'crm/object_info') # CRM取得対象オブジェクトの情報のファイル名 OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME] # CRMデータ取得結果を格納するフォルダパス -PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER] +PROCESS_RESULT_FOLDER = os.environ.get(constants.PROCESS_RESULT_FOLDER, 'data_import') # CRMデータ取得結果を格納するファイル名 -PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME] +PROCESS_RESULT_FILENAME = os.environ.get(constants.PROCESS_RESULT_FILENAME, 'process_result.json') # CRMからの最終取得日時ファイルを格納するフォルダパス -LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER] +LAST_FETCH_DATE_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_FOLDER, 'crm/last_fetch_datetime') # CRMから取得し、取込用に変換したデータを格納するフォルダ -CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER] +CRM_IMPORT_DATA_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_FOLDER, 'crm/target') # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER] +LAST_FETCH_DATE_BACKUP_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_BACKUP_FOLDER, 'last_fetch_datetime') # CRMから取得した生データのバックアップを格納するフォルダパス -RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER] +RESPONSE_JSON_BACKUP_FOLDER = os.environ.get(constants.RESPONSE_JSON_BACKUP_FOLDER, 'response_json') # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ -CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER] +CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_BACKUP_FOLDER, 'data_import') diff --git a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py index f606c9ad..e83e58ab 100644 --- a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py +++ b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py @@ -1,12 +1,33 @@ import json from src.aws.s3 import ConfigBucket -from src.system_var.constants import UPD_JP_NAME +from src.config.objects import TargetObject, LastFetchDatetime from src.error.exceptions import FileUploadException +from src.system_var.constants import UPD_JP_NAME from src.util.logger import logger_instance as logger -def updload_last_fetch_datetime(target_object, last_fetch_datetime): +def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): + """ + 前回取得日時ファイル更新 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + last_fetch_datetime : LastFetchDatetime + 取得対象オブジェクト情報インスタンス + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + # ① 前回取得日時ファイル更新処理の開始ログを出力する logger.info( f'I-UPD-01 [{target_object.object_name}] の前回取得日時ファイルの更新処理を開始します') @@ -16,20 +37,21 @@ def updload_last_fetch_datetime(target_object, last_fetch_datetime): # ② オブジェクト情報.is_update_last_fetch_datetimeがfalseの場合、以降の処理をスキップする logger.info( f'I-UPD-02 [{target_object.object_name}] の前回取得日時ファイルの更新処理をスキップします') - else: - # ③ 前回取得日時ファイル.last_fetch_datetime_fromに取得処理開始年月日時分秒を設定する - # 前回取得日時ファイル.last_fetch_datetime_toに空文字を設定する - last_fetch_datetime_dict = { - 'last_fetch_datetime_from': last_fetch_datetime.last_fetch_datetime_to, - 'last_fetch_datetime_to': '' - } + return - config_bucket = ConfigBucket() - config_bucket.put_last_fetch_datetime_file( - target_object.last_fetch_datetime_file_name, json.dumps(last_fetch_datetime_dict)) + # ③ 前回取得日時ファイル.last_fetch_datetime_fromに取得処理開始年月日時分秒を設定する + # 前回取得日時ファイル.last_fetch_datetime_toに空文字を設定する + last_fetch_datetime_dict = { + 'last_fetch_datetime_from': last_fetch_datetime.last_fetch_datetime_to, + 'last_fetch_datetime_to': '' + } - logger.info( - f'D-UPD-03 [{target_object.object_name}] の前回取得日時ファイル更新処理 正常終了') + config_bucket = ConfigBucket() + config_bucket.put_last_fetch_datetime_file( + target_object.last_fetch_datetime_file_name, json.dumps(last_fetch_datetime_dict)) + + logger.info( + f'D-UPD-03 [{target_object.object_name}] の前回取得日時ファイル更新処理 正常終了') except Exception as e: raise FileUploadException( diff --git a/ecs/crm-datafetch/src/updload_result_data_process.py b/ecs/crm-datafetch/src/upload_result_data_process.py similarity index 64% rename from ecs/crm-datafetch/src/updload_result_data_process.py rename to ecs/crm-datafetch/src/upload_result_data_process.py index e70f4679..855ec900 100644 --- a/ecs/crm-datafetch/src/updload_result_data_process.py +++ b/ecs/crm-datafetch/src/upload_result_data_process.py @@ -1,11 +1,32 @@ from src.aws.s3 import BackupBucket +from src.error.exceptions import FileUploadException from src.system_var.constants import END_JP_NAME from src.system_var.environments import PROCESS_RESULT_FILENAME -from src.error.exceptions import FileUploadException +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def updload_result_data(process_result, date_path): +def upload_result_data_process(process_result: dict, execute_datetime: ExecuteDateTime): + """ + 取得処理実施結果アップロード処理 + + Parameters + ---------- + process_result : dict + 取得処理実行結果辞書オブジェクト + last_fetch_datetime : LastFetchDatetime + 取得対象オブジェクト情報インスタンス + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + # ① 取得処理実施結果アップロード処理のログを出力する logger.info( f'I-END-01 取得処理実施結果アップロード処理を開始します') @@ -14,7 +35,7 @@ def updload_result_data(process_result, date_path): # ② CRMバックアップ保管用バケットに、取得処理実施結果のJSONデータを保管する backup_bucket = BackupBucket() backup_bucket.put_result_json( - f'{date_path}/{PROCESS_RESULT_FILENAME}', process_result) + f'{execute_datetime.to_path()}/{PROCESS_RESULT_FILENAME}', process_result) logger.debug(f'D-END-02 取得処理実施結果アップロード 正常終了') diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py index a8bfed62..9d498b5e 100644 --- a/ecs/crm-datafetch/src/util/dict_checker.py +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -1,64 +1,39 @@ import re -class DictCheck: - def __init__(self) -> None: - pass +class DictChecker: + def __init__(self, object_dict: dict) -> None: + self.__object_dict = object_dict - def check_key_exist(self, object_dict: dict, check_key: str) -> bool: - ''' - 辞書型キー存在チェック - ''' - return True if check_key in object_dict and object_dict[check_key] != '' else False + def check_key_exist(self, check_key: str) -> bool: + """辞書型キー存在チェック""" + return check_key in self.__object_dict and self.__object_dict[check_key] != '' - def check_value_type(self, object_dict: dict, check_key: str, check_type: type) -> bool: - ''' - 辞書型バリュー型チェック - ''' - return True if isinstance(object_dict[check_key], check_type) else False + def check_data_type(self, check_key: str, check_type: type) -> bool: + """辞書型バリュー型チェック""" + return isinstance(self.__object_dict[check_key], check_type) - def check_regex(self, regex_str: str, object_dict: dict, check_key: str) -> bool: - ''' - 辞書型バリュー正規表現チェック - ''' - return True if re.fullmatch(regex_str, object_dict[check_key]) else False + def check_match_pattern(self, regex_str: str, check_key: str) -> bool: + """辞書型バリュー正規表現チェック""" + return True if re.fullmatch(regex_str, self.__object_dict[check_key]) else False - def check_key_exist_and_value_type(self, object_dict: dict, check_key: str, check_type: type) -> None: - ''' - 辞書型キー存在チェック＆バリュー型チェック - ''' - if not self.check_key_exist(object_dict, check_key): + def assert_key_exist(self, check_key: str, check_type: type) -> None: + """辞書型キー存在検査""" + if not self.check_key_exist(check_key): raise Exception(f'「{check_key}」キーは必須です') - elif not self.check_value_type(object_dict, check_key, check_type): + return + + def assert_data_type(self, check_key: str, check_type: type) -> None: + """バリュー型検査""" + if not self.check_data_type(check_key, check_type): raise Exception(f'「{check_key}」キーの値は「{check_type}」でなければなりません') - def check_key_exist_case_value_type(self, object_dict: dict, check_key: str, check_type: type): - ''' - 辞書型キー存在した場合のバリュー型チェック - ''' - if not self.check_key_exist(object_dict, check_key): - pass + return - elif not self.check_value_type(object_dict, check_key, check_type): - raise Exception(f'「{check_key}」キーの値は「{check_type}」でなければなりません') - - def check_key_exsit_and_regex(self, object_dict: dict, check_key: str, regex_str: str): - ''' - 辞書型キー存在チェック＆バリュー正規表現チェック - ''' - if not self.check_key_exist(object_dict, check_key): - raise Exception(f'「{check_key}」キーは必須です') - - elif not self.check_regex(regex_str, object_dict, check_key): + def assert_match_pattern(self, check_key: str, regex_str: str): + """正規表現検査""" + if not self.check_match_pattern(regex_str, check_key): raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') - def check_key_exsit_case_regex(self, object_dict: dict, check_key: str, regex_str: str): - ''' - 辞書型キー存在した場合のバリュー正規表現チェック - ''' - if not self.check_key_exist(object_dict, check_key): - pass - - elif not self.check_regex(regex_str, object_dict, check_key): - raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') + return diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py new file mode 100644 index 00000000..2f07736a --- /dev/null +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -0,0 +1,17 @@ +from datetime import datetime + +from src.system_var.constants import( + YYYYMMDDTHHMMSSTZ, + MILLISEC_FORMAT +) + + +class ExecuteDateTime: + def __init__(self): + self.__execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) + + def __str__(self) -> str: + return self.__execute_datetime + + def to_path(self) -> str: + return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index 689d9c36..ec1df277 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -2,6 +2,7 @@ import logging from src.system_var.environments import LOG_LEVEL +"""boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する""" for name in ["boto3", "botocore", "s3transfer", "urllib3"]: logging.getLogger(name).setLevel(logging.WARNING) diff --git a/s3/config/crm/object_info/crm_object_list_all.json b/s3/config/crm/object_info/crm_object_list_all.json new file mode 100644 index 00000000..a65cf1bb --- /dev/null +++ b/s3/config/crm/object_info/crm_object_list_all.json @@ -0,0 +1,47 @@ +{ + "objects": [ + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": false, + "last_fetch_datetime_file_name": "Territory2_ALL.json", + "upload_file_name": "CRM_Territory2_ALL_{execute_datetime}.csv" + }, + { + "object_name": "UserTerritory2Association", + "columns": [ + "Id", + "UserId", + "Territory2Id", + "IsActive", + "RoleInTerritory2", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": false, + "last_fetch_datetime_file_name": "UserTerritory2Association_ALL.json", + "upload_file_name": "CRM_UserTerritory2Association_ALL_{execute_datetime}.csv" + } + ] +} diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index 78ec31a7..00f7fe89 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -1,41 +1,3008 @@ { "objects": [ - { - # これはコメント行です - "object_name": "Account", - "columns": [ - "Id", - "Name", - "SystemModStamp", - "NumberOfEmployees", - "IsDeleted" - ] - }, - { - "object_name": "Territory2", - "is_skip": false, - "can_update_last_update": false, - "upload_file_name": "CRM_Territory2_All_{execute_datetime}_{split_csv_suffix_number:03}", - "last_fetch_datetime_file_name": "Territory2.json", - "columns": [ - "Id", - "Name", - "Territory2TypeId", - "SystemModStamp" - ] - }, - { - "object_name": "AccountShare", - "is_skip": false, - "datetime_column": "LastModifiedDate", - "columns": [ - "Id", - "Name", - "Territory2TypeId", - "LastModifiedDate" - ] - } + { + "object_name": "Clm_Presentation_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Mobile_ID_vod__c", + "Presentation_Id_vod__c", + "Product_vod__c", + "Default_Presentation_vod__c", + "Training_vod__c", + "ParentId_vod__c", + "Hidden_vod__c", + "Type_vod__c", + "Approved_vod__c", + "Copied_From_vod__c", + "Copy_Date_vod__c", + "Survey_vod__c", + "Original_Record_ID_vod__c", + "Directory_vod__c", + "End_Date_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "VExternal_Id_vod__c", + "Vault_DNS_vod__c", + "Vault_Doc_Id_vod__c", + "Vault_External_Id_vod__c", + "Vault_GUID_vod__c", + "Vault_Last_Modified_Date_Time_vod__c", + "Version_vod__c", + "Enable_Survey_Overlay_vod__c", + "Description_vod__c", + "Keywords_vod__c", + "Content_Channel_vod__c", + "original_material_approved_in_veritas__c", + "keywords__c", + "trade_team__c", + "ewizard_link__c", + "business_function__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Clm_Presentation_Slide_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Clm_Presentation_vod__c", + "Key_Message_vod__c", + "Display_Order_vod__c", + "Sub_Presentation_vod__c", + "Mobile_ID_vod__c", + "External_ID_vod__c", + "VExternal_Id_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Insight_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Clinical_Trial_vod__c", + "Date_vod__c", + "Description_vod__c", + "Entity_Reference_Id_vod__c", + "Interaction_vod__c", + "Medical_Event_vod__c", + "Mobile_ID_vod__c", + "Other_Source_vod__c", + "Override_Lock_vod__c", + "Publication_vod__c", + "Status_vod__c", + "Summary_vod__c", + "Unlock_vod__c", + "Commercial_Medical__c", + "MSJ_Level_1A__c", + "MSJ_Level_1B__c", + "MSJ_Level_2A__c", + "MSJ_Level_2B__c", + "MSJ_Level_3A__c", + "MSJ_Level_3B__c", + "MSJ_Level_4A__c", + "MSJ_Level_4B__c", + "MSJ_SubStatus__c", + "MSJ_Type_A__c", + "MSJ_Type_B__c", + "MSJ_Description_Backup__c", + "MSJ_Country__c", + "MSJ_Received_at_Boomi__c", + "MSJ_Level_1A_Value__c", + "MSJ_Level_1B_Value__c", + "MSJ_Level_2A_Value__c", + "MSJ_Level_2B_Value__c", + "MSJ_Level_3A_Value__c", + "MSJ_Level_3B_Value__c", + "MSJ_Level_4A_Value__c", + "MSJ_Level_4B_Value__c", + "MSJ_Hospital_ID__c", + "MSJ_Hospital_Name__c", + "MSJ_Hospital__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_MR_Weekly_Report__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Account_Name__c", + "MSJ_Activity_Results_Summary__c", + "MSJ_MUID__c", + "MSJ_Next_Week_Action_What__c", + "MSJ_Next_Week_Action_When__c", + "MSJ_Next_Week_Action_Where__c", + "MSJ_Next_Week_Action_Who__c", + "MSJ_Report_Week__c", + "MSJ_Target_Patient_Count__c", + "Mobile_ID_vod__c", + "MSJ_Activity_Results_Summary_HN__c", + "MSJ_Next_Week_Action_Where_HN__c", + "MSJ_Next_Week_Action_Who_HN__c", + "MSJ_Next_Week_Action_What_HN__c", + "MSJ_Next_Week_Action_When_HN__c", + "MSJ_Target_Patient_Count_HN__c", + "MSJ_Activity_Results_Summary_MCC__c", + "MSJ_Next_Week_Action_Where_MCC__c", + "MSJ_Next_Week_Action_Who_MCC__c", + "MSJ_Next_Week_Action_What_MCC__c", + "MSJ_Next_Week_Action_When_MCC__c", + "MSJ_Target_Patient_Count_MCC__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Account_Territory_Loader_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "External_ID_vod__c", + "Territory_vod__c", + "Mobile_ID_vod__c", + "Territory_To_Add_vod__c", + "Territory_to_Drop_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Event_Attendee_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Attendee_vod__c", + "User_vod__c", + "Medical_Event_vod__c", + "Attendee_Type_vod__c", + "Status_vod__c", + "Contact_vod__c", + "Attendee_Name_vod__c", + "Account_vod__c", + "Start_Date_vod__c", + "Signature_vod__c", + "Signature_Datetime_vod__c", + "MSJ_Copy_Account_Type__c", + "MSJ_Evaluation__c", + "MSJ_Hospital__c", + "MSJ_Role__c", + "Mobile_ID_vod__c", + "MSJ_Evaluation_Comment__c", + "Position_vod__c", + "Talk_Title_vod__c", + "MSJ_Attendee_Reaction__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "ObjectTerritory2Association", + "columns": [ + "Id", + "ObjectId", + "Territory2Id", + "AssociationCause", + "SobjectType", + "IsDeleted", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Key_Message_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Description_vod__c", + "Product_vod__c", + "Product_Strategy_vod__c", + "Display_Order_vod__c", + "Active_vod__c", + "Category_vod__c", + "Vehicle_vod__c", + "CLM_ID_vod__c", + "Custom_Reaction_vod__c", + "Slide_Version_vod__c", + "Language_vod__c", + "Media_File_CRC_vod__c", + "Media_File_Name_vod__c", + "Media_File_Size_vod__c", + "Segment_vod__c", + "Disable_Actions_vod__c", + "VExternal_Id_vod__c", + "CDN_Path_vod__c", + "Status_vod__c", + "Vault_DNS_vod__c", + "Vault_Doc_Id_vod__c", + "Vault_External_Id_vod__c", + "Vault_GUID_vod__c", + "Vault_Last_Modified_Date_Time_vod__c", + "Is_Shared_Resource_vod__c", + "iOS_Viewer_vod__c", + "iOS_Resolution_vod__c", + "approved_for_distribution_date__c", + "approved_for_use_date__c", + "ewizard_link__c", + "expiration_date__c", + "keywords__c", + "trade_team__c", + "business_function__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Group", + "columns": [ + "Id", + "Name", + "DeveloperName", + "RelatedId", + "Type", + "Email", + "OwnerId", + "DoesSendEmailToMembers", + "DoesIncludeBosses", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Event_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Primary_Product__c", + "Description_vod__c", + "Start_Date_vod__c", + "Location__c", + "End_Date_vod__c", + "Secondary_Product__c", + "Website__c", + "Active_vod__c", + "Event_Type__c", + "MSJ_Area__c", + "MSJ_Business_Unit__c", + "MSJ_Comment__c", + "MSJ_Company__c", + "MSJ_Expense_App_No__c", + "MSJ_Form__c", + "MSJ_HQ_Area__c", + "MSJ_Location__c", + "MSJ_MR__c", + "MSJ_Number_of_Attendee__c", + "MSJ_Product__c", + "MSJ_Site__c", + "MSJ_Type__c", + "MSJ_Number_of_Attendee_Auto_Calc__c", + "MSJ_Number_of_Attendee_Invited__c", + "Account_vod__c", + "MSJ_MUID__c", + "Country_Name_vod__c", + "MSJ_CE_SIPAGL_Updater__c", + "MSJ_CE_SIPAGL_1A__c", + "MSJ_CE_SIPAGL_1B__c", + "MSJ_CE_SIPAGL_2__c", + "MSJ_CE_SIPAGL_3__c", + "MSJ_CE_SIPAGL_4__c", + "MSJ_CE_SIPAGL_5__c", + "MSJ_CE_SIPAGL_Comment__c", + "MSJ_CE_SIPAGL_1A_date__c", + "MSJ_CE_SIPAGL_6__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Medical_Event_Evaluation__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Medical_Event__c", + "MSJ_Evaluation_Comment__c", + "MSJ_Evaluation__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Coaching_Report_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Mobile_ID_vod__c", + "Manager_vod__c", + "Employee_vod__c", + "Review_Date__c", + "Review_Period__c", + "Status__c", + "Comments__c", + "Strategic_Planning__c", + "Customer_Focus__c", + "Knowledge_Expertise__c", + "Business_Account_Planning__c", + "Call_Productivity__c", + "Overall_Rating__c", + "MSJ_A01__c", + "MSJ_A02__c", + "MSJ_A03__c", + "MSJ_AM_Memo__c", + "MSJ_Aid_Total__c", + "MSJ_C0_GC__c", + "MSJ_C1_GC__c", + "MSJ_C2_GC__c", + "MSJ_Countermeasure__c", + "MSJ_Deadline__c", + "MSJ_Double_Visit_Time__c", + "MSJ_Hospital__c", + "MSJ_K01_FE__c", + "MSJ_K01_ONC__c", + "MSJ_K02_FE__c", + "MSJ_K02_ONC__c", + "MSJ_K03_FE__c", + "MSJ_K03_ONC__c", + "MSJ_K04_FE__c", + "MSJ_K04_ONC__c", + "MSJ_K05_FE__c", + "MSJ_K05_ONC__c", + "MSJ_K06_FE__c", + "MSJ_K06_ONC__c", + "MSJ_K0_GC__c", + "MSJ_K1_GC__c", + "MSJ_K2_GC__c", + "MSJ_Knowledge_Total__c", + "MSJ_L0_GC__c", + "MSJ_L1_GC__c", + "MSJ_L2_GC__c", + "MSJ_MR_GC__c", + "MSJ_MR_Problems__c", + "MSJ_N0_GC__c", + "MSJ_N1_GC__c", + "MSJ_N2_GC__c", + "MSJ_Num_of_DTL__c", + "MSJ_P01__c", + "MSJ_P02__c", + "MSJ_P03__c", + "MSJ_P04__c", + "MSJ_P05__c", + "MSJ_P0_GC__c", + "MSJ_P1_GC__c", + "MSJ_P2_GC__c", + "MSJ_PlanningTotal__c", + "MSJ_R0_GC__c", + "MSJ_R1_GC__c", + "MSJ_R2_GC__c", + "MSJ_S01__c", + "MSJ_S02__c", + "MSJ_S03__c", + "MSJ_S04__c", + "MSJ_S05__c", + "MSJ_S06__c", + "MSJ_S07__c", + "MSJ_S08__c", + "MSJ_S09__c", + "MSJ_S10__c", + "MSJ_S11__c", + "MSJ_S12__c", + "MSJ_Skill_Total__c", + "MSJ_After_Call_01__c", + "MSJ_After_Call_02__c", + "MSJ_After_Call_03__c", + "MSJ_After_Call_04__c", + "MSJ_Closing__c", + "MSJ_Comment_by_MR__c", + "MSJ_Confirmed_by_MR__c", + "MSJ_Createdby__c", + "MSJ_FT_AM_Name__c", + "MSJ_Interview_Preparation__c", + "MSJ_Interview_Reflection__c", + "MSJ_Notify_To_MR__c", + "MSJ_Opening__c", + "MSJ_Others_01_Result__c", + "MSJ_Others_01__c", + "MSJ_Others_02_Result__c", + "MSJ_Others_02__c", + "MSJ_Patient_Thinking__c", + "MSJ_Probing__c", + "MSJ_Supporting__c", + "MSJ_Patient_Thinking_for_FE__c", + "MSJ_After_Call_05__c", + "MSJ_After_Call_06__c", + "MSJ_After_Call_07__c", + "MSJ_After_Call_08__c", + "MSJ_Createdby_FE__c", + "MSJ_Createdby_ONC__c", + "MSJ_Development_Level__c", + "MSJ_Interview_Prep_01__c", + "MSJ_Interview_Prep_02__c", + "MSJ_Leadership_Style__c", + "MSJ_Overcome_01__c", + "MSJ_Overcome_02__c", + "MSJ_Overcome_03__c", + "MSJ_Overcome_04__c", + "MSJ_Review_01__c", + "MSJ_Review_02__c", + "MSJ_SK_01__c", + "MSJ_SK_02__c", + "MSJ_SK_03__c", + "MSJ_SK_04__c", + "MSJ_SK_05__c", + "MSJ_SK_06__c", + "MSJ_SK_07__c", + "MSJ_SK_08__c", + "MSJ_SK_09__c", + "MSJ_SK_10__c", + "MSJ_Specific_Action__c", + "MSJ_Training_Point__c", + "MSJ_Efforts_of_Year__c", + "MSJ_Efforts_of_Month__c", + "MSJ_Skill_Task__c", + "MSJ_Action_of_This_Month__c", + "MSJ_Achievement_of_This_Month__c", + "MSJ_Comment_from_AM__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Call_Comments_vod__c", + "Sample_Card_vod__c", + "Add_Detail_vod__c", + "Property_vod__c", + "Account_vod__c", + "zvod_Product_Discussion_vod__c", + "Status_vod__c", + "Parent_Address_vod__c", + "Account_Plan_vod__c", + "zvod_SaveNew_vod__c", + "Next_Call_Notes_vod__c", + "Pre_Call_Notes_vod__c", + "Mobile_ID_vod__c", + "zvod_Account_Credentials_vod_c_vod__c", + "zvod_Account_Preferred_Name_vod_c_vod__c", + "zvod_Account_Sample_Status_vod_c_vod__c", + "zvod_Attendees_vod__c", + "zvod_Key_Messages_vod__c", + "zvod_Detailing_vod__c", + "zvod_Expenses_vod__c", + "zvod_Followup_vod__c", + "zvod_Samples_vod__c", + "zvod_Save_vod__c", + "zvod_Submit_vod__c", + "zvod_Delete_vod__c", + "Activity_Type__c", + "Significant_Event__c", + "Location_vod__c", + "Subject_vod__c", + "Unlock_vod__c", + "Call_Datetime_vod__c", + "Disbursed_To_vod__c", + "Disclaimer_vod__c", + "Request_Receipt_vod__c", + "Signature_Date_vod__c", + "Signature_vod__c", + "Territory_vod__c", + "Submitted_By_Mobile_vod__c", + "Call_Type_vod__c", + "Add_Key_Message_vod__c", + "Address_vod__c", + "Attendees_vod__c", + "Attendee_Type_vod__c", + "Call_Date_vod__c", + "Detailed_Products_vod__c", + "No_Disbursement_vod__c", + "Parent_Call_vod__c", + "User_vod__c", + "Contact_vod__c", + "zvod_Entity_vod__c", + "Medical_Event_vod__c", + "Mobile_Created_Datetime_vod__c", + "Mobile_Last_Modified_Datetime_vod__c", + "License_vod__c", + "Is_Parent_Call_vod__c", + "Entity_Display_Name_vod__c", + "Override_Lock_vod__c", + "Last_Device_vod__c", + "Ship_Address_Line_1_vod__c", + "Ship_Address_Line_2_vod__c", + "Ship_City_vod__c", + "Ship_Country_vod__c", + "Ship_License_Expiration_Date_vod__c", + "Ship_License_Status_vod__c", + "Ship_License_vod__c", + "Ship_State_vod__c", + "Ship_To_Address_vod__c", + "Ship_Zip_vod__c", + "Ship_To_Address_Text_vod__c", + "CLM_vod__c", + "zvod_CLMDetails_vod__c", + "Is_Sampled_Call_vod__c", + "zvod_Surveys_vod__c", + "Presentations_vod__c", + "Entity_Reference_Id_vod__c", + "Error_Reference_Call_vod__c", + "Duration_vod__c", + "Color_vod__c", + "Allowed_Products_vod__c", + "zvod_Attachments_vod__c", + "Sample_Card_Reason_vod__c", + "ASSMCA_vod__c", + "Address_Line_1_vod__c", + "Address_Line_2_vod__c", + "City_vod__c", + "DEA_Address_Line_1_vod__c", + "DEA_Address_Line_2_vod__c", + "DEA_Address_vod__c", + "DEA_City_vod__c", + "DEA_Expiration_Date_vod__c", + "DEA_State_vod__c", + "DEA_Zip_4_vod__c", + "DEA_Zip_vod__c", + "DEA_vod__c", + "Ship_Zip_4_vod__c", + "State_vod__c", + "Zip_4_vod__c", + "Zip_vod__c", + "Sample_Send_Card_vod__c", + "zvod_Address_vod_c_DEA_Status_vod_c_vod__c", + "Signature_Page_Image_vod__c", + "Credentials_vod__c", + "Salutation_vod__c", + "zvod_Account_Call_Reminder_vod_c_vod__c", + "MSJ_Meeting_Duration__c", + "MSJ_Double_Visit_AM__c", + "zvod_Business_Account_vod__c", + "Product_Priority_1_vod__c", + "Product_Priority_2_vod__c", + "Product_Priority_3_vod__c", + "Product_Priority_4_vod__c", + "Product_Priority_5_vod__c", + "zvod_More_Actions_vod__c", + "zvod_Call_Conflict_Status_vod__c", + "Signature_Timestamp_vod__c", + "Expense_Amount_vod__c", + "Total_Expense_Attendees_Count_vod__c", + "Attendee_list_vod__c", + "Expense_Post_Status_vod__c", + "Attendee_Post_Status_vod__c", + "Expense_System_External_ID_vod__c", + "Incurred_Expense_vod__c", + "Assigner_vod__c", + "Assignment_Datetime_vod__c", + "zvod_Call_Objective_vod__c", + "Signature_Location_Longitude_vod__c", + "Signature_Location_Latitude_vod__c", + "Location_Services_Status_vod__c", + "MSJ_Double_Visit_Other__c", + "MSJ_Comment__c", + "MSJ_For_Reporting__c", + "MSJ_Number_of_Attendees__c", + "MSJ_Main_Dept__c", + "Planned_Type_vjh__c", + "Cobrowse_URL_Participant_vod__c", + "MSJ_Activity_Method_Text__c", + "MSJ_Activity_Method__c", + "MSJ_Classification__c", + "MSJ_Double_Visit_MSL__c", + "MSJ_MSL_Comment_for_MR__c", + "MSJ_APD__c", + "Medical_Inquiry_vod__c", + "MSJ_Call_Type_MSJ__c", + "MSJ_Prescription_Request__c", + "MSJ_Patient_Follow__c", + "Child_Account_Id_vod__c", + "Child_Account_vod__c", + "Location_Id_vod__c", + "Location_Name_vod__c", + "MSJ_Comments_about_technology__c", + "Remote_Meeting_vod__c", + "Veeva_Remote_Meeting_Id_vod__c", + "MSJ_Activity_Type_Report__c", + "MSJ_Activity_Type__c", + "MSJ_Activity__c", + "MSJ_Comments__c", + "MSJ_Therapy__c", + "MSJ_Time_Hrs__c", + "EMDS_CO_Reference__c", + "EMDS_Call_Sub_Type__c", + "EMDS_Call_Type__c", + "EMDS_Call_Unsuccessful__c", + "EMDS_Congress_Type__c", + "EMDS_Date_of_Service__c", + "EMDS_Fertility_DisInterest__c", + "EMDS_Fertility_Interest__c", + "EMDS_Installed_Equipment__c", + "EMDS_Pipeline_Stage_Value__c", + "EMDS_Pipeline_Stage__c", + "EMDS_Pipeline__c", + "EMDS_Reason_for_Call__c", + "EMDS_Training_Completed__c", + "MSJ_BrainStorming__c", + "MSJ_SIPAGL_1A__c", + "MSJ_SIPAGL_1B__c", + "MSJ_SIPAGL_2__c", + "MSJ_SIPAGL_3__c", + "MSJ_SIPAGL_4A__c", + "MSJ_SIPAGL_5A__c", + "MSJ_SIPAGL_comment__c", + "MSJ_SIPAGL_4B__c", + "MSJ_SIPAGL_5B__c", + "Location_Text_vod__c", + "Call_Channel_vod__c", + "MSJ_Scientific_Interaction__c", + "MSJ_Activity_Email_Reply__c", + "MSJ_Interaction_Duration__c", + "MSJ_SIPAGL_1A_date__c", + "MSJ_CoPromotion__c", + "Call_Channel_Formula_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Detail_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Is_Parent_Call_vod__c", + "Call2_vod__c", + "Product_vod__c", + "Detail_Priority_vod__c", + "Mobile_ID_vod__c", + "Override_Lock_vod__c", + "Type_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Key_Message_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Call2_vod__c", + "Reaction_vod__c", + "Product_vod__c", + "Key_Message_vod__c", + "Mobile_ID_vod__c", + "Contact_vod__c", + "Call_Date_vod__c", + "User_vod__c", + "Category_vod__c", + "Vehicle_vod__c", + "Is_Parent_Call_vod__c", + "Override_Lock_vod__c", + "CLM_ID_vod__c", + "Slide_Version_vod__c", + "Duration_vod__c", + "Presentation_ID_vod__c", + "Start_Time_vod__c", + "Attendee_Type_vod__c", + "Entity_Reference_Id_vod__c", + "Segment_vod__c", + "Display_Order_vod__c", + "Clm_Presentation_Name_vod__c", + "Clm_Presentation_Version_vod__c", + "Clm_Presentation_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call_Clickstream_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Answer_vod__c", + "Call_vod__c", + "Key_Message_vod__c", + "Mobile_ID_vod__c", + "Popup_Opened_vod__c", + "Possible_Answers_vod__c", + "Presentation_ID_vod__c", + "Product_vod__c", + "Range_Value_vod__c", + "Rollover_Entered_vod__c", + "Selected_Items_vod__c", + "CLM_ID_vod__c", + "Question_vod__c", + "Survey_Type_vod__c", + "Text_Entered_vod__c", + "Toggle_Button_On_vod__c", + "Track_Element_Description_vod__c", + "Track_Element_Id_vod__c", + "Track_Element_Type_vod__c", + "Usage_Duration_vod__c", + "Usage_Start_Time_vod__c", + "AuxillaryId_vod__c", + "ParentId_vod__c", + "Revision_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Discussion_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Call2_vod__c", + "Activity__c", + "Comments__c", + "Contact_vod__c", + "Call_Date_vod__c", + "Product_Strategy_vod__c", + "Product_Tactic_vod__c", + "Restricted_Comments__c", + "Product_vod__c", + "Presentation__c", + "Discussion_Topics__c", + "Slides__c", + "User_vod__c", + "Indication__c", + "Mobile_ID_vod__c", + "Medical_Event_vod__c", + "Is_Parent_Call_vod__c", + "Override_Lock_vod__c", + "zvod_Product_Map_vod__c", + "Attendee_Type_vod__c", + "Entity_Reference_Id_vod__c", + "Account_Tactic_vod__c", + "MSJ_Material_Type__c", + "MSJ_Discussion_Contents__c", + "MSJ_IST_Minutes__c", + "MSJ_Off_Label_Minutes__c", + "MSJ_Discussion_Objectives__c", + "MSJ_Insight__c", + "EMDS_Materials__c", + "EMDS_Topic__c", + "MSJ_Visit_Purpose__c", + "MSJ_Insight_Count__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Time_Off_Territory_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Reason_vod__c", + "Territory_vod__c", + "Date_vod__c", + "Status_vod__c", + "Time_vod__c", + "Hours_vod__c", + "Mobile_ID_vod__c", + "Hours_off_vod__c", + "Start_Time_vod__c", + "MSJ_Day__c", + "MSJ_Comment__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Dynamic_Attribute_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Active_vod__c", + "Dynamic_Attribute_Configuration_vod__c", + "Dynamic_Attribute_Description_vod__c", + "Dynamic_Attribute_Help_Text_vod__c", + "Dynamic_Attribute_Label_vod__c", + "Dynamic_Attribute_Name_vod__c", + "Dynamic_Attribute_Record_Type_vod__c", + "Dynamic_Attribute_Value_Checkbox_vod__c", + "Dynamic_Attribute_Value_Date_Time_vod__c", + "Dynamic_Attribute_Value_Date_vod__c", + "Dynamic_Attribute_Value_Number_vod__c", + "Dynamic_Attribute_Value_Text_Area_vod__c", + "Dynamic_Attribute_Value_Text_vod__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Dynamic_Attribute_Configuration_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Applies_To_vod__c", + "Attribute_Label_vod__c", + "Attribute_Name_vod__c", + "Available_Values_vod__c", + "Description_vod__c", + "Detail_Group_vod__c", + "Display_Order_vod__c", + "External_ID_vod__c", + "Help_Text_vod__c", + "Product_vod__c", + "Read_Only_vod__c", + "Section_Name_vod__c", + "Sharing_Group_vod__c", + "Status_vod__c", + "Track_Changes_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Profile", + "columns": [ + "Id", + "Name", + "PermissionsEmailSingle", + "PermissionsEmailMass", + "PermissionsEditTask", + "PermissionsEditEvent", + "PermissionsExportReport", + "PermissionsImportPersonal", + "PermissionsDataExport", + "PermissionsManageUsers", + "PermissionsEditPublicFilters", + "PermissionsEditPublicTemplates", + "PermissionsModifyAllData", + "PermissionsManageCases", + "PermissionsManageSolutions", + "PermissionsCustomizeApplication", + "PermissionsEditReadonlyFields", + "PermissionsRunReports", + "PermissionsViewSetup", + "PermissionsTransferAnyEntity", + "PermissionsNewReportBuilder", + "PermissionsManageSelfService", + "PermissionsManageCssUsers", + "PermissionsActivateContract", + "PermissionsApproveContract", + "PermissionsImportLeads", + "PermissionsManageLeads", + "PermissionsTransferAnyLead", + "PermissionsViewAllData", + "PermissionsEditPublicDocuments", + "PermissionsViewEncryptedData", + "PermissionsEditBrandTemplates", + "PermissionsEditHtmlTemplates", + "PermissionsManageTranslation", + "PermissionsDeleteActivatedContract", + "PermissionsSendSitRequests", + "PermissionsApiUserOnly", + "PermissionsManageRemoteAccess", + "PermissionsCanUseNewDashboardBuilder", + "PermissionsManageCategories", + "PermissionsConvertLeads", + "PermissionsTestInstanceCreate", + "PermissionsPasswordNeverExpires", + "PermissionsUseTeamReassignWizards", + "PermissionsInstallMultiforce", + "PermissionsPublishMultiforce", + "PermissionsEditOppLineItemUnitPrice", + "PermissionsManageTerritories", + "PermissionsCreateMultiforce", + "PermissionsBulkApiHardDelete", + "PermissionsInboundMigrationToolsUser", + "PermissionsSolutionImport", + "PermissionsManageCallCenters", + "PermissionsManageSynonyms", + "PermissionsOutboundMigrationToolsUser", + "PermissionsViewContent", + "PermissionsManageEmailClientConfig", + "PermissionsEnableNotifications", + "PermissionsManageDataIntegrations", + "PermissionsDistributeFromPersWksp", + "PermissionsViewDataCategories", + "PermissionsManageDataCategories", + "PermissionsAuthorApex", + "PermissionsManageMobile", + "PermissionsApiEnabled", + "PermissionsManageCustomReportTypes", + "PermissionsEditCaseComments", + "PermissionsTransferAnyCase", + "PermissionsContentAdministrator", + "PermissionsCreateWorkspaces", + "PermissionsManageContentPermissions", + "PermissionsManageContentProperties", + "PermissionsManageContentTypes", + "PermissionsScheduleJob", + "PermissionsManageExchangeConfig", + "PermissionsManageAnalyticSnapshots", + "PermissionsScheduleReports", + "PermissionsManageBusinessHourHolidays", + "PermissionsManageDynamicDashboards", + "PermissionsManageInteraction", + "PermissionsViewMyTeamsDashboards", + "PermissionsResetPasswords", + "PermissionsFlowUFLRequired", + "PermissionsActivitiesAccess", + "PermissionsEmailTemplateManagement", + "PermissionsEmailAdministration", + "PermissionsChatterFileLink", + "PermissionsForceTwoFactor", + "PermissionsViewEventLogFiles", + "PermissionsManageNetworks", + "PermissionsManageAuthProviders", + "PermissionsRunFlow", + "PermissionsCreateCustomizeDashboards", + "PermissionsCreateDashboardFolders", + "PermissionsViewPublicDashboards", + "PermissionsManageDashbdsInPubFolders", + "PermissionsCreateCustomizeReports", + "PermissionsCreateReportFolders", + "PermissionsViewPublicReports", + "PermissionsManageReportsInPubFolders", + "PermissionsEditMyDashboards", + "PermissionsEditMyReports", + "PermissionsViewAllUsers", + "PermissionsConnectOrgToEnvironmentHub", + "PermissionsCreateCustomizeFilters", + "PermissionsContentHubUser", + "PermissionsGovernNetworks", + "PermissionsSalesConsole", + "PermissionsTwoFactorApi", + "PermissionsDeleteTopics", + "PermissionsEditTopics", + "PermissionsCreateTopics", + "PermissionsAssignTopics", + "PermissionsIdentityEnabled", + "PermissionsIdentityConnect", + "PermissionsContentWorkspaces", + "PermissionsCustomMobileAppsAccess", + "PermissionsViewHelpLink", + "PermissionsManageProfilesPermissionsets", + "PermissionsAssignPermissionSets", + "PermissionsManageRoles", + "PermissionsManageIpAddresses", + "PermissionsManageSharing", + "PermissionsManageInternalUsers", + "PermissionsManagePasswordPolicies", + "PermissionsManageLoginAccessPolicies", + "PermissionsManageCustomPermissions", + "PermissionsStdAutomaticActivityCapture", + "PermissionsManageTwoFactor", + "PermissionsDebugApex", + "PermissionsLightningExperienceUser", + "PermissionsConfigCustomRecs", + "PermissionsSubmitMacrosAllowed", + "PermissionsBulkMacrosAllowed", + "PermissionsManageSessionPermissionSets", + "PermissionsCreateAuditFields", + "PermissionsUpdateWithInactiveOwner", + "PermissionsManageSandboxes", + "PermissionsAutomaticActivityCapture", + "PermissionsImportCustomObjects", + "PermissionsDelegatedTwoFactor", + "PermissionsSelectFilesFromSalesforce", + "PermissionsModerateNetworkUsers", + "PermissionsMergeTopics", + "PermissionsSubscribeToLightningReports", + "PermissionsManagePvtRptsAndDashbds", + "PermissionsAllowLightningLogin", + "PermissionsCampaignInfluence2", + "PermissionsViewDataAssessment", + "PermissionsCanApproveFeedPost", + "PermissionsAllowViewEditConvertedLeads", + "PermissionsShowCompanyNameAsUserBadge", + "PermissionsAccessCMC", + "PermissionsViewHealthCheck", + "PermissionsManageHealthCheck", + "PermissionsPackaging2", + "PermissionsManageCertificates", + "PermissionsCreateReportInLightning", + "PermissionsPreventClassicExperience", + "PermissionsListEmailSend", + "PermissionsChangeDashboardColors", + "PermissionsManageRecommendationStrategies", + "PermissionsManagePropositions", + "PermissionsSubscribeReportRolesGrps", + "PermissionsSubscribeDashboardRolesGrps", + "PermissionsUseWebLink", + "PermissionsHasUnlimitedNBAExecutions", + "PermissionsViewOnlyEmbeddedAppUser", + "PermissionsViewAllActivities", + "PermissionsSubscribeReportToOtherUsers", + "PermissionsLightningConsoleAllowedForUser", + "PermissionsSubscribeReportsRunAsUser", + "PermissionsSubscribeToLightningDashboards", + "PermissionsSubscribeDashboardToOtherUsers", + "PermissionsCreateLtngTempInPub", + "PermissionsTransactionalEmailSend", + "PermissionsViewPrivateStaticResources", + "PermissionsCreateLtngTempFolder", + "PermissionsApexRestServices", + "PermissionsEnableCommunityAppLauncher", + "PermissionsGiveRecognitionBadge", + "PermissionsUseMySearch", + "PermissionsLtngPromoReserved01UserPerm", + "PermissionsManageSubscriptions", + "PermissionsManageSurveys", + "PermissionsUseAssistantDialog", + "PermissionsUseQuerySuggestions", + "PermissionsViewRoles", + "PermissionsLMOutboundMessagingUserPerm", + "PermissionsModifyDataClassification", + "PermissionsPrivacyDataAccess", + "PermissionsQueryAllFiles", + "PermissionsModifyMetadata", + "PermissionsManageCMS", + "PermissionsSandboxTestingInCommunityApp", + "PermissionsCanEditPrompts", + "PermissionsViewUserPII", + "PermissionsManageHubConnections", + "PermissionsB2BMarketingAnalyticsUser", + "PermissionsTraceXdsQueries", + "PermissionsViewAllCustomSettings", + "PermissionsViewAllForeignKeyNames", + "PermissionsHeadlessCMSAccess", + "PermissionsLMEndMessagingSessionUserPerm", + "PermissionsConsentApiUpdate", + "PermissionsAccessContentBuilder", + "PermissionsAccountSwitcherUser", + "PermissionsManageC360AConnections", + "PermissionsManageReleaseUpdates", + "PermissionsViewAllProfiles", + "PermissionsSkipIdentityConfirmation", + "PermissionsSendCustomNotifications", + "PermissionsPackaging2Delete", + "PermissionsFSCComprehensiveUserAccess", + "PermissionsManageTrustMeasures", + "PermissionsViewTrustMeasures", + "PermissionsIsotopeCToCUser", + "PermissionsIsotopeAccess", + "PermissionsIsotopeLEX", + "PermissionsQuipMetricsAccess", + "PermissionsQuipUserEngagementMetrics", + "PermissionsManageExternalConnections", + "PermissionsAIViewInsightObjects", + "PermissionsAICreateInsightObjects", + "PermissionsNativeWebviewScrolling", + "PermissionsViewDeveloperName", + "Type", + "UserLicenseId", + "UserType", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "Description", + "LastViewedDate", + "LastReferencedDate" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "My_Setup_Products_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Product_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Activity_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_External_ID_Map_vod__c", + "Account_vod__c", + "Call_vod__c", + "City_vod__c", + "Client_Name_vod__c", + "Client_OS_vod__c", + "Client_Type_vod__c", + "Country_vod__c", + "Debug_vod__c", + "Device_vod__c", + "IP_Address_vod__c", + "Multichannel_Activity_vod__c", + "Referring_Site_vod__c", + "Region_vod__c", + "Sent_Email_vod__c", + "Session_Id_vod__c", + "Site_vod__c", + "Start_DateTime_vod__c", + "Total_Duration_vod__c", + "URL_vod__c", + "User_Agent_vod__c", + "VExternal_Id_vod__c", + "Viewport_Height_vod__c", + "Viewport_Width_vod__c", + "Color_vod__c", + "Icon_vod__c", + "MCD_Primary_Key_vod__c", + "Record_Type_Name_vod__c", + "MSJ_Date_Opened__c", + "MSJ_Sent_Date__c", + "MSJ_Email_Subject__c", + "MSJ_Opens__c", + "MSJ_Email_Status__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Activity_Line_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Multichannel_Activity_vod__c", + "Call_vod__c", + "Custom_vod__c", + "DateTime_vod__c", + "Debug_vod__c", + "Detail_Group_VExternal_Id_vod__c", + "Detail_Group_vod__c", + "Duration_vod__c", + "Event_Subtype_vod__c", + "Event_Type_vod__c", + "Key_Message_VExternal_Id_vod__c", + "Key_Message_vod__c", + "Multichannel_Content_Asset_Id_vod__c", + "Multichannel_Content_Asset_Version_vod__c", + "Multichannel_Content_Asset_vod__c", + "Multichannel_Content_vod__c", + "Product_VExternal_Id_vod__c", + "Product_vod__c", + "Sent_Email_vod__c", + "VExternal_Id_vod__c", + "Video_Last_Viewed_Time_vod__c", + "Video_Length_vod__c", + "Video_Total_Time_Spent_vod__c", + "View_Order_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Consent_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Capture_Datetime_vod__c", + "Channel_Value_vod__c", + "Detail_Group_vod__c", + "External_ID_vod__c", + "Last_Device_vod__c", + "Mobile_ID_vod__c", + "Opt_Expiration_Date_vod__c", + "Opt_Type_vod__c", + "Optout_Event_Type_vod__c", + "Product_vod__c", + "Signature_Datetime_vod__c", + "Signature_ID_vod__c", + "Signature_vod__c", + "Sample_Consent_Template_Data_vod__c", + "Sample_Consent_Template_vod__c", + "Consent_Line_vod__c", + "Consent_Type_vod__c", + "Default_Consent_Text_vod__c", + "Disclaimer_Text_vod__c", + "Sub_Channel_Key_vod__c", + "Consent_Confirm_Datetime_vod__c", + "Related_Transaction_Id_vod__c", + "Sent_Email_vod__c", + "Content_Type_vod__c", + "Receipt_Email_vod__c", + "Receipt_Sent_Email_Transaction_Id_vod__c", + "Receipt_Sent_Email_vod__c", + "Captured_By_vod__c", + "Opt_Out_Disclaimer_Text_vod__c", + "Channel_Source_vod__c", + "Union_Id_vod__c", + "User_Last_Notified_vod__c", + "Sub_Channel_Display_Name__c", + "MSJ_Consent_Source__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Inquiry_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Address_Line_1_vod__c", + "Address_Line_2_vod__c", + "City_vod__c", + "Delivery_Method_vod__c", + "Email_vod__c", + "Fax_Number_vod__c", + "Inquiry_Text__c", + "Lock_vod__c", + "Mobile_ID_vod__c", + "Phone_Number_vod__c", + "Product__c", + "Rush_Delivery__c", + "Signature_Date_vod__c", + "Signature_vod__c", + "State_vod__c", + "Status_vod__c", + "Zip_vod__c", + "zvod_Delivery_Method_vod__c", + "zvod_Disclaimer_vod__c", + "Submitted_By_Mobile_vod__c", + "Disclaimer_vod__c", + "Entity_Reference_Id_vod__c", + "Call2_vod__c", + "Country_vod__c", + "Override_Lock_vod__c", + "MSJ_Department__c", + "MSJ_Doctor_Name__c", + "MSJ_Hospital_Name__c", + "MSJ_Indication__c", + "MSJ_Inquiry_Assignment__c", + "MSJ_Inquiry_Date__c", + "MSJ_Inquiry_Input_Manager__c", + "MSJ_Inquiry_Input_User__c", + "MSJ_MSL_Manager__c", + "MSJ_Notice_to_MR__c", + "MSJ_Person_in_charge_1__c", + "MSJ_Person_in_charge_2__c", + "MSJ_Product_for_MEC__c", + "MSJ_Product_for_MR__c", + "MSJ_Reply_Date__c", + "MSJ_Reply_User__c", + "MSJ_Reply__c", + "MSJ_Title__c", + "MSJ_AE_Infomation__c", + "MSJ_FAQ_Number_Report__c", + "MSJ_Return_Call_Report__c", + "MSJ_Inquiry_Origin_Report__c", + "MSJ_AE_Report__c", + "MSJ_Background__c", + "MSJ_MSL_Support__c", + "MSJ_Material_Requirement__c", + "MSJ_Hospital_Name_Disp__c", + "MSJ_Hospital__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Email_Activity_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Sent_Email_vod__c", + "Activity_DateTime_vod__c", + "City_vod__c", + "Click_URL_vod__c", + "Client_Name_vod__c", + "Client_OS_vod__c", + "Client_Type_vod__c", + "Country_vod__c", + "Device_Type_vod__c", + "Event_Msg_vod__c", + "Event_type_vod__c", + "IP_Address_vod__c", + "Region_vod__c", + "User_Agent_vod__c", + "Vault_Doc_ID_vod__c", + "Vault_Doc_Name_vod__c", + "Vault_Document_Major_Version_vod__c", + "Vault_Document_Minor_Version_vod__c", + "Vault_Document_Number_vod__c", + "Vault_Document_Title_vod__c", + "Vault_Instance_ID_vod__c", + "Preference_Modification_vod__c", + "Approved_Document_vod__c", + "Link_Name_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "User", + "columns": [ + "Id", + "Username", + "LastName", + "FirstName", + "Name", + "CompanyName", + "Division", + "Department", + "Title", + "Street", + "City", + "State", + "PostalCode", + "Country", + "Latitude", + "Longitude", + "GeocodeAccuracy", + "Address", + "Email", + "EmailPreferencesAutoBcc", + "EmailPreferencesAutoBccStayInTouch", + "EmailPreferencesStayInTouchReminder", + "SenderEmail", + "SenderName", + "Signature", + "StayInTouchSubject", + "StayInTouchSignature", + "StayInTouchNote", + "Phone", + "Fax", + "MobilePhone", + "Alias", + "CommunityNickname", + "BadgeText", + "IsActive", + "TimeZoneSidKey", + "UserRoleId", + "LocaleSidKey", + "ReceivesInfoEmails", + "ReceivesAdminInfoEmails", + "EmailEncodingKey", + "ProfileId", + "UserType", + "LanguageLocaleKey", + "EmployeeNumber", + "DelegatedApproverId", + "ManagerId", + "LastLoginDate", + "LastPasswordChangeDate", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "NumberOfFailedLogins", + "OfflineTrialExpirationDate", + "OfflinePdaTrialExpirationDate", + "UserPermissionsMarketingUser", + "UserPermissionsOfflineUser", + "UserPermissionsWirelessUser", + "UserPermissionsAvantgoUser", + "UserPermissionsCallCenterAutoLogin", + "UserPermissionsSFContentUser", + "UserPermissionsInteractionUser", + "UserPermissionsSupportUser", + "UserPermissionsChatterAnswersUser", + "ForecastEnabled", + "UserPreferencesActivityRemindersPopup", + "UserPreferencesEventRemindersCheckboxDefault", + "UserPreferencesTaskRemindersCheckboxDefault", + "UserPreferencesReminderSoundOff", + "UserPreferencesDisableAllFeedsEmail", + "UserPreferencesApexPagesDeveloperMode", + "UserPreferencesReceiveNoNotificationsAsApprover", + "UserPreferencesReceiveNotificationsAsDelegatedApprover", + "UserPreferencesHideCSNGetChatterMobileTask", + "UserPreferencesHideCSNDesktopTask", + "UserPreferencesHideChatterOnboardingSplash", + "UserPreferencesHideSecondChatterOnboardingSplash", + "UserPreferencesShowTitleToExternalUsers", + "UserPreferencesShowManagerToExternalUsers", + "UserPreferencesShowEmailToExternalUsers", + "UserPreferencesShowWorkPhoneToExternalUsers", + "UserPreferencesShowMobilePhoneToExternalUsers", + "UserPreferencesShowFaxToExternalUsers", + "UserPreferencesShowStreetAddressToExternalUsers", + "UserPreferencesShowCityToExternalUsers", + "UserPreferencesShowStateToExternalUsers", + "UserPreferencesShowPostalCodeToExternalUsers", + "UserPreferencesShowCountryToExternalUsers", + "UserPreferencesShowProfilePicToGuestUsers", + "UserPreferencesShowTitleToGuestUsers", + "UserPreferencesShowCityToGuestUsers", + "UserPreferencesShowStateToGuestUsers", + "UserPreferencesShowPostalCodeToGuestUsers", + "UserPreferencesShowCountryToGuestUsers", + "UserPreferencesHideInvoicesRedirectConfirmation", + "UserPreferencesHideStatementsRedirectConfirmation", + "UserPreferencesPathAssistantCollapsed", + "UserPreferencesCacheDiagnostics", + "UserPreferencesShowEmailToGuestUsers", + "UserPreferencesShowManagerToGuestUsers", + "UserPreferencesShowWorkPhoneToGuestUsers", + "UserPreferencesShowMobilePhoneToGuestUsers", + "UserPreferencesShowFaxToGuestUsers", + "UserPreferencesShowStreetAddressToGuestUsers", + "UserPreferencesLightningExperiencePreferred", + "UserPreferencesPreviewLightning", + "UserPreferencesHideEndUserOnboardingAssistantModal", + "UserPreferencesHideLightningMigrationModal", + "UserPreferencesHideSfxWelcomeMat", + "UserPreferencesHideBiggerPhotoCallout", + "UserPreferencesGlobalNavBarWTShown", + "UserPreferencesGlobalNavGridMenuWTShown", + "UserPreferencesCreateLEXAppsWTShown", + "UserPreferencesFavoritesWTShown", + "UserPreferencesRecordHomeSectionCollapseWTShown", + "UserPreferencesRecordHomeReservedWTShown", + "UserPreferencesFavoritesShowTopFavorites", + "UserPreferencesExcludeMailAppAttachments", + "UserPreferencesSuppressTaskSFXReminders", + "UserPreferencesSuppressEventSFXReminders", + "UserPreferencesPreviewCustomTheme", + "UserPreferencesHasCelebrationBadge", + "UserPreferencesUserDebugModePref", + "UserPreferencesSRHOverrideActivities", + "UserPreferencesNewLightningReportRunPageEnabled", + "UserPreferencesReverseOpenActivitiesView", + "UserPreferencesNativeEmailClient", + "UserPreferencesHideBrowseProductRedirectConfirmation", + "UserPreferencesHideOnlineSalesAppWelcomeMat", + "ContactId", + "AccountId", + "CallCenterId", + "Extension", + "FederationIdentifier", + "AboutMe", + "FullPhotoUrl", + "SmallPhotoUrl", + "IsExtIndicatorVisible", + "OutOfOfficeMessage", + "MediumPhotoUrl", + "DigestFrequency", + "DefaultGroupNotificationFrequency", + "LastViewedDate", + "LastReferencedDate", + "BannerPhotoUrl", + "SmallBannerPhotoUrl", + "MediumBannerPhotoUrl", + "IsProfilePhotoActive", + "IndividualId", + "Last_Mobile_Connect_vod__c", + "Last_Tablet_Connect_vod__c", + "Last_Mobile_Connect_Version_vod__c", + "Last_Tablet_Connect_Version_vod__c", + "Last_Mobile_Sync_vod__c", + "Last_Tablet_Sync_vod__c", + "RaiseLoggingLevel_vod__c", + "SendDetailedLog_vod__c", + "Last_Blackberry_Connect_vod__c", + "Last_Blackberry_Connect_Version_vod__c", + "Last_Blackberry_Sync_vod__c", + "Force_Full_Refresh_vod__c", + "Override_SystemModstamp_Timestamp_vod__c", + "Facetime_Email_vod__c", + "Facetime_Phone_vod__c", + "Product_Expertise_vod__c", + "Available_vod__c", + "Available_Last_Update_vod__c", + "Last_iPad_Connect_Version_vod__c", + "Last_iPad_Connect_vod__c", + "Last_iPad_Sync_vod__c", + "Inventory_Order_Allocation_Group_vod__c", + "Concur_User_Id_vod__c", + "Last_iPad_iOS_Version_vod__c", + "Approved_Email_Admin_vod__c", + "Last_WinModern_Connect_Version_vod__c", + "Last_WinModern_Connect_vod__c", + "Last_WinModern_Sync_vod__c", + "Primary_Territory_vod__c", + "Analytics_Admin_vod__c", + "Content_Admin_vod__c", + "Last_WinModern_Windows_Version_vod__c", + "Upload_VTrans_vod__c", + "Sync_Frequency_vjh__c", + "Clear_Client_Sync_Errors_vod__c", + "Remote_Meeting_Host_Id_vod__c", + "Remote_Meeting_Host_Token_vod__c", + "Last_iPhone_Connect_Version_vod__c", + "Last_iPhone_Connect_vod__c", + "Last_iPhone_Sync_vod__c", + "Last_iPhone_iOS_Version_vod__c", + "Remote_Meeting_Start_From_CRM_Online_vod__c", + "Country_Code_vod__c", + "User_Type_vod__c", + "Engage_Group_Provisioning_Status_vod__c", + "Engage_Group_Request_vod__c", + "Engage_Group_vod__c", + "Last_CRMDesktop_Mac_Sync_vod__c", + "Last_CRMDesktop_Mac_Version_vod__c", + "Last_CRMDesktop_Windows_Sync_vod__c", + "Last_CRMDesktop_Windows_Version_vod__c", + "MSJ_Test_User__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "UserTerritory2Association", + "columns": [ + "Id", + "UserId", + "Territory2Id", + "IsActive", + "RoleInTerritory2", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Remote_Meeting_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Meeting_Id_vod__c", + "Meeting_Name_vod__c", + "Mobile_ID_vod__c", + "Scheduled_DateTime_vod__c", + "Scheduled_vod__c", + "Attendance_Report_Process_Status_vod__c", + "Latest_Meeting_Start_Datetime_vod__c", + "Meeting_Password_vod__c", + "Meeting_Outcome_Status_vod__c", + "Allow_for_Joining_via_Zoom_vod__c", + "Zoom_Join_Token_vod__c", + "VExternal_Id_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "RecordType", + "columns": [ + "Id", + "Name", + "DeveloperName", + "NamespacePrefix", + "Description", + "BusinessProcessId", + "SobjectType", + "IsActive", + "IsPersonType", + "CreatedById", + "CreatedDate", + "LastModifiedById", + "LastModifiedDate", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "UserRole", + "columns": [ + "Id", + "Name", + "ParentRoleId", + "RollupDescription", + "OpportunityAccessForAccountOwner", + "CaseAccessForAccountOwner", + "ContactAccessForAccountOwner", + "ForecastUserId", + "MayForecastManagerShare", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "PortalAccountId", + "PortalType", + "PortalAccountOwnerId" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Account", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "Name", + "LastName", + "FirstName", + "Salutation", + "RecordTypeId", + "Phone", + "Fax", + "Website", + "PhotoUrl", + "NumberOfEmployees", + "Ownership", + "OwnerId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "IsExcludedFromRealign", + "PersonContactId", + "IsPersonAccount", + "PersonMailingStreet", + "PersonMailingCity", + "PersonMailingState", + "PersonMailingPostalCode", + "PersonMailingCountry", + "PersonMailingLatitude", + "PersonMailingLongitude", + "PersonMailingGeocodeAccuracy", + "PersonMailingAddress", + "PersonOtherStreet", + "PersonOtherCity", + "PersonOtherState", + "PersonOtherPostalCode", + "PersonOtherCountry", + "PersonOtherLatitude", + "PersonOtherLongitude", + "PersonOtherGeocodeAccuracy", + "PersonOtherAddress", + "PersonMobilePhone", + "PersonHomePhone", + "PersonOtherPhone", + "PersonAssistantPhone", + "PersonEmail", + "PersonTitle", + "PersonDepartment", + "PersonAssistantName", + "PersonBirthdate", + "PersonHasOptedOutOfEmail", + "PersonHasOptedOutOfFax", + "PersonDoNotCall", + "PersonLastCURequestDate", + "PersonLastCUUpdateDate", + "PersonEmailBouncedReason", + "PersonEmailBouncedDate", + "PersonIndividualId", + "Jigsaw", + "JigsawCompanyId", + "AccountSource", + "SicDesc", + "External_ID_vod__c", + "Credentials_vod__c", + "Territory_vod__c", + "Exclude_from_Zip_to_Terr_Processing_vod__c", + "Group_Specialty_1_vod__c", + "Group_Specialty_2_vod__c", + "Specialty_1_vod__c", + "Specialty_2_vod__c", + "Formatted_Name_vod__c", + "Territory_Test_vod__c", + "Mobile_ID_vod__c", + "Gender_vod__c", + "ID_vod__c", + "Do_Not_Sync_Sales_Data_vod__c", + "ID2_vod__c", + "Preferred_Name_vod__c", + "Sample_Default_vod__c", + "Segmentations_vod__c", + "Restricted_Products_vod__c", + "Payer_Id_vod__c", + "Alternate_Name_vod__c", + "Do_Not_Call_vod__c", + "MSJ_Beds__c", + "Spend_Amount__c", + "PDRP_Opt_Out_vod__c", + "Spend_Status_Value_vod__c", + "PDRP_Opt_Out_Date_vod__c", + "Spend_Status_vod__c", + "Enable_Restricted_Products_vod__c", + "Call_Reminder_vod__c", + "Account_Group_vod__c", + "Primary_Parent_vod__c", + "Color_vod__c", + "Middle_vod__c", + "Suffix_vod__c", + "MSJ_Type__c", + "No_Orders_vod__c", + "MSJ_BU_ONC__c", + "MSJ_BU_FE__c", + "Account_Search_FirstLast_vod__c", + "Account_Search_LastFirst_vod__c", + "MSJ_Operation__c", + "Practice_at_Hospital_vod__c", + "Practice_Near_Hospital_vod__c", + "Do_Not_Create_Child_Account_vod__c", + "Total_MDs_DOs__c", + "AHA__c", + "Order_Type_vod__c", + "NPI_vod__c", + "ME__c", + "Speaker__c", + "Investigator_vod__c", + "Default_Order_Type_vod__c", + "Tax_Status__c", + "Model__c", + "Offerings__c", + "Departments__c", + "Account_Type__c", + "MSJ_ONC_Tier__c", + "Account_Search_Business_vod__c", + "Business_Professional_Person_vod__c", + "Hospital_Type_vod__c", + "Account_Class_vod__c", + "Furigana_vod__c", + "MSJ_JISART__c", + "Total_Revenue_000__c", + "Net_Income_Loss_000__c", + "PMPM_Income_Loss_000__c", + "Commercial_Premiums_PMPM__c", + "Medical_Loss_Ratio__c", + "Medical_Expenses_PMPM__c", + "Commercial_Patient_Days_1000__c", + "HMO_Market_Shr__c", + "HMO__c", + "HMO_POS__c", + "PPO__c", + "PPO_POS__c", + "Medicare__c", + "Medicaid__c", + "MSJ_HP_Name_E__c", + "MSJ_Department__c", + "MSJ_Date_Of_Birth__c", + "MSJ_FE_GF_Potential__c", + "MSJ_FE_SZ_Potential__c", + "MSJ_EB_CRC_Ladder__c", + "MSJ_EB_CRC_Segment__c", + "MSJ_EB_HN_Segment__c", + "Business_Description__c", + "Regional_Strategy__c", + "Contracts_Process__c", + "MSJ_GF_segment__c", + "MSJ_DCF_DR_Code__c", + "MSJ_SZ_Segment__c", + "MSJ_Remark__c", + "MSJ_Title__c", + "MSJ_Role__c", + "MSJ_Kana__c", + "MSJ_Specialism__c", + "MSJ_Graduated_from__c", + "MSJ_Year_Graduation__c", + "Target__c", + "KOL_vod__c", + "MSJ_EPPV_Code__c", + "MSJ_DCF_HP_Code__c", + "Total_Lives__c", + "Total_Physicians_Enrolled__c", + "MSJ_Delete__c", + "MSJ_KOL_LOL__c", + "MSJ_ONC_Status__c", + "Account_Identifier_vod__c", + "Approved_Email_Opt_Type_vod__c", + "Language_vod__c", + "MSJ_KRAS_Routine_Date__c", + "MSJ_KRAS_Routine__c", + "MSJ_DRP_Target__c", + "MSJ_Fertility_Evaluation_Score__c", + "MSJ_Fertility_Tracking_Last_Modify_Date__c", + "Total_Pharmacists__c", + "MSJ_Number_of_Gonadotropin__c", + "MSJ_Number_of_IUI_cycle__c", + "MSJ_Number_of_OI_monthly_cycle__c", + "MSJ_OI_Protocol_learning_level__c", + "MSJ_H_N_Tier__c", + "MSJ_XLK_Segment__c", + "MSJ_XLK_Tier__c", + "Career_Status_vod__c", + "Photo_vod__c", + "MSJ_EB_H_N_LA_Segment__c", + "MSJ_EB_H_N_RM_Segment__c", + "MSJ_FE_CE_Potential__c", + "MSJ_FE_1C_potential__c", + "MSJ_FE_OV_potential__c", + "MSJ_FE_Tech_potential__c", + "MSJ_CE_segment__c", + "MSJ_1C_segment__c", + "MSJ_OV_segment__c", + "MSJ_Tech_segment__c", + "MSJ_Target_Call_Num__c", + "MSJ_DR_Change_Log__c", + "MSJ_Global_scientific_exposure__c", + "MSJ_H_index__c", + "MSJ_Num_of_Article_3Y__c", + "MSJ_Num_of_Article__c", + "MSJ_Num_of_Article_as_1st_Author_3Y__c", + "MSJ_Num_of_article_growth_rate_3Y__c", + "MSJ_Num_of_cited_3Y__c", + "MSJ_Num_of_impact_factor_3Y__c", + "MSJ_impact_factor_as_1st_Author_3Y__c", + "EMDS_Has_Pipeline_Opportunity__c", + "EMDS_Pipeline_Count__c", + "MSJ_BVC_Segment__c", + "MSJ_BVC_Tier__c", + "MSJ_BVC_AcctOpen__c", + "MSJ_BVC_MCC_Patients__c", + "MSJ_ONC_HP_Segment__c", + "MSJ_AE_Department__c", + "MSJ_AE_Facility__c", + "MSJ_AE_Name__c", + "MSJ_AE_Title__c", + "MSJ_Email__c", + "MSJ_FE_GF2_Potential__c", + "MSJ_FE_Location_potential__c", + "MSJ_GF2_segment__c", + "MSJ_OPTIN_target__c", + "MSJ_Merck_Specialty1__c", + "MSJ_Merck_Specialty2__c", + "MSJ_Marketing_Cloud_Integration__c", + "MSJ_Marketing_Cloud1__c", + "MSJ_Marketing_Cloud2__c", + "MSJ_Marketing_Cloud3__c", + "MSJ_Marketing_Cloud4__c", + "MSJ_Medical_Department__c", + "MSJ_Marketing_Cloud0__c", + "Mobile_ID_vod__pc", + "H1Insights__H1_NPI_Value_for_Testing__pc", + "H1Insights__H1_Person_ID__pc", + "H1Insights__H1_Request_Status__pc", + "H1Insights__H1_URL__pc", + "H1Insights__NPI_Number__pc", + "H1Insights__NPI_Number_for_H1_Insights__pc", + "MSJ_Marketing_Cloud_Integration__pc" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Type_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Header_vod__c", + "Channel_Label_vod__c", + "Channel_Source_vod__c", + "Consent_Expires_In_vod__c", + "Default_Consent_Type_vod__c", + "Disclaimer_Text_vod__c", + "Display_Order_vod__c", + "Product_Preference_vod__c", + "zvod_Consent_Default_Consent_Text_vod__c", + "zvod_Consent_Line_vod__c", + "zvod_Signature_Capture_vod__c", + "Double_Opt_In_vod__c", + "zvod_Consent_Activity_Tracking_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Header_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Header_Help_Text_vod__c", + "Country_vod__c", + "Inactive_Datetime_vod__c", + "Language_vod__c", + "Status_vod__c", + "Signature_Required_On_Opt_Out_vod__c", + "Request_Receipt_vod__c", + "Subscription_Option_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Line_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Type_vod__c", + "Detail_Group_Display_Name_vod__c", + "Detail_Group_vod__c", + "Display_Order_vod__c", + "End_Date_vod__c", + "Group_By_vod__c", + "Product_Display_Name_vod__c", + "Product_vod__c", + "Start_Date_vod__c", + "Sub_Channel_Description_vod__c", + "Sub_Channel_Display_Name_vod__c", + "Sub_Channel_Key_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Inquiry_Assignment__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Medical_Inquiry__c", + "MSJ_Close__c", + "MSJ_Doctor_Name__c", + "MSJ_Hospital_Name__c", + "MSJ_Indication__c", + "MSJ_Inquiry_Text__c", + "MSJ_MEC_User__c", + "MSJ_MSL_Manager__c", + "MSJ_MSL_User__c", + "MSJ_Notice_to_MR__c", + "MSJ_Product_for_MEC__c", + "MSJ_Product_for_MR__c", + "MSJ_Reply_Date__c", + "MSJ_Reply__c", + "MSJ_AE_Infomation__c", + "MSJ_Cancel__c", + "MSJ_FAQ_number_c__c", + "MSJ_Return_Call__c", + "MSJ_Inquiry_Origin__c", + "First_Response__c", + "Inquiry_Created_Date__c", + "Inquiry_Type_1__c", + "Inquiry_Type_2__c", + "MSJ_First_User__c", + "MSJ_MEC_Comment__c", + "MSJ_Send_Email__c", + "MSJ_Temp_Aggregated_Info__c", + "MSJ_AE_Report__c", + "MSJ_Background__c", + "MSJ_Inquiry_Date__c", + "MSJ_MSL_Support__c", + "MSJ_Handover_Comment__c", + "MSJ_Handover_Email__c", + "MSJ_Material_Requirement__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Approved_Document_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Detail_Group_vod__c", + "Document_Description_vod__c", + "Document_Host_URL_vod__c", + "Document_ID_vod__c", + "Document_Last_Mod_DateTime_vod__c", + "Email_Allows_Documents_vod__c", + "Email_Domain_vod__c", + "Email_Fragment_HTML_vod__c", + "Email_From_Address_vod__c", + "Email_From_Name_vod__c", + "Email_HTML_1_vod__c", + "Email_HTML_2_vod__c", + "Email_ReplyTo_Address_vod__c", + "Email_ReplyTo_Name_vod__c", + "Email_Subject_vod__c", + "Email_Template_Fragment_Document_ID_vod__c", + "Email_Template_Fragment_HTML_vod__c", + "ISI_Document_ID_vod__c", + "Language_vod__c", + "Other_Document_ID_List_vod__c", + "PI_Document_ID_vod__c", + "Piece_Document_ID_vod__c", + "Product_vod__c", + "Status_vod__c", + "Territory_vod__c", + "Vault_Instance_ID_vod__c", + "Allow_Any_Product_Fragment_vod__c", + "Allowed_Document_IDs_vod__c", + "Engage_Document_Id_vod__c", + "Vault_Document_ID_vod__c", + "Key_Message_vod__c", + "Events_Management_Subtype_vod__c", + "Survey_vod__c", + "Content_Type_vod__c", + "Bcc_vod__c", + "Audience_vod__c", + "WeChat_Template_ID_vod__c", + "Check_Consent_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Child_Account_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Parent_Account_vod__c", + "Child_Account_vod__c", + "External_ID_vod__c", + "Mobile_ID_vod__c", + "Primary_vod__c", + "Copy_Address_vod__c", + "Child_Name_vod__c", + "Parent_Name_vod__c", + "Parent_Child_Name_vod__c", + "Account_Code__c", + "Child_Department__c", + "Child_Role__c", + "Child_Title__c", + "Child_Remark__c", + "MSJ_1C_segment__c", + "MSJ_BU_FE__c", + "MSJ_BU_ONC__c", + "MSJ_BVC_Segment__c", + "MSJ_CE_segment__c", + "MSJ_Child_Account_Link__c", + "MSJ_DCF_DR_Code__c", + "MSJ_DCF_HP_Code__c", + "MSJ_DR_Change_Log__c", + "MSJ_Delete__c", + "MSJ_Department__c", + "MSJ_EB_CRC_Segment__c", + "MSJ_EB_HN_Segment__c", + "MSJ_EB_H_N_LA_Segment__c", + "MSJ_EB_H_N_RM_Segment__c", + "MSJ_External_ID__c", + "MSJ_Fax__c", + "MSJ_GF2_segment__c", + "MSJ_GF_segment__c", + "MSJ_KOL_LOL__c", + "MSJ_KOL__c", + "MSJ_ONC_HP_Segment__c", + "MSJ_OPTIN_target__c", + "MSJ_OV_segment__c", + "MSJ_Parent_Child_Name__c", + "MSJ_Phone__c", + "MSJ_Remark__c", + "MSJ_Target_Call_Num__c", + "MSJ_Tech_segment__c", + "MSJ_Title__c", + "MSJ_XLK_Segment__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Hospital_Medical_Regimen__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Account_Name__c", + "MSJ_Delete_Date__c", + "MSJ_Delete_Flag__c", + "MSJ_Indication__c", + "MSJ_Line__c", + "MSJ_Medical_Regimen__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Medical_Regimen__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Delete_Date__c", + "MSJ_Delete_Flag__c", + "MSJ_Indication__c", + "MSJ_Remark__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Patient__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Account_Name__c", + "MSJ_CRC_Group__c", + "MSJ_Casus_or_Transfer_Point__c", + "MSJ_Entry_Date__c", + "MSJ_IST_Name__c", + "MSJ_Indication__c", + "MSJ_Line__c", + "MSJ_MR_Comments__c", + "MSJ_MUID__c", + "MSJ_Medical_Regimen__c", + "MSJ_Month__c", + "MSJ_Report_Comments__c", + "MSJ_Start_Date_Of_Administration__c", + "MSJ_Year__c", + "Mobile_ID_vod__c", + "MSJ_CRC_RAS_KRAS__c", + "MSJ_End_Date_Of_Administration__c", + "MSJ_End_Date_of_Stop_Administration__c", + "MSJ_HN_Hospitalized_Type__c", + "MSJ_Start_Date_of_Stop_Administration__c", + "MSJ_Patient_Status__c", + "MSJ_Patient_TA__c", + "MSJ_Child_Account_Name__c", + "MSJ_Child_Account__c", + "MSJ_Parent_Account_Name__c", + "MSJ_Parent_Child_Name__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consumer_site__c", + "Product_info__c", + "Therapeutic_Class_vod__c", + "Parent_Product_vod__c", + "Therapeutic_Area_vod__c", + "Product_Type_vod__c", + "Require_Key_Message_vod__c", + "Cost_vod__c", + "External_ID_vod__c", + "Manufacturer_vod__c", + "Company_Product_vod__c", + "Controlled_Substance_vod__c", + "Description_vod__c", + "Sample_Quantity_Picklist_vod__c", + "Display_Order_vod__c", + "No_Metrics_vod__c", + "Distributor_vod__c", + "Sample_Quantity_Bound_vod__c", + "Sample_U_M_vod__c", + "No_Details_vod__c", + "Quantity_Per_Case_vod__c", + "Schedule_vod__c", + "Restricted_vod__c", + "Pricing_Rule_Quantity_Bound_vod__c", + "No_Promo_Items_vod__c", + "User_Aligned_vod__c", + "Restricted_States_vod__c", + "Sort_Code_vod__c", + "No_Cycle_Plans_vod__c", + "Inventory_Order_UOM_vod__c", + "Inventory_Quantity_Per_Case_vod__c", + "VExternal_Id_vod__c", + "Country__c", + "MSJ_Product_Classification__c", + "MSJ_Indication__c", + "MSJ_Therapeutic_Area__c", + "MSJ_Global_Brand__c", + "MSJ_Global_Business_Unit__c", + "MSJ_Molecules__c", + "MSJ_SBU__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_Group_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Description_vod__c", + "Product_vod__c", + "Product_Catalog_vod__c", + "Start_Date_vod__c", + "End_Date_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_Metrics_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Awareness__c", + "Selling_Stage__c", + "Formulary_Status__c", + "Movement__c", + "Products_vod__c", + "Segment__c", + "X12_mo_trx_chg__c", + "Speaker_Skills__c", + "Investigator_Readiness__c", + "Engagements__c", + "Mobile_ID_vod__c", + "External_ID_vod__c", + "MSJ_Patient__c", + "Detail_Group_vod__c", + "MSJ_EB_1st_Line_Liver_Meta__c", + "MSJ_EB_1st_Line_Multi_Meta__c", + "MSJ_EB_2nd_Line_Mono__c", + "MSJ_EB_2nd_Line_Combination__c", + "MSJ_EB_3rd_Line_Mono__c", + "MSJ_EB_3rd_Line_Combination__c", + "EMDS_Ability__c", + "EMDS_Brand_Loyalty__c", + "EMDS_Decision_Maker__c", + "EMDS_Early_Tech_Adopter__c", + "EMDS_Influence__c", + "EMDS_Main_Driver__c", + "EMDS_Priority__c", + "EMDS_Willingness__c", + "MSJ_KTL_Type__c", + "MSJ_KTL_Tier__c", + "MSJ_Publications__c", + "MSJ_Clinical_Trials__c", + "MSJ_Speaker_for_Medical_Events__c", + "MSJ_Advisor_to_Medical_Affairs__c", + "MSJ_Guidelines_Treatment_Standards__c", + "MSJ_Therapeutic_Area_Expertise__c", + "MSJ_MAP_GAP__c", + "MSJ_Associations__c", + "MSJ_Tier_Score__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Assignment_Type_vod__c", + "Channels_vod__c", + "End_Date_vod__c", + "Expired_vod__c", + "External_ID_vod__c", + "Language_vod__c", + "Lock_vod__c", + "Open_vod__c", + "Product_vod__c", + "Region_vod__c", + "Segment_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "Territory_vod__c", + "zvod_Questions_vod__c", + "zvod_Segments_vod__c", + "zvod_Targets_vod__c", + "Max_Score_vod__c", + "Min_Score_vod__c", + "Autotarget_vod__c", + "Territories_vod__c", + "Target_Type_vod__c", + "MSJ_External_ID__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_Target_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_Display_Name_vod__c", + "Account_vod__c", + "Channels_vod__c", + "End_Date_vod__c", + "Entity_Reference_Id_vod__c", + "External_ID_vod__c", + "Language_vod__c", + "Lock_vod__c", + "Mobile_ID_vod__c", + "No_Autoassign_vod__c", + "Not_Completed_vod__c", + "Region_vod__c", + "Segment_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "Survey_vod__c", + "Territory_vod__c", + "zvod_Address_vod__c", + "zvod_Specialty_vod__c", + "Score_vod__c", + "User_vod__c", + "Child_Account_vod__c", + "Location_Entity_Reference_Id_vod__c", + "Location_vod__c", + "Target_Type_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_Question_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Survey_vod__c", + "Answer_Choice_vod__c", + "External_ID_vod__c", + "Max_Score_vod__c", + "Min_Score_vod__c", + "Order_vod__c", + "Question_vod__c", + "Required_vod__c", + "Text_vod__c", + "Condition_vod__c", + "Source_ID_vod__c", + "MSJ_External_ID__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Question_Response_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Survey_Target_vod__c", + "Answer_Choice_vod__c", + "Date_vod__c", + "Datetime_vod__c", + "External_ID_vod__c", + "Mobile_ID_vod__c", + "Number_vod__c", + "Order_vod__c", + "Question_Text_vod__c", + "Required_vod__c", + "Response_Hash_vod__c", + "Response_vod__c", + "Score_vod__c", + "Survey_Question_vod__c", + "Text_vod__c", + "Type_vod__c", + "Condition_vod__c", + "Inactive_Condition_vod__c", + "Source_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Sent_Fragment_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Sent_Email_vod__c", + "Account_vod__c", + "Email_Template_vod__c", + "Sent_Fragment_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Sent_Email_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Account_Email_vod__c", + "Account_vod__c", + "Approved_Email_Template_vod__c", + "Capture_Datetime_vod__c", + "Detail_Group_vod__c", + "Email_Config_Values_vod__c", + "Email_Content2_vod__c", + "Email_Content_vod__c", + "Email_Fragments_vod__c", + "Email_Sent_Date_vod__c", + "Failure_Msg_vod__c", + "Last_Activity_Date_vod__c", + "Last_Device_vod__c", + "MC_Capture_Datetime_vod__c", + "Mobile_ID_vod__c", + "Opened_vod__c", + "Product_Display_vod__c", + "Product_vod__c", + "Sender_Email_vod__c", + "Status_vod__c", + "Valid_Consent_Exists_vod__c", + "Approved_Document_Views_vod__c", + "Click_Count_vod__c", + "Last_Click_Date_vod__c", + "Last_Open_Date_vod__c", + "Open_Count_vod__c", + "Receipt_Entity_Type_vod__c", + "Receipt_Record_Id_vod__c", + "Territory_vod__c", + "Call2_vod__c", + "Medical_Inquiry_vod__c", + "Parent_Email_vod__c", + "Related_Transaction_ID_vod__c", + "Case_vod__c", + "Key_Message_vod__c", + "Suggestion_vod__c", + "EM_Attendee_vod__c", + "EM_Event_Speaker_vod__c", + "EM_Event_Team_Member_vod__c", + "Event_Attendee_vod__c", + "Event_vod__c", + "Medical_Event_vod__c", + "Scheduled_Send_Datetime_vod__c", + "User_vod__c", + "Content_Type_vod__c", + "Bcc_vod__c", + "Event_Attendee_Mobile_Id_vod__c", + "Event_Mobile_Id_vod__c", + "Activity_Tracking_Mode_vod__c", + "Email_Source_vod__c", + "Subject_vod__c", + "User_Input_Text_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + } ] - } - - \ No newline at end of file +} \ No newline at end of file From afcca943ca451fc1eaa34c7762d6d2f239d88b76 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 28 Jul 2022 22:43:32 +0900 Subject: [PATCH 14/21] =?UTF-8?q?fix:=20=E3=82=BD=E3=83=BC=E3=82=B9?= =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=89=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E3=81=AE=E8=BF=BD=E5=8A=A0=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/aws/s3.py | 32 ++-- .../src/backup_crm_csv_data_process.py | 27 +--- .../src/backup_crm_data_process.py | 25 +-- .../src/check_object_info_process.py | 26 ++-- ecs/crm-datafetch/src/config/objects.py | 60 +++---- ecs/crm-datafetch/src/controller.py | 59 +++---- .../src/convert_crm_csv_data_process.py | 29 ++-- .../src/converter/convert_factory.py | 29 ++-- ecs/crm-datafetch/src/converter/converter.py | 3 +- .../src/copy_crm_csv_data_process.py | 25 +-- .../src/fetch_crm_data_process.py | 146 +++++++++--------- ecs/crm-datafetch/src/parser/json_parse.py | 2 +- .../src/prepare_data_fetch_process.py | 36 ++--- .../src/salesforce/salesforce_api.py | 12 +- .../src/set_datetime_period_process.py | 36 ++--- .../src/system_var/environments.py | 30 ++-- .../src/upload_last_fetch_datetime_process.py | 24 +-- .../src/upload_result_data_process.py | 22 +-- ecs/crm-datafetch/src/util/counter_object.py | 14 ++ ecs/crm-datafetch/src/util/dict_checker.py | 2 +- .../src/util/execute_datetime.py | 5 +- ecs/crm-datafetch/src/util/logger.py | 2 +- .../crm/last_fetch_datetime/Account.json | 4 + .../crm/last_fetch_datetime/AccountShare.json | 4 + .../Account_Territory_Loader_vod__c.json | 4 + .../Approved_Document_vod__c.json | 4 + .../Call2_Detail_vod__c.json | 4 + .../Call2_Discussion_vod__c.json | 4 + .../Call2_Key_Message_vod__c.json | 4 + .../crm/last_fetch_datetime/Call2_vod__c.json | 4 + .../Call_Clickstream_vod__c.json | 4 + .../Child_Account_vod__c.json | 4 + .../Clm_Presentation_Slide_vod__c.json | 4 + .../Clm_Presentation_vod__c.json | 4 + .../Coaching_Report_vod__c.json | 4 + .../Consent_Header_vod__c.json | 4 + .../Consent_Line_vod__c.json | 4 + .../Consent_Type_vod__c.json | 4 + .../crm/last_fetch_datetime/Contact.json | 4 + ...ynamic_Attribute_Configuration_vod__c.json | 4 + .../Dynamic_Attribute_vod__c.json | 4 + .../Email_Activity_vod__c.json | 4 + .../Event_Attendee_vod__c.json | 4 + s3/config/crm/last_fetch_datetime/Group.json | 4 + .../Key_Message_vod__c.json | 4 + .../MSJ_Hospital_Medical_Regimen__c.json | 4 + .../MSJ_Inquiry_Assignment__c.json | 4 + .../MSJ_MR_Weekly_Report__c.json | 4 + .../MSJ_Medical_Event_Evaluation__c.json | 4 + .../MSJ_Medical_Regimen__c.json | 4 + .../last_fetch_datetime/MSJ_Patient__c.json | 4 + .../Medical_Event_vod__c.json | 4 + .../Medical_Inquiry_vod__c.json | 4 + .../Medical_Insight_vod__c.json | 4 + .../Multichannel_Activity_Line_vod__c.json | 4 + .../Multichannel_Activity_vod__c.json | 4 + .../Multichannel_Consent_vod__c.json | 4 + .../My_Setup_Products_vod__c.json | 4 + .../ObjectTerritory2Association.json | 4 + .../Product_Group_vod__c.json | 4 + .../Product_Metrics_vod__c.json | 4 + .../last_fetch_datetime/Product_vod__c.json | 4 + .../crm/last_fetch_datetime/Profile.json | 4 + .../Question_Response_vod__c.json | 4 + .../crm/last_fetch_datetime/RecordType.json | 4 + .../Remote_Meeting_vod__c.json | 4 + .../Sent_Email_vod__c.json | 4 + .../Sent_Fragment_vod__c.json | 4 + .../Survey_Question_vod__c.json | 4 + .../Survey_Target_vod__c.json | 4 + .../last_fetch_datetime/Survey_vod__c.json | 4 + .../crm/last_fetch_datetime/Territory2.json | 4 + .../Time_Off_Territory_vod__c.json | 4 + s3/config/crm/last_fetch_datetime/User.json | 4 + .../crm/last_fetch_datetime/UserRole.json | 4 + .../UserTerritory2Association.json | 4 + .../crm/object_info/crm_object_list_all.json | 4 +- .../crm/object_info/crm_object_list_diff.json | 3 +- 78 files changed, 491 insertions(+), 378 deletions(-) create mode 100644 ecs/crm-datafetch/src/util/counter_object.py create mode 100644 s3/config/crm/last_fetch_datetime/Account.json create mode 100644 s3/config/crm/last_fetch_datetime/AccountShare.json create mode 100644 s3/config/crm/last_fetch_datetime/Account_Territory_Loader_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Approved_Document_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Call2_Detail_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Call2_Discussion_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Call2_Key_Message_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Call2_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Call_Clickstream_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Child_Account_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Clm_Presentation_Slide_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Clm_Presentation_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Coaching_Report_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Consent_Header_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Consent_Line_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Consent_Type_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Contact.json create mode 100644 s3/config/crm/last_fetch_datetime/Dynamic_Attribute_Configuration_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Dynamic_Attribute_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Email_Activity_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Event_Attendee_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Group.json create mode 100644 s3/config/crm/last_fetch_datetime/Key_Message_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/MSJ_Hospital_Medical_Regimen__c.json create mode 100644 s3/config/crm/last_fetch_datetime/MSJ_Inquiry_Assignment__c.json create mode 100644 s3/config/crm/last_fetch_datetime/MSJ_MR_Weekly_Report__c.json create mode 100644 s3/config/crm/last_fetch_datetime/MSJ_Medical_Event_Evaluation__c.json create mode 100644 s3/config/crm/last_fetch_datetime/MSJ_Medical_Regimen__c.json create mode 100644 s3/config/crm/last_fetch_datetime/MSJ_Patient__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Medical_Inquiry_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Medical_Insight_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Multichannel_Activity_Line_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Multichannel_Activity_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Multichannel_Consent_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/My_Setup_Products_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/ObjectTerritory2Association.json create mode 100644 s3/config/crm/last_fetch_datetime/Product_Group_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Product_Metrics_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Product_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Profile.json create mode 100644 s3/config/crm/last_fetch_datetime/Question_Response_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/RecordType.json create mode 100644 s3/config/crm/last_fetch_datetime/Remote_Meeting_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Sent_Email_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Sent_Fragment_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Survey_Question_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Survey_Target_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Survey_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/Territory2.json create mode 100644 s3/config/crm/last_fetch_datetime/Time_Off_Territory_vod__c.json create mode 100644 s3/config/crm/last_fetch_datetime/User.json create mode 100644 s3/config/crm/last_fetch_datetime/UserRole.json create mode 100644 s3/config/crm/last_fetch_datetime/UserTerritory2Association.json diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index e1d0dc70..d5bd03f0 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -1,13 +1,17 @@ import json import boto3 -from src.system_var.constants import AWS_RESOURCE_S3, S3_RESPONSE_BODY, S3_CHAR_CODE +from src.system_var.constants import (AWS_RESOURCE_S3, S3_CHAR_CODE, + S3_RESPONSE_BODY) from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, - CRM_IMPORT_DATA_BACKUP_FOLDER, - CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET, - LAST_FETCH_DATE_FOLDER, OBJECT_INFO_FILENAME, - OBJECT_INFO_FOLDER, PROCESS_RESULT_FOLDER, - RESPONSE_JSON_BACKUP_FOLDER) + CRM_IMPORT_DATA_BACKUP_FOLDER, + CRM_IMPORT_DATA_FOLDER, + IMPORT_DATA_BUCKET, + LAST_FETCH_DATE_FOLDER, + OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER, + PROCESS_RESULT_FOLDER, + RESPONSE_JSON_BACKUP_FOLDER) class S3Resource: @@ -31,16 +35,6 @@ class S3Resource: return -#class S3ResourceNonBucket: -# def __init__(self) -> None: -# self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) -# -# def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: -# copy_source = {'Bucket': src_bucket, 'Key': src_key} -# self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) -# return - - class ConfigBucket: __s3_resource: S3Resource = None @@ -77,11 +71,11 @@ class DataBucket: return def put_csv_from(self, src_bucket: str, src_key: str): - self.__s3_resource.copy(src_bucket, src_key, str(self), CRM_IMPORT_DATA_FOLDER) + dest_filename = src_key.split('/')[-1] + self.__s3_resource.copy(src_bucket, src_key, str(self), f'{CRM_IMPORT_DATA_FOLDER}/{dest_filename}') return - class BackupBucket: __s3_resource: S3Resource = None @@ -104,4 +98,4 @@ class BackupBucket: def put_result_json(self, file_path: str, data: dict) -> None: object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, json.dumps(data)) - return \ No newline at end of file + return diff --git a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py index f14490cf..d40c7a13 100644 --- a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py @@ -1,32 +1,21 @@ from src.aws.s3 import BackupBucket from src.config.objects import TargetObject -from src.system_var.constants import CSVBK_JP_NAME from src.error.exceptions import FileUploadException +from src.system_var.constants import CSVBK_JP_NAME from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger def backup_crm_csv_data_process(target_object: TargetObject, exetute_datetime: ExecuteDateTime, csv_string: str): - """ - CSVバックアップ処理 + """CSVバックアップ処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス - csv_string : str - csvデータ + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + exetute_datetime (ExecuteDateTime): 実行日次取得インスタンス + csv_string (str): csvデータ - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① CSVバックアップ処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/backup_crm_data_process.py b/ecs/crm-datafetch/src/backup_crm_data_process.py index a37eb583..0375d927 100644 --- a/ecs/crm-datafetch/src/backup_crm_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_data_process.py @@ -6,26 +6,15 @@ from src.util.logger import logger_instance as logger def backup_crm_data_process(object_name: str, sf_object_dict: dict, execute_datetime: ExecuteDateTime): - """ - CRM電文データバックアップ処理 + """CRM電文データバックアップ処理 - Parameters - ---------- - object_name : str - 取得対象オブジェクト情報インスタンス - sf_object_dict : dict - Salesforceオブジェクトデータ - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス + Args: + object_name (str): 取得対象オブジェクト名 + sf_object_dict (dict): Salesforceオブジェクトデータ + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① CRM電文データバックアップ処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/check_object_info_process.py b/ecs/crm-datafetch/src/check_object_info_process.py index ed2a34da..84f0db40 100644 --- a/ecs/crm-datafetch/src/check_object_info_process.py +++ b/ecs/crm-datafetch/src/check_object_info_process.py @@ -1,30 +1,22 @@ from src.config.objects import TargetObject from src.error.exceptions import InvalidConfigException -from src.util.execute_datetime import ExecuteDateTime from src.system_var.constants import CHK_JP_NAME +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger def check_object_info_process(object_info: dict, execute_datetime: ExecuteDateTime): - """ - オブジェクト情報形式チェック処理 + """オブジェクト情報形式チェック処理 - Parameters - ---------- - object_info : dict - 取得対象オブジェクト情報 - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス + Args: + object_info (dict): 取得対象オブジェクト情報 + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス - Returns - ------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス + Raises: + InvalidConfigException: オブジェクト情報定義が不正だった場合 - Raises - ------ - InvalidConfigException - オブジェクト情報定義が不正だった場合 + Returns: + target_object: 取得対象オブジェクト情報インスタンス """ # ① オブジェクト情報形式チェック処理開始ログを出力する diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 464f7b0e..3ac08851 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,25 +1,19 @@ -from src.system_var.constants import ( DATE_PATTERN_YYYYMMDDTHHMMSSTZ, - OBJECTS_KEY, - OBJECTS_TYPE, - OBJECT_NAME_KEY, - OBJECT_NAME_TYPE, - COLUMNS_KEY, - COLUMNS_TYPE, - IS_SKIP_KEY, - IS_SKIP_TYPE, - IS_UPDATE_LAST_FETCH_DATETIME_KEY, - IS_UPDATE_LAST_FETCH_DATETIME_TYPE, - LAST_FETCH_DATETIME_FILE_NAME_KEY, - LAST_FETCH_DATETIME_FILE_NAME_TYPE, - UPLOAD_FILE_NAME_KEY, - UPLOAD_FILE_NAME_TYPE, - DATETIME_COLUMN_KEY, - DATETIME_COLUMN_TYPE, - LAST_FETCH_DATETIME_FROM_KEY, - LAST_FETCH_DATETIME_TO_KEY, - DATETIME_COLUMN_DEFAULT_VALUE -) - +from src.system_var.constants import (COLUMNS_KEY, COLUMNS_TYPE, + DATE_PATTERN_YYYYMMDDTHHMMSSTZ, + DATETIME_COLUMN_DEFAULT_VALUE, + DATETIME_COLUMN_KEY, + DATETIME_COLUMN_TYPE, IS_SKIP_KEY, + IS_SKIP_TYPE, + IS_UPDATE_LAST_FETCH_DATETIME_KEY, + IS_UPDATE_LAST_FETCH_DATETIME_TYPE, + LAST_FETCH_DATETIME_FILE_NAME_KEY, + LAST_FETCH_DATETIME_FILE_NAME_TYPE, + LAST_FETCH_DATETIME_FROM_KEY, + LAST_FETCH_DATETIME_TO_KEY, + OBJECT_NAME_KEY, OBJECT_NAME_TYPE, + OBJECTS_KEY, OBJECTS_TYPE, + UPLOAD_FILE_NAME_KEY, + UPLOAD_FILE_NAME_TYPE) from src.util.dict_checker import DictChecker @@ -98,16 +92,24 @@ class TargetObject(): @property def is_update_last_fetch_datetime(self) -> bool: - return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY) else False + if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY): + return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] + else: + return False @property def last_fetch_datetime_file_name(self) -> str: - return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY] if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}.json' - + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY): + return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY] + else: + return f'{self.__object_info[OBJECT_NAME_KEY]}.json' @property def upload_file_name(self) -> str: - return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' + if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): + return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) + else: + return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' @property def datetime_column(self) -> str: @@ -126,7 +128,6 @@ class LastFetchDatetime(): self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) - return @property @@ -135,4 +136,7 @@ class LastFetchDatetime(): @property def last_fetch_datetime_to(self) -> str: - return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FROM_KEY) else self.__execute_datetime + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): + return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY] + else: + return self.__execute_datetime diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index acbd2383..3c92277b 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -7,13 +7,14 @@ from src.config.objects import FetchTargetObjects from src.convert_crm_csv_data_process import convert_crm_csv_data_process from src.copy_crm_csv_data_process import copy_crm_csv_data_process from src.error.exceptions import MeDaCaCRMDataFetchException -from src.util.execute_datetime import ExecuteDateTime from src.fetch_crm_data_process import fetch_crm_data_process from src.prepare_data_fetch_process import prepare_data_fetch_process from src.set_datetime_period_process import set_datetime_period_process from src.system_var.constants import OBJECT_NAME_KEY -from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime_process +from src.upload_last_fetch_datetime_process import \ + upload_last_fetch_datetime_process from src.upload_result_data_process import upload_result_data_process +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger @@ -63,29 +64,22 @@ def controller() -> None: def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: ExecuteDateTime, process_result: dict): + """取得対象オブジェクト情報をループし、１オブジェクトごとのデータを取得する + + Args: + fetch_target_objects (FetchTargetObjects): CRMオブジェクト情報インスタンス + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス + process_result (dict): 取得処理実行結果辞書オブジェクト + + Returns: + process_result: 取得処理実行結果辞書オブジェクト """ - 取得対象オブジェクト情報をループし、１オブジェクトごとのデータを取得する - Parameters - ---------- - fetch_target_objects : FetchTargetObjects - CRMオブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス - process_result : dict - 取得処理実行結果辞書オブジェクト - - Returns - ------- - process_result : dict - 取得処理実行結果辞書オブジェクト - - """ for object_info in fetch_target_objects: try: process_result[object_info.get(OBJECT_NAME_KEY)] = 'fail' - fetch_crm_data_per_object(object_info, execute_datetime, process_result) + fetch_crm_data_per_object(object_info, execute_datetime) process_result[object_info.get(OBJECT_NAME_KEY)] = 'success' @@ -102,28 +96,13 @@ def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: E return process_result + def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTime) -> None: - """ - オブジェクトごとにCRMのデータを取得し、取込フォルダにアップロードする - - Parameters - ---------- - object_info : dict - 取得対象オブジェクト情報 - execute_datetime : object - 実行日次取得インスタンス - - Returns - ------- - なし - - Raises - ------ - FileNotFoundException - S3上のファイルが存在しない場合 - InvalidConfigException - オブジェクト情報定義が不正だった場合 + """オブジェクトごとにCRMのデータを取得し、取込フォルダにアップロードする + Args: + object_info (dict): 取得対象オブジェクト情報 + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス """ # 1. オブジェクト処理結果の初期化 @@ -194,4 +173,4 @@ def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTi # 14. オブジェクトのアップロードが完了した旨をログに出力する logger.info(f'I-CTRL-16 [{target_object_name}] 処理正常終了') - return \ No newline at end of file + return diff --git a/ecs/crm-datafetch/src/convert_crm_csv_data_process.py b/ecs/crm-datafetch/src/convert_crm_csv_data_process.py index 1ed05c3a..fd6138b1 100644 --- a/ecs/crm-datafetch/src/convert_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/convert_crm_csv_data_process.py @@ -6,25 +6,17 @@ from src.util.logger import logger_instance as logger def convert_crm_csv_data_process(target_object: TargetObject, crm_data_response: dict): - """ - CSV変換処理 + """CSV変換処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - crm_data_response : dict - Salesforceオブジェクトデータ + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + crm_data_response (dict): Salesforceオブジェクトデータ - Returns - ------- - csv_string : str - csvデータ + Raises: + DataConvertException: データ変換が失敗した場合 - Raises - ------ - DataConvertException - データ変換が失敗した場合 + Returns: + csv_string: csvデータ """ # ① CSV変換処理の開始ログを出力する @@ -34,9 +26,8 @@ def convert_crm_csv_data_process(target_object: TargetObject, crm_data_response: try: # ② CSV変換 - CSVStringConverter(target_object, crm_data_response) - csv_string = CSVStringConverter.convert() - + csv_string_converter = CSVStringConverter(target_object, crm_data_response) + csv_string = csv_string_converter.convert() logger.debug(f'D-CONV-02 [{target_object_name}] のCSV変換処理 正常終了') diff --git a/ecs/crm-datafetch/src/converter/convert_factory.py b/ecs/crm-datafetch/src/converter/convert_factory.py index 48673f7e..bf949a6e 100644 --- a/ecs/crm-datafetch/src/converter/convert_factory.py +++ b/ecs/crm-datafetch/src/converter/convert_factory.py @@ -1,9 +1,10 @@ import re from datetime import datetime -from src.config.objects import TargetObject from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, - CSV_TRUE_VALUE, YYYYMMDDHHMMSS, DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC) + CSV_TRUE_VALUE, + DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, + YYYYMMDDHHMMSS) class ConvertStrategyFactory: @@ -12,30 +13,30 @@ class ConvertStrategyFactory: self.__float_convert_strategy = FloatConvertStrategy() self.__boolean_convert_strategy = BooleanConvertStrategy() self.__datetime_convert_strategy = DatatimeConvertStrategy() + self.__non_convert_strategy = NonConvertStrategy() def create(self, value): - converted_value = value - if value is None: - converted_value = self.__none_value_convert_strategy.convert_value() + convert_strategy = self.__none_value_convert_strategy - # 指数表記で取得できるパターン。指数表記を整数表記に変換する。 elif type(value) == float: - converted_value = self.__float_convert_strategy.convert_value(value) + convert_strategy = self.__float_convert_strategy - # SQLの真偽値に対応するために変換する elif type(value) == bool: - converted_value = self.__boolean_convert_strategy.convert_value(value) + convert_strategy = self.__boolean_convert_strategy elif type(value) == str and re.fullmatch(DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, value): - converted_value = self.__datetime_convert_strategy.convert_value(value) + convert_strategy = self.__datetime_convert_strategy - return converted_value + else: + convert_strategy = self.__non_convert_strategy + + return convert_strategy class NoneValueConvertStrategy: - def convert_value(self) -> str: + def convert_value(self, convert_value: None) -> str: return '' @@ -53,3 +54,7 @@ class FloatConvertStrategy: def convert_value(self, convert_value: str) -> int: return int(convert_value) + +class NonConvertStrategy: + def convert_value(self, convert_value: str): + return convert_value diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index bf7faaee..9c223182 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -52,7 +52,8 @@ class CSVStringConverter: for column in columns: v = json_object[column.upper()] - converted_value = self.__convert_strategy_factory.create(v) + convert_strategy = self.__convert_strategy_factory.create(v) + converted_value = convert_strategy.convert_value(v) csv_row.append(converted_value) diff --git a/ecs/crm-datafetch/src/copy_crm_csv_data_process.py b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py index 57a3659f..ba2831ea 100644 --- a/ecs/crm-datafetch/src/copy_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py @@ -2,31 +2,20 @@ from src.aws.s3 import BackupBucket, DataBucket from src.config.objects import TargetObject from src.error.exceptions import FileUploadException from src.system_var.constants import UPLD_JP_NAME -from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, - CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) +from src.system_var.environments import CRM_IMPORT_DATA_BACKUP_FOLDER from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger def copy_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime): - """ - CSVアップロード処理 + """CSVアップロード処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① CSVデータアップロード処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/fetch_crm_data_process.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py index 3a89da1e..d22efb7e 100644 --- a/ecs/crm-datafetch/src/fetch_crm_data_process.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -2,102 +2,91 @@ from requests.exceptions import ConnectTimeout, ReadTimeout from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential -from src.config.objects import TargetObject, LastFetchDatetime +from src.config.objects import LastFetchDatetime, TargetObject from src.error.exceptions import DataConvertException, SalesforceAPIException from src.salesforce.salesforce_api import SalesforceApiClient from src.salesforce.soql_builder import SOQLBuilder from src.system_var.constants import FETCH_JP_NAME -from src.system_var.environments import (CRM_AUTH_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, - CRM_GET_RECORD_COUNT_TIMEOUT) +from src.system_var.environments import ( + CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, CRM_FETCH_RECORD_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_TIMEOUT) +from src.util.counter_object import CounterObject from src.util.logger import logger_instance as logger def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): - """ - CRMデータ取得処理 + """CRMデータ取得処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - last_fetch_datetime : LastFetchDatetime - 取得対象オブジェクト情報インスタンス + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + last_fetch_datetime (LastFetchDatetime): データ取得期間設定インスタンス - Returns - ------- - sf_object_dict : dict - Salesforceオブジェクトデータ + Raises: + SalesforceAPIException: SalseforceのAPI実行失敗が発生した場合 + DataConvertException: データ変換が失敗した場合 - Raises - ------ - SalesforceAPIException - SalseforceのAPI実行失敗が発生した場合 - DataConvertException - データ変換が失敗した場合 + Returns: + crm_data_response: Salesforceオブジェクトデータ """ # ① CRMデータ取得処理開始ログを出力する logger.info( f'I-FETCH-01 [{target_object.object_name}] のCRMからのデータ取得処理を開始します') - object_name = target_object.object_name + target_object_name = target_object.object_name - global count_contime_counter, count_readtime_counter, count_counter, data_contime_counter, data_readtime_counter, data_counter - count_contime_counter = 1 - count_readtime_counter = 1 - count_counter = 1 - data_contime_counter = 1 - data_readtime_counter = 1 - data_counter = 1 + # リトライ回数判定用のカウンタオブジェクトを生成(@retryデコレータを利用したことによるリトライ対象の関数内でのカウント変数保持不可の対策のためオブジェクト化する) + count_contime_counter = CounterObject(1) + count_readtime_counter = CounterObject(1) + count_counter = CounterObject(1) + data_contime_counter = CounterObject(1) + data_readtime_counter = CounterObject(1) + data_counter = CounterObject(1) try: # ② 取得対象オブジェクトの取得期間内のレコード件数を取得する - logger.info(f'I-FETCH-02 [{object_name}] の件数取得を開始します') + logger.info(f'I-FETCH-02 [{target_object_name}] の件数取得を開始します') soql_builder = SOQLBuilder(target_object, last_fetch_datetime) count_soql = soql_builder.create_count_soql() - record_count = fetch_record_count_retry(count_soql, object_name) + record_count = fetch_record_count_retry(count_soql, target_object_name, count_contime_counter, count_readtime_counter, count_counter) - logger.info(f'I-FETCH-03 [{object_name}] の件数：[{record_count}]') + logger.info(f'I-FETCH-03 [{target_object_name}] の件数：[{record_count}]') except Exception as e: raise SalesforceAPIException( - 'E-FETCH-01', FETCH_JP_NAME, f'[{object_name}] の件数取得に失敗しました エラー内容：[{e}]') + 'E-FETCH-01', FETCH_JP_NAME, f'[{target_object_name}] の件数取得に失敗しました エラー内容：[{e}]') try: # ③ 取得対象オブジェクトのレコードを取得する - logger.info(f'I-FETCH-04 [{object_name}] のレコード取得を開始します') + logger.info(f'I-FETCH-04 [{target_object_name}] のレコード取得を開始します') fetch_soql = soql_builder.create_fetch_soql() - record_all = fetch_sf_data_retry(fetch_soql, object_name) + record_all = fetch_sf_data_retry(fetch_soql, target_object_name, data_contime_counter, data_readtime_counter, data_counter) except Exception as e: raise SalesforceAPIException( - 'E-FETCH-02', FETCH_JP_NAME, f'[{object_name}] のレコード取得に失敗しました エラー内容：[{e}]') + 'E-FETCH-02', FETCH_JP_NAME, f'[{target_object_name}] のレコード取得に失敗しました エラー内容：[{e}]') try: # ④ 取得対象オブジェクトをJSONに変換 - logger.info(f'I-FETCH-05 [{object_name}] のレコードをJSONに変換します') + logger.info(f'I-FETCH-05 [{target_object_name}] のレコードをJSONに変換します') crm_data_response = [record for record in record_all] except Exception as e: raise DataConvertException( - 'E-FETCH-03', FETCH_JP_NAME, f'[{object_name}] のレコードのJSON変換に失敗しました エラー内容：[{e}]') + 'E-FETCH-03', FETCH_JP_NAME, f'[{target_object_name}] のレコードのJSON変換に失敗しました エラー内容：[{e}]') # ⑤ CRMデータ取得処理終了ログを出力する - logger.info(f'I-FETCH-06 [{object_name}] のレコード取得が成功しました') + logger.info(f'I-FETCH-06 [{target_object_name}] のレコード取得が成功しました') # ⑥ 次の処理へ移行する return crm_data_response @@ -107,31 +96,34 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) -def fetch_record_count_retry(soql: str, object_name: str): +def fetch_record_count_retry(soql: str, target_object_name: str, + count_contime_counter: CounterObject, count_readtime_counter: CounterObject, count_counter: CounterObject): try: - global count_contime_counter, count_readtime_counter, count_counter - salesforce_api_client = SalesforceApiClient() return salesforce_api_client.fetch_sf_count(soql) except ConnectTimeout as e: - if count_contime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_contime_counter += 1 - logger.warn(f'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if count_contime_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_contime_counter.increment(1) + logger.warning(f'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') raise e except ReadTimeout as e: - if count_readtime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_readtime_counter += 1 - logger.warn( - f'W-FETCH-02 [{object_name}] の件数取得処理がタイムアウトしたため、リトライします：[{CRM_GET_RECORD_COUNT_TIMEOUT}] エラー内容：[{e}]') + + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if count_readtime_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_readtime_counter.increment(1) + logger.warning( + f'W-FETCH-02 [{target_object_name}] の件数取得処理がタイムアウトしたため、リトライします：[{CRM_GET_RECORD_COUNT_TIMEOUT}] エラー内容：[{e}]') raise e except Exception as e: - if count_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_counter += 1 - logger.warn( - f'W-FETCH-03 [{object_name}] の件数取得に失敗したため、リトライします エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if count_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_counter.increment(1) + logger.warning( + f'W-FETCH-03 [{target_object_name}] の件数取得に失敗したため、リトライします エラー内容：[{e}]') raise e @@ -139,29 +131,31 @@ def fetch_record_count_retry(soql: str, object_name: str): wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) -def fetch_sf_data_retry(soql: str, object_name: str): +def fetch_sf_data_retry(soql: str, target_object_name: str, + data_contime_counter: CounterObject, data_readtime_counter: CounterObject, data_counter: CounterObject): try: - global data_contime_counter, data_readtime_counter, data_counter - salesforce_api_client = SalesforceApiClient() return salesforce_api_client.fetch_sf_data(soql) except ConnectTimeout as e: - if data_contime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_contime_counter += 1 - logger.warn(f'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if data_contime_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_contime_counter.increment(1) + logger.warning(f'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') raise e except ReadTimeout as e: - if data_readtime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_readtime_counter += 1 - logger.warn( - f'W-FETCH-05 [{object_name}] のレコード取得処理がタイムアウトしたため、リトライします：[{CRM_FETCH_RECORD_TIMEOUT}] エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if data_readtime_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_readtime_counter.increment(1) + logger.warning( + f'W-FETCH-05 [{target_object_name}] のレコード取得処理がタイムアウトしたため、リトライします：[{CRM_FETCH_RECORD_TIMEOUT}] エラー内容：[{e}]') raise e except Exception as e: - if data_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_counter += 1 - logger.warn( - f'W-FETCH-06 [{object_name}] のレコード取得に失敗したため、リトライします エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if data_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_counter.increment(1) + logger.warning( + f'W-FETCH-06 [{target_object_name}] のレコード取得に失敗したため、リトライします エラー内容：[{e}]') raise e diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parse.py index 5e7249b9..ca1d8ed2 100644 --- a/ecs/crm-datafetch/src/parser/json_parse.py +++ b/ecs/crm-datafetch/src/parser/json_parse.py @@ -8,7 +8,7 @@ class JsonParser(): def __init__(self, json_str) -> None: self.__json_str = json_str - def json_parser(self) -> dict: + def parse(self) -> dict: for symbol in EXCLUDE_SYMBOL: # コメントアウトシンボルを含む部分を置き換える正規表現 replace_comment_regex = rf'\s(?!\"){symbol}[\s\S]*?.*' diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index 409651ad..3488c71f 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -3,36 +3,24 @@ from src.config.objects import FetchTargetObjects from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser from src.system_var.constants import PRE_JP_NAME -from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, - OBJECT_INFO_FOLDER) +from src.system_var.environments import (CRM_CONFIG_BUCKET, + OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER) from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger def prepare_data_fetch_process(): - """ - データ取得準備処理 + """データ取得準備処理 - Parameters - ---------- - なし - - Returns - ------- - fetch_target_objects : FetchTargetObjects - CRMオブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス - process_result : dict - 取得処理実行結果辞書オブジェクト - - Raises - ------ - FileNotFoundException - S3上のファイルが存在しない場合 - InvalidConfigException - オブジェクト情報定義が不正だった場合 + Raises: + FileNotFoundException: S3上のファイルが存在しない場合 + InvalidConfigException: オブジェクト情報定義が不正だった場合 + Returns: + fetch_target_objects : CRMオブジェクト情報インスタンス + execute_datetime : 実行日次取得インスタンス + process_result : 取得処理実行結果辞書オブジェクト """ # ① データ取得準備処理の開始ログを出力する @@ -64,7 +52,7 @@ def prepare_data_fetch_process(): logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします') json_parser = JsonParser(object_info_file_str) - object_info_file_dict = json_parser.json_parser() + object_info_file_dict = json_parser.parse() logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました') diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index 0c073b9a..4e4bb267 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,10 +1,9 @@ from simple_salesforce import Salesforce - from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_TIMEOUT, - CRM_FETCH_RECORD_TIMEOUT, - CRM_GET_RECORD_COUNT_TIMEOUT, CRM_USER_NAME, - CRM_USER_PASSWORD, CRM_USER_SECURITY_TOKEN) - + CRM_FETCH_RECORD_TIMEOUT, + CRM_GET_RECORD_COUNT_TIMEOUT, + CRM_USER_NAME, CRM_USER_PASSWORD, + CRM_USER_SECURITY_TOKEN) class SalesforceApiClient(): @@ -25,4 +24,5 @@ class SalesforceApiClient(): return count_res.get('records')[0].get('expr0') def fetch_sf_data(self, soql: str): - return self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + data_res = self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + return data_res.get('records') diff --git a/ecs/crm-datafetch/src/set_datetime_period_process.py b/ecs/crm-datafetch/src/set_datetime_period_process.py index 9795ab2e..bfebd5c2 100644 --- a/ecs/crm-datafetch/src/set_datetime_period_process.py +++ b/ecs/crm-datafetch/src/set_datetime_period_process.py @@ -1,37 +1,27 @@ from src.aws.s3 import ConfigBucket -from src.config.objects import TargetObject, LastFetchDatetime +from src.config.objects import LastFetchDatetime, TargetObject from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser from src.system_var.constants import DATE_JP_NAME -from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.system_var.environments import (CRM_CONFIG_BUCKET, + LAST_FETCH_DATE_FOLDER) from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger - - def set_datetime_period_process(target_object: TargetObject, execute_datetime: ExecuteDateTime): - """ - データ取得期間設定処理 + """データ取得期間設定処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス - Returns - ------- - last_fetch_datetime : LastFetchDatetime - 取得対象オブジェクト情報インスタンス + Raises: + FileNotFoundException: S3上のファイルが存在しない場合 + InvalidConfigException: オブジェクト情報定義が不正だった場合 - Raises - ------ - FileNotFoundException - S3上のファイルが存在しない場合 - InvalidConfigException - オブジェクト情報定義が不正だった場合 + Returns: + last_fetch_datetime: データ取得期間設定インスタンス """ # ① データ取得期間設定処理の開始ログを出力する @@ -59,7 +49,7 @@ def set_datetime_period_process(target_object: TargetObject, execute_datetime: E logger.debug(f'D-DATE-04 前回取得日時ファイルの形式チェックを開始します') json_parser = JsonParser(last_fetch_datetime_file_str) - last_fetch_datetime_file_dict = json_parser.json_parser() + last_fetch_datetime_file_dict = json_parser.parse() last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_file_dict, execute_datetime) diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index 5ff41929..7dc62c1b 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -6,35 +6,35 @@ import src.system_var.constants as constants # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_TIMEOUT = os.environ.get(constants.CRM_AUTH_TIMEOUT, 100) +CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)) # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3) +CRM_AUTH_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3)) # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5) +CRM_AUTH_RETRY_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5)) # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5) +CRM_AUTH_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5)) # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50) +CRM_AUTH_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50)) # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300) +CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)) # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3) +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3)) # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5) +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5)) # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, 5) +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, 5)) # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50) +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50)) # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300) +CRM_FETCH_RECORD_TIMEOUT = int(os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300)) # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3) +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3)) # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5) +CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5) +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50) +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50)) # environments(ECS Task Environment) # CRMのAPI実行のための認証エンドポイントのドメイン diff --git a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py index e83e58ab..e6b34c53 100644 --- a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py +++ b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py @@ -1,31 +1,21 @@ import json from src.aws.s3 import ConfigBucket -from src.config.objects import TargetObject, LastFetchDatetime +from src.config.objects import LastFetchDatetime, TargetObject from src.error.exceptions import FileUploadException from src.system_var.constants import UPD_JP_NAME from src.util.logger import logger_instance as logger def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): - """ - 前回取得日時ファイル更新 + """前回取得日時ファイル更新 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - last_fetch_datetime : LastFetchDatetime - 取得対象オブジェクト情報インスタンス + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + last_fetch_datetime (LastFetchDatetime): データ取得期間設定インスタンス - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① 前回取得日時ファイル更新処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/upload_result_data_process.py b/ecs/crm-datafetch/src/upload_result_data_process.py index 855ec900..8248fb78 100644 --- a/ecs/crm-datafetch/src/upload_result_data_process.py +++ b/ecs/crm-datafetch/src/upload_result_data_process.py @@ -7,24 +7,14 @@ from src.util.logger import logger_instance as logger def upload_result_data_process(process_result: dict, execute_datetime: ExecuteDateTime): - """ - 取得処理実施結果アップロード処理 + """取得処理実施結果アップロード処理 - Parameters - ---------- - process_result : dict - 取得処理実行結果辞書オブジェクト - last_fetch_datetime : LastFetchDatetime - 取得対象オブジェクト情報インスタンス + Args: + process_result (dict): 取得処理実行結果辞書オブジェクト + execute_datetime (ExecuteDateTime): データ取得期間設定インスタンス - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① 取得処理実施結果アップロード処理のログを出力する diff --git a/ecs/crm-datafetch/src/util/counter_object.py b/ecs/crm-datafetch/src/util/counter_object.py new file mode 100644 index 00000000..23da4f82 --- /dev/null +++ b/ecs/crm-datafetch/src/util/counter_object.py @@ -0,0 +1,14 @@ +class CounterObject: + def __init__(self, base_num=1) -> None: + self.__counter = base_num + + def describe(self) -> int: + return self.__counter + + def increment(self, num=1) -> int: + self.__counter += num + return self.__counter + + def decrement(self, num=1) -> int: + self.__counter -= num + return self.__counter diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py index 9d498b5e..1f97dded 100644 --- a/ecs/crm-datafetch/src/util/dict_checker.py +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -17,7 +17,7 @@ class DictChecker: """辞書型バリュー正規表現チェック""" return True if re.fullmatch(regex_str, self.__object_dict[check_key]) else False - def assert_key_exist(self, check_key: str, check_type: type) -> None: + def assert_key_exist(self, check_key: str) -> None: """辞書型キー存在検査""" if not self.check_key_exist(check_key): raise Exception(f'「{check_key}」キーは必須です') diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py index 2f07736a..8a804291 100644 --- a/ecs/crm-datafetch/src/util/execute_datetime.py +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -1,9 +1,6 @@ from datetime import datetime -from src.system_var.constants import( - YYYYMMDDTHHMMSSTZ, - MILLISEC_FORMAT -) +from src.system_var.constants import MILLISEC_FORMAT, YYYYMMDDTHHMMSSTZ class ExecuteDateTime: diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index ec1df277..47fd2e45 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -2,7 +2,7 @@ import logging from src.system_var.environments import LOG_LEVEL -"""boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する""" +# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する for name in ["boto3", "botocore", "s3transfer", "urllib3"]: logging.getLogger(name).setLevel(logging.WARNING) diff --git a/s3/config/crm/last_fetch_datetime/Account.json b/s3/config/crm/last_fetch_datetime/Account.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Account.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/AccountShare.json b/s3/config/crm/last_fetch_datetime/AccountShare.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/AccountShare.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Account_Territory_Loader_vod__c.json b/s3/config/crm/last_fetch_datetime/Account_Territory_Loader_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Account_Territory_Loader_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Approved_Document_vod__c.json b/s3/config/crm/last_fetch_datetime/Approved_Document_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Approved_Document_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call2_Detail_vod__c.json b/s3/config/crm/last_fetch_datetime/Call2_Detail_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call2_Detail_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call2_Discussion_vod__c.json b/s3/config/crm/last_fetch_datetime/Call2_Discussion_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call2_Discussion_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call2_Key_Message_vod__c.json b/s3/config/crm/last_fetch_datetime/Call2_Key_Message_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call2_Key_Message_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call2_vod__c.json b/s3/config/crm/last_fetch_datetime/Call2_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call2_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call_Clickstream_vod__c.json b/s3/config/crm/last_fetch_datetime/Call_Clickstream_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call_Clickstream_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Child_Account_vod__c.json b/s3/config/crm/last_fetch_datetime/Child_Account_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Child_Account_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Clm_Presentation_Slide_vod__c.json b/s3/config/crm/last_fetch_datetime/Clm_Presentation_Slide_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Clm_Presentation_Slide_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Clm_Presentation_vod__c.json b/s3/config/crm/last_fetch_datetime/Clm_Presentation_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Clm_Presentation_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Coaching_Report_vod__c.json b/s3/config/crm/last_fetch_datetime/Coaching_Report_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Coaching_Report_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Consent_Header_vod__c.json b/s3/config/crm/last_fetch_datetime/Consent_Header_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Consent_Header_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Consent_Line_vod__c.json b/s3/config/crm/last_fetch_datetime/Consent_Line_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Consent_Line_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Consent_Type_vod__c.json b/s3/config/crm/last_fetch_datetime/Consent_Type_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Consent_Type_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Contact.json b/s3/config/crm/last_fetch_datetime/Contact.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Contact.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_Configuration_vod__c.json b/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_Configuration_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_Configuration_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_vod__c.json b/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Email_Activity_vod__c.json b/s3/config/crm/last_fetch_datetime/Email_Activity_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Email_Activity_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Event_Attendee_vod__c.json b/s3/config/crm/last_fetch_datetime/Event_Attendee_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Event_Attendee_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Group.json b/s3/config/crm/last_fetch_datetime/Group.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Group.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Key_Message_vod__c.json b/s3/config/crm/last_fetch_datetime/Key_Message_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Key_Message_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Hospital_Medical_Regimen__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Hospital_Medical_Regimen__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Hospital_Medical_Regimen__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Inquiry_Assignment__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Inquiry_Assignment__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Inquiry_Assignment__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_MR_Weekly_Report__c.json b/s3/config/crm/last_fetch_datetime/MSJ_MR_Weekly_Report__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_MR_Weekly_Report__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Medical_Event_Evaluation__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Medical_Event_Evaluation__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Medical_Event_Evaluation__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Medical_Regimen__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Medical_Regimen__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Medical_Regimen__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Patient__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Patient__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Patient__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json b/s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Medical_Inquiry_vod__c.json b/s3/config/crm/last_fetch_datetime/Medical_Inquiry_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Medical_Inquiry_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Medical_Insight_vod__c.json b/s3/config/crm/last_fetch_datetime/Medical_Insight_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Medical_Insight_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Multichannel_Activity_Line_vod__c.json b/s3/config/crm/last_fetch_datetime/Multichannel_Activity_Line_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Multichannel_Activity_Line_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Multichannel_Activity_vod__c.json b/s3/config/crm/last_fetch_datetime/Multichannel_Activity_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Multichannel_Activity_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Multichannel_Consent_vod__c.json b/s3/config/crm/last_fetch_datetime/Multichannel_Consent_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Multichannel_Consent_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/My_Setup_Products_vod__c.json b/s3/config/crm/last_fetch_datetime/My_Setup_Products_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/My_Setup_Products_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/ObjectTerritory2Association.json b/s3/config/crm/last_fetch_datetime/ObjectTerritory2Association.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/ObjectTerritory2Association.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Product_Group_vod__c.json b/s3/config/crm/last_fetch_datetime/Product_Group_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Product_Group_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Product_Metrics_vod__c.json b/s3/config/crm/last_fetch_datetime/Product_Metrics_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Product_Metrics_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Product_vod__c.json b/s3/config/crm/last_fetch_datetime/Product_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Product_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Profile.json b/s3/config/crm/last_fetch_datetime/Profile.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Profile.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Question_Response_vod__c.json b/s3/config/crm/last_fetch_datetime/Question_Response_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Question_Response_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/RecordType.json b/s3/config/crm/last_fetch_datetime/RecordType.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/RecordType.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Remote_Meeting_vod__c.json b/s3/config/crm/last_fetch_datetime/Remote_Meeting_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Remote_Meeting_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Sent_Email_vod__c.json b/s3/config/crm/last_fetch_datetime/Sent_Email_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Sent_Email_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Sent_Fragment_vod__c.json b/s3/config/crm/last_fetch_datetime/Sent_Fragment_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Sent_Fragment_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Survey_Question_vod__c.json b/s3/config/crm/last_fetch_datetime/Survey_Question_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Survey_Question_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Survey_Target_vod__c.json b/s3/config/crm/last_fetch_datetime/Survey_Target_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Survey_Target_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Survey_vod__c.json b/s3/config/crm/last_fetch_datetime/Survey_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Survey_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Territory2.json b/s3/config/crm/last_fetch_datetime/Territory2.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Territory2.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Time_Off_Territory_vod__c.json b/s3/config/crm/last_fetch_datetime/Time_Off_Territory_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Time_Off_Territory_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/User.json b/s3/config/crm/last_fetch_datetime/User.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/User.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/UserRole.json b/s3/config/crm/last_fetch_datetime/UserRole.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/UserRole.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/UserTerritory2Association.json b/s3/config/crm/last_fetch_datetime/UserTerritory2Association.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/UserTerritory2Association.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/object_info/crm_object_list_all.json b/s3/config/crm/object_info/crm_object_list_all.json index a65cf1bb..0792da37 100644 --- a/s3/config/crm/object_info/crm_object_list_all.json +++ b/s3/config/crm/object_info/crm_object_list_all.json @@ -24,7 +24,7 @@ "is_skip": false, "is_update_last_fetch_datetime": false, "last_fetch_datetime_file_name": "Territory2_ALL.json", - "upload_file_name": "CRM_Territory2_ALL_{execute_datetime}.csv" + "upload_file_name": "CRM_Territory2_ALL_{execute_datetime}" }, { "object_name": "UserTerritory2Association", @@ -41,7 +41,7 @@ "is_skip": false, "is_update_last_fetch_datetime": false, "last_fetch_datetime_file_name": "UserTerritory2Association_ALL.json", - "upload_file_name": "CRM_UserTerritory2Association_ALL_{execute_datetime}.csv" + "upload_file_name": "CRM_UserTerritory2Association_ALL_{execute_datetime}" } ] } diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index 00f7fe89..aa20d187 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -2164,7 +2164,8 @@ "IsDeleted" ], "is_skip": false, - "is_update_last_fetch_datetime": true + "is_update_last_fetch_datetime": true, + "datetime_column": "LastModifiedDate" }, { "object_name": "Contact", From 939bdd6f0ac05439c05ee6a4cf2b3df4c2bad720 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Thu, 28 Jul 2022 22:52:12 +0900 Subject: [PATCH 15/21] =?UTF-8?q?feat:=20=E5=85=A8=E4=BB=B6=E5=8F=96?= =?UTF-8?q?=E5=BE=97=E3=82=AA=E3=83=96=E3=82=B8=E3=82=A7=E3=82=AF=E3=83=88?= =?UTF-8?q?=E7=94=A8=E3=81=AE=E6=9C=80=E7=B5=82=E5=8F=96=E5=BE=97=E6=97=A5?= =?UTF-8?q?=E6=99=82=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=82=92=E4=BD=9C?= =?UTF-8?q?=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/config/crm/last_fetch_datetime/Territory2_ALL.json | 4 ++++ .../last_fetch_datetime/UserTerritory2Association_ALL.json | 4 ++++ 2 files changed, 8 insertions(+) create mode 100644 s3/config/crm/last_fetch_datetime/Territory2_ALL.json create mode 100644 s3/config/crm/last_fetch_datetime/UserTerritory2Association_ALL.json diff --git a/s3/config/crm/last_fetch_datetime/Territory2_ALL.json b/s3/config/crm/last_fetch_datetime/Territory2_ALL.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Territory2_ALL.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/UserTerritory2Association_ALL.json b/s3/config/crm/last_fetch_datetime/UserTerritory2Association_ALL.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/UserTerritory2Association_ALL.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} From fcac3d7b23a7da41e29ad832820823b1fe896ac8 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 1 Aug 2022 19:28:15 +0900 Subject: [PATCH 16/21] =?UTF-8?q?fix:=20=E6=8C=87=E6=91=98=E5=86=85?= =?UTF-8?q?=E5=AE=B9=E3=81=AE=E5=8F=8D=E6=98=A0=E3=80=81=E3=83=AD=E3=82=B0?= =?UTF-8?q?=E5=87=BA=E5=8A=9B=E3=81=AE=E5=BE=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/backup_crm_csv_data_process.py | 6 +-- ecs/crm-datafetch/src/config/objects.py | 12 ++--- ecs/crm-datafetch/src/controller.py | 10 ++-- ...convert_factory.py => convert_strategy.py} | 9 ++-- ecs/crm-datafetch/src/converter/converter.py | 2 +- ecs/crm-datafetch/src/error/exceptions.py | 2 +- .../src/fetch_crm_data_process.py | 54 +++++++------------ ecs/crm-datafetch/src/util/dict_checker.py | 6 ++- .../src/util/execute_datetime.py | 4 +- 9 files changed, 45 insertions(+), 60 deletions(-) rename ecs/crm-datafetch/src/converter/{convert_factory.py => convert_strategy.py} (83%) diff --git a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py index d40c7a13..ba7bb8ea 100644 --- a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py @@ -6,12 +6,12 @@ from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def backup_crm_csv_data_process(target_object: TargetObject, exetute_datetime: ExecuteDateTime, csv_string: str): +def backup_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime, csv_string: str): """CSVバックアップ処理 Args: target_object (TargetObject): 取得対象オブジェクト情報インスタンス - exetute_datetime (ExecuteDateTime): 実行日次取得インスタンス + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス csv_string (str): csvデータ Raises: @@ -29,7 +29,7 @@ def backup_crm_csv_data_process(target_object: TargetObject, exetute_datetime: E # ② CRMバックアップ保管用バケットに、変換後のCSVデータのバックアップを保管する backup_bucket = BackupBucket() backup_bucket.put_csv( - f'{exetute_datetime.to_path()}/{upload_file_name}.csv', csv_string) + f'{execute_datetime.to_path()}/{upload_file_name}.csv', csv_string) logger.debug( f'D-CSVBK-02 [{target_object_name}] のCSVデータバックアップ 正常終了') diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 3ac08851..3368a1db 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -94,22 +94,19 @@ class TargetObject(): def is_update_last_fetch_datetime(self) -> bool: if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY): return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] - else: - return False + return False @property def last_fetch_datetime_file_name(self) -> str: if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY): return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY] - else: - return f'{self.__object_info[OBJECT_NAME_KEY]}.json' + return f'{self.__object_info[OBJECT_NAME_KEY]}.json' @property def upload_file_name(self) -> str: if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) - else: - return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' + return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' @property def datetime_column(self) -> str: @@ -138,5 +135,4 @@ class LastFetchDatetime(): def last_fetch_datetime_to(self) -> str: if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY] - else: - return self.__execute_datetime + return self.__execute_datetime diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index 3c92277b..c531f113 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -39,17 +39,17 @@ def controller() -> None: logger.info(f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result}]') # ⑤ 取得処理実施結果アップロード処理を呼び出す - logger.info('I-CTRL-19 CRM_取得処理実施結果ファイルアップロード処理開始') + logger.info('I-CTRL-18 CRM_取得処理実施結果ファイルアップロード処理開始') upload_result_data_process(process_result, execute_datetime) # ⑥ 最終結果をチェックし、チェック結果をログに出力 if not all([v == 'success' for v in process_result.values()]): logger.error('E-CTRL-01 一部のデータ取得に失敗しています。詳細はログをご確認ください。') else: - logger.info('I-CTRL-20 すべてのデータの取得に成功しました。') + logger.info('I-CTRL-19 すべてのデータの取得に成功しました。') # ⑦ CRMデータ取得処理終了ログを出力する - logger.info('I-CTRL-21 CRMデータ取得処理を終了します') + logger.info('I-CTRL-20 CRMデータ取得処理を終了します') return exit(0) @@ -106,7 +106,7 @@ def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTi """ # 1. オブジェクト処理結果の初期化 - logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：{object_info}') + logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：[{object_info}]') # 2. オブジェクト情報形式チェック処理を呼び出す logger.info('I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') @@ -138,7 +138,7 @@ def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTi # 7. 出力ファイル名をログ出力する logger.info( - f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{target_object.upload_file_name}]となります') + f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{target_object.upload_file_name}] となります') # 8. CRM電文データバックアップ処理を呼び出す logger.info( diff --git a/ecs/crm-datafetch/src/converter/convert_factory.py b/ecs/crm-datafetch/src/converter/convert_strategy.py similarity index 83% rename from ecs/crm-datafetch/src/converter/convert_factory.py rename to ecs/crm-datafetch/src/converter/convert_strategy.py index bf949a6e..92500588 100644 --- a/ecs/crm-datafetch/src/converter/convert_factory.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -1,5 +1,5 @@ import re -from datetime import datetime +from datetime import datetime, timedelta, timezone from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, CSV_TRUE_VALUE, @@ -12,7 +12,7 @@ class ConvertStrategyFactory: self.__none_value_convert_strategy = NoneValueConvertStrategy() self.__float_convert_strategy = FloatConvertStrategy() self.__boolean_convert_strategy = BooleanConvertStrategy() - self.__datetime_convert_strategy = DatatimeConvertStrategy() + self.__datetime_convert_strategy = DatetimeConvertStrategy() self.__non_convert_strategy = NonConvertStrategy() def create(self, value): @@ -45,9 +45,10 @@ class BooleanConvertStrategy: return CSV_TRUE_VALUE if convert_value is True else CSV_FALSE_VALUE -class DatatimeConvertStrategy: +class DatetimeConvertStrategy: def convert_value(self, convert_value: str) -> str: - return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).strftime(YYYYMMDDHHMMSS) + # データ登録処理がJSTとして登録するため、変換処理内で事前にJSTの日次文字列に変換する + return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(timezone(timedelta(hours=+9))).strftime(YYYYMMDDHHMMSS) class FloatConvertStrategy: diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 9c223182..761f550e 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -2,7 +2,7 @@ import csv import io from src.config.objects import TargetObject -from src.converter.convert_factory import ConvertStrategyFactory +from src.converter.convert_strategy import ConvertStrategyFactory class CSVStringConverter: diff --git a/ecs/crm-datafetch/src/error/exceptions.py b/ecs/crm-datafetch/src/error/exceptions.py index f231804b..5085dbca 100644 --- a/ecs/crm-datafetch/src/error/exceptions.py +++ b/ecs/crm-datafetch/src/error/exceptions.py @@ -31,5 +31,5 @@ class DataConvertException(MeDaCaCRMDataFetchException): class SalesforceAPIException(MeDaCaCRMDataFetchException): - """SalseforceのAPI実行失敗が発生した場合の例外""" + """SalesforceのAPI実行失敗が発生した場合の例外""" pass diff --git a/ecs/crm-datafetch/src/fetch_crm_data_process.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py index d22efb7e..1c37f860 100644 --- a/ecs/crm-datafetch/src/fetch_crm_data_process.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -3,7 +3,7 @@ from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential from src.config.objects import LastFetchDatetime, TargetObject -from src.error.exceptions import DataConvertException, SalesforceAPIException +from src.error.exceptions import SalesforceAPIException from src.salesforce.salesforce_api import SalesforceApiClient from src.salesforce.soql_builder import SOQLBuilder from src.system_var.constants import FETCH_JP_NAME @@ -27,8 +27,7 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las last_fetch_datetime (LastFetchDatetime): データ取得期間設定インスタンス Raises: - SalesforceAPIException: SalseforceのAPI実行失敗が発生した場合 - DataConvertException: データ変換が失敗した場合 + SalesforceAPIException: SalesforceのAPI実行失敗が発生した場合 Returns: crm_data_response: Salesforceオブジェクトデータ @@ -40,12 +39,9 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las target_object_name = target_object.object_name - # リトライ回数判定用のカウンタオブジェクトを生成(@retryデコレータを利用したことによるリトライ対象の関数内でのカウント変数保持不可の対策のためオブジェクト化する) - count_contime_counter = CounterObject(1) - count_readtime_counter = CounterObject(1) + # リトライ回数判定用のカウンタオブジェクトを生成 + # @retryデコレータを利用した関数のリトライ処理で、基本データ型だとリトライ回数をカウントすることができないため、オブジェクト化する count_counter = CounterObject(1) - data_contime_counter = CounterObject(1) - data_readtime_counter = CounterObject(1) data_counter = CounterObject(1) try: @@ -55,7 +51,7 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las soql_builder = SOQLBuilder(target_object, last_fetch_datetime) count_soql = soql_builder.create_count_soql() - record_count = fetch_record_count_retry(count_soql, target_object_name, count_contime_counter, count_readtime_counter, count_counter) + record_count = fetch_record_count_retry(count_soql, target_object_name, count_counter) logger.info(f'I-FETCH-03 [{target_object_name}] の件数：[{record_count}]') @@ -69,26 +65,16 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las fetch_soql = soql_builder.create_fetch_soql() - record_all = fetch_sf_data_retry(fetch_soql, target_object_name, data_contime_counter, data_readtime_counter, data_counter) + crm_data_response = fetch_sf_data_retry(fetch_soql, target_object_name, data_counter) except Exception as e: raise SalesforceAPIException( 'E-FETCH-02', FETCH_JP_NAME, f'[{target_object_name}] のレコード取得に失敗しました エラー内容：[{e}]') - try: - # ④ 取得対象オブジェクトをJSONに変換 - logger.info(f'I-FETCH-05 [{target_object_name}] のレコードをJSONに変換します') + # ④ CRMデータ取得処理終了ログを出力する + logger.info(f'I-FETCH-05 [{target_object_name}] のレコード取得が成功しました') - crm_data_response = [record for record in record_all] - - except Exception as e: - raise DataConvertException( - 'E-FETCH-03', FETCH_JP_NAME, f'[{target_object_name}] のレコードのJSON変換に失敗しました エラー内容：[{e}]') - - # ⑤ CRMデータ取得処理終了ログを出力する - logger.info(f'I-FETCH-06 [{target_object_name}] のレコード取得が成功しました') - - # ⑥ 次の処理へ移行する + # ⑤ 次の処理へ移行する return crm_data_response @@ -96,24 +82,23 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) -def fetch_record_count_retry(soql: str, target_object_name: str, - count_contime_counter: CounterObject, count_readtime_counter: CounterObject, count_counter: CounterObject): +def fetch_record_count_retry(soql: str, target_object_name: str, count_counter: CounterObject): try: salesforce_api_client = SalesforceApiClient() return salesforce_api_client.fetch_sf_count(soql) except ConnectTimeout as e: # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う - if count_contime_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_contime_counter.increment(1) + if count_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_counter.increment(1) logger.warning(f'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') raise e except ReadTimeout as e: # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う - if count_readtime_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_readtime_counter.increment(1) + if count_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_counter.increment(1) logger.warning( f'W-FETCH-02 [{target_object_name}] の件数取得処理がタイムアウトしたため、リトライします：[{CRM_GET_RECORD_COUNT_TIMEOUT}] エラー内容：[{e}]') raise e @@ -131,23 +116,22 @@ def fetch_record_count_retry(soql: str, target_object_name: str, wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) -def fetch_sf_data_retry(soql: str, target_object_name: str, - data_contime_counter: CounterObject, data_readtime_counter: CounterObject, data_counter: CounterObject): +def fetch_sf_data_retry(soql: str, target_object_name: str, data_counter: CounterObject): try: salesforce_api_client = SalesforceApiClient() return salesforce_api_client.fetch_sf_data(soql) except ConnectTimeout as e: # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う - if data_contime_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_contime_counter.increment(1) + if data_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_counter.increment(1) logger.warning(f'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') raise e except ReadTimeout as e: # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う - if data_readtime_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_readtime_counter.increment(1) + if data_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_counter.increment(1) logger.warning( f'W-FETCH-05 [{target_object_name}] のレコード取得処理がタイムアウトしたため、リトライします：[{CRM_FETCH_RECORD_TIMEOUT}] エラー内容：[{e}]') raise e diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py index 1f97dded..39ebf9f0 100644 --- a/ecs/crm-datafetch/src/util/dict_checker.py +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -5,9 +5,13 @@ class DictChecker: def __init__(self, object_dict: dict) -> None: self.__object_dict = object_dict + def is_empty(self, check_key): + """辞書型バリュー空文字チェック""" + return self.__object_dict[check_key] != '' and self.__object_dict[check_key] is not None + def check_key_exist(self, check_key: str) -> bool: """辞書型キー存在チェック""" - return check_key in self.__object_dict and self.__object_dict[check_key] != '' + return check_key in self.__object_dict and self.is_empty(check_key) def check_data_type(self, check_key: str, check_type: type) -> bool: """辞書型バリュー型チェック""" diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py index 8a804291..bdf7f053 100644 --- a/ecs/crm-datafetch/src/util/execute_datetime.py +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -1,11 +1,11 @@ -from datetime import datetime +from datetime import datetime, timezone from src.system_var.constants import MILLISEC_FORMAT, YYYYMMDDTHHMMSSTZ class ExecuteDateTime: def __init__(self): - self.__execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) + self.__execute_datetime = datetime.now(timezone.utc).strftime(YYYYMMDDTHHMMSSTZ) def __str__(self) -> str: return self.__execute_datetime From 96487ccb2513a0e4c90e968cd9b59ace3d7472ee Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Mon, 1 Aug 2022 19:30:23 +0900 Subject: [PATCH 17/21] =?UTF-8?q?fix:=20=E6=8C=87=E6=91=98=E5=86=85?= =?UTF-8?q?=E5=AE=B9=E3=81=AE=E8=BF=BD=E5=8A=A0=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/system_var/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index b84d8f1d..56f49681 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -49,7 +49,7 @@ S3_CHAR_CODE = 'utf-8' # 正規表現チェック EXCLUDE_SYMBOL = ['#', '/'] DATE_PATTERN_YYYYMMDDTHHMMSSTZ = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' -DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.000\+0000' +DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.000\+0000' # logger LOG_FORMAT = '[%(levelname)s]\t%(asctime)s\t%(message)s\n' From 75187e54e9b13b1f54934ed9a8ab8049c4f9218d Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 2 Aug 2022 09:55:30 +0900 Subject: [PATCH 18/21] =?UTF-8?q?fix:=20=E3=83=95=E3=82=A1=E3=82=A4?= =?UTF-8?q?=E3=83=AB=E5=90=8D=E3=81=AE=E6=97=A5=E4=BB=98=E3=83=95=E3=82=A9?= =?UTF-8?q?=E3=83=BC=E3=83=9E=E3=83=83=E3=83=88=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/config/objects.py | 7 ++++--- ecs/crm-datafetch/src/util/execute_datetime.py | 3 +++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 3368a1db..65a7bc9a 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -15,6 +15,7 @@ from src.system_var.constants import (COLUMNS_KEY, COLUMNS_TYPE, UPLOAD_FILE_NAME_KEY, UPLOAD_FILE_NAME_TYPE) from src.util.dict_checker import DictChecker +from src.util.execute_datetime import ExecuteDateTime class FetchTargetObjects(): @@ -40,7 +41,7 @@ class FetchTargetObjects(): class TargetObject(): - def __init__(self, object_info, execute_datetime) -> None: + def __init__(self, object_info, execute_datetime: ExecuteDateTime) -> None: self.__dict_checker = DictChecker(object_info) self.__object_info = object_info self.__execute_datetime = execute_datetime @@ -105,8 +106,8 @@ class TargetObject(): @property def upload_file_name(self) -> str: if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): - return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) - return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' + return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime.to_YYYYMMDDHHMMSS) + return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.to_YYYYMMDDHHMMSS}' @property def datetime_column(self) -> str: diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py index bdf7f053..e87ef861 100644 --- a/ecs/crm-datafetch/src/util/execute_datetime.py +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -12,3 +12,6 @@ class ExecuteDateTime: def to_path(self) -> str: return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) + + def to_YYYYMMDDHHMMSS(self) -> str: + return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': None, 'T': None, ':': None, '.': None})) \ No newline at end of file From 070763ec8e4393f0821528623c2efeb5511804ad Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 2 Aug 2022 15:10:14 +0900 Subject: [PATCH 19/21] =?UTF-8?q?fix:=20=E3=83=AD=E3=82=B0=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=80=81=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E5=90=8D=E3=81=AE=E6=97=A5=E4=BB=98=E5=8F=96=E5=BE=97=E3=81=AE?= =?UTF-8?q?=E4=B8=8D=E5=85=B7=E5=90=88=E4=BF=AE=E6=AD=A3=E3=80=81=E3=82=BF?= =?UTF-8?q?=E3=82=A4=E3=83=A0=E3=82=BE=E3=83=BC=E3=83=B3=E5=88=A9=E7=94=A8?= =?UTF-8?q?=E5=87=A6=E7=90=86=E3=81=AE=E5=8F=96=E3=82=8A=E8=BE=BC=E3=81=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/config/objects.py | 4 ++-- ecs/crm-datafetch/src/controller.py | 4 ++-- ecs/crm-datafetch/src/converter/convert_strategy.py | 9 +++++---- ecs/crm-datafetch/src/fetch_crm_data_process.py | 4 +++- ecs/crm-datafetch/src/system_var/constants.py | 1 + ecs/crm-datafetch/src/system_var/environments.py | 2 ++ ecs/crm-datafetch/src/util/execute_datetime.py | 2 +- s3/config/crm/task_settings/task_settings.env | 3 ++- 8 files changed, 18 insertions(+), 11 deletions(-) diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 65a7bc9a..20673d70 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -106,8 +106,8 @@ class TargetObject(): @property def upload_file_name(self) -> str: if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): - return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime.to_YYYYMMDDHHMMSS) - return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.to_YYYYMMDDHHMMSS}' + return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime.to_YYYYMMDDHHMMSS()) + return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.to_YYYYMMDDHHMMSS()}' @property def datetime_column(self) -> str: diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index c531f113..3d0537b4 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -44,9 +44,9 @@ def controller() -> None: # ⑥ 最終結果をチェックし、チェック結果をログに出力 if not all([v == 'success' for v in process_result.values()]): - logger.error('E-CTRL-01 一部のデータ取得に失敗しています。詳細はログをご確認ください。') + logger.error('E-CTRL-01 一部のデータ取得に失敗しています 詳細はログをご確認ください') else: - logger.info('I-CTRL-19 すべてのデータの取得に成功しました。') + logger.info('I-CTRL-19 すべてのデータの取得に成功しました') # ⑦ CRMデータ取得処理終了ログを出力する logger.info('I-CTRL-20 CRMデータ取得処理を終了します') diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 92500588..307fbb9a 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -1,8 +1,9 @@ import re -from datetime import datetime, timedelta, timezone +from datetime import datetime -from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, - CSV_TRUE_VALUE, +from dateutil.tz import gettz +from src.system_var.constants import (CONVERT_TZ, CRM_DATETIME_FORMAT, + CSV_FALSE_VALUE, CSV_TRUE_VALUE, DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, YYYYMMDDHHMMSS) @@ -48,7 +49,7 @@ class BooleanConvertStrategy: class DatetimeConvertStrategy: def convert_value(self, convert_value: str) -> str: # データ登録処理がJSTとして登録するため、変換処理内で事前にJSTの日次文字列に変換する - return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(timezone(timedelta(hours=+9))).strftime(YYYYMMDDHHMMSS) + return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS) class FloatConvertStrategy: diff --git a/ecs/crm-datafetch/src/fetch_crm_data_process.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py index 1c37f860..8c66bf59 100644 --- a/ecs/crm-datafetch/src/fetch_crm_data_process.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -67,12 +67,14 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las crm_data_response = fetch_sf_data_retry(fetch_soql, target_object_name, data_counter) + logger.info(f'I-FETCH-05 [{target_object_name}] のレコード取得が成功しました') + except Exception as e: raise SalesforceAPIException( 'E-FETCH-02', FETCH_JP_NAME, f'[{target_object_name}] のレコード取得に失敗しました エラー内容：[{e}]') # ④ CRMデータ取得処理終了ログを出力する - logger.info(f'I-FETCH-05 [{target_object_name}] のレコード取得が成功しました') + logger.info(f'I-FETCH-06 [{target_object_name}] のCRMからのデータ取得処理を終了します') # ⑤ 次の処理へ移行する return crm_data_response diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index 56f49681..bf65010c 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -15,6 +15,7 @@ CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = 'CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT' CRM_FETCH_RECORD_RETRY_INTERVAL = 'CRM_FETCH_RECORD_RETRY_INTERVAL' # CRMのレコード取得処理のリトライ時の初回待ち秒数 CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MIN_INTERVAL' # CRMのレコード取得処理のリトライ時の最小待ち秒数 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MAX_INTERVAL' # CRMのレコード取得処理のリトライ時の最大待ち秒数 +CONVERT_TZ = 'CONVERT_TZ' # CRMデータの日時を変換するときのタイムゾーン # environments(ECS Task Environment) CRM_AUTH_DOMAIN = 'CRM_AUTH_DOMAIN' # CRMのAPI実行のための認証エンドポイントのドメイン diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index 7dc62c1b..b290cc7b 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -35,6 +35,8 @@ CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_ CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最大待ち秒数 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50)) +# CRMデータの日時を変換するときのタイムゾーン +CONVERT_TZ = os.environ.get(constants.CONVERT_TZ, 'Asia/Tokyo') # environments(ECS Task Environment) # CRMのAPI実行のための認証エンドポイントのドメイン diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py index e87ef861..82a451e2 100644 --- a/ecs/crm-datafetch/src/util/execute_datetime.py +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -14,4 +14,4 @@ class ExecuteDateTime: return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) def to_YYYYMMDDHHMMSS(self) -> str: - return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': None, 'T': None, ':': None, '.': None})) \ No newline at end of file + return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': None, 'T': None, ':': None, '.': None})) diff --git a/s3/config/crm/task_settings/task_settings.env b/s3/config/crm/task_settings/task_settings.env index d48ff51a..1998012c 100644 --- a/s3/config/crm/task_settings/task_settings.env +++ b/s3/config/crm/task_settings/task_settings.env @@ -13,4 +13,5 @@ CRM_FETCH_RECORD_TIMEOUT=300 CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3 CRM_FETCH_RECORD_RETRY_INTERVAL=5 CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 \ No newline at end of file +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50 +CONVERT_TZ='Asia/Tokyo' \ No newline at end of file From f1f7cb6bbfb277980e4d75e38698fd2abf2dd1f1 Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 2 Aug 2022 15:55:39 +0900 Subject: [PATCH 20/21] =?UTF-8?q?fix:=20=E9=96=A2=E6=95=B0=E5=90=8D?= =?UTF-8?q?=E3=81=AE=E4=BF=AE=E6=AD=A3=E3=80=81=E3=82=BF=E3=82=A4=E3=83=A0?= =?UTF-8?q?=E3=82=BE=E3=83=BC=E3=83=B3=E3=82=92=E7=92=B0=E5=A2=83=E5=A4=89?= =?UTF-8?q?=E6=95=B0=E3=81=8B=E3=82=89=E8=AA=AD=E3=82=80=E3=82=88=E3=81=86?= =?UTF-8?q?=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/config/objects.py | 4 ++-- ecs/crm-datafetch/src/converter/convert_strategy.py | 5 +++-- ecs/crm-datafetch/src/util/execute_datetime.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 20673d70..41ca753b 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -106,8 +106,8 @@ class TargetObject(): @property def upload_file_name(self) -> str: if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): - return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime.to_YYYYMMDDHHMMSS()) - return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.to_YYYYMMDDHHMMSS()}' + return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime.format_date()) + return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime.format_date()}' @property def datetime_column(self) -> str: diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 307fbb9a..89ad2a2e 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -2,10 +2,11 @@ import re from datetime import datetime from dateutil.tz import gettz -from src.system_var.constants import (CONVERT_TZ, CRM_DATETIME_FORMAT, - CSV_FALSE_VALUE, CSV_TRUE_VALUE, +from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, + CSV_TRUE_VALUE, DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, YYYYMMDDHHMMSS) +from src.system_var.environments import CONVERT_TZ class ConvertStrategyFactory: diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py index 82a451e2..5a608013 100644 --- a/ecs/crm-datafetch/src/util/execute_datetime.py +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -13,5 +13,5 @@ class ExecuteDateTime: def to_path(self) -> str: return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) - def to_YYYYMMDDHHMMSS(self) -> str: + def format_date(self) -> str: return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': None, 'T': None, ':': None, '.': None})) From dd9c2209533b8d10a26876da9798f1809486717b Mon Sep 17 00:00:00 2001 From: Y_SAKAI Date: Tue, 2 Aug 2022 17:58:00 +0900 Subject: [PATCH 21/21] =?UTF-8?q?docs:=20=E3=82=BD=E3=83=BC=E3=82=B9?= =?UTF-8?q?=E5=86=85=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=81=AE=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/crm-datafetch/src/converter/convert_strategy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/crm-datafetch/src/converter/convert_strategy.py b/ecs/crm-datafetch/src/converter/convert_strategy.py index 89ad2a2e..8664d7cd 100644 --- a/ecs/crm-datafetch/src/converter/convert_strategy.py +++ b/ecs/crm-datafetch/src/converter/convert_strategy.py @@ -49,7 +49,7 @@ class BooleanConvertStrategy: class DatetimeConvertStrategy: def convert_value(self, convert_value: str) -> str: - # データ登録処理がJSTとして登録するため、変換処理内で事前にJSTの日次文字列に変換する + # データ登録処理がJSTとして登録するため、変換処理内で事前にJSTの日時文字列に変換する return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS)