diff --git a/ecs/crm-datafetch/main.py b/ecs/crm-datafetch/main.py index 6e197d30..dcae9825 100644 --- a/ecs/crm-datafetch/main.py +++ b/ecs/crm-datafetch/main.py @@ -1,4 +1,5 @@ -import src.controller as controller +from src.controller import controller +"""CRMデータ取得処理のエントリーポイント""" if __name__ == '__main__': - controller.main() + controller() diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index dec0ec62..e1d0dc70 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -1,7 +1,7 @@ import json import boto3 -from src.system_var.constants import AWS_CLINET_S3, AWS_RESOURCE_S3, S3_RESPONSE_BODY +from src.system_var.constants import AWS_RESOURCE_S3, S3_RESPONSE_BODY, S3_CHAR_CODE from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET, @@ -10,16 +10,6 @@ from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, RESPONSE_JSON_BACKUP_FOLDER) -class S3Clinet: - def __init__(self) -> None: - self.__s3_client = boto3.client(AWS_CLINET_S3) - - def copy_object(self, src_bucket, src_key, dest_bucket, dest_key): - self.__s3_client.copy( - {"Bucket": src_bucket, "key": src_key}, dest_bucket, dest_key) - return - - class S3Resource: def __init__(self, bucket_name: str) -> None: self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) @@ -28,41 +18,47 @@ class S3Resource: def get_object(self, object_key: str) -> str: response = self.__s3_bucket.Object(object_key).get() body = response[S3_RESPONSE_BODY].read() + return body.decode(S3_CHAR_CODE) - return body.decode('utf-8') - - def put_object(self, object_key: str, data) -> str: + def put_object(self, object_key: str, data: str) -> None: s3_object = self.__s3_bucket.Object(object_key) - s3_object.put(Body=data.encode('utf-8'), ContentEncoding='utf-8') + s3_object.put(Body=data.encode(S3_CHAR_CODE), ContentEncoding=S3_CHAR_CODE) return - -class S3ResourceNonBucket: - def __init__(self) -> None: - self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) - - def copy(self, src_bucket, src_key, dest_bucket, dest_key) -> None: + def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: copy_source = {'Bucket': src_bucket, 'Key': src_key} self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) - return +#class S3ResourceNonBucket: +# def __init__(self) -> None: +# self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) +# +# def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: +# copy_source = {'Bucket': src_bucket, 'Key': src_key} +# self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) +# return + + class ConfigBucket: __s3_resource: S3Resource = None def __init__(self) -> None: self.__s3_resource = S3Resource(CRM_CONFIG_BUCKET) + def __str__(self) -> str: + return CRM_CONFIG_BUCKET + def get_object_info_file(self) -> str: return self.__s3_resource.get_object(f'{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}') - def get_last_fetch_datetime_file(self, file_name) -> str: - return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_name}') + def get_last_fetch_datetime_file(self, file_path: str) -> str: + return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_path}') - def put_last_fetch_datetime_file(self, file_name, data) -> None: + def put_last_fetch_datetime_file(self, file_path: str, data: str) -> None: self.__s3_resource.put_object( - f'{LAST_FETCH_DATE_FOLDER}/{file_name}', data) + f'{LAST_FETCH_DATE_FOLDER}/{file_path}', data) return @@ -72,11 +68,19 @@ class DataBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(IMPORT_DATA_BUCKET) - def put_csv(self, file_name, data) -> None: - object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_name}' + def __str__(self) -> str: + return IMPORT_DATA_BUCKET + + def put_csv(self, file_path: str, data: str) -> None: + object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, data) return + def put_csv_from(self, src_bucket: str, src_key: str): + self.__s3_resource.copy(src_bucket, src_key, str(self), CRM_IMPORT_DATA_FOLDER) + return + + class BackupBucket: __s3_resource: S3Resource = None @@ -84,16 +88,20 @@ class BackupBucket: def __init__(self) -> None: self.__s3_resource = S3Resource(CRM_BACKUP_BUCKET) - def put_response_json(self, file_name, data) -> None: - object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_name}' + def __str__(self) -> str: + return CRM_BACKUP_BUCKET + + def put_response_json(self, file_path: str, data: dict) -> None: + object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, json.dumps(data)) return - def put_csv_bk(self, file_name, data) -> None: - object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_name}' + def put_csv(self, file_path: str, data: str) -> None: + object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, data) return - def put_result_json(self, file_name, data) -> None: - object_key = f'{PROCESS_RESULT_FOLDER}/{file_name}' + def put_result_json(self, file_path: str, data: dict) -> None: + object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, json.dumps(data)) + return \ No newline at end of file diff --git a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py new file mode 100644 index 00000000..f14490cf --- /dev/null +++ b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py @@ -0,0 +1,58 @@ +from src.aws.s3 import BackupBucket +from src.config.objects import TargetObject +from src.system_var.constants import CSVBK_JP_NAME +from src.error.exceptions import FileUploadException +from src.util.execute_datetime import ExecuteDateTime +from src.util.logger import logger_instance as logger + + +def backup_crm_csv_data_process(target_object: TargetObject, exetute_datetime: ExecuteDateTime, csv_string: str): + """ + CSVバックアップ処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + csv_string : str + csvデータ + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + + # ① CSVバックアップ処理の開始ログを出力する + target_object_name = target_object.object_name + upload_file_name = target_object.upload_file_name + + logger.info( + f'I-CSVBK-01 [{target_object_name}] のCSVデータのバックアップ処理を開始します ファイル名：[{upload_file_name}.csv]') + + try: + # ② CRMバックアップ保管用バケットに、変換後のCSVデータのバックアップを保管する + backup_bucket = BackupBucket() + backup_bucket.put_csv( + f'{exetute_datetime.to_path()}/{upload_file_name}.csv', csv_string) + + logger.debug( + f'D-CSVBK-02 [{target_object_name}] のCSVデータバックアップ 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-CSVBK-01', + CSVBK_JP_NAME, f'[{target_object_name}] CSVデータのバックアップに失敗しました ファイル名：[{upload_file_name}.csv] エラー内容:[{e}]') + + # ③ CSVバックアップ処理の終了ログを出力する + logger.info( + f'I-CSVBK-03 [{target_object_name}] のCSVデータのバックアップ処理を終了します') + + # ④ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/backup_crm_csvdata_process.py b/ecs/crm-datafetch/src/backup_crm_csvdata_process.py deleted file mode 100644 index 1960e7b0..00000000 --- a/ecs/crm-datafetch/src/backup_crm_csvdata_process.py +++ /dev/null @@ -1,31 +0,0 @@ -from src.aws.s3 import BackupBucket -from src.system_var.constants import CSVBK_JP_NAME -from src.error.exceptions import FileUploadException -from src.util.logger import logger_instance as logger - - -def backup_crm_csvdata(target_object, date_path, csv_object): - # ① CSVバックアップ処理の開始ログを出力する - logger.info( - f'I-CSVBK-01 [{target_object.object_name}] のCSVデータのバックアップ処理を開始します ファイル名：[{target_object.upload_file_name}.csv]') - - try: - # ② CRMバックアップ保管用バケットに、変換後のCSVデータのバックアップを保管する - backup_bucket = BackupBucket() - backup_bucket.put_csv_bk( - f'{date_path}/{target_object.upload_file_name}.csv', csv_object.csv_buffer) - - logger.debug( - f'D-CSVBK-02 [{target_object.object_name}] のCSVデータバックアップ 正常終了') - - except Exception as e: - raise FileUploadException( - 'E-CSVBK-01', - CSVBK_JP_NAME, f'[{target_object.object_name}] CSVデータのバックアップに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') - - # ③ CSVバックアップ処理の終了ログを出力する - logger.info( - f'I-CSVBK-03 [{target_object.object_name}] のCSVデータのバックアップ処理を終了します') - - # ④ 次の処理へ移行する - return diff --git a/ecs/crm-datafetch/src/backup_crm_data_process.py b/ecs/crm-datafetch/src/backup_crm_data_process.py index 130da769..a37eb583 100644 --- a/ecs/crm-datafetch/src/backup_crm_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_data_process.py @@ -1,19 +1,42 @@ from src.aws.s3 import BackupBucket -from src.system_var.constants import RESBK_JP_NAME from src.error.exceptions import FileUploadException +from src.system_var.constants import RESBK_JP_NAME +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def backup_crm_data(object_name, sf_object_jsons, date_path): +def backup_crm_data_process(object_name: str, sf_object_dict: dict, execute_datetime: ExecuteDateTime): + """ + CRM電文データバックアップ処理 + + Parameters + ---------- + object_name : str + 取得対象オブジェクト情報インスタンス + sf_object_dict : dict + Salesforceオブジェクトデータ + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + # ① CRM電文データバックアップ処理の開始ログを出力する logger.info(f'I-RESBK-01 [{object_name}] のCRM電文データバックアップ処理を開始します') try: # ② CRMバックアップ保管用バケットに、CRMから取得したJSONの電文データのバックアップを保管する - file_name = f'{date_path}/{object_name}.json' + file_name = f'{execute_datetime.to_path()}/{object_name}.json' backup_bucket = BackupBucket() - backup_bucket.put_response_json(file_name, sf_object_jsons) + backup_bucket.put_response_json(file_name, sf_object_dict) logger.debug(f'D-RESBK-02 [{object_name}] のJSONデータバックアップ 正常終了') diff --git a/ecs/crm-datafetch/src/check_object_info_process.py b/ecs/crm-datafetch/src/check_object_info_process.py index 91551d21..ed2a34da 100644 --- a/ecs/crm-datafetch/src/check_object_info_process.py +++ b/ecs/crm-datafetch/src/check_object_info_process.py @@ -1,10 +1,32 @@ from src.config.objects import TargetObject -from src.system_var.constants import CHK_JP_NAME from src.error.exceptions import InvalidConfigException +from src.util.execute_datetime import ExecuteDateTime +from src.system_var.constants import CHK_JP_NAME from src.util.logger import logger_instance as logger -def check_object_info(object_info, execute_datetime): +def check_object_info_process(object_info: dict, execute_datetime: ExecuteDateTime): + """ + オブジェクト情報形式チェック処理 + + Parameters + ---------- + object_info : dict + 取得対象オブジェクト情報 + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + + Returns + ------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + + Raises + ------ + InvalidConfigException + オブジェクト情報定義が不正だった場合 + """ + # ① オブジェクト情報形式チェック処理開始ログを出力する logger.info('I-CHK-01 オブジェクト情報形式チェック処理を開始します') diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index c00274bf..464f7b0e 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,193 +1,138 @@ -from src.system_var.constants import DATE_PATTERN_YYYYMMDDTHHMMSSTZ -from src.util.dict_checker import DictCheck +from src.system_var.constants import ( DATE_PATTERN_YYYYMMDDTHHMMSSTZ, + OBJECTS_KEY, + OBJECTS_TYPE, + OBJECT_NAME_KEY, + OBJECT_NAME_TYPE, + COLUMNS_KEY, + COLUMNS_TYPE, + IS_SKIP_KEY, + IS_SKIP_TYPE, + IS_UPDATE_LAST_FETCH_DATETIME_KEY, + IS_UPDATE_LAST_FETCH_DATETIME_TYPE, + LAST_FETCH_DATETIME_FILE_NAME_KEY, + LAST_FETCH_DATETIME_FILE_NAME_TYPE, + UPLOAD_FILE_NAME_KEY, + UPLOAD_FILE_NAME_TYPE, + DATETIME_COLUMN_KEY, + DATETIME_COLUMN_TYPE, + LAST_FETCH_DATETIME_FROM_KEY, + LAST_FETCH_DATETIME_TO_KEY, + DATETIME_COLUMN_DEFAULT_VALUE +) + +from src.util.dict_checker import DictChecker class FetchTargetObjects(): def __init__(self, object_info_file_dict) -> None: - self.__dict_check = DictCheck() self.__objects = object_info_file_dict - self.__key = 'objects' - self.check_key_objects() + self.__dict_checker = DictChecker(self.__objects) + self.validate() self.__i = 0 def __iter__(self): return self def __next__(self): - if self.__i == len(self.__objects[self.__key]): + if self.__i == len(self.__objects[OBJECTS_KEY]): raise StopIteration() - value = self.__objects['objects'][self.__i] + value = self.__objects[OBJECTS_KEY][self.__i] self.__i += 1 return value - def check_key_objects(self) -> None: - __check_key = self.__key - __check_type = list - self.__dict_check.check_key_exist_and_value_type( - self.__objects, __check_key, __check_type) + def validate(self) -> None: + self.__dict_checker.assert_key_exist(OBJECTS_KEY) + self.__dict_checker.assert_data_type(OBJECTS_KEY, OBJECTS_TYPE) class TargetObject(): def __init__(self, object_info, execute_datetime) -> None: - self.__dict_check = DictCheck() + self.__dict_checker = DictChecker(object_info) self.__object_info = object_info - self.execute_datetime = execute_datetime - self.check_key_object_name() - self.check_key_columns() - self.check_key_is_skip() - self.check_key_is_update_last_fetch_datetime() - self.check_key_last_fetch_datetime_file_name() - self.check_key_upload_file_name() - self.object_name = self.__object_info['object_name'] - self.columns = self.__object_info['columns'] - self.is_skip = self.set_is_skip() - self.is_update_last_fetch_datetime = self.set_is_update_last_fetch_datetime() - self.last_fetch_datetime_file_name = self.set_fetch_datetime_file_name() - self.upload_file_name = self.set_upload_file_name() + self.__execute_datetime = execute_datetime + self.__validate() - def check_key_object_name(self) -> None: - ''' - オブジェクト名チェック - ''' - __check_key = 'object_name' - __check_type = str - self.__dict_check.check_key_exist_and_value_type( - self.__object_info, __check_key, __check_type) + def __validate(self) -> None: + self.__validate_required_properties() + self.__validate_optional_properties() return - def check_key_columns(self) -> None: - ''' - カラム情報チェック - ''' - __key = 'columns' - __type = list - self.__dict_check.check_key_exist_and_value_type( - self.__object_info, __key, __type) + def __validate_required_properties(self) -> None: + self.__dict_checker.assert_key_exist(OBJECT_NAME_KEY) + self.__dict_checker.assert_data_type(OBJECT_NAME_KEY, OBJECT_NAME_TYPE) + self.__dict_checker.assert_key_exist(COLUMNS_KEY) + self.__dict_checker.assert_data_type(COLUMNS_KEY, COLUMNS_TYPE) return - def check_key_is_skip(self,) -> None: - ''' - スキップフラグ型チェック - ''' - __check_key = 'is_skip' - __check_type = bool - self.__dict_check.check_key_exist_case_value_type( - self.__object_info, __check_key, __check_type) + def __validate_optional_properties(self) -> None: + if self.__dict_checker.check_key_exist(IS_SKIP_KEY): + self.__dict_checker.assert_data_type(IS_SKIP_KEY, IS_SKIP_TYPE) + + if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY): + self.__dict_checker.assert_data_type(IS_UPDATE_LAST_FETCH_DATETIME_KEY, IS_UPDATE_LAST_FETCH_DATETIME_TYPE) + + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY): + self.__dict_checker.assert_data_type(LAST_FETCH_DATETIME_FILE_NAME_KEY, LAST_FETCH_DATETIME_FILE_NAME_TYPE) + + if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): + self.__dict_checker.assert_data_type(UPLOAD_FILE_NAME_KEY, UPLOAD_FILE_NAME_TYPE) + + if self.__dict_checker.check_key_exist(DATETIME_COLUMN_KEY): + self.__dict_checker.assert_data_type(DATETIME_COLUMN_KEY, DATETIME_COLUMN_TYPE) return - def check_key_is_update_last_fetch_datetime(self) -> None: - ''' - 前回取得日時ファイル更新フラグチェック - ''' - __check_key = 'is_update_last_fetch_datetime' - __check_type = bool - self.__dict_check.check_key_exist_case_value_type( - self.__object_info, __check_key, __check_type) + @property + def object_name(self) -> str: + return self.__object_info[OBJECT_NAME_KEY] - return + @property + def columns(self) -> list: + return self.__object_info[COLUMNS_KEY] - def check_key_last_fetch_datetime_file_name(self) -> None: - ''' - 前回取得日時ファイル名型チェック - ''' - __check_key = 'last_fetch_datetime_file_name' - __check_type = str - self.__dict_check.check_key_exist_case_value_type( - self.__object_info, __check_key, __check_type) + @property + def is_skip(self) -> bool: + return self.__object_info[IS_SKIP_KEY] if self.__dict_checker.check_key_exist(IS_SKIP_KEY) else False - return + @property + def is_update_last_fetch_datetime(self) -> bool: + return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY) else False - def check_key_upload_file_name(self) -> None: - ''' - アップロードファイル名称型チェック - ''' - __check_key = 'upload_file_name' - __check_type = str - self.__dict_check.check_key_exist_case_value_type( - self.__object_info, __check_key, __check_type) + @property + def last_fetch_datetime_file_name(self) -> str: + return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY] if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}.json' - return - def set_is_skip(self) -> bool: - ''' - スキップフラグ設定 - ''' - __check_key = 'is_skip' - if self.__dict_check.check_key_exist(self.__object_info, __check_key): - return self.__object_info[__check_key] - else: - return False + @property + def upload_file_name(self) -> str: + return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' - def set_is_update_last_fetch_datetime(self) -> bool: - ''' - 前回取得日時ファイル更新フラグ設定 - ''' - __check_key = 'is_update_last_fetch_datetime' - if self.__dict_check.check_key_exist(self.__object_info, __check_key): - return self.__object_info[__check_key] - else: - return False - - def set_fetch_datetime_file_name(self) -> str: - ''' - 前回取得日時ファイル名設定 - ''' - __check_key = 'last_fetch_datetime_file_name' - if self.__dict_check.check_key_exist(self.__object_info, __check_key): - return self.__object_info[__check_key] - else: - return self.__object_info['object_name'] + '.json' - - def set_upload_file_name(self) -> str: - ''' - アップロードファイル名称設定 - ''' - __check_key = 'upload_file_name' - if self.__dict_check.check_key_exist(self.__object_info, __check_key): - return self.__object_info[__check_key].format(execute_datetime=self.execute_datetime) - else: - return 'CRM_' + self.__object_info['object_name'] + '_' + self.execute_datetime + @property + def datetime_column(self) -> str: + return self.__object_info[DATETIME_COLUMN_KEY] if self.__dict_checker.check_key_exist(DATETIME_COLUMN_KEY) else DATETIME_COLUMN_DEFAULT_VALUE class LastFetchDatetime(): - def __init__(self, last_fetch_datetime_file_name, last_fetch_datetime_file_dict, execute_datetime) -> None: - self.__dict_check = DictCheck() - self.execute_datetime = execute_datetime + def __init__(self, last_fetch_datetime_file_dict, execute_datetime) -> None: + self.__dict_checker = DictChecker(last_fetch_datetime_file_dict) + self.__execute_datetime = execute_datetime self.__last_fetch_datetime_file_dict = last_fetch_datetime_file_dict - self.last_fetch_datetime_file_name = last_fetch_datetime_file_name - self.check_key_last_fetch_datetime_from - self.check_key_last_fetch_datetime_to - self.last_fetch_datetime_from = self.__last_fetch_datetime_file_dict[ - 'last_fetch_datetime_from'] - self.last_fetch_datetime_to = self.set_last_fetch_datetime_to() + self.__validate() - def check_key_last_fetch_datetime_from(self) -> None: - ''' - データ取得開始日時チェック - ''' - __check_key = 'last_fetch_datetime_from' - __regex_str = DATE_PATTERN_YYYYMMDDTHHMMSSTZ - self.__dict_check.check_key_exsit_and_regex( - self.__last_fetch_datetime_file_dict, __check_key, __regex_str) + def __validate(self) -> None: + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FROM_KEY): + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): + self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) - def check_key_last_fetch_datetime_to(self) -> None: - ''' - データ取得終了日時チェック - ''' - __check_key = 'last_fetch_datetime_to' - __regex_str = DATE_PATTERN_YYYYMMDDTHHMMSSTZ - self.__dict_check.check_key_exsit_case_regex( - self.__last_fetch_datetime_file_dict, __check_key, __regex_str) + return - def set_last_fetch_datetime_to(self) -> str: - ''' - データ取得終了日時設定 - ''' - __check_key = 'last_fetch_datetime_to' - if self.__dict_check.check_key_exist(self.__last_fetch_datetime_file_dict, __check_key): - return self.__last_fetch_datetime_file_dict[__check_key].format(execute_datetime=self.execute_datetime) - else: - return self.execute_datetime + @property + def last_fetch_datetime_from(self) -> str: + return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_FROM_KEY] + + @property + def last_fetch_datetime_to(self) -> str: + return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FROM_KEY) else self.__execute_datetime diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index ebd3b3cd..acbd2383 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -1,125 +1,45 @@ -from src.check_object_info_process import check_object_info # オブジェクト情報形式チェック処理 -from src.convert_crm_csvdata_process import convert_crm_csvdata # CSV変換処理 -from src.backup_crm_csvdata_process import backup_crm_csvdata # CSVバックアップ処理 -from src.set_datetime_period_process import set_datetime_period # データ取得期間設定処理 -from src.updload_result_data_process import updload_result_data # 取得処理実施結果アップロード処理 +import gc + +from src.backup_crm_csv_data_process import backup_crm_csv_data_process +from src.backup_crm_data_process import backup_crm_data_process +from src.check_object_info_process import check_object_info_process +from src.config.objects import FetchTargetObjects +from src.convert_crm_csv_data_process import convert_crm_csv_data_process +from src.copy_crm_csv_data_process import copy_crm_csv_data_process from src.error.exceptions import MeDaCaCRMDataFetchException -from src.fetch_crm_data_process import fetch_crm_data # CRMデータ取得処理 -from src.prepare_get_data_process import prepare_get_data # データ取得準備処理 -from src.backup_crm_data_process import backup_crm_data # CRM電文データバックアップ処理 -from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime # 前回取得日時ファイル更新 -from src.copy_crm_csvdata_process import copy_crm_csvdata # CSVアップロード処理 +from src.util.execute_datetime import ExecuteDateTime +from src.fetch_crm_data_process import fetch_crm_data_process +from src.prepare_data_fetch_process import prepare_data_fetch_process +from src.set_datetime_period_process import set_datetime_period_process +from src.system_var.constants import OBJECT_NAME_KEY +from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime_process +from src.upload_result_data_process import upload_result_data_process from src.util.logger import logger_instance as logger -def main() -> None: +def controller() -> None: + """コントロール処理""" + try: # ① CRMデータ取得処理開始ログを出力する logger.info('I-CTRL-01 CRMデータ取得処理を開始します') - fetch_target_objects = None # オブジェクト情報ファイル用オブジェクト - execute_datetime = None # 実行日次文字列 - date_path = None # 実行日次のパス文字列 - process_result = None # オブジェクトごとの実行結果JSON - # ② データ取得準備処理を呼び出す logger.info('I-CTRL-02 データ取得準備処理呼び出し') - fetch_target_objects, execute_datetime, date_path, process_result = prepare_get_data() + fetch_target_objects, execute_datetime, process_result = prepare_data_fetch_process() # ③ object_infoのobjectsキーの値の件数分ループする logger.info('I-CTRL-03 取得対象オブジェクトのループ処理開始') - for object_info in fetch_target_objects: - try: - # 1. オブジェクト処理結果の初期化 - target_object = None # オブジェクトごとの情報格納用オブジェクト - last_fetch_datetime = None # オブジェクトごとの取得日付格納用オブジェクト - sf_object_jsons = None # オブジェクトごとのSalesforce取得変数JSON - csv_object = None # CSVオブジェクト - process_result[object_info.get('object_name')] = 'fail' # オブジェクト処理結果 - - logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：{object_info}') - - # 2. オブジェクト情報形式チェック処理を呼び出す - logger.info('I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') - - target_object = check_object_info(object_info, execute_datetime) - - # 3. 処理対象のオブジェクト名をログ出力する - logger.info( - f'I-CTRL-06 [{target_object.object_name}]のデータ取得を開始します') - - # 4. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する - if target_object.is_skip is True: - logger.info( - f'I-CTRL-07 [{target_object.object_name}]のデータ取得処理をスキップします') - continue - - # 5. データ取得期間設定処理を呼び出す - logger.info( - f'I-CTRL-08 [{target_object.object_name}]のデータ取得期間設定処理呼び出し') - - last_fetch_datetime = set_datetime_period(target_object, execute_datetime) - - # 6. CRMデータ取得処理を呼び出す - logger.info( - f'I-CTRL-09 [{target_object.object_name}]のデータ取得処理呼び出し') - - sf_object_jsons = fetch_crm_data(target_object, last_fetch_datetime) - - # 7. 出力ファイル名をログ出力する - logger.info( - f'I-CTRL-10 [{target_object.object_name}] の出力ファイル名は [{target_object.upload_file_name}]となります') - - # 8. CRM電文データバックアップ処理を呼び出す - logger.info( - f'I-CTRL-11 [{target_object.object_name}] CRM電文データバックアップ処理呼び出し') - backup_crm_data(target_object.object_name, sf_object_jsons, date_path) - - # 9. CSV変換処理を呼び出す - logger.info( - f'I-CTRL-12 [{target_object.object_name}] CSV変換処理呼び出し') - csv_object = convert_crm_csvdata(target_object, sf_object_jsons) - - # 10. CSVバックアップ処理を呼び出す - logger.info( - f'I-CTRL-13 [{target_object.object_name}] CSVデータバックアップ処理呼び出し') - backup_crm_csvdata(target_object, date_path, csv_object) - - # 11. CSVアップロード処理を呼び出す - logger.info( - f'I-CTRL-14 [{target_object.object_name}] CSVデータアップロード処理呼び出し') - copy_crm_csvdata(target_object, date_path) - - # 12. 前回取得日時ファイル更新処理を呼びだす - logger.info( - f'I-CTRL-15 [{target_object.object_name}] 前回取得日時ファイル更新処理呼び出し') - upload_last_fetch_datetime(target_object, last_fetch_datetime) - - # 13. オブジェクト処理結果の更新 - process_result[target_object.object_name] = 'success' - - # 14. オブジェクトのアップロードが完了した旨をログに出力する - logger.info(f'I-CTRL-16 [{target_object.object_name}] 処理正常終了') - - except MeDaCaCRMDataFetchException as e: - logger.info(f'{e.error_id} {e}') - logger.info( - f'I-ERR-03 [{object_info.get("object_name")}] の[{e.func_name}]でエラーが発生しました 次のオブジェクトの処理に移行します', exc_info=True) - continue - - except Exception as e: - logger.info( - f'I-ERR-04 [{object_info.get("object_name")}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e, exc_info=True) - continue + process_result = fetch_crm_data(fetch_target_objects, execute_datetime, process_result) # ④ すべてのオブジェクトの処理が完了したことと、オブジェクト毎の処理結果をログに出力する logger.info(f'I-CTRL-17 すべてのオブジェクトの処理が終了しました 実行結果：[{process_result}]') # ⑤ 取得処理実施結果アップロード処理を呼び出す logger.info('I-CTRL-19 CRM_取得処理実施結果ファイルアップロード処理開始') - updload_result_data(process_result, date_path) + upload_result_data_process(process_result, execute_datetime) # ⑥ 最終結果をチェックし、チェック結果をログに出力 if not all([v == 'success' for v in process_result.values()]): @@ -142,5 +62,136 @@ def main() -> None: return exit(0) -if __name__ == '__main__': - main() +def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: ExecuteDateTime, process_result: dict): + """ + 取得対象オブジェクト情報をループし、１オブジェクトごとのデータを取得する + + Parameters + ---------- + fetch_target_objects : FetchTargetObjects + CRMオブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + process_result : dict + 取得処理実行結果辞書オブジェクト + + Returns + ------- + process_result : dict + 取得処理実行結果辞書オブジェクト + + """ + for object_info in fetch_target_objects: + try: + process_result[object_info.get(OBJECT_NAME_KEY)] = 'fail' + + fetch_crm_data_per_object(object_info, execute_datetime, process_result) + + process_result[object_info.get(OBJECT_NAME_KEY)] = 'success' + + except MeDaCaCRMDataFetchException as e: + logger.info(f'{e.error_id} {e}') + logger.info( + f'I-ERR-03 [{object_info.get(OBJECT_NAME_KEY)}] の[{e.func_name}]でエラーが発生しました 次のオブジェクトの処理に移行します', exc_info=True) + continue + + except Exception as e: + logger.info( + f'I-ERR-04 [{object_info.get(OBJECT_NAME_KEY)}] の処理中に予期せぬエラーが発生しました 次のオブジェクトの処理に移行します', e, exc_info=True) + continue + + return process_result + +def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTime) -> None: + """ + オブジェクトごとにCRMのデータを取得し、取込フォルダにアップロードする + + Parameters + ---------- + object_info : dict + 取得対象オブジェクト情報 + execute_datetime : object + 実行日次取得インスタンス + + Returns + ------- + なし + + Raises + ------ + FileNotFoundException + S3上のファイルが存在しない場合 + InvalidConfigException + オブジェクト情報定義が不正だった場合 + + """ + + # 1. オブジェクト処理結果の初期化 + logger.debug(f'D-CTRL-04 対象のオブジェクト情報を出力します　オブジェクト情報：{object_info}') + + # 2. オブジェクト情報形式チェック処理を呼び出す + logger.info('I-CTRL-05 オブジェクト情報形式チェック処理呼び出し') + + target_object = check_object_info_process(object_info, execute_datetime) + target_object_name = target_object.object_name + + # 3. 処理対象のオブジェクト名をログ出力する + logger.info( + f'I-CTRL-06 [{target_object_name}]のデータ取得を開始します') + + # 4. オブジェクト情報.is_skipがTrueの場合、次のオブジェクトの処理に移行する + if target_object.is_skip is True: + logger.info( + f'I-CTRL-07 [{target_object_name}]のデータ取得処理をスキップします') + return + + # 5. データ取得期間設定処理を呼び出す + logger.info( + f'I-CTRL-08 [{target_object_name}]のデータ取得期間設定処理呼び出し') + + last_fetch_datetime = set_datetime_period_process(target_object, execute_datetime) + + # 6. CRMデータ取得処理を呼び出す + logger.info( + f'I-CTRL-09 [{target_object_name}]のデータ取得処理呼び出し') + + crm_data_response = fetch_crm_data_process(target_object, last_fetch_datetime) + + # 7. 出力ファイル名をログ出力する + logger.info( + f'I-CTRL-10 [{target_object_name}] の出力ファイル名は [{target_object.upload_file_name}]となります') + + # 8. CRM電文データバックアップ処理を呼び出す + logger.info( + f'I-CTRL-11 [{target_object_name}] CRM電文データバックアップ処理呼び出し') + backup_crm_data_process(target_object_name, crm_data_response, execute_datetime) + + # 9. CSV変換処理を呼び出す + logger.info( + f'I-CTRL-12 [{target_object.object_name}] CSV変換処理呼び出し') + csv_string = convert_crm_csv_data_process(target_object, crm_data_response) + + # 10. CSVバックアップ処理を呼び出す + logger.info( + f'I-CTRL-13 [{target_object_name}] CSVデータバックアップ処理呼び出し') + backup_crm_csv_data_process(target_object, execute_datetime, csv_string) + + # 11. CSVアップロード処理を呼び出す + logger.info( + f'I-CTRL-14 [{target_object_name}] CSVデータアップロード処理呼び出し') + copy_crm_csv_data_process(target_object, execute_datetime) + + # 12. メモリ解放 + del crm_data_response + del csv_string + gc.collect() + + # 13. 前回取得日時ファイル更新処理を呼びだす + logger.info( + f'I-CTRL-15 [{target_object_name}] 前回取得日時ファイル更新処理呼び出し') + upload_last_fetch_datetime_process(target_object, last_fetch_datetime) + + # 14. オブジェクトのアップロードが完了した旨をログに出力する + logger.info(f'I-CTRL-16 [{target_object_name}] 処理正常終了') + + return \ No newline at end of file diff --git a/ecs/crm-datafetch/src/convert_crm_csv_data_process.py b/ecs/crm-datafetch/src/convert_crm_csv_data_process.py new file mode 100644 index 00000000..1ed05c3a --- /dev/null +++ b/ecs/crm-datafetch/src/convert_crm_csv_data_process.py @@ -0,0 +1,51 @@ +from src.config.objects import TargetObject +from src.converter.converter import CSVStringConverter +from src.error.exceptions import DataConvertException +from src.system_var.constants import CONV_JP_NAME +from src.util.logger import logger_instance as logger + + +def convert_crm_csv_data_process(target_object: TargetObject, crm_data_response: dict): + """ + CSV変換処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + crm_data_response : dict + Salesforceオブジェクトデータ + + Returns + ------- + csv_string : str + csvデータ + + Raises + ------ + DataConvertException + データ変換が失敗した場合 + """ + + # ① CSV変換処理の開始ログを出力する + target_object_name = target_object.object_name + + logger.info(f'I-CONV-01 [{target_object_name}] のCSV変換処理を開始します') + + try: + # ② CSV変換 + CSVStringConverter(target_object, crm_data_response) + csv_string = CSVStringConverter.convert() + + + logger.debug(f'D-CONV-02 [{target_object_name}] のCSV変換処理 正常終了') + + except Exception as e: + raise DataConvertException( + 'E-CONV-01', CONV_JP_NAME, f'[{target_object_name}] CSV変換に失敗しました エラー内容:[{e}]') + + # ③ CSV変換処理の終了ログを出力する + logger.info(f'I-CONV-03 [{target_object_name}] のCSV変換処理を終了します') + + # ④ 次の処理へ移行する + return csv_string diff --git a/ecs/crm-datafetch/src/convert_crm_csvdata_process.py b/ecs/crm-datafetch/src/convert_crm_csvdata_process.py deleted file mode 100644 index 4ef6dd11..00000000 --- a/ecs/crm-datafetch/src/convert_crm_csvdata_process.py +++ /dev/null @@ -1,25 +0,0 @@ -from src.system_var.constants import CONV_JP_NAME -from src.converter.converter import CSVStringConverter -from src.error.exceptions import DataConvertException -from src.util.logger import logger_instance as logger - - -def convert_crm_csvdata(target_object, sf_object_jsons): - # ① CSV変換処理の開始ログを出力する - logger.info(f'I-CONV-01 [{target_object.object_name}] のCSV変換処理を開始します') - - try: - # ② CSV変換 - csv_object = CSVStringConverter(target_object, sf_object_jsons) - - logger.debug(f'D-CONV-02 [{target_object.object_name}] のCSV変換処理 正常終了') - - except Exception as e: - raise DataConvertException( - 'E-CONV-01', CONV_JP_NAME, f'[{target_object.object_name}] CSV変換に失敗しました エラー内容:[{e}]') - - # ③ CSV変換処理の終了ログを出力する - logger.info(f'I-CONV-03 [{target_object.object_name}] のCSV変換処理を終了します') - - # ④ 次の処理へ移行する - return csv_object diff --git a/ecs/crm-datafetch/src/converter/convert_factory.py b/ecs/crm-datafetch/src/converter/convert_factory.py new file mode 100644 index 00000000..48673f7e --- /dev/null +++ b/ecs/crm-datafetch/src/converter/convert_factory.py @@ -0,0 +1,55 @@ +import re +from datetime import datetime + +from src.config.objects import TargetObject +from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, + CSV_TRUE_VALUE, YYYYMMDDHHMMSS, DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC) + + +class ConvertStrategyFactory: + def __init__(self) -> None: + self.__none_value_convert_strategy = NoneValueConvertStrategy() + self.__float_convert_strategy = FloatConvertStrategy() + self.__boolean_convert_strategy = BooleanConvertStrategy() + self.__datetime_convert_strategy = DatatimeConvertStrategy() + + def create(self, value): + + converted_value = value + + if value is None: + converted_value = self.__none_value_convert_strategy.convert_value() + + # 指数表記で取得できるパターン。指数表記を整数表記に変換する。 + elif type(value) == float: + converted_value = self.__float_convert_strategy.convert_value(value) + + # SQLの真偽値に対応するために変換する + elif type(value) == bool: + converted_value = self.__boolean_convert_strategy.convert_value(value) + + elif type(value) == str and re.fullmatch(DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, value): + converted_value = self.__datetime_convert_strategy.convert_value(value) + + return converted_value + + +class NoneValueConvertStrategy: + def convert_value(self) -> str: + return '' + + +class BooleanConvertStrategy: + def convert_value(self, convert_value: str) -> bool: + return CSV_TRUE_VALUE if convert_value is True else CSV_FALSE_VALUE + + +class DatatimeConvertStrategy: + def convert_value(self, convert_value: str) -> str: + return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).strftime(YYYYMMDDHHMMSS) + + +class FloatConvertStrategy: + def convert_value(self, convert_value: str) -> int: + return int(convert_value) + diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 7b4adb1e..bf7faaee 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -1,33 +1,35 @@ import csv import io -import re -from datetime import datetime -from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, - CSV_TRUE_VALUE, YYYYMMDDHHMMSS) +from src.config.objects import TargetObject +from src.converter.convert_factory import ConvertStrategyFactory class CSVStringConverter: - def __init__(self, target_object, sf_object_jsons) -> None: + def __init__(self, target_object: TargetObject, sf_object_jsons: dict) -> None: self.__target_object = target_object self.__sf_object_jsons = sf_object_jsons - self.__extracted_sf_object_jsons = self.extract_sf_object_jsons() - self.csv_data = self.convert_to_csv() - self.csv_buffer = self.write_csv() + self.__convert_strategy_factory = ConvertStrategyFactory() - def extract_sf_object_jsons(self) -> list: + def convert(self) -> str: + extracted_sf_object_jsons = self.__extract_sf_object_jsons() + csv_data = self.__convert_to_csv(extracted_sf_object_jsons) + csv_string = self.__write_csv_string(csv_data) + return csv_string + + def __extract_sf_object_jsons(self) -> list: try: extracted_sf_object_jsons = [] for sf_object_json in self.__sf_object_jsons: extracted_sf_object_jsons.append( - self.extract_necessary_props_from(sf_object_json)) + self.__extract_necessary_props_from(sf_object_json)) return extracted_sf_object_jsons except Exception as e: - raise Exception('必要なjsonのデータ抽出に失敗しました') + raise Exception('必要なjsonのデータ抽出に失敗しました', e) - def extract_necessary_props_from(self, sf_object_json) -> dict: + def __extract_necessary_props_from(self, sf_object_json) -> dict: try: clone_sf_object = {**sf_object_json} @@ -39,19 +41,18 @@ class CSVStringConverter: return uppercase_key_sf_object except Exception as e: - raise Exception('必要なjsonのデータ成形に失敗しました') + raise Exception('必要なjsonのデータ成形に失敗しました', e) - def convert_to_csv(self) -> list: + def __convert_to_csv(self, extracted_sf_object_jsons) -> list: try: columns = self.__target_object.columns csv_data = [] - for i, json_object in enumerate(self.__extracted_sf_object_jsons, 1): + for i, json_object in enumerate(extracted_sf_object_jsons, 1): csv_row = [] for column in columns: v = json_object[column.upper()] - converted_value = CSVStringConverterFactory( - v).value_convert() + converted_value = self.__convert_strategy_factory.create(v) csv_row.append(converted_value) @@ -62,74 +63,16 @@ class CSVStringConverter: raise Exception( f'CSV変換に失敗しました カラム名：[{column}] 行番号: [{i}] エラー内容:[{e}]') - def write_csv(self) -> str: + def __write_csv_string(self, csv_data) -> str: try: with io.StringIO(newline='') as string_stream: writer = csv.writer(string_stream, delimiter=',', lineterminator='\r\n', doublequote=True, quotechar='"', quoting=csv.QUOTE_ALL, strict=True) writer.writerow(self.__target_object.columns) - writer.writerows(self.csv_data) + writer.writerows(csv_data) csv_value = string_stream.getvalue() return csv_value except Exception as e: - raise Exception('csvデータの取得に失敗しました') - - -class NoneValueConverter: - def __init__(self, convert_value) -> None: - self.__convert_value = convert_value - self.value = self.convert_value() - - def convert_value(self) -> str: - return '' - - -class BooleanConverter: - def __init__(self, convert_value) -> None: - self.__convert_value = convert_value - self.value = self.convert_value() - - def convert_value(self) -> bool: - return CSV_TRUE_VALUE if self.__convert_value is True else CSV_FALSE_VALUE - - -class DatatimeConverter: - def __init__(self, convert_value) -> None: - self.__convert_value = convert_value - self.value = self.convert_value() - - def convert_value(self) -> str: - return datetime.strptime(self.__convert_value, CRM_DATETIME_FORMAT).strftime(YYYYMMDDHHMMSS) - - -class FloatConverter: - def __init__(self, convert_value) -> None: - self.__convert_value = convert_value - self.value = self.convert_value() - - def convert_value(self) -> int: - return int(self.__convert_value) - - -class CSVStringConverterFactory: - def __init__(self, v) -> None: - self.__value = v - - def value_convert(self): - - converted_value = self.__value - - if self.__value is None: - converted_value = NoneValueConverter(self.__value).value - # 指数表記で取得できるパターン。指数表記を整数表記に変換する。 - elif type(self.__value) == float: - converted_value = FloatConverter(self.__value).value - # SQLの真偽値に対応するために変換する - elif type(self.__value) == bool: - converted_value = BooleanConverter(self.__value).value - elif type(self.__value) == str and re.fullmatch(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.000\+0000', self.__value): - converted_value = DatatimeConverter(self.__value).value - - return converted_value + raise Exception('csvデータの取得に失敗しました', e) diff --git a/ecs/crm-datafetch/src/copy_crm_csv_data_process.py b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py new file mode 100644 index 00000000..57a3659f --- /dev/null +++ b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py @@ -0,0 +1,57 @@ +from src.aws.s3 import BackupBucket, DataBucket +from src.config.objects import TargetObject +from src.error.exceptions import FileUploadException +from src.system_var.constants import UPLD_JP_NAME +from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, + CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) +from src.util.execute_datetime import ExecuteDateTime +from src.util.logger import logger_instance as logger + + +def copy_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime): + """ + CSVアップロード処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + + # ① CSVデータアップロード処理の開始ログを出力する + target_object_name = target_object.object_name + upload_file_name = target_object.upload_file_name + + logger.info( + f'I-UPLD-01 [{target_object_name}] のCSVデータアップロード処理を開始します ファイル名：[{upload_file_name}.csv]') + + try: + # ② CRMバックアップ保管用バケットに保管した変換後のCSVデータをデータ取込バケットにコピーする + data_bucket = DataBucket() + backup_bucket = BackupBucket() + data_bucket.put_csv_from(str(backup_bucket), f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{execute_datetime.to_path()}/{upload_file_name}.csv') + + logger.debug( + f'D-UPLD-02 [{target_object_name}] のCSVデータアップロード 正常終了') + + except Exception as e: + raise FileUploadException( + 'E-UPLD-01', UPLD_JP_NAME, f'[{target_object_name}] CSVデータのアップロードに失敗しました ファイル名：[{upload_file_name}.csv] エラー内容:[{e}]') + + # ③ CSVデータアップロード処理の終了ログを出力する + logger.info( + f'I-UPLD-03 [{target_object_name}] のCSVデータのアップロード処理を終了します') + + # ④ 次の処理へ移行する + return diff --git a/ecs/crm-datafetch/src/copy_crm_csvdata_process.py b/ecs/crm-datafetch/src/copy_crm_csvdata_process.py deleted file mode 100644 index ecebe9ea..00000000 --- a/ecs/crm-datafetch/src/copy_crm_csvdata_process.py +++ /dev/null @@ -1,32 +0,0 @@ -from src.aws.s3 import S3ResourceNonBucket -from src.system_var.constants import UPLD_JP_NAME -from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, - CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) -from src.error.exceptions import FileUploadException -from src.util.logger import logger_instance as logger - - -def copy_crm_csvdata(target_object, date_path): - # ① CSVデータアップロード処理の開始ログを出力する - logger.info( - f'I-UPLD-01 [{target_object.object_name}] のCSVデータアップロード処理を開始します ファイル名：[{target_object.upload_file_name}.csv]') - - try: - # ② CRMバックアップ保管用バケットに保管した変換後のCSVデータをデータ取込バケットにコピーする - s3_resource_non_bucket = S3ResourceNonBucket() - s3_resource_non_bucket.copy(CRM_BACKUP_BUCKET, f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{date_path}/{target_object.upload_file_name}.csv', - IMPORT_DATA_BUCKET, f'{CRM_IMPORT_DATA_FOLDER}/{target_object.upload_file_name}.csv') - - logger.debug( - f'D-UPLD-02 [{target_object.object_name}] のCSVデータアップロード 正常終了') - - except Exception as e: - raise FileUploadException( - 'E-UPLD-01', UPLD_JP_NAME, f'[{target_object.object_name}] CSVデータのアップロードに失敗しました ファイル名：[{target_object.upload_file_name}.csv] エラー内容:[{e}]') - - # ③ CSVデータアップロード処理の終了ログを出力する - logger.info( - f'I-UPLD-03 [{target_object.object_name}] のCSVデータのアップロード処理を終了します') - - # ④ 次の処理へ移行する - return diff --git a/ecs/crm-datafetch/src/error/exceptions.py b/ecs/crm-datafetch/src/error/exceptions.py index 71a174c0..f231804b 100644 --- a/ecs/crm-datafetch/src/error/exceptions.py +++ b/ecs/crm-datafetch/src/error/exceptions.py @@ -4,7 +4,7 @@ from abc import ABCMeta class MeDaCaCRMDataFetchException(Exception, metaclass=ABCMeta): """MeDaCaシステム固有のカスタムエラークラス""" - def __init__(self, error_id: str, func_name, message) -> None: + def __init__(self, error_id: str, func_name: str, message: str) -> None: super().__init__(message) self.func_name = func_name self.error_id = error_id diff --git a/ecs/crm-datafetch/src/fetch_crm_data_process.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py index eac5478b..3a89da1e 100644 --- a/ecs/crm-datafetch/src/fetch_crm_data_process.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -2,6 +2,10 @@ from requests.exceptions import ConnectTimeout, ReadTimeout from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential +from src.config.objects import TargetObject, LastFetchDatetime +from src.error.exceptions import DataConvertException, SalesforceAPIException +from src.salesforce.salesforce_api import SalesforceApiClient +from src.salesforce.soql_builder import SOQLBuilder from src.system_var.constants import FETCH_JP_NAME from src.system_var.environments import (CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, @@ -14,20 +18,38 @@ from src.system_var.environments import (CRM_AUTH_TIMEOUT, CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_TIMEOUT) -from src.error.exceptions import DataConvertException, SalesforceAPIException -from src.salesforce.salesforce_api import SalesForceCount, SalesForceData from src.util.logger import logger_instance as logger -def fetch_crm_data(target_object, last_fetch_datetime): +def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): + """ + CRMデータ取得処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + last_fetch_datetime : LastFetchDatetime + 取得対象オブジェクト情報インスタンス + + Returns + ------- + sf_object_dict : dict + Salesforceオブジェクトデータ + + Raises + ------ + SalesforceAPIException + SalseforceのAPI実行失敗が発生した場合 + DataConvertException + データ変換が失敗した場合 + """ + # ① CRMデータ取得処理開始ログを出力する logger.info( f'I-FETCH-01 [{target_object.object_name}] のCRMからのデータ取得処理を開始します') object_name = target_object.object_name - columns = ','.join(target_object.columns) - last_fetch_datetime_from = last_fetch_datetime.last_fetch_datetime_from - last_fetch_datetime_to = last_fetch_datetime.last_fetch_datetime_to global count_contime_counter, count_readtime_counter, count_counter, data_contime_counter, data_readtime_counter, data_counter count_contime_counter = 1 @@ -41,8 +63,10 @@ def fetch_crm_data(target_object, last_fetch_datetime): # ② 取得対象オブジェクトの取得期間内のレコード件数を取得する logger.info(f'I-FETCH-02 [{object_name}] の件数取得を開始します') - record_count = fetch_sf_count_retry( - object_name, last_fetch_datetime_from, last_fetch_datetime_to) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + count_soql = soql_builder.create_count_soql() + + record_count = fetch_record_count_retry(count_soql, object_name) logger.info(f'I-FETCH-03 [{object_name}] の件数：[{record_count}]') @@ -54,8 +78,9 @@ def fetch_crm_data(target_object, last_fetch_datetime): # ③ 取得対象オブジェクトのレコードを取得する logger.info(f'I-FETCH-04 [{object_name}] のレコード取得を開始します') - record_generator = fetch_sf_data_retry( - columns, object_name, last_fetch_datetime_from, last_fetch_datetime_to) + fetch_soql = soql_builder.create_fetch_soql() + + record_all = fetch_sf_data_retry(fetch_soql, object_name) except Exception as e: raise SalesforceAPIException( @@ -65,10 +90,7 @@ def fetch_crm_data(target_object, last_fetch_datetime): # ④ 取得対象オブジェクトをJSONに変換 logger.info(f'I-FETCH-05 [{object_name}] のレコードをJSONに変換します') - sf_object_jsons = [] - - for record in record_generator: - sf_object_jsons.append(record) + crm_data_response = [record for record in record_all] except Exception as e: raise DataConvertException( @@ -78,19 +100,19 @@ def fetch_crm_data(target_object, last_fetch_datetime): logger.info(f'I-FETCH-06 [{object_name}] のレコード取得が成功しました') # ⑥ 次の処理へ移行する - return sf_object_jsons + return crm_data_response @retry( wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) -def fetch_sf_count_retry(object_name, last_update_datetime_from, last_update_datetime_to): +def fetch_record_count_retry(soql: str, object_name: str): try: global count_contime_counter, count_readtime_counter, count_counter - saleforce_count = SalesForceCount() - return saleforce_count.fetch_sf_count(object_name, last_update_datetime_from, last_update_datetime_to) + salesforce_api_client = SalesforceApiClient() + return salesforce_api_client.fetch_sf_count(soql) except ConnectTimeout as e: if count_contime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: @@ -117,12 +139,12 @@ def fetch_sf_count_retry(object_name, last_update_datetime_from, last_update_dat wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) -def fetch_sf_data_retry(columns, object_name, last_update_datetime_from, last_update_datetime_to): +def fetch_sf_data_retry(soql: str, object_name: str): try: global data_contime_counter, data_readtime_counter, data_counter - saleforce_data = SalesForceData() - return saleforce_data.fetch_sf_data(columns, object_name, last_update_datetime_from, last_update_datetime_to) + salesforce_api_client = SalesforceApiClient() + return salesforce_api_client.fetch_sf_data(soql) except ConnectTimeout as e: if data_contime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: diff --git a/ecs/crm-datafetch/src/prepare_get_data_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py similarity index 76% rename from ecs/crm-datafetch/src/prepare_get_data_process.py rename to ecs/crm-datafetch/src/prepare_data_fetch_process.py index e96de09c..409651ad 100644 --- a/ecs/crm-datafetch/src/prepare_get_data_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -1,24 +1,46 @@ -from datetime import datetime - from src.aws.s3 import ConfigBucket from src.config.objects import FetchTargetObjects -from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ -from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, - OBJECT_INFO_FOLDER) from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser +from src.system_var.constants import PRE_JP_NAME +from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER) +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def prepare_get_data(): +def prepare_data_fetch_process(): + """ + データ取得準備処理 + + Parameters + ---------- + なし + + Returns + ------- + fetch_target_objects : FetchTargetObjects + CRMオブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + process_result : dict + 取得処理実行結果辞書オブジェクト + + Raises + ------ + FileNotFoundException + S3上のファイルが存在しない場合 + InvalidConfigException + オブジェクト情報定義が不正だった場合 + + """ # ① データ取得準備処理の開始ログを出力する logger.info('I-PRE-01 データ取得準備処理を開始します') # ② 取得処理開始年月日時分秒を控える - execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) - date_path = execute_datetime.rstrip('000Z').translate( - str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) + + execute_datetime = ExecuteDateTime() logger.info(f'I-PRE-02 データ取得処理開始日時：{execute_datetime}') @@ -29,7 +51,7 @@ def prepare_get_data(): f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]') config_bucket = ConfigBucket() - object_info_file_json = config_bucket.get_object_info_file() + object_info_file_str = config_bucket.get_object_info_file() logger.debug('D-PRE-04 CRM_取得オブジェクト情報ファイルの取得成功しました') @@ -41,7 +63,7 @@ def prepare_get_data(): # ④ CRM_取得オブジェクト情報ファイルをパースし、メモリ上に展開する logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします') - json_parser = JsonParser(object_info_file_json) + json_parser = JsonParser(object_info_file_str) object_info_file_dict = json_parser.json_parser() logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました') @@ -69,4 +91,4 @@ def prepare_get_data(): logger.info('I-PRE-09 データ取得準備処理を終了します') # ⑧ 次の処理へ移行する - return(fetch_target_objects, execute_datetime, date_path, process_result) + return(fetch_target_objects, execute_datetime, process_result) diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index a6b9c458..0c073b9a 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,61 +1,28 @@ from simple_salesforce import Salesforce -from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_MAX_RETRY_ATTEMPT, - CRM_AUTH_RETRY_INTERVAL, - CRM_AUTH_RETRY_MAX_INTERVAL, - CRM_AUTH_RETRY_MIN_INTERVAL, CRM_AUTH_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, + +from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_TIMEOUT, CRM_USER_NAME, CRM_USER_PASSWORD, CRM_USER_SECURITY_TOKEN) -FETCH_SOQL = """SELECT {column_or_expression} FROM {object_name} - WHERE SystemModStamp > {last_update_datetime_from} - AND SystemModStamp <= {last_update_datetime_to} -""" -class SalesfoeceApi(): +class SalesforceApiClient(): def __init__(self) -> None: self.__sf = Salesforce(username=CRM_USER_NAME, password=CRM_USER_PASSWORD, security_token=CRM_USER_SECURITY_TOKEN, domain=CRM_AUTH_DOMAIN ) - def sf_query(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + def query(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): return self.__sf.query(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) - def sf_query_all_iter(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): - return self.__sf.query_all_iter(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) + def query_all(self, soql, include_deleted=True, conn_timeout=100, read_timeout=300): + return self.__sf.query_all(soql, include_deleted, timeout=(float(conn_timeout), float(read_timeout))) - -class SalesForceCount(): - def fetch_sf_count(self, object_name, last_update_datetime_from, last_update_datetime_to): - count_soql = FETCH_SOQL.format( - column_or_expression='COUNT(Id)', - object_name=object_name, - last_update_datetime_from=last_update_datetime_from, - last_update_datetime_to=last_update_datetime_to - ) - self.__sf = SalesfoeceApi() - count_res = self.__sf.sf_query( - count_soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) + def fetch_sf_count(self, soql: str): + count_res = self.query(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_GET_RECORD_COUNT_TIMEOUT) return count_res.get('records')[0].get('expr0') - -class SalesForceData(): - def fetch_sf_data(self, columns, object_name, last_update_datetime_from, last_update_datetime_to): - soql = FETCH_SOQL.format( - column_or_expression=columns, - object_name=object_name, - last_update_datetime_from=last_update_datetime_from, - last_update_datetime_to=last_update_datetime_to - ) - self.__sf = SalesfoeceApi() - return self.__sf.sf_query_all_iter(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + def fetch_sf_data(self, soql: str): + return self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) diff --git a/ecs/crm-datafetch/src/salesforce/soql_builder.py b/ecs/crm-datafetch/src/salesforce/soql_builder.py new file mode 100644 index 00000000..a84744a8 --- /dev/null +++ b/ecs/crm-datafetch/src/salesforce/soql_builder.py @@ -0,0 +1,34 @@ +from src.config.objects import TargetObject, LastFetchDatetime + + +class SOQLBuilder: + def __init__(self, target_object: TargetObject, last_fetch_datetime: LastFetchDatetime) -> None: + self.__SELECT_SOQL = """SELECT {column_or_expression} FROM {object_name} + WHERE {datetime_column} > {last_fetch_datetime_from} + AND {datetime_column} <= {last_fetch_datetime_to} + """ + self.__target_object = target_object + self.__last_fetch_datetime = last_fetch_datetime + + def create_count_soql(self): + count_soql = self.__SELECT_SOQL.format( + column_or_expression='COUNT(Id)', + object_name=self.__target_object.object_name, + last_fetch_datetime_from=self.__last_fetch_datetime.last_fetch_datetime_from, + last_fetch_datetime_to=self.__last_fetch_datetime.last_fetch_datetime_to, + datetime_column=self.__target_object.datetime_column + ) + + return count_soql + + def create_fetch_soql(self): + columns = ','.join(self.__target_object.columns) + fetch_soql = self.__SELECT_SOQL.format( + column_or_expression=columns, + object_name=self.__target_object.object_name, + last_fetch_datetime_from=self.__last_fetch_datetime.last_fetch_datetime_from, + last_fetch_datetime_to=self.__last_fetch_datetime.last_fetch_datetime_to, + datetime_column=self.__target_object.datetime_column + ) + + return fetch_soql diff --git a/ecs/crm-datafetch/src/set_datetime_period_process.py b/ecs/crm-datafetch/src/set_datetime_period_process.py index ea281fba..9795ab2e 100644 --- a/ecs/crm-datafetch/src/set_datetime_period_process.py +++ b/ecs/crm-datafetch/src/set_datetime_period_process.py @@ -1,13 +1,39 @@ from src.aws.s3 import ConfigBucket -from src.config.objects import LastFetchDatetime -from src.system_var.constants import DATE_JP_NAME -from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.config.objects import TargetObject, LastFetchDatetime from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser +from src.system_var.constants import DATE_JP_NAME +from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def set_datetime_period(target_object, execute_datetime): + + +def set_datetime_period_process(target_object: TargetObject, execute_datetime: ExecuteDateTime): + """ + データ取得期間設定処理 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + execute_datetime : ExecuteDateTime + 実行日次取得インスタンス + + Returns + ------- + last_fetch_datetime : LastFetchDatetime + 取得対象オブジェクト情報インスタンス + + Raises + ------ + FileNotFoundException + S3上のファイルが存在しない場合 + InvalidConfigException + オブジェクト情報定義が不正だった場合 + """ + # ① データ取得期間設定処理の開始ログを出力する logger.info( f'I-DATE-01 [{target_object.object_name}] のデータ取得期間設定処理を開始します') @@ -18,7 +44,7 @@ def set_datetime_period(target_object, execute_datetime): f'I-DATE-02 前回取得日時ファイルの取得開始します ファイルパス：[s3://{CRM_CONFIG_BUCKET}/{LAST_FETCH_DATE_FOLDER}/{target_object.last_fetch_datetime_file_name}]') s3_config_bucket = ConfigBucket() - last_fetch_datetime_file_json = s3_config_bucket.get_last_fetch_datetime_file( + last_fetch_datetime_file_str = s3_config_bucket.get_last_fetch_datetime_file( target_object.last_fetch_datetime_file_name) logger.info(f'I-DATE-03 前回取得日時ファイルの取得成功しました') @@ -32,11 +58,10 @@ def set_datetime_period(target_object, execute_datetime): # ④ データの取得期間を設定する logger.debug(f'D-DATE-04 前回取得日時ファイルの形式チェックを開始します') - json_parser = JsonParser(last_fetch_datetime_file_json) + json_parser = JsonParser(last_fetch_datetime_file_str) last_fetch_datetime_file_dict = json_parser.json_parser() - last_fetch_datetime = LastFetchDatetime(target_object.last_fetch_datetime_file_name, - last_fetch_datetime_file_dict, execute_datetime) + last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_file_dict, execute_datetime) logger.debug(f'D-DATE-05 前回取得日時ファイルの形式チェック 正常終了') logger.info( diff --git a/ecs/crm-datafetch/src/system_var/constants.py b/ecs/crm-datafetch/src/system_var/constants.py index fde2ec65..b84d8f1d 100644 --- a/ecs/crm-datafetch/src/system_var/constants.py +++ b/ecs/crm-datafetch/src/system_var/constants.py @@ -1,6 +1,5 @@ # environments(task settings file) -# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する -LOG_LEVEL = "LOG_LEVEL" +LOG_LEVEL = 'LOG_LEVEL' # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する CRM_AUTH_TIMEOUT = 'CRM_AUTH_TIMEOUT' # CRMへの認証処理のタイムアウト秒数 CRM_AUTH_MAX_RETRY_ATTEMPT = 'CRM_AUTH_MAX_RETRY_ATTEMPT' # CRMへの認証処理の最大リトライ試行回数 CRM_AUTH_RETRY_INTERVAL = 'CRM_AUTH_RETRY_INTERVAL' # CRMへの認証処理のリトライ時の初回待ち秒数 @@ -17,7 +16,7 @@ CRM_FETCH_RECORD_RETRY_INTERVAL = 'CRM_FETCH_RECORD_RETRY_INTERVAL' CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MIN_INTERVAL' # CRMのレコード取得処理のリトライ時の最小待ち秒数 CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = 'CRM_FETCH_RECORD_RETRY_MAX_INTERVAL' # CRMのレコード取得処理のリトライ時の最大待ち秒数 -# environments(ECS Task Enviroment) +# environments(ECS Task Environment) CRM_AUTH_DOMAIN = 'CRM_AUTH_DOMAIN' # CRMのAPI実行のための認証エンドポイントのドメイン CRM_USER_NAME = 'CRM_USER_NAME' # CRMのAPI実行用ユーザ名 CRM_USER_PASSWORD = 'CRM_USER_PASSWORD' # CRMのAPI実行用ユーザパスワード @@ -40,16 +39,17 @@ CRM_IMPORT_DATA_BACKUP_FOLDER = 'CRM_IMPORT_DATA_BACKUP_FOLDER' YYYYMMDDTHHMMSSTZ = '%Y-%m-%dT%H:%M:%S.000Z' CRM_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S.000%z' YYYYMMDDHHMMSS = '%Y-%m-%d %H:%M:%S' +MILLISEC_FORMAT = '000Z' # aws AWS_RESOURCE_S3 = 's3' -AWS_CLINET_S3 = 's3' S3_RESPONSE_BODY = 'Body' - +S3_CHAR_CODE = 'utf-8' # 正規表現チェック EXCLUDE_SYMBOL = ['#', '/'] -DATE_PATTERN_YYYYMMDDTHHMMSSTZ = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9].000Z' +DATE_PATTERN_YYYYMMDDTHHMMSSTZ = r'[12]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[12][0-9]|3[01])T([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]\.000Z' +DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC = r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.000\+0000' # logger LOG_FORMAT = '[%(levelname)s]\t%(asctime)s\t%(message)s\n' @@ -72,3 +72,24 @@ END_JP_NAME = '取得処理実施結果アップロード処理' # CSVチェック CSV_TRUE_VALUE = '1' CSV_FALSE_VALUE = '0' + +# オブジェクト変数 +OBJECTS_KEY = 'objects' +OBJECTS_TYPE = list +OBJECT_NAME_KEY = 'object_name' +OBJECT_NAME_TYPE = str +COLUMNS_KEY = 'columns' +COLUMNS_TYPE = list +IS_SKIP_KEY = 'is_skip' +IS_SKIP_TYPE = bool +IS_UPDATE_LAST_FETCH_DATETIME_KEY = 'is_update_last_fetch_datetime' +IS_UPDATE_LAST_FETCH_DATETIME_TYPE = bool +LAST_FETCH_DATETIME_FILE_NAME_KEY = 'last_fetch_datetime_file_name' +LAST_FETCH_DATETIME_FILE_NAME_TYPE = str +UPLOAD_FILE_NAME_KEY = 'upload_file_name' +UPLOAD_FILE_NAME_TYPE = str +DATETIME_COLUMN_KEY = 'datetime_column' +DATETIME_COLUMN_TYPE = str +DATETIME_COLUMN_DEFAULT_VALUE = 'SystemModstamp' +LAST_FETCH_DATETIME_TO_KEY = 'last_fetch_datetime_to' +LAST_FETCH_DATETIME_FROM_KEY = 'last_fetch_datetime_from' diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index dfa87ae3..5ff41929 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -6,37 +6,37 @@ import src.system_var.constants as constants # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_TIMEOUT = os.environ[constants.CRM_AUTH_TIMEOUT] +CRM_AUTH_TIMEOUT = os.environ.get(constants.CRM_AUTH_TIMEOUT, 100) # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_AUTH_MAX_RETRY_ATTEMPT] +CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3) # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_INTERVAL] +CRM_AUTH_RETRY_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5) # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MIN_INTERVAL] +CRM_AUTH_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5) # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ[constants.CRM_AUTH_RETRY_MAX_INTERVAL] +CRM_AUTH_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50) # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ[constants.CRM_GET_RECORD_COUNT_TIMEOUT] +CRM_GET_RECORD_COUNT_TIMEOUT = os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300) # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT] +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3) # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL] +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5) # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL] +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, 5) # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ[constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL] +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50) # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ[constants.CRM_FETCH_RECORD_TIMEOUT] +CRM_FETCH_RECORD_TIMEOUT = os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300) # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ[constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT] +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3) # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_INTERVAL] +CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5) # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL] +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5) # CRMのレコード取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ[constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL] +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50) -# environments(ECS Task Enviroment) +# environments(ECS Task Environment) # CRMのAPI実行のための認証エンドポイントのドメイン CRM_AUTH_DOMAIN = os.environ[constants.CRM_AUTH_DOMAIN] # CRMのAPI実行用ユーザ名 @@ -52,20 +52,20 @@ CRM_BACKUP_BUCKET = os.environ[constants.CRM_BACKUP_BUCKET] # CRMの取込データを格納するバケット名 IMPORT_DATA_BUCKET = os.environ[constants.IMPORT_DATA_BUCKET] # CRM取得対象オブジェクトの情報を格納するフォルダパス -OBJECT_INFO_FOLDER = os.environ[constants.OBJECT_INFO_FOLDER] +OBJECT_INFO_FOLDER = os.environ.get(constants.OBJECT_INFO_FOLDER, 'crm/object_info') # CRM取得対象オブジェクトの情報のファイル名 OBJECT_INFO_FILENAME = os.environ[constants.OBJECT_INFO_FILENAME] # CRMデータ取得結果を格納するフォルダパス -PROCESS_RESULT_FOLDER = os.environ[constants.PROCESS_RESULT_FOLDER] +PROCESS_RESULT_FOLDER = os.environ.get(constants.PROCESS_RESULT_FOLDER, 'data_import') # CRMデータ取得結果を格納するファイル名 -PROCESS_RESULT_FILENAME = os.environ[constants.PROCESS_RESULT_FILENAME] +PROCESS_RESULT_FILENAME = os.environ.get(constants.PROCESS_RESULT_FILENAME, 'process_result.json') # CRMからの最終取得日時ファイルを格納するフォルダパス -LAST_FETCH_DATE_FOLDER = os.environ[constants.LAST_FETCH_DATE_FOLDER] +LAST_FETCH_DATE_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_FOLDER, 'crm/last_fetch_datetime') # CRMから取得し、取込用に変換したデータを格納するフォルダ -CRM_IMPORT_DATA_FOLDER = os.environ[constants.CRM_IMPORT_DATA_FOLDER] +CRM_IMPORT_DATA_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_FOLDER, 'crm/target') # CRMからの最終取得日時ファイルのバックアップを格納するフォルダパス -LAST_FETCH_DATE_BACKUP_FOLDER = os.environ[constants.LAST_FETCH_DATE_BACKUP_FOLDER] +LAST_FETCH_DATE_BACKUP_FOLDER = os.environ.get(constants.LAST_FETCH_DATE_BACKUP_FOLDER, 'last_fetch_datetime') # CRMから取得した生データのバックアップを格納するフォルダパス -RESPONSE_JSON_BACKUP_FOLDER = os.environ[constants.RESPONSE_JSON_BACKUP_FOLDER] +RESPONSE_JSON_BACKUP_FOLDER = os.environ.get(constants.RESPONSE_JSON_BACKUP_FOLDER, 'response_json') # CRMから取得し、取込用に変換したデータのバックアップを格納するフォルダ -CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ[constants.CRM_IMPORT_DATA_BACKUP_FOLDER] +CRM_IMPORT_DATA_BACKUP_FOLDER = os.environ.get(constants.CRM_IMPORT_DATA_BACKUP_FOLDER, 'data_import') diff --git a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py index f606c9ad..e83e58ab 100644 --- a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py +++ b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py @@ -1,12 +1,33 @@ import json from src.aws.s3 import ConfigBucket -from src.system_var.constants import UPD_JP_NAME +from src.config.objects import TargetObject, LastFetchDatetime from src.error.exceptions import FileUploadException +from src.system_var.constants import UPD_JP_NAME from src.util.logger import logger_instance as logger -def updload_last_fetch_datetime(target_object, last_fetch_datetime): +def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): + """ + 前回取得日時ファイル更新 + + Parameters + ---------- + target_object : TargetObject + 取得対象オブジェクト情報インスタンス + last_fetch_datetime : LastFetchDatetime + 取得対象オブジェクト情報インスタンス + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + # ① 前回取得日時ファイル更新処理の開始ログを出力する logger.info( f'I-UPD-01 [{target_object.object_name}] の前回取得日時ファイルの更新処理を開始します') @@ -16,20 +37,21 @@ def updload_last_fetch_datetime(target_object, last_fetch_datetime): # ② オブジェクト情報.is_update_last_fetch_datetimeがfalseの場合、以降の処理をスキップする logger.info( f'I-UPD-02 [{target_object.object_name}] の前回取得日時ファイルの更新処理をスキップします') - else: - # ③ 前回取得日時ファイル.last_fetch_datetime_fromに取得処理開始年月日時分秒を設定する - # 前回取得日時ファイル.last_fetch_datetime_toに空文字を設定する - last_fetch_datetime_dict = { - 'last_fetch_datetime_from': last_fetch_datetime.last_fetch_datetime_to, - 'last_fetch_datetime_to': '' - } + return - config_bucket = ConfigBucket() - config_bucket.put_last_fetch_datetime_file( - target_object.last_fetch_datetime_file_name, json.dumps(last_fetch_datetime_dict)) + # ③ 前回取得日時ファイル.last_fetch_datetime_fromに取得処理開始年月日時分秒を設定する + # 前回取得日時ファイル.last_fetch_datetime_toに空文字を設定する + last_fetch_datetime_dict = { + 'last_fetch_datetime_from': last_fetch_datetime.last_fetch_datetime_to, + 'last_fetch_datetime_to': '' + } - logger.info( - f'D-UPD-03 [{target_object.object_name}] の前回取得日時ファイル更新処理 正常終了') + config_bucket = ConfigBucket() + config_bucket.put_last_fetch_datetime_file( + target_object.last_fetch_datetime_file_name, json.dumps(last_fetch_datetime_dict)) + + logger.info( + f'D-UPD-03 [{target_object.object_name}] の前回取得日時ファイル更新処理 正常終了') except Exception as e: raise FileUploadException( diff --git a/ecs/crm-datafetch/src/updload_result_data_process.py b/ecs/crm-datafetch/src/upload_result_data_process.py similarity index 64% rename from ecs/crm-datafetch/src/updload_result_data_process.py rename to ecs/crm-datafetch/src/upload_result_data_process.py index e70f4679..855ec900 100644 --- a/ecs/crm-datafetch/src/updload_result_data_process.py +++ b/ecs/crm-datafetch/src/upload_result_data_process.py @@ -1,11 +1,32 @@ from src.aws.s3 import BackupBucket +from src.error.exceptions import FileUploadException from src.system_var.constants import END_JP_NAME from src.system_var.environments import PROCESS_RESULT_FILENAME -from src.error.exceptions import FileUploadException +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger -def updload_result_data(process_result, date_path): +def upload_result_data_process(process_result: dict, execute_datetime: ExecuteDateTime): + """ + 取得処理実施結果アップロード処理 + + Parameters + ---------- + process_result : dict + 取得処理実行結果辞書オブジェクト + last_fetch_datetime : LastFetchDatetime + 取得対象オブジェクト情報インスタンス + + Returns + ------- + なし + + Raises + ------ + FileUploadException + S3のファイルアップロード失敗 + """ + # ① 取得処理実施結果アップロード処理のログを出力する logger.info( f'I-END-01 取得処理実施結果アップロード処理を開始します') @@ -14,7 +35,7 @@ def updload_result_data(process_result, date_path): # ② CRMバックアップ保管用バケットに、取得処理実施結果のJSONデータを保管する backup_bucket = BackupBucket() backup_bucket.put_result_json( - f'{date_path}/{PROCESS_RESULT_FILENAME}', process_result) + f'{execute_datetime.to_path()}/{PROCESS_RESULT_FILENAME}', process_result) logger.debug(f'D-END-02 取得処理実施結果アップロード 正常終了') diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py index a8bfed62..9d498b5e 100644 --- a/ecs/crm-datafetch/src/util/dict_checker.py +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -1,64 +1,39 @@ import re -class DictCheck: - def __init__(self) -> None: - pass +class DictChecker: + def __init__(self, object_dict: dict) -> None: + self.__object_dict = object_dict - def check_key_exist(self, object_dict: dict, check_key: str) -> bool: - ''' - 辞書型キー存在チェック - ''' - return True if check_key in object_dict and object_dict[check_key] != '' else False + def check_key_exist(self, check_key: str) -> bool: + """辞書型キー存在チェック""" + return check_key in self.__object_dict and self.__object_dict[check_key] != '' - def check_value_type(self, object_dict: dict, check_key: str, check_type: type) -> bool: - ''' - 辞書型バリュー型チェック - ''' - return True if isinstance(object_dict[check_key], check_type) else False + def check_data_type(self, check_key: str, check_type: type) -> bool: + """辞書型バリュー型チェック""" + return isinstance(self.__object_dict[check_key], check_type) - def check_regex(self, regex_str: str, object_dict: dict, check_key: str) -> bool: - ''' - 辞書型バリュー正規表現チェック - ''' - return True if re.fullmatch(regex_str, object_dict[check_key]) else False + def check_match_pattern(self, regex_str: str, check_key: str) -> bool: + """辞書型バリュー正規表現チェック""" + return True if re.fullmatch(regex_str, self.__object_dict[check_key]) else False - def check_key_exist_and_value_type(self, object_dict: dict, check_key: str, check_type: type) -> None: - ''' - 辞書型キー存在チェック＆バリュー型チェック - ''' - if not self.check_key_exist(object_dict, check_key): + def assert_key_exist(self, check_key: str, check_type: type) -> None: + """辞書型キー存在検査""" + if not self.check_key_exist(check_key): raise Exception(f'「{check_key}」キーは必須です') - elif not self.check_value_type(object_dict, check_key, check_type): + return + + def assert_data_type(self, check_key: str, check_type: type) -> None: + """バリュー型検査""" + if not self.check_data_type(check_key, check_type): raise Exception(f'「{check_key}」キーの値は「{check_type}」でなければなりません') - def check_key_exist_case_value_type(self, object_dict: dict, check_key: str, check_type: type): - ''' - 辞書型キー存在した場合のバリュー型チェック - ''' - if not self.check_key_exist(object_dict, check_key): - pass + return - elif not self.check_value_type(object_dict, check_key, check_type): - raise Exception(f'「{check_key}」キーの値は「{check_type}」でなければなりません') - - def check_key_exsit_and_regex(self, object_dict: dict, check_key: str, regex_str: str): - ''' - 辞書型キー存在チェック＆バリュー正規表現チェック - ''' - if not self.check_key_exist(object_dict, check_key): - raise Exception(f'「{check_key}」キーは必須です') - - elif not self.check_regex(regex_str, object_dict, check_key): + def assert_match_pattern(self, check_key: str, regex_str: str): + """正規表現検査""" + if not self.check_match_pattern(regex_str, check_key): raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') - def check_key_exsit_case_regex(self, object_dict: dict, check_key: str, regex_str: str): - ''' - 辞書型キー存在した場合のバリュー正規表現チェック - ''' - if not self.check_key_exist(object_dict, check_key): - pass - - elif not self.check_regex(regex_str, object_dict, check_key): - raise Exception(f'「{check_key}」キーの値の正規表現「{regex_str}」チェックに失敗しました') + return diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py new file mode 100644 index 00000000..2f07736a --- /dev/null +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -0,0 +1,17 @@ +from datetime import datetime + +from src.system_var.constants import( + YYYYMMDDTHHMMSSTZ, + MILLISEC_FORMAT +) + + +class ExecuteDateTime: + def __init__(self): + self.__execute_datetime = datetime.now().strftime(YYYYMMDDTHHMMSSTZ) + + def __str__(self) -> str: + return self.__execute_datetime + + def to_path(self) -> str: + return self.__execute_datetime.rstrip(MILLISEC_FORMAT).translate(str.maketrans({'-': '/', 'T': '/', ':': None, '.': None})) diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index 689d9c36..ec1df277 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -2,6 +2,7 @@ import logging from src.system_var.environments import LOG_LEVEL +"""boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する""" for name in ["boto3", "botocore", "s3transfer", "urllib3"]: logging.getLogger(name).setLevel(logging.WARNING) diff --git a/s3/config/crm/object_info/crm_object_list_all.json b/s3/config/crm/object_info/crm_object_list_all.json new file mode 100644 index 00000000..a65cf1bb --- /dev/null +++ b/s3/config/crm/object_info/crm_object_list_all.json @@ -0,0 +1,47 @@ +{ + "objects": [ + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": false, + "last_fetch_datetime_file_name": "Territory2_ALL.json", + "upload_file_name": "CRM_Territory2_ALL_{execute_datetime}.csv" + }, + { + "object_name": "UserTerritory2Association", + "columns": [ + "Id", + "UserId", + "Territory2Id", + "IsActive", + "RoleInTerritory2", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": false, + "last_fetch_datetime_file_name": "UserTerritory2Association_ALL.json", + "upload_file_name": "CRM_UserTerritory2Association_ALL_{execute_datetime}.csv" + } + ] +} diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index 78ec31a7..00f7fe89 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -1,41 +1,3008 @@ { "objects": [ - { - # これはコメント行です - "object_name": "Account", - "columns": [ - "Id", - "Name", - "SystemModStamp", - "NumberOfEmployees", - "IsDeleted" - ] - }, - { - "object_name": "Territory2", - "is_skip": false, - "can_update_last_update": false, - "upload_file_name": "CRM_Territory2_All_{execute_datetime}_{split_csv_suffix_number:03}", - "last_fetch_datetime_file_name": "Territory2.json", - "columns": [ - "Id", - "Name", - "Territory2TypeId", - "SystemModStamp" - ] - }, - { - "object_name": "AccountShare", - "is_skip": false, - "datetime_column": "LastModifiedDate", - "columns": [ - "Id", - "Name", - "Territory2TypeId", - "LastModifiedDate" - ] - } + { + "object_name": "Clm_Presentation_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Mobile_ID_vod__c", + "Presentation_Id_vod__c", + "Product_vod__c", + "Default_Presentation_vod__c", + "Training_vod__c", + "ParentId_vod__c", + "Hidden_vod__c", + "Type_vod__c", + "Approved_vod__c", + "Copied_From_vod__c", + "Copy_Date_vod__c", + "Survey_vod__c", + "Original_Record_ID_vod__c", + "Directory_vod__c", + "End_Date_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "VExternal_Id_vod__c", + "Vault_DNS_vod__c", + "Vault_Doc_Id_vod__c", + "Vault_External_Id_vod__c", + "Vault_GUID_vod__c", + "Vault_Last_Modified_Date_Time_vod__c", + "Version_vod__c", + "Enable_Survey_Overlay_vod__c", + "Description_vod__c", + "Keywords_vod__c", + "Content_Channel_vod__c", + "original_material_approved_in_veritas__c", + "keywords__c", + "trade_team__c", + "ewizard_link__c", + "business_function__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Clm_Presentation_Slide_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Clm_Presentation_vod__c", + "Key_Message_vod__c", + "Display_Order_vod__c", + "Sub_Presentation_vod__c", + "Mobile_ID_vod__c", + "External_ID_vod__c", + "VExternal_Id_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Insight_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Clinical_Trial_vod__c", + "Date_vod__c", + "Description_vod__c", + "Entity_Reference_Id_vod__c", + "Interaction_vod__c", + "Medical_Event_vod__c", + "Mobile_ID_vod__c", + "Other_Source_vod__c", + "Override_Lock_vod__c", + "Publication_vod__c", + "Status_vod__c", + "Summary_vod__c", + "Unlock_vod__c", + "Commercial_Medical__c", + "MSJ_Level_1A__c", + "MSJ_Level_1B__c", + "MSJ_Level_2A__c", + "MSJ_Level_2B__c", + "MSJ_Level_3A__c", + "MSJ_Level_3B__c", + "MSJ_Level_4A__c", + "MSJ_Level_4B__c", + "MSJ_SubStatus__c", + "MSJ_Type_A__c", + "MSJ_Type_B__c", + "MSJ_Description_Backup__c", + "MSJ_Country__c", + "MSJ_Received_at_Boomi__c", + "MSJ_Level_1A_Value__c", + "MSJ_Level_1B_Value__c", + "MSJ_Level_2A_Value__c", + "MSJ_Level_2B_Value__c", + "MSJ_Level_3A_Value__c", + "MSJ_Level_3B_Value__c", + "MSJ_Level_4A_Value__c", + "MSJ_Level_4B_Value__c", + "MSJ_Hospital_ID__c", + "MSJ_Hospital_Name__c", + "MSJ_Hospital__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_MR_Weekly_Report__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Account_Name__c", + "MSJ_Activity_Results_Summary__c", + "MSJ_MUID__c", + "MSJ_Next_Week_Action_What__c", + "MSJ_Next_Week_Action_When__c", + "MSJ_Next_Week_Action_Where__c", + "MSJ_Next_Week_Action_Who__c", + "MSJ_Report_Week__c", + "MSJ_Target_Patient_Count__c", + "Mobile_ID_vod__c", + "MSJ_Activity_Results_Summary_HN__c", + "MSJ_Next_Week_Action_Where_HN__c", + "MSJ_Next_Week_Action_Who_HN__c", + "MSJ_Next_Week_Action_What_HN__c", + "MSJ_Next_Week_Action_When_HN__c", + "MSJ_Target_Patient_Count_HN__c", + "MSJ_Activity_Results_Summary_MCC__c", + "MSJ_Next_Week_Action_Where_MCC__c", + "MSJ_Next_Week_Action_Who_MCC__c", + "MSJ_Next_Week_Action_What_MCC__c", + "MSJ_Next_Week_Action_When_MCC__c", + "MSJ_Target_Patient_Count_MCC__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Account_Territory_Loader_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "External_ID_vod__c", + "Territory_vod__c", + "Mobile_ID_vod__c", + "Territory_To_Add_vod__c", + "Territory_to_Drop_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Event_Attendee_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Attendee_vod__c", + "User_vod__c", + "Medical_Event_vod__c", + "Attendee_Type_vod__c", + "Status_vod__c", + "Contact_vod__c", + "Attendee_Name_vod__c", + "Account_vod__c", + "Start_Date_vod__c", + "Signature_vod__c", + "Signature_Datetime_vod__c", + "MSJ_Copy_Account_Type__c", + "MSJ_Evaluation__c", + "MSJ_Hospital__c", + "MSJ_Role__c", + "Mobile_ID_vod__c", + "MSJ_Evaluation_Comment__c", + "Position_vod__c", + "Talk_Title_vod__c", + "MSJ_Attendee_Reaction__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "ObjectTerritory2Association", + "columns": [ + "Id", + "ObjectId", + "Territory2Id", + "AssociationCause", + "SobjectType", + "IsDeleted", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Key_Message_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Description_vod__c", + "Product_vod__c", + "Product_Strategy_vod__c", + "Display_Order_vod__c", + "Active_vod__c", + "Category_vod__c", + "Vehicle_vod__c", + "CLM_ID_vod__c", + "Custom_Reaction_vod__c", + "Slide_Version_vod__c", + "Language_vod__c", + "Media_File_CRC_vod__c", + "Media_File_Name_vod__c", + "Media_File_Size_vod__c", + "Segment_vod__c", + "Disable_Actions_vod__c", + "VExternal_Id_vod__c", + "CDN_Path_vod__c", + "Status_vod__c", + "Vault_DNS_vod__c", + "Vault_Doc_Id_vod__c", + "Vault_External_Id_vod__c", + "Vault_GUID_vod__c", + "Vault_Last_Modified_Date_Time_vod__c", + "Is_Shared_Resource_vod__c", + "iOS_Viewer_vod__c", + "iOS_Resolution_vod__c", + "approved_for_distribution_date__c", + "approved_for_use_date__c", + "ewizard_link__c", + "expiration_date__c", + "keywords__c", + "trade_team__c", + "business_function__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Group", + "columns": [ + "Id", + "Name", + "DeveloperName", + "RelatedId", + "Type", + "Email", + "OwnerId", + "DoesSendEmailToMembers", + "DoesIncludeBosses", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Event_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Primary_Product__c", + "Description_vod__c", + "Start_Date_vod__c", + "Location__c", + "End_Date_vod__c", + "Secondary_Product__c", + "Website__c", + "Active_vod__c", + "Event_Type__c", + "MSJ_Area__c", + "MSJ_Business_Unit__c", + "MSJ_Comment__c", + "MSJ_Company__c", + "MSJ_Expense_App_No__c", + "MSJ_Form__c", + "MSJ_HQ_Area__c", + "MSJ_Location__c", + "MSJ_MR__c", + "MSJ_Number_of_Attendee__c", + "MSJ_Product__c", + "MSJ_Site__c", + "MSJ_Type__c", + "MSJ_Number_of_Attendee_Auto_Calc__c", + "MSJ_Number_of_Attendee_Invited__c", + "Account_vod__c", + "MSJ_MUID__c", + "Country_Name_vod__c", + "MSJ_CE_SIPAGL_Updater__c", + "MSJ_CE_SIPAGL_1A__c", + "MSJ_CE_SIPAGL_1B__c", + "MSJ_CE_SIPAGL_2__c", + "MSJ_CE_SIPAGL_3__c", + "MSJ_CE_SIPAGL_4__c", + "MSJ_CE_SIPAGL_5__c", + "MSJ_CE_SIPAGL_Comment__c", + "MSJ_CE_SIPAGL_1A_date__c", + "MSJ_CE_SIPAGL_6__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Medical_Event_Evaluation__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Medical_Event__c", + "MSJ_Evaluation_Comment__c", + "MSJ_Evaluation__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Coaching_Report_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Mobile_ID_vod__c", + "Manager_vod__c", + "Employee_vod__c", + "Review_Date__c", + "Review_Period__c", + "Status__c", + "Comments__c", + "Strategic_Planning__c", + "Customer_Focus__c", + "Knowledge_Expertise__c", + "Business_Account_Planning__c", + "Call_Productivity__c", + "Overall_Rating__c", + "MSJ_A01__c", + "MSJ_A02__c", + "MSJ_A03__c", + "MSJ_AM_Memo__c", + "MSJ_Aid_Total__c", + "MSJ_C0_GC__c", + "MSJ_C1_GC__c", + "MSJ_C2_GC__c", + "MSJ_Countermeasure__c", + "MSJ_Deadline__c", + "MSJ_Double_Visit_Time__c", + "MSJ_Hospital__c", + "MSJ_K01_FE__c", + "MSJ_K01_ONC__c", + "MSJ_K02_FE__c", + "MSJ_K02_ONC__c", + "MSJ_K03_FE__c", + "MSJ_K03_ONC__c", + "MSJ_K04_FE__c", + "MSJ_K04_ONC__c", + "MSJ_K05_FE__c", + "MSJ_K05_ONC__c", + "MSJ_K06_FE__c", + "MSJ_K06_ONC__c", + "MSJ_K0_GC__c", + "MSJ_K1_GC__c", + "MSJ_K2_GC__c", + "MSJ_Knowledge_Total__c", + "MSJ_L0_GC__c", + "MSJ_L1_GC__c", + "MSJ_L2_GC__c", + "MSJ_MR_GC__c", + "MSJ_MR_Problems__c", + "MSJ_N0_GC__c", + "MSJ_N1_GC__c", + "MSJ_N2_GC__c", + "MSJ_Num_of_DTL__c", + "MSJ_P01__c", + "MSJ_P02__c", + "MSJ_P03__c", + "MSJ_P04__c", + "MSJ_P05__c", + "MSJ_P0_GC__c", + "MSJ_P1_GC__c", + "MSJ_P2_GC__c", + "MSJ_PlanningTotal__c", + "MSJ_R0_GC__c", + "MSJ_R1_GC__c", + "MSJ_R2_GC__c", + "MSJ_S01__c", + "MSJ_S02__c", + "MSJ_S03__c", + "MSJ_S04__c", + "MSJ_S05__c", + "MSJ_S06__c", + "MSJ_S07__c", + "MSJ_S08__c", + "MSJ_S09__c", + "MSJ_S10__c", + "MSJ_S11__c", + "MSJ_S12__c", + "MSJ_Skill_Total__c", + "MSJ_After_Call_01__c", + "MSJ_After_Call_02__c", + "MSJ_After_Call_03__c", + "MSJ_After_Call_04__c", + "MSJ_Closing__c", + "MSJ_Comment_by_MR__c", + "MSJ_Confirmed_by_MR__c", + "MSJ_Createdby__c", + "MSJ_FT_AM_Name__c", + "MSJ_Interview_Preparation__c", + "MSJ_Interview_Reflection__c", + "MSJ_Notify_To_MR__c", + "MSJ_Opening__c", + "MSJ_Others_01_Result__c", + "MSJ_Others_01__c", + "MSJ_Others_02_Result__c", + "MSJ_Others_02__c", + "MSJ_Patient_Thinking__c", + "MSJ_Probing__c", + "MSJ_Supporting__c", + "MSJ_Patient_Thinking_for_FE__c", + "MSJ_After_Call_05__c", + "MSJ_After_Call_06__c", + "MSJ_After_Call_07__c", + "MSJ_After_Call_08__c", + "MSJ_Createdby_FE__c", + "MSJ_Createdby_ONC__c", + "MSJ_Development_Level__c", + "MSJ_Interview_Prep_01__c", + "MSJ_Interview_Prep_02__c", + "MSJ_Leadership_Style__c", + "MSJ_Overcome_01__c", + "MSJ_Overcome_02__c", + "MSJ_Overcome_03__c", + "MSJ_Overcome_04__c", + "MSJ_Review_01__c", + "MSJ_Review_02__c", + "MSJ_SK_01__c", + "MSJ_SK_02__c", + "MSJ_SK_03__c", + "MSJ_SK_04__c", + "MSJ_SK_05__c", + "MSJ_SK_06__c", + "MSJ_SK_07__c", + "MSJ_SK_08__c", + "MSJ_SK_09__c", + "MSJ_SK_10__c", + "MSJ_Specific_Action__c", + "MSJ_Training_Point__c", + "MSJ_Efforts_of_Year__c", + "MSJ_Efforts_of_Month__c", + "MSJ_Skill_Task__c", + "MSJ_Action_of_This_Month__c", + "MSJ_Achievement_of_This_Month__c", + "MSJ_Comment_from_AM__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Call_Comments_vod__c", + "Sample_Card_vod__c", + "Add_Detail_vod__c", + "Property_vod__c", + "Account_vod__c", + "zvod_Product_Discussion_vod__c", + "Status_vod__c", + "Parent_Address_vod__c", + "Account_Plan_vod__c", + "zvod_SaveNew_vod__c", + "Next_Call_Notes_vod__c", + "Pre_Call_Notes_vod__c", + "Mobile_ID_vod__c", + "zvod_Account_Credentials_vod_c_vod__c", + "zvod_Account_Preferred_Name_vod_c_vod__c", + "zvod_Account_Sample_Status_vod_c_vod__c", + "zvod_Attendees_vod__c", + "zvod_Key_Messages_vod__c", + "zvod_Detailing_vod__c", + "zvod_Expenses_vod__c", + "zvod_Followup_vod__c", + "zvod_Samples_vod__c", + "zvod_Save_vod__c", + "zvod_Submit_vod__c", + "zvod_Delete_vod__c", + "Activity_Type__c", + "Significant_Event__c", + "Location_vod__c", + "Subject_vod__c", + "Unlock_vod__c", + "Call_Datetime_vod__c", + "Disbursed_To_vod__c", + "Disclaimer_vod__c", + "Request_Receipt_vod__c", + "Signature_Date_vod__c", + "Signature_vod__c", + "Territory_vod__c", + "Submitted_By_Mobile_vod__c", + "Call_Type_vod__c", + "Add_Key_Message_vod__c", + "Address_vod__c", + "Attendees_vod__c", + "Attendee_Type_vod__c", + "Call_Date_vod__c", + "Detailed_Products_vod__c", + "No_Disbursement_vod__c", + "Parent_Call_vod__c", + "User_vod__c", + "Contact_vod__c", + "zvod_Entity_vod__c", + "Medical_Event_vod__c", + "Mobile_Created_Datetime_vod__c", + "Mobile_Last_Modified_Datetime_vod__c", + "License_vod__c", + "Is_Parent_Call_vod__c", + "Entity_Display_Name_vod__c", + "Override_Lock_vod__c", + "Last_Device_vod__c", + "Ship_Address_Line_1_vod__c", + "Ship_Address_Line_2_vod__c", + "Ship_City_vod__c", + "Ship_Country_vod__c", + "Ship_License_Expiration_Date_vod__c", + "Ship_License_Status_vod__c", + "Ship_License_vod__c", + "Ship_State_vod__c", + "Ship_To_Address_vod__c", + "Ship_Zip_vod__c", + "Ship_To_Address_Text_vod__c", + "CLM_vod__c", + "zvod_CLMDetails_vod__c", + "Is_Sampled_Call_vod__c", + "zvod_Surveys_vod__c", + "Presentations_vod__c", + "Entity_Reference_Id_vod__c", + "Error_Reference_Call_vod__c", + "Duration_vod__c", + "Color_vod__c", + "Allowed_Products_vod__c", + "zvod_Attachments_vod__c", + "Sample_Card_Reason_vod__c", + "ASSMCA_vod__c", + "Address_Line_1_vod__c", + "Address_Line_2_vod__c", + "City_vod__c", + "DEA_Address_Line_1_vod__c", + "DEA_Address_Line_2_vod__c", + "DEA_Address_vod__c", + "DEA_City_vod__c", + "DEA_Expiration_Date_vod__c", + "DEA_State_vod__c", + "DEA_Zip_4_vod__c", + "DEA_Zip_vod__c", + "DEA_vod__c", + "Ship_Zip_4_vod__c", + "State_vod__c", + "Zip_4_vod__c", + "Zip_vod__c", + "Sample_Send_Card_vod__c", + "zvod_Address_vod_c_DEA_Status_vod_c_vod__c", + "Signature_Page_Image_vod__c", + "Credentials_vod__c", + "Salutation_vod__c", + "zvod_Account_Call_Reminder_vod_c_vod__c", + "MSJ_Meeting_Duration__c", + "MSJ_Double_Visit_AM__c", + "zvod_Business_Account_vod__c", + "Product_Priority_1_vod__c", + "Product_Priority_2_vod__c", + "Product_Priority_3_vod__c", + "Product_Priority_4_vod__c", + "Product_Priority_5_vod__c", + "zvod_More_Actions_vod__c", + "zvod_Call_Conflict_Status_vod__c", + "Signature_Timestamp_vod__c", + "Expense_Amount_vod__c", + "Total_Expense_Attendees_Count_vod__c", + "Attendee_list_vod__c", + "Expense_Post_Status_vod__c", + "Attendee_Post_Status_vod__c", + "Expense_System_External_ID_vod__c", + "Incurred_Expense_vod__c", + "Assigner_vod__c", + "Assignment_Datetime_vod__c", + "zvod_Call_Objective_vod__c", + "Signature_Location_Longitude_vod__c", + "Signature_Location_Latitude_vod__c", + "Location_Services_Status_vod__c", + "MSJ_Double_Visit_Other__c", + "MSJ_Comment__c", + "MSJ_For_Reporting__c", + "MSJ_Number_of_Attendees__c", + "MSJ_Main_Dept__c", + "Planned_Type_vjh__c", + "Cobrowse_URL_Participant_vod__c", + "MSJ_Activity_Method_Text__c", + "MSJ_Activity_Method__c", + "MSJ_Classification__c", + "MSJ_Double_Visit_MSL__c", + "MSJ_MSL_Comment_for_MR__c", + "MSJ_APD__c", + "Medical_Inquiry_vod__c", + "MSJ_Call_Type_MSJ__c", + "MSJ_Prescription_Request__c", + "MSJ_Patient_Follow__c", + "Child_Account_Id_vod__c", + "Child_Account_vod__c", + "Location_Id_vod__c", + "Location_Name_vod__c", + "MSJ_Comments_about_technology__c", + "Remote_Meeting_vod__c", + "Veeva_Remote_Meeting_Id_vod__c", + "MSJ_Activity_Type_Report__c", + "MSJ_Activity_Type__c", + "MSJ_Activity__c", + "MSJ_Comments__c", + "MSJ_Therapy__c", + "MSJ_Time_Hrs__c", + "EMDS_CO_Reference__c", + "EMDS_Call_Sub_Type__c", + "EMDS_Call_Type__c", + "EMDS_Call_Unsuccessful__c", + "EMDS_Congress_Type__c", + "EMDS_Date_of_Service__c", + "EMDS_Fertility_DisInterest__c", + "EMDS_Fertility_Interest__c", + "EMDS_Installed_Equipment__c", + "EMDS_Pipeline_Stage_Value__c", + "EMDS_Pipeline_Stage__c", + "EMDS_Pipeline__c", + "EMDS_Reason_for_Call__c", + "EMDS_Training_Completed__c", + "MSJ_BrainStorming__c", + "MSJ_SIPAGL_1A__c", + "MSJ_SIPAGL_1B__c", + "MSJ_SIPAGL_2__c", + "MSJ_SIPAGL_3__c", + "MSJ_SIPAGL_4A__c", + "MSJ_SIPAGL_5A__c", + "MSJ_SIPAGL_comment__c", + "MSJ_SIPAGL_4B__c", + "MSJ_SIPAGL_5B__c", + "Location_Text_vod__c", + "Call_Channel_vod__c", + "MSJ_Scientific_Interaction__c", + "MSJ_Activity_Email_Reply__c", + "MSJ_Interaction_Duration__c", + "MSJ_SIPAGL_1A_date__c", + "MSJ_CoPromotion__c", + "Call_Channel_Formula_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Detail_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Is_Parent_Call_vod__c", + "Call2_vod__c", + "Product_vod__c", + "Detail_Priority_vod__c", + "Mobile_ID_vod__c", + "Override_Lock_vod__c", + "Type_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Key_Message_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Call2_vod__c", + "Reaction_vod__c", + "Product_vod__c", + "Key_Message_vod__c", + "Mobile_ID_vod__c", + "Contact_vod__c", + "Call_Date_vod__c", + "User_vod__c", + "Category_vod__c", + "Vehicle_vod__c", + "Is_Parent_Call_vod__c", + "Override_Lock_vod__c", + "CLM_ID_vod__c", + "Slide_Version_vod__c", + "Duration_vod__c", + "Presentation_ID_vod__c", + "Start_Time_vod__c", + "Attendee_Type_vod__c", + "Entity_Reference_Id_vod__c", + "Segment_vod__c", + "Display_Order_vod__c", + "Clm_Presentation_Name_vod__c", + "Clm_Presentation_Version_vod__c", + "Clm_Presentation_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call_Clickstream_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Answer_vod__c", + "Call_vod__c", + "Key_Message_vod__c", + "Mobile_ID_vod__c", + "Popup_Opened_vod__c", + "Possible_Answers_vod__c", + "Presentation_ID_vod__c", + "Product_vod__c", + "Range_Value_vod__c", + "Rollover_Entered_vod__c", + "Selected_Items_vod__c", + "CLM_ID_vod__c", + "Question_vod__c", + "Survey_Type_vod__c", + "Text_Entered_vod__c", + "Toggle_Button_On_vod__c", + "Track_Element_Description_vod__c", + "Track_Element_Id_vod__c", + "Track_Element_Type_vod__c", + "Usage_Duration_vod__c", + "Usage_Start_Time_vod__c", + "AuxillaryId_vod__c", + "ParentId_vod__c", + "Revision_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Call2_Discussion_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Call2_vod__c", + "Activity__c", + "Comments__c", + "Contact_vod__c", + "Call_Date_vod__c", + "Product_Strategy_vod__c", + "Product_Tactic_vod__c", + "Restricted_Comments__c", + "Product_vod__c", + "Presentation__c", + "Discussion_Topics__c", + "Slides__c", + "User_vod__c", + "Indication__c", + "Mobile_ID_vod__c", + "Medical_Event_vod__c", + "Is_Parent_Call_vod__c", + "Override_Lock_vod__c", + "zvod_Product_Map_vod__c", + "Attendee_Type_vod__c", + "Entity_Reference_Id_vod__c", + "Account_Tactic_vod__c", + "MSJ_Material_Type__c", + "MSJ_Discussion_Contents__c", + "MSJ_IST_Minutes__c", + "MSJ_Off_Label_Minutes__c", + "MSJ_Discussion_Objectives__c", + "MSJ_Insight__c", + "EMDS_Materials__c", + "EMDS_Topic__c", + "MSJ_Visit_Purpose__c", + "MSJ_Insight_Count__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Time_Off_Territory_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Reason_vod__c", + "Territory_vod__c", + "Date_vod__c", + "Status_vod__c", + "Time_vod__c", + "Hours_vod__c", + "Mobile_ID_vod__c", + "Hours_off_vod__c", + "Start_Time_vod__c", + "MSJ_Day__c", + "MSJ_Comment__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Dynamic_Attribute_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Account_vod__c", + "Active_vod__c", + "Dynamic_Attribute_Configuration_vod__c", + "Dynamic_Attribute_Description_vod__c", + "Dynamic_Attribute_Help_Text_vod__c", + "Dynamic_Attribute_Label_vod__c", + "Dynamic_Attribute_Name_vod__c", + "Dynamic_Attribute_Record_Type_vod__c", + "Dynamic_Attribute_Value_Checkbox_vod__c", + "Dynamic_Attribute_Value_Date_Time_vod__c", + "Dynamic_Attribute_Value_Date_vod__c", + "Dynamic_Attribute_Value_Number_vod__c", + "Dynamic_Attribute_Value_Text_Area_vod__c", + "Dynamic_Attribute_Value_Text_vod__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Dynamic_Attribute_Configuration_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Applies_To_vod__c", + "Attribute_Label_vod__c", + "Attribute_Name_vod__c", + "Available_Values_vod__c", + "Description_vod__c", + "Detail_Group_vod__c", + "Display_Order_vod__c", + "External_ID_vod__c", + "Help_Text_vod__c", + "Product_vod__c", + "Read_Only_vod__c", + "Section_Name_vod__c", + "Sharing_Group_vod__c", + "Status_vod__c", + "Track_Changes_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Territory2", + "columns": [ + "Id", + "Name", + "Territory2TypeId", + "Territory2ModelId", + "ParentTerritory2Id", + "Description", + "ForecastUserId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "MSJ_Territory_Type__c", + "MSJ_Level__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Profile", + "columns": [ + "Id", + "Name", + "PermissionsEmailSingle", + "PermissionsEmailMass", + "PermissionsEditTask", + "PermissionsEditEvent", + "PermissionsExportReport", + "PermissionsImportPersonal", + "PermissionsDataExport", + "PermissionsManageUsers", + "PermissionsEditPublicFilters", + "PermissionsEditPublicTemplates", + "PermissionsModifyAllData", + "PermissionsManageCases", + "PermissionsManageSolutions", + "PermissionsCustomizeApplication", + "PermissionsEditReadonlyFields", + "PermissionsRunReports", + "PermissionsViewSetup", + "PermissionsTransferAnyEntity", + "PermissionsNewReportBuilder", + "PermissionsManageSelfService", + "PermissionsManageCssUsers", + "PermissionsActivateContract", + "PermissionsApproveContract", + "PermissionsImportLeads", + "PermissionsManageLeads", + "PermissionsTransferAnyLead", + "PermissionsViewAllData", + "PermissionsEditPublicDocuments", + "PermissionsViewEncryptedData", + "PermissionsEditBrandTemplates", + "PermissionsEditHtmlTemplates", + "PermissionsManageTranslation", + "PermissionsDeleteActivatedContract", + "PermissionsSendSitRequests", + "PermissionsApiUserOnly", + "PermissionsManageRemoteAccess", + "PermissionsCanUseNewDashboardBuilder", + "PermissionsManageCategories", + "PermissionsConvertLeads", + "PermissionsTestInstanceCreate", + "PermissionsPasswordNeverExpires", + "PermissionsUseTeamReassignWizards", + "PermissionsInstallMultiforce", + "PermissionsPublishMultiforce", + "PermissionsEditOppLineItemUnitPrice", + "PermissionsManageTerritories", + "PermissionsCreateMultiforce", + "PermissionsBulkApiHardDelete", + "PermissionsInboundMigrationToolsUser", + "PermissionsSolutionImport", + "PermissionsManageCallCenters", + "PermissionsManageSynonyms", + "PermissionsOutboundMigrationToolsUser", + "PermissionsViewContent", + "PermissionsManageEmailClientConfig", + "PermissionsEnableNotifications", + "PermissionsManageDataIntegrations", + "PermissionsDistributeFromPersWksp", + "PermissionsViewDataCategories", + "PermissionsManageDataCategories", + "PermissionsAuthorApex", + "PermissionsManageMobile", + "PermissionsApiEnabled", + "PermissionsManageCustomReportTypes", + "PermissionsEditCaseComments", + "PermissionsTransferAnyCase", + "PermissionsContentAdministrator", + "PermissionsCreateWorkspaces", + "PermissionsManageContentPermissions", + "PermissionsManageContentProperties", + "PermissionsManageContentTypes", + "PermissionsScheduleJob", + "PermissionsManageExchangeConfig", + "PermissionsManageAnalyticSnapshots", + "PermissionsScheduleReports", + "PermissionsManageBusinessHourHolidays", + "PermissionsManageDynamicDashboards", + "PermissionsManageInteraction", + "PermissionsViewMyTeamsDashboards", + "PermissionsResetPasswords", + "PermissionsFlowUFLRequired", + "PermissionsActivitiesAccess", + "PermissionsEmailTemplateManagement", + "PermissionsEmailAdministration", + "PermissionsChatterFileLink", + "PermissionsForceTwoFactor", + "PermissionsViewEventLogFiles", + "PermissionsManageNetworks", + "PermissionsManageAuthProviders", + "PermissionsRunFlow", + "PermissionsCreateCustomizeDashboards", + "PermissionsCreateDashboardFolders", + "PermissionsViewPublicDashboards", + "PermissionsManageDashbdsInPubFolders", + "PermissionsCreateCustomizeReports", + "PermissionsCreateReportFolders", + "PermissionsViewPublicReports", + "PermissionsManageReportsInPubFolders", + "PermissionsEditMyDashboards", + "PermissionsEditMyReports", + "PermissionsViewAllUsers", + "PermissionsConnectOrgToEnvironmentHub", + "PermissionsCreateCustomizeFilters", + "PermissionsContentHubUser", + "PermissionsGovernNetworks", + "PermissionsSalesConsole", + "PermissionsTwoFactorApi", + "PermissionsDeleteTopics", + "PermissionsEditTopics", + "PermissionsCreateTopics", + "PermissionsAssignTopics", + "PermissionsIdentityEnabled", + "PermissionsIdentityConnect", + "PermissionsContentWorkspaces", + "PermissionsCustomMobileAppsAccess", + "PermissionsViewHelpLink", + "PermissionsManageProfilesPermissionsets", + "PermissionsAssignPermissionSets", + "PermissionsManageRoles", + "PermissionsManageIpAddresses", + "PermissionsManageSharing", + "PermissionsManageInternalUsers", + "PermissionsManagePasswordPolicies", + "PermissionsManageLoginAccessPolicies", + "PermissionsManageCustomPermissions", + "PermissionsStdAutomaticActivityCapture", + "PermissionsManageTwoFactor", + "PermissionsDebugApex", + "PermissionsLightningExperienceUser", + "PermissionsConfigCustomRecs", + "PermissionsSubmitMacrosAllowed", + "PermissionsBulkMacrosAllowed", + "PermissionsManageSessionPermissionSets", + "PermissionsCreateAuditFields", + "PermissionsUpdateWithInactiveOwner", + "PermissionsManageSandboxes", + "PermissionsAutomaticActivityCapture", + "PermissionsImportCustomObjects", + "PermissionsDelegatedTwoFactor", + "PermissionsSelectFilesFromSalesforce", + "PermissionsModerateNetworkUsers", + "PermissionsMergeTopics", + "PermissionsSubscribeToLightningReports", + "PermissionsManagePvtRptsAndDashbds", + "PermissionsAllowLightningLogin", + "PermissionsCampaignInfluence2", + "PermissionsViewDataAssessment", + "PermissionsCanApproveFeedPost", + "PermissionsAllowViewEditConvertedLeads", + "PermissionsShowCompanyNameAsUserBadge", + "PermissionsAccessCMC", + "PermissionsViewHealthCheck", + "PermissionsManageHealthCheck", + "PermissionsPackaging2", + "PermissionsManageCertificates", + "PermissionsCreateReportInLightning", + "PermissionsPreventClassicExperience", + "PermissionsListEmailSend", + "PermissionsChangeDashboardColors", + "PermissionsManageRecommendationStrategies", + "PermissionsManagePropositions", + "PermissionsSubscribeReportRolesGrps", + "PermissionsSubscribeDashboardRolesGrps", + "PermissionsUseWebLink", + "PermissionsHasUnlimitedNBAExecutions", + "PermissionsViewOnlyEmbeddedAppUser", + "PermissionsViewAllActivities", + "PermissionsSubscribeReportToOtherUsers", + "PermissionsLightningConsoleAllowedForUser", + "PermissionsSubscribeReportsRunAsUser", + "PermissionsSubscribeToLightningDashboards", + "PermissionsSubscribeDashboardToOtherUsers", + "PermissionsCreateLtngTempInPub", + "PermissionsTransactionalEmailSend", + "PermissionsViewPrivateStaticResources", + "PermissionsCreateLtngTempFolder", + "PermissionsApexRestServices", + "PermissionsEnableCommunityAppLauncher", + "PermissionsGiveRecognitionBadge", + "PermissionsUseMySearch", + "PermissionsLtngPromoReserved01UserPerm", + "PermissionsManageSubscriptions", + "PermissionsManageSurveys", + "PermissionsUseAssistantDialog", + "PermissionsUseQuerySuggestions", + "PermissionsViewRoles", + "PermissionsLMOutboundMessagingUserPerm", + "PermissionsModifyDataClassification", + "PermissionsPrivacyDataAccess", + "PermissionsQueryAllFiles", + "PermissionsModifyMetadata", + "PermissionsManageCMS", + "PermissionsSandboxTestingInCommunityApp", + "PermissionsCanEditPrompts", + "PermissionsViewUserPII", + "PermissionsManageHubConnections", + "PermissionsB2BMarketingAnalyticsUser", + "PermissionsTraceXdsQueries", + "PermissionsViewAllCustomSettings", + "PermissionsViewAllForeignKeyNames", + "PermissionsHeadlessCMSAccess", + "PermissionsLMEndMessagingSessionUserPerm", + "PermissionsConsentApiUpdate", + "PermissionsAccessContentBuilder", + "PermissionsAccountSwitcherUser", + "PermissionsManageC360AConnections", + "PermissionsManageReleaseUpdates", + "PermissionsViewAllProfiles", + "PermissionsSkipIdentityConfirmation", + "PermissionsSendCustomNotifications", + "PermissionsPackaging2Delete", + "PermissionsFSCComprehensiveUserAccess", + "PermissionsManageTrustMeasures", + "PermissionsViewTrustMeasures", + "PermissionsIsotopeCToCUser", + "PermissionsIsotopeAccess", + "PermissionsIsotopeLEX", + "PermissionsQuipMetricsAccess", + "PermissionsQuipUserEngagementMetrics", + "PermissionsManageExternalConnections", + "PermissionsAIViewInsightObjects", + "PermissionsAICreateInsightObjects", + "PermissionsNativeWebviewScrolling", + "PermissionsViewDeveloperName", + "Type", + "UserLicenseId", + "UserType", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "Description", + "LastViewedDate", + "LastReferencedDate" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "My_Setup_Products_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Product_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Activity_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_External_ID_Map_vod__c", + "Account_vod__c", + "Call_vod__c", + "City_vod__c", + "Client_Name_vod__c", + "Client_OS_vod__c", + "Client_Type_vod__c", + "Country_vod__c", + "Debug_vod__c", + "Device_vod__c", + "IP_Address_vod__c", + "Multichannel_Activity_vod__c", + "Referring_Site_vod__c", + "Region_vod__c", + "Sent_Email_vod__c", + "Session_Id_vod__c", + "Site_vod__c", + "Start_DateTime_vod__c", + "Total_Duration_vod__c", + "URL_vod__c", + "User_Agent_vod__c", + "VExternal_Id_vod__c", + "Viewport_Height_vod__c", + "Viewport_Width_vod__c", + "Color_vod__c", + "Icon_vod__c", + "MCD_Primary_Key_vod__c", + "Record_Type_Name_vod__c", + "MSJ_Date_Opened__c", + "MSJ_Sent_Date__c", + "MSJ_Email_Subject__c", + "MSJ_Opens__c", + "MSJ_Email_Status__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Activity_Line_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Multichannel_Activity_vod__c", + "Call_vod__c", + "Custom_vod__c", + "DateTime_vod__c", + "Debug_vod__c", + "Detail_Group_VExternal_Id_vod__c", + "Detail_Group_vod__c", + "Duration_vod__c", + "Event_Subtype_vod__c", + "Event_Type_vod__c", + "Key_Message_VExternal_Id_vod__c", + "Key_Message_vod__c", + "Multichannel_Content_Asset_Id_vod__c", + "Multichannel_Content_Asset_Version_vod__c", + "Multichannel_Content_Asset_vod__c", + "Multichannel_Content_vod__c", + "Product_VExternal_Id_vod__c", + "Product_vod__c", + "Sent_Email_vod__c", + "VExternal_Id_vod__c", + "Video_Last_Viewed_Time_vod__c", + "Video_Length_vod__c", + "Video_Total_Time_Spent_vod__c", + "View_Order_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Multichannel_Consent_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Capture_Datetime_vod__c", + "Channel_Value_vod__c", + "Detail_Group_vod__c", + "External_ID_vod__c", + "Last_Device_vod__c", + "Mobile_ID_vod__c", + "Opt_Expiration_Date_vod__c", + "Opt_Type_vod__c", + "Optout_Event_Type_vod__c", + "Product_vod__c", + "Signature_Datetime_vod__c", + "Signature_ID_vod__c", + "Signature_vod__c", + "Sample_Consent_Template_Data_vod__c", + "Sample_Consent_Template_vod__c", + "Consent_Line_vod__c", + "Consent_Type_vod__c", + "Default_Consent_Text_vod__c", + "Disclaimer_Text_vod__c", + "Sub_Channel_Key_vod__c", + "Consent_Confirm_Datetime_vod__c", + "Related_Transaction_Id_vod__c", + "Sent_Email_vod__c", + "Content_Type_vod__c", + "Receipt_Email_vod__c", + "Receipt_Sent_Email_Transaction_Id_vod__c", + "Receipt_Sent_Email_vod__c", + "Captured_By_vod__c", + "Opt_Out_Disclaimer_Text_vod__c", + "Channel_Source_vod__c", + "Union_Id_vod__c", + "User_Last_Notified_vod__c", + "Sub_Channel_Display_Name__c", + "MSJ_Consent_Source__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Medical_Inquiry_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Address_Line_1_vod__c", + "Address_Line_2_vod__c", + "City_vod__c", + "Delivery_Method_vod__c", + "Email_vod__c", + "Fax_Number_vod__c", + "Inquiry_Text__c", + "Lock_vod__c", + "Mobile_ID_vod__c", + "Phone_Number_vod__c", + "Product__c", + "Rush_Delivery__c", + "Signature_Date_vod__c", + "Signature_vod__c", + "State_vod__c", + "Status_vod__c", + "Zip_vod__c", + "zvod_Delivery_Method_vod__c", + "zvod_Disclaimer_vod__c", + "Submitted_By_Mobile_vod__c", + "Disclaimer_vod__c", + "Entity_Reference_Id_vod__c", + "Call2_vod__c", + "Country_vod__c", + "Override_Lock_vod__c", + "MSJ_Department__c", + "MSJ_Doctor_Name__c", + "MSJ_Hospital_Name__c", + "MSJ_Indication__c", + "MSJ_Inquiry_Assignment__c", + "MSJ_Inquiry_Date__c", + "MSJ_Inquiry_Input_Manager__c", + "MSJ_Inquiry_Input_User__c", + "MSJ_MSL_Manager__c", + "MSJ_Notice_to_MR__c", + "MSJ_Person_in_charge_1__c", + "MSJ_Person_in_charge_2__c", + "MSJ_Product_for_MEC__c", + "MSJ_Product_for_MR__c", + "MSJ_Reply_Date__c", + "MSJ_Reply_User__c", + "MSJ_Reply__c", + "MSJ_Title__c", + "MSJ_AE_Infomation__c", + "MSJ_FAQ_Number_Report__c", + "MSJ_Return_Call_Report__c", + "MSJ_Inquiry_Origin_Report__c", + "MSJ_AE_Report__c", + "MSJ_Background__c", + "MSJ_MSL_Support__c", + "MSJ_Material_Requirement__c", + "MSJ_Hospital_Name_Disp__c", + "MSJ_Hospital__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Email_Activity_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Sent_Email_vod__c", + "Activity_DateTime_vod__c", + "City_vod__c", + "Click_URL_vod__c", + "Client_Name_vod__c", + "Client_OS_vod__c", + "Client_Type_vod__c", + "Country_vod__c", + "Device_Type_vod__c", + "Event_Msg_vod__c", + "Event_type_vod__c", + "IP_Address_vod__c", + "Region_vod__c", + "User_Agent_vod__c", + "Vault_Doc_ID_vod__c", + "Vault_Doc_Name_vod__c", + "Vault_Document_Major_Version_vod__c", + "Vault_Document_Minor_Version_vod__c", + "Vault_Document_Number_vod__c", + "Vault_Document_Title_vod__c", + "Vault_Instance_ID_vod__c", + "Preference_Modification_vod__c", + "Approved_Document_vod__c", + "Link_Name_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "User", + "columns": [ + "Id", + "Username", + "LastName", + "FirstName", + "Name", + "CompanyName", + "Division", + "Department", + "Title", + "Street", + "City", + "State", + "PostalCode", + "Country", + "Latitude", + "Longitude", + "GeocodeAccuracy", + "Address", + "Email", + "EmailPreferencesAutoBcc", + "EmailPreferencesAutoBccStayInTouch", + "EmailPreferencesStayInTouchReminder", + "SenderEmail", + "SenderName", + "Signature", + "StayInTouchSubject", + "StayInTouchSignature", + "StayInTouchNote", + "Phone", + "Fax", + "MobilePhone", + "Alias", + "CommunityNickname", + "BadgeText", + "IsActive", + "TimeZoneSidKey", + "UserRoleId", + "LocaleSidKey", + "ReceivesInfoEmails", + "ReceivesAdminInfoEmails", + "EmailEncodingKey", + "ProfileId", + "UserType", + "LanguageLocaleKey", + "EmployeeNumber", + "DelegatedApproverId", + "ManagerId", + "LastLoginDate", + "LastPasswordChangeDate", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "NumberOfFailedLogins", + "OfflineTrialExpirationDate", + "OfflinePdaTrialExpirationDate", + "UserPermissionsMarketingUser", + "UserPermissionsOfflineUser", + "UserPermissionsWirelessUser", + "UserPermissionsAvantgoUser", + "UserPermissionsCallCenterAutoLogin", + "UserPermissionsSFContentUser", + "UserPermissionsInteractionUser", + "UserPermissionsSupportUser", + "UserPermissionsChatterAnswersUser", + "ForecastEnabled", + "UserPreferencesActivityRemindersPopup", + "UserPreferencesEventRemindersCheckboxDefault", + "UserPreferencesTaskRemindersCheckboxDefault", + "UserPreferencesReminderSoundOff", + "UserPreferencesDisableAllFeedsEmail", + "UserPreferencesApexPagesDeveloperMode", + "UserPreferencesReceiveNoNotificationsAsApprover", + "UserPreferencesReceiveNotificationsAsDelegatedApprover", + "UserPreferencesHideCSNGetChatterMobileTask", + "UserPreferencesHideCSNDesktopTask", + "UserPreferencesHideChatterOnboardingSplash", + "UserPreferencesHideSecondChatterOnboardingSplash", + "UserPreferencesShowTitleToExternalUsers", + "UserPreferencesShowManagerToExternalUsers", + "UserPreferencesShowEmailToExternalUsers", + "UserPreferencesShowWorkPhoneToExternalUsers", + "UserPreferencesShowMobilePhoneToExternalUsers", + "UserPreferencesShowFaxToExternalUsers", + "UserPreferencesShowStreetAddressToExternalUsers", + "UserPreferencesShowCityToExternalUsers", + "UserPreferencesShowStateToExternalUsers", + "UserPreferencesShowPostalCodeToExternalUsers", + "UserPreferencesShowCountryToExternalUsers", + "UserPreferencesShowProfilePicToGuestUsers", + "UserPreferencesShowTitleToGuestUsers", + "UserPreferencesShowCityToGuestUsers", + "UserPreferencesShowStateToGuestUsers", + "UserPreferencesShowPostalCodeToGuestUsers", + "UserPreferencesShowCountryToGuestUsers", + "UserPreferencesHideInvoicesRedirectConfirmation", + "UserPreferencesHideStatementsRedirectConfirmation", + "UserPreferencesPathAssistantCollapsed", + "UserPreferencesCacheDiagnostics", + "UserPreferencesShowEmailToGuestUsers", + "UserPreferencesShowManagerToGuestUsers", + "UserPreferencesShowWorkPhoneToGuestUsers", + "UserPreferencesShowMobilePhoneToGuestUsers", + "UserPreferencesShowFaxToGuestUsers", + "UserPreferencesShowStreetAddressToGuestUsers", + "UserPreferencesLightningExperiencePreferred", + "UserPreferencesPreviewLightning", + "UserPreferencesHideEndUserOnboardingAssistantModal", + "UserPreferencesHideLightningMigrationModal", + "UserPreferencesHideSfxWelcomeMat", + "UserPreferencesHideBiggerPhotoCallout", + "UserPreferencesGlobalNavBarWTShown", + "UserPreferencesGlobalNavGridMenuWTShown", + "UserPreferencesCreateLEXAppsWTShown", + "UserPreferencesFavoritesWTShown", + "UserPreferencesRecordHomeSectionCollapseWTShown", + "UserPreferencesRecordHomeReservedWTShown", + "UserPreferencesFavoritesShowTopFavorites", + "UserPreferencesExcludeMailAppAttachments", + "UserPreferencesSuppressTaskSFXReminders", + "UserPreferencesSuppressEventSFXReminders", + "UserPreferencesPreviewCustomTheme", + "UserPreferencesHasCelebrationBadge", + "UserPreferencesUserDebugModePref", + "UserPreferencesSRHOverrideActivities", + "UserPreferencesNewLightningReportRunPageEnabled", + "UserPreferencesReverseOpenActivitiesView", + "UserPreferencesNativeEmailClient", + "UserPreferencesHideBrowseProductRedirectConfirmation", + "UserPreferencesHideOnlineSalesAppWelcomeMat", + "ContactId", + "AccountId", + "CallCenterId", + "Extension", + "FederationIdentifier", + "AboutMe", + "FullPhotoUrl", + "SmallPhotoUrl", + "IsExtIndicatorVisible", + "OutOfOfficeMessage", + "MediumPhotoUrl", + "DigestFrequency", + "DefaultGroupNotificationFrequency", + "LastViewedDate", + "LastReferencedDate", + "BannerPhotoUrl", + "SmallBannerPhotoUrl", + "MediumBannerPhotoUrl", + "IsProfilePhotoActive", + "IndividualId", + "Last_Mobile_Connect_vod__c", + "Last_Tablet_Connect_vod__c", + "Last_Mobile_Connect_Version_vod__c", + "Last_Tablet_Connect_Version_vod__c", + "Last_Mobile_Sync_vod__c", + "Last_Tablet_Sync_vod__c", + "RaiseLoggingLevel_vod__c", + "SendDetailedLog_vod__c", + "Last_Blackberry_Connect_vod__c", + "Last_Blackberry_Connect_Version_vod__c", + "Last_Blackberry_Sync_vod__c", + "Force_Full_Refresh_vod__c", + "Override_SystemModstamp_Timestamp_vod__c", + "Facetime_Email_vod__c", + "Facetime_Phone_vod__c", + "Product_Expertise_vod__c", + "Available_vod__c", + "Available_Last_Update_vod__c", + "Last_iPad_Connect_Version_vod__c", + "Last_iPad_Connect_vod__c", + "Last_iPad_Sync_vod__c", + "Inventory_Order_Allocation_Group_vod__c", + "Concur_User_Id_vod__c", + "Last_iPad_iOS_Version_vod__c", + "Approved_Email_Admin_vod__c", + "Last_WinModern_Connect_Version_vod__c", + "Last_WinModern_Connect_vod__c", + "Last_WinModern_Sync_vod__c", + "Primary_Territory_vod__c", + "Analytics_Admin_vod__c", + "Content_Admin_vod__c", + "Last_WinModern_Windows_Version_vod__c", + "Upload_VTrans_vod__c", + "Sync_Frequency_vjh__c", + "Clear_Client_Sync_Errors_vod__c", + "Remote_Meeting_Host_Id_vod__c", + "Remote_Meeting_Host_Token_vod__c", + "Last_iPhone_Connect_Version_vod__c", + "Last_iPhone_Connect_vod__c", + "Last_iPhone_Sync_vod__c", + "Last_iPhone_iOS_Version_vod__c", + "Remote_Meeting_Start_From_CRM_Online_vod__c", + "Country_Code_vod__c", + "User_Type_vod__c", + "Engage_Group_Provisioning_Status_vod__c", + "Engage_Group_Request_vod__c", + "Engage_Group_vod__c", + "Last_CRMDesktop_Mac_Sync_vod__c", + "Last_CRMDesktop_Mac_Version_vod__c", + "Last_CRMDesktop_Windows_Sync_vod__c", + "Last_CRMDesktop_Windows_Version_vod__c", + "MSJ_Test_User__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "UserTerritory2Association", + "columns": [ + "Id", + "UserId", + "Territory2Id", + "IsActive", + "RoleInTerritory2", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Remote_Meeting_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Meeting_Id_vod__c", + "Meeting_Name_vod__c", + "Mobile_ID_vod__c", + "Scheduled_DateTime_vod__c", + "Scheduled_vod__c", + "Attendance_Report_Process_Status_vod__c", + "Latest_Meeting_Start_Datetime_vod__c", + "Meeting_Password_vod__c", + "Meeting_Outcome_Status_vod__c", + "Allow_for_Joining_via_Zoom_vod__c", + "Zoom_Join_Token_vod__c", + "VExternal_Id_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "RecordType", + "columns": [ + "Id", + "Name", + "DeveloperName", + "NamespacePrefix", + "Description", + "BusinessProcessId", + "SobjectType", + "IsActive", + "IsPersonType", + "CreatedById", + "CreatedDate", + "LastModifiedById", + "LastModifiedDate", + "SystemModstamp" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "UserRole", + "columns": [ + "Id", + "Name", + "ParentRoleId", + "RollupDescription", + "OpportunityAccessForAccountOwner", + "CaseAccessForAccountOwner", + "ContactAccessForAccountOwner", + "ForecastUserId", + "MayForecastManagerShare", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "DeveloperName", + "PortalAccountId", + "PortalType", + "PortalAccountOwnerId" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Account", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "Name", + "LastName", + "FirstName", + "Salutation", + "RecordTypeId", + "Phone", + "Fax", + "Website", + "PhotoUrl", + "NumberOfEmployees", + "Ownership", + "OwnerId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "IsExcludedFromRealign", + "PersonContactId", + "IsPersonAccount", + "PersonMailingStreet", + "PersonMailingCity", + "PersonMailingState", + "PersonMailingPostalCode", + "PersonMailingCountry", + "PersonMailingLatitude", + "PersonMailingLongitude", + "PersonMailingGeocodeAccuracy", + "PersonMailingAddress", + "PersonOtherStreet", + "PersonOtherCity", + "PersonOtherState", + "PersonOtherPostalCode", + "PersonOtherCountry", + "PersonOtherLatitude", + "PersonOtherLongitude", + "PersonOtherGeocodeAccuracy", + "PersonOtherAddress", + "PersonMobilePhone", + "PersonHomePhone", + "PersonOtherPhone", + "PersonAssistantPhone", + "PersonEmail", + "PersonTitle", + "PersonDepartment", + "PersonAssistantName", + "PersonBirthdate", + "PersonHasOptedOutOfEmail", + "PersonHasOptedOutOfFax", + "PersonDoNotCall", + "PersonLastCURequestDate", + "PersonLastCUUpdateDate", + "PersonEmailBouncedReason", + "PersonEmailBouncedDate", + "PersonIndividualId", + "Jigsaw", + "JigsawCompanyId", + "AccountSource", + "SicDesc", + "External_ID_vod__c", + "Credentials_vod__c", + "Territory_vod__c", + "Exclude_from_Zip_to_Terr_Processing_vod__c", + "Group_Specialty_1_vod__c", + "Group_Specialty_2_vod__c", + "Specialty_1_vod__c", + "Specialty_2_vod__c", + "Formatted_Name_vod__c", + "Territory_Test_vod__c", + "Mobile_ID_vod__c", + "Gender_vod__c", + "ID_vod__c", + "Do_Not_Sync_Sales_Data_vod__c", + "ID2_vod__c", + "Preferred_Name_vod__c", + "Sample_Default_vod__c", + "Segmentations_vod__c", + "Restricted_Products_vod__c", + "Payer_Id_vod__c", + "Alternate_Name_vod__c", + "Do_Not_Call_vod__c", + "MSJ_Beds__c", + "Spend_Amount__c", + "PDRP_Opt_Out_vod__c", + "Spend_Status_Value_vod__c", + "PDRP_Opt_Out_Date_vod__c", + "Spend_Status_vod__c", + "Enable_Restricted_Products_vod__c", + "Call_Reminder_vod__c", + "Account_Group_vod__c", + "Primary_Parent_vod__c", + "Color_vod__c", + "Middle_vod__c", + "Suffix_vod__c", + "MSJ_Type__c", + "No_Orders_vod__c", + "MSJ_BU_ONC__c", + "MSJ_BU_FE__c", + "Account_Search_FirstLast_vod__c", + "Account_Search_LastFirst_vod__c", + "MSJ_Operation__c", + "Practice_at_Hospital_vod__c", + "Practice_Near_Hospital_vod__c", + "Do_Not_Create_Child_Account_vod__c", + "Total_MDs_DOs__c", + "AHA__c", + "Order_Type_vod__c", + "NPI_vod__c", + "ME__c", + "Speaker__c", + "Investigator_vod__c", + "Default_Order_Type_vod__c", + "Tax_Status__c", + "Model__c", + "Offerings__c", + "Departments__c", + "Account_Type__c", + "MSJ_ONC_Tier__c", + "Account_Search_Business_vod__c", + "Business_Professional_Person_vod__c", + "Hospital_Type_vod__c", + "Account_Class_vod__c", + "Furigana_vod__c", + "MSJ_JISART__c", + "Total_Revenue_000__c", + "Net_Income_Loss_000__c", + "PMPM_Income_Loss_000__c", + "Commercial_Premiums_PMPM__c", + "Medical_Loss_Ratio__c", + "Medical_Expenses_PMPM__c", + "Commercial_Patient_Days_1000__c", + "HMO_Market_Shr__c", + "HMO__c", + "HMO_POS__c", + "PPO__c", + "PPO_POS__c", + "Medicare__c", + "Medicaid__c", + "MSJ_HP_Name_E__c", + "MSJ_Department__c", + "MSJ_Date_Of_Birth__c", + "MSJ_FE_GF_Potential__c", + "MSJ_FE_SZ_Potential__c", + "MSJ_EB_CRC_Ladder__c", + "MSJ_EB_CRC_Segment__c", + "MSJ_EB_HN_Segment__c", + "Business_Description__c", + "Regional_Strategy__c", + "Contracts_Process__c", + "MSJ_GF_segment__c", + "MSJ_DCF_DR_Code__c", + "MSJ_SZ_Segment__c", + "MSJ_Remark__c", + "MSJ_Title__c", + "MSJ_Role__c", + "MSJ_Kana__c", + "MSJ_Specialism__c", + "MSJ_Graduated_from__c", + "MSJ_Year_Graduation__c", + "Target__c", + "KOL_vod__c", + "MSJ_EPPV_Code__c", + "MSJ_DCF_HP_Code__c", + "Total_Lives__c", + "Total_Physicians_Enrolled__c", + "MSJ_Delete__c", + "MSJ_KOL_LOL__c", + "MSJ_ONC_Status__c", + "Account_Identifier_vod__c", + "Approved_Email_Opt_Type_vod__c", + "Language_vod__c", + "MSJ_KRAS_Routine_Date__c", + "MSJ_KRAS_Routine__c", + "MSJ_DRP_Target__c", + "MSJ_Fertility_Evaluation_Score__c", + "MSJ_Fertility_Tracking_Last_Modify_Date__c", + "Total_Pharmacists__c", + "MSJ_Number_of_Gonadotropin__c", + "MSJ_Number_of_IUI_cycle__c", + "MSJ_Number_of_OI_monthly_cycle__c", + "MSJ_OI_Protocol_learning_level__c", + "MSJ_H_N_Tier__c", + "MSJ_XLK_Segment__c", + "MSJ_XLK_Tier__c", + "Career_Status_vod__c", + "Photo_vod__c", + "MSJ_EB_H_N_LA_Segment__c", + "MSJ_EB_H_N_RM_Segment__c", + "MSJ_FE_CE_Potential__c", + "MSJ_FE_1C_potential__c", + "MSJ_FE_OV_potential__c", + "MSJ_FE_Tech_potential__c", + "MSJ_CE_segment__c", + "MSJ_1C_segment__c", + "MSJ_OV_segment__c", + "MSJ_Tech_segment__c", + "MSJ_Target_Call_Num__c", + "MSJ_DR_Change_Log__c", + "MSJ_Global_scientific_exposure__c", + "MSJ_H_index__c", + "MSJ_Num_of_Article_3Y__c", + "MSJ_Num_of_Article__c", + "MSJ_Num_of_Article_as_1st_Author_3Y__c", + "MSJ_Num_of_article_growth_rate_3Y__c", + "MSJ_Num_of_cited_3Y__c", + "MSJ_Num_of_impact_factor_3Y__c", + "MSJ_impact_factor_as_1st_Author_3Y__c", + "EMDS_Has_Pipeline_Opportunity__c", + "EMDS_Pipeline_Count__c", + "MSJ_BVC_Segment__c", + "MSJ_BVC_Tier__c", + "MSJ_BVC_AcctOpen__c", + "MSJ_BVC_MCC_Patients__c", + "MSJ_ONC_HP_Segment__c", + "MSJ_AE_Department__c", + "MSJ_AE_Facility__c", + "MSJ_AE_Name__c", + "MSJ_AE_Title__c", + "MSJ_Email__c", + "MSJ_FE_GF2_Potential__c", + "MSJ_FE_Location_potential__c", + "MSJ_GF2_segment__c", + "MSJ_OPTIN_target__c", + "MSJ_Merck_Specialty1__c", + "MSJ_Merck_Specialty2__c", + "MSJ_Marketing_Cloud_Integration__c", + "MSJ_Marketing_Cloud1__c", + "MSJ_Marketing_Cloud2__c", + "MSJ_Marketing_Cloud3__c", + "MSJ_Marketing_Cloud4__c", + "MSJ_Medical_Department__c", + "MSJ_Marketing_Cloud0__c", + "Mobile_ID_vod__pc", + "H1Insights__H1_NPI_Value_for_Testing__pc", + "H1Insights__H1_Person_ID__pc", + "H1Insights__H1_Request_Status__pc", + "H1Insights__H1_URL__pc", + "H1Insights__NPI_Number__pc", + "H1Insights__NPI_Number_for_H1_Insights__pc", + "MSJ_Marketing_Cloud_Integration__pc" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "AccountShare", + "columns": [ + "Id", + "AccountId", + "UserOrGroupId", + "AccountAccessLevel", + "OpportunityAccessLevel", + "CaseAccessLevel", + "ContactAccessLevel", + "RowCause", + "LastModifiedDate", + "LastModifiedById", + "IsDeleted" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Contact", + "columns": [ + "Id", + "IsDeleted", + "MasterRecordId", + "AccountId", + "IsPersonAccount", + "LastName", + "FirstName", + "Salutation", + "Name", + "OtherStreet", + "OtherCity", + "OtherState", + "OtherPostalCode", + "OtherCountry", + "OtherLatitude", + "OtherLongitude", + "OtherGeocodeAccuracy", + "OtherAddress", + "MailingStreet", + "MailingCity", + "MailingState", + "MailingPostalCode", + "MailingCountry", + "MailingLatitude", + "MailingLongitude", + "MailingGeocodeAccuracy", + "MailingAddress", + "Phone", + "Fax", + "MobilePhone", + "HomePhone", + "OtherPhone", + "AssistantPhone", + "ReportsToId", + "Email", + "Title", + "Department", + "AssistantName", + "Birthdate", + "Description", + "OwnerId", + "HasOptedOutOfEmail", + "HasOptedOutOfFax", + "DoNotCall", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "LastCURequestDate", + "LastCUUpdateDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "EmailBouncedReason", + "EmailBouncedDate", + "IsEmailBounced", + "PhotoUrl", + "Jigsaw", + "JigsawContactId", + "IndividualId", + "Mobile_ID_vod__c", + "H1Insights__H1_NPI_Value_for_Testing__c", + "H1Insights__H1_Person_ID__c", + "H1Insights__H1_Request_Status__c", + "H1Insights__H1_URL__c", + "H1Insights__NPI_Number__c", + "H1Insights__NPI_Number_for_H1_Insights__c", + "MSJ_Marketing_Cloud_Integration__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Type_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Header_vod__c", + "Channel_Label_vod__c", + "Channel_Source_vod__c", + "Consent_Expires_In_vod__c", + "Default_Consent_Type_vod__c", + "Disclaimer_Text_vod__c", + "Display_Order_vod__c", + "Product_Preference_vod__c", + "zvod_Consent_Default_Consent_Text_vod__c", + "zvod_Consent_Line_vod__c", + "zvod_Signature_Capture_vod__c", + "Double_Opt_In_vod__c", + "zvod_Consent_Activity_Tracking_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Header_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Header_Help_Text_vod__c", + "Country_vod__c", + "Inactive_Datetime_vod__c", + "Language_vod__c", + "Status_vod__c", + "Signature_Required_On_Opt_Out_vod__c", + "Request_Receipt_vod__c", + "Subscription_Option_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Consent_Line_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consent_Type_vod__c", + "Detail_Group_Display_Name_vod__c", + "Detail_Group_vod__c", + "Display_Order_vod__c", + "End_Date_vod__c", + "Group_By_vod__c", + "Product_Display_Name_vod__c", + "Product_vod__c", + "Start_Date_vod__c", + "Sub_Channel_Description_vod__c", + "Sub_Channel_Display_Name_vod__c", + "Sub_Channel_Key_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Inquiry_Assignment__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Medical_Inquiry__c", + "MSJ_Close__c", + "MSJ_Doctor_Name__c", + "MSJ_Hospital_Name__c", + "MSJ_Indication__c", + "MSJ_Inquiry_Text__c", + "MSJ_MEC_User__c", + "MSJ_MSL_Manager__c", + "MSJ_MSL_User__c", + "MSJ_Notice_to_MR__c", + "MSJ_Product_for_MEC__c", + "MSJ_Product_for_MR__c", + "MSJ_Reply_Date__c", + "MSJ_Reply__c", + "MSJ_AE_Infomation__c", + "MSJ_Cancel__c", + "MSJ_FAQ_number_c__c", + "MSJ_Return_Call__c", + "MSJ_Inquiry_Origin__c", + "First_Response__c", + "Inquiry_Created_Date__c", + "Inquiry_Type_1__c", + "Inquiry_Type_2__c", + "MSJ_First_User__c", + "MSJ_MEC_Comment__c", + "MSJ_Send_Email__c", + "MSJ_Temp_Aggregated_Info__c", + "MSJ_AE_Report__c", + "MSJ_Background__c", + "MSJ_Inquiry_Date__c", + "MSJ_MSL_Support__c", + "MSJ_Handover_Comment__c", + "MSJ_Handover_Email__c", + "MSJ_Material_Requirement__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Approved_Document_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Detail_Group_vod__c", + "Document_Description_vod__c", + "Document_Host_URL_vod__c", + "Document_ID_vod__c", + "Document_Last_Mod_DateTime_vod__c", + "Email_Allows_Documents_vod__c", + "Email_Domain_vod__c", + "Email_Fragment_HTML_vod__c", + "Email_From_Address_vod__c", + "Email_From_Name_vod__c", + "Email_HTML_1_vod__c", + "Email_HTML_2_vod__c", + "Email_ReplyTo_Address_vod__c", + "Email_ReplyTo_Name_vod__c", + "Email_Subject_vod__c", + "Email_Template_Fragment_Document_ID_vod__c", + "Email_Template_Fragment_HTML_vod__c", + "ISI_Document_ID_vod__c", + "Language_vod__c", + "Other_Document_ID_List_vod__c", + "PI_Document_ID_vod__c", + "Piece_Document_ID_vod__c", + "Product_vod__c", + "Status_vod__c", + "Territory_vod__c", + "Vault_Instance_ID_vod__c", + "Allow_Any_Product_Fragment_vod__c", + "Allowed_Document_IDs_vod__c", + "Engage_Document_Id_vod__c", + "Vault_Document_ID_vod__c", + "Key_Message_vod__c", + "Events_Management_Subtype_vod__c", + "Survey_vod__c", + "Content_Type_vod__c", + "Bcc_vod__c", + "Audience_vod__c", + "WeChat_Template_ID_vod__c", + "Check_Consent_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Child_Account_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Parent_Account_vod__c", + "Child_Account_vod__c", + "External_ID_vod__c", + "Mobile_ID_vod__c", + "Primary_vod__c", + "Copy_Address_vod__c", + "Child_Name_vod__c", + "Parent_Name_vod__c", + "Parent_Child_Name_vod__c", + "Account_Code__c", + "Child_Department__c", + "Child_Role__c", + "Child_Title__c", + "Child_Remark__c", + "MSJ_1C_segment__c", + "MSJ_BU_FE__c", + "MSJ_BU_ONC__c", + "MSJ_BVC_Segment__c", + "MSJ_CE_segment__c", + "MSJ_Child_Account_Link__c", + "MSJ_DCF_DR_Code__c", + "MSJ_DCF_HP_Code__c", + "MSJ_DR_Change_Log__c", + "MSJ_Delete__c", + "MSJ_Department__c", + "MSJ_EB_CRC_Segment__c", + "MSJ_EB_HN_Segment__c", + "MSJ_EB_H_N_LA_Segment__c", + "MSJ_EB_H_N_RM_Segment__c", + "MSJ_External_ID__c", + "MSJ_Fax__c", + "MSJ_GF2_segment__c", + "MSJ_GF_segment__c", + "MSJ_KOL_LOL__c", + "MSJ_KOL__c", + "MSJ_ONC_HP_Segment__c", + "MSJ_OPTIN_target__c", + "MSJ_OV_segment__c", + "MSJ_Parent_Child_Name__c", + "MSJ_Phone__c", + "MSJ_Remark__c", + "MSJ_Target_Call_Num__c", + "MSJ_Tech_segment__c", + "MSJ_Title__c", + "MSJ_XLK_Segment__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Hospital_Medical_Regimen__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Account_Name__c", + "MSJ_Delete_Date__c", + "MSJ_Delete_Flag__c", + "MSJ_Indication__c", + "MSJ_Line__c", + "MSJ_Medical_Regimen__c", + "Mobile_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Medical_Regimen__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "MSJ_Delete_Date__c", + "MSJ_Delete_Flag__c", + "MSJ_Indication__c", + "MSJ_Remark__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "MSJ_Patient__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "MSJ_Account_Name__c", + "MSJ_CRC_Group__c", + "MSJ_Casus_or_Transfer_Point__c", + "MSJ_Entry_Date__c", + "MSJ_IST_Name__c", + "MSJ_Indication__c", + "MSJ_Line__c", + "MSJ_MR_Comments__c", + "MSJ_MUID__c", + "MSJ_Medical_Regimen__c", + "MSJ_Month__c", + "MSJ_Report_Comments__c", + "MSJ_Start_Date_Of_Administration__c", + "MSJ_Year__c", + "Mobile_ID_vod__c", + "MSJ_CRC_RAS_KRAS__c", + "MSJ_End_Date_Of_Administration__c", + "MSJ_End_Date_of_Stop_Administration__c", + "MSJ_HN_Hospitalized_Type__c", + "MSJ_Start_Date_of_Stop_Administration__c", + "MSJ_Patient_Status__c", + "MSJ_Patient_TA__c", + "MSJ_Child_Account_Name__c", + "MSJ_Child_Account__c", + "MSJ_Parent_Account_Name__c", + "MSJ_Parent_Child_Name__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Consumer_site__c", + "Product_info__c", + "Therapeutic_Class_vod__c", + "Parent_Product_vod__c", + "Therapeutic_Area_vod__c", + "Product_Type_vod__c", + "Require_Key_Message_vod__c", + "Cost_vod__c", + "External_ID_vod__c", + "Manufacturer_vod__c", + "Company_Product_vod__c", + "Controlled_Substance_vod__c", + "Description_vod__c", + "Sample_Quantity_Picklist_vod__c", + "Display_Order_vod__c", + "No_Metrics_vod__c", + "Distributor_vod__c", + "Sample_Quantity_Bound_vod__c", + "Sample_U_M_vod__c", + "No_Details_vod__c", + "Quantity_Per_Case_vod__c", + "Schedule_vod__c", + "Restricted_vod__c", + "Pricing_Rule_Quantity_Bound_vod__c", + "No_Promo_Items_vod__c", + "User_Aligned_vod__c", + "Restricted_States_vod__c", + "Sort_Code_vod__c", + "No_Cycle_Plans_vod__c", + "Inventory_Order_UOM_vod__c", + "Inventory_Quantity_Per_Case_vod__c", + "VExternal_Id_vod__c", + "Country__c", + "MSJ_Product_Classification__c", + "MSJ_Indication__c", + "MSJ_Therapeutic_Area__c", + "MSJ_Global_Brand__c", + "MSJ_Global_Business_Unit__c", + "MSJ_Molecules__c", + "MSJ_SBU__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_Group_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Description_vod__c", + "Product_vod__c", + "Product_Catalog_vod__c", + "Start_Date_vod__c", + "End_Date_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Product_Metrics_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_vod__c", + "Awareness__c", + "Selling_Stage__c", + "Formulary_Status__c", + "Movement__c", + "Products_vod__c", + "Segment__c", + "X12_mo_trx_chg__c", + "Speaker_Skills__c", + "Investigator_Readiness__c", + "Engagements__c", + "Mobile_ID_vod__c", + "External_ID_vod__c", + "MSJ_Patient__c", + "Detail_Group_vod__c", + "MSJ_EB_1st_Line_Liver_Meta__c", + "MSJ_EB_1st_Line_Multi_Meta__c", + "MSJ_EB_2nd_Line_Mono__c", + "MSJ_EB_2nd_Line_Combination__c", + "MSJ_EB_3rd_Line_Mono__c", + "MSJ_EB_3rd_Line_Combination__c", + "EMDS_Ability__c", + "EMDS_Brand_Loyalty__c", + "EMDS_Decision_Maker__c", + "EMDS_Early_Tech_Adopter__c", + "EMDS_Influence__c", + "EMDS_Main_Driver__c", + "EMDS_Priority__c", + "EMDS_Willingness__c", + "MSJ_KTL_Type__c", + "MSJ_KTL_Tier__c", + "MSJ_Publications__c", + "MSJ_Clinical_Trials__c", + "MSJ_Speaker_for_Medical_Events__c", + "MSJ_Advisor_to_Medical_Affairs__c", + "MSJ_Guidelines_Treatment_Standards__c", + "MSJ_Therapeutic_Area_Expertise__c", + "MSJ_MAP_GAP__c", + "MSJ_Associations__c", + "MSJ_Tier_Score__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Assignment_Type_vod__c", + "Channels_vod__c", + "End_Date_vod__c", + "Expired_vod__c", + "External_ID_vod__c", + "Language_vod__c", + "Lock_vod__c", + "Open_vod__c", + "Product_vod__c", + "Region_vod__c", + "Segment_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "Territory_vod__c", + "zvod_Questions_vod__c", + "zvod_Segments_vod__c", + "zvod_Targets_vod__c", + "Max_Score_vod__c", + "Min_Score_vod__c", + "Autotarget_vod__c", + "Territories_vod__c", + "Target_Type_vod__c", + "MSJ_External_ID__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_Target_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "LastViewedDate", + "LastReferencedDate", + "Account_Display_Name_vod__c", + "Account_vod__c", + "Channels_vod__c", + "End_Date_vod__c", + "Entity_Reference_Id_vod__c", + "External_ID_vod__c", + "Language_vod__c", + "Lock_vod__c", + "Mobile_ID_vod__c", + "No_Autoassign_vod__c", + "Not_Completed_vod__c", + "Region_vod__c", + "Segment_vod__c", + "Start_Date_vod__c", + "Status_vod__c", + "Survey_vod__c", + "Territory_vod__c", + "zvod_Address_vod__c", + "zvod_Specialty_vod__c", + "Score_vod__c", + "User_vod__c", + "Child_Account_vod__c", + "Location_Entity_Reference_Id_vod__c", + "Location_vod__c", + "Target_Type_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Survey_Question_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Survey_vod__c", + "Answer_Choice_vod__c", + "External_ID_vod__c", + "Max_Score_vod__c", + "Min_Score_vod__c", + "Order_vod__c", + "Question_vod__c", + "Required_vod__c", + "Text_vod__c", + "Condition_vod__c", + "Source_ID_vod__c", + "MSJ_External_ID__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Question_Response_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "MayEdit", + "IsLocked", + "Survey_Target_vod__c", + "Answer_Choice_vod__c", + "Date_vod__c", + "Datetime_vod__c", + "External_ID_vod__c", + "Mobile_ID_vod__c", + "Number_vod__c", + "Order_vod__c", + "Question_Text_vod__c", + "Required_vod__c", + "Response_Hash_vod__c", + "Response_vod__c", + "Score_vod__c", + "Survey_Question_vod__c", + "Text_vod__c", + "Type_vod__c", + "Condition_vod__c", + "Inactive_Condition_vod__c", + "Source_ID_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Sent_Fragment_vod__c", + "columns": [ + "Id", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Sent_Email_vod__c", + "Account_vod__c", + "Email_Template_vod__c", + "Sent_Fragment_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + }, + { + "object_name": "Sent_Email_vod__c", + "columns": [ + "Id", + "OwnerId", + "IsDeleted", + "Name", + "RecordTypeId", + "CreatedDate", + "CreatedById", + "LastModifiedDate", + "LastModifiedById", + "SystemModstamp", + "LastActivityDate", + "MayEdit", + "IsLocked", + "Account_Email_vod__c", + "Account_vod__c", + "Approved_Email_Template_vod__c", + "Capture_Datetime_vod__c", + "Detail_Group_vod__c", + "Email_Config_Values_vod__c", + "Email_Content2_vod__c", + "Email_Content_vod__c", + "Email_Fragments_vod__c", + "Email_Sent_Date_vod__c", + "Failure_Msg_vod__c", + "Last_Activity_Date_vod__c", + "Last_Device_vod__c", + "MC_Capture_Datetime_vod__c", + "Mobile_ID_vod__c", + "Opened_vod__c", + "Product_Display_vod__c", + "Product_vod__c", + "Sender_Email_vod__c", + "Status_vod__c", + "Valid_Consent_Exists_vod__c", + "Approved_Document_Views_vod__c", + "Click_Count_vod__c", + "Last_Click_Date_vod__c", + "Last_Open_Date_vod__c", + "Open_Count_vod__c", + "Receipt_Entity_Type_vod__c", + "Receipt_Record_Id_vod__c", + "Territory_vod__c", + "Call2_vod__c", + "Medical_Inquiry_vod__c", + "Parent_Email_vod__c", + "Related_Transaction_ID_vod__c", + "Case_vod__c", + "Key_Message_vod__c", + "Suggestion_vod__c", + "EM_Attendee_vod__c", + "EM_Event_Speaker_vod__c", + "EM_Event_Team_Member_vod__c", + "Event_Attendee_vod__c", + "Event_vod__c", + "Medical_Event_vod__c", + "Scheduled_Send_Datetime_vod__c", + "User_vod__c", + "Content_Type_vod__c", + "Bcc_vod__c", + "Event_Attendee_Mobile_Id_vod__c", + "Event_Mobile_Id_vod__c", + "Activity_Tracking_Mode_vod__c", + "Email_Source_vod__c", + "Subject_vod__c", + "User_Input_Text_vod__c" + ], + "is_skip": false, + "is_update_last_fetch_datetime": true + } ] - } - - \ No newline at end of file +} \ No newline at end of file