diff --git a/ecs/crm-datafetch/src/aws/s3.py b/ecs/crm-datafetch/src/aws/s3.py index e1d0dc70..d5bd03f0 100644 --- a/ecs/crm-datafetch/src/aws/s3.py +++ b/ecs/crm-datafetch/src/aws/s3.py @@ -1,13 +1,17 @@ import json import boto3 -from src.system_var.constants import AWS_RESOURCE_S3, S3_RESPONSE_BODY, S3_CHAR_CODE +from src.system_var.constants import (AWS_RESOURCE_S3, S3_CHAR_CODE, + S3_RESPONSE_BODY) from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET, - CRM_IMPORT_DATA_BACKUP_FOLDER, - CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET, - LAST_FETCH_DATE_FOLDER, OBJECT_INFO_FILENAME, - OBJECT_INFO_FOLDER, PROCESS_RESULT_FOLDER, - RESPONSE_JSON_BACKUP_FOLDER) + CRM_IMPORT_DATA_BACKUP_FOLDER, + CRM_IMPORT_DATA_FOLDER, + IMPORT_DATA_BUCKET, + LAST_FETCH_DATE_FOLDER, + OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER, + PROCESS_RESULT_FOLDER, + RESPONSE_JSON_BACKUP_FOLDER) class S3Resource: @@ -31,16 +35,6 @@ class S3Resource: return -#class S3ResourceNonBucket: -# def __init__(self) -> None: -# self.__s3_resource = boto3.resource(AWS_RESOURCE_S3) -# -# def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: -# copy_source = {'Bucket': src_bucket, 'Key': src_key} -# self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key) -# return - - class ConfigBucket: __s3_resource: S3Resource = None @@ -77,11 +71,11 @@ class DataBucket: return def put_csv_from(self, src_bucket: str, src_key: str): - self.__s3_resource.copy(src_bucket, src_key, str(self), CRM_IMPORT_DATA_FOLDER) + dest_filename = src_key.split('/')[-1] + self.__s3_resource.copy(src_bucket, src_key, str(self), f'{CRM_IMPORT_DATA_FOLDER}/{dest_filename}') return - class BackupBucket: __s3_resource: S3Resource = None @@ -104,4 +98,4 @@ class BackupBucket: def put_result_json(self, file_path: str, data: dict) -> None: object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}' self.__s3_resource.put_object(object_key, json.dumps(data)) - return \ No newline at end of file + return diff --git a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py index f14490cf..d40c7a13 100644 --- a/ecs/crm-datafetch/src/backup_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_csv_data_process.py @@ -1,32 +1,21 @@ from src.aws.s3 import BackupBucket from src.config.objects import TargetObject -from src.system_var.constants import CSVBK_JP_NAME from src.error.exceptions import FileUploadException +from src.system_var.constants import CSVBK_JP_NAME from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger def backup_crm_csv_data_process(target_object: TargetObject, exetute_datetime: ExecuteDateTime, csv_string: str): - """ - CSVバックアップ処理 + """CSVバックアップ処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス - csv_string : str - csvデータ + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + exetute_datetime (ExecuteDateTime): 実行日次取得インスタンス + csv_string (str): csvデータ - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① CSVバックアップ処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/backup_crm_data_process.py b/ecs/crm-datafetch/src/backup_crm_data_process.py index a37eb583..0375d927 100644 --- a/ecs/crm-datafetch/src/backup_crm_data_process.py +++ b/ecs/crm-datafetch/src/backup_crm_data_process.py @@ -6,26 +6,15 @@ from src.util.logger import logger_instance as logger def backup_crm_data_process(object_name: str, sf_object_dict: dict, execute_datetime: ExecuteDateTime): - """ - CRM電文データバックアップ処理 + """CRM電文データバックアップ処理 - Parameters - ---------- - object_name : str - 取得対象オブジェクト情報インスタンス - sf_object_dict : dict - Salesforceオブジェクトデータ - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス + Args: + object_name (str): 取得対象オブジェクト名 + sf_object_dict (dict): Salesforceオブジェクトデータ + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① CRM電文データバックアップ処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/check_object_info_process.py b/ecs/crm-datafetch/src/check_object_info_process.py index ed2a34da..84f0db40 100644 --- a/ecs/crm-datafetch/src/check_object_info_process.py +++ b/ecs/crm-datafetch/src/check_object_info_process.py @@ -1,30 +1,22 @@ from src.config.objects import TargetObject from src.error.exceptions import InvalidConfigException -from src.util.execute_datetime import ExecuteDateTime from src.system_var.constants import CHK_JP_NAME +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger def check_object_info_process(object_info: dict, execute_datetime: ExecuteDateTime): - """ - オブジェクト情報形式チェック処理 + """オブジェクト情報形式チェック処理 - Parameters - ---------- - object_info : dict - 取得対象オブジェクト情報 - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス + Args: + object_info (dict): 取得対象オブジェクト情報 + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス - Returns - ------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス + Raises: + InvalidConfigException: オブジェクト情報定義が不正だった場合 - Raises - ------ - InvalidConfigException - オブジェクト情報定義が不正だった場合 + Returns: + target_object: 取得対象オブジェクト情報インスタンス """ # ① オブジェクト情報形式チェック処理開始ログを出力する diff --git a/ecs/crm-datafetch/src/config/objects.py b/ecs/crm-datafetch/src/config/objects.py index 464f7b0e..3ac08851 100644 --- a/ecs/crm-datafetch/src/config/objects.py +++ b/ecs/crm-datafetch/src/config/objects.py @@ -1,25 +1,19 @@ -from src.system_var.constants import ( DATE_PATTERN_YYYYMMDDTHHMMSSTZ, - OBJECTS_KEY, - OBJECTS_TYPE, - OBJECT_NAME_KEY, - OBJECT_NAME_TYPE, - COLUMNS_KEY, - COLUMNS_TYPE, - IS_SKIP_KEY, - IS_SKIP_TYPE, - IS_UPDATE_LAST_FETCH_DATETIME_KEY, - IS_UPDATE_LAST_FETCH_DATETIME_TYPE, - LAST_FETCH_DATETIME_FILE_NAME_KEY, - LAST_FETCH_DATETIME_FILE_NAME_TYPE, - UPLOAD_FILE_NAME_KEY, - UPLOAD_FILE_NAME_TYPE, - DATETIME_COLUMN_KEY, - DATETIME_COLUMN_TYPE, - LAST_FETCH_DATETIME_FROM_KEY, - LAST_FETCH_DATETIME_TO_KEY, - DATETIME_COLUMN_DEFAULT_VALUE -) - +from src.system_var.constants import (COLUMNS_KEY, COLUMNS_TYPE, + DATE_PATTERN_YYYYMMDDTHHMMSSTZ, + DATETIME_COLUMN_DEFAULT_VALUE, + DATETIME_COLUMN_KEY, + DATETIME_COLUMN_TYPE, IS_SKIP_KEY, + IS_SKIP_TYPE, + IS_UPDATE_LAST_FETCH_DATETIME_KEY, + IS_UPDATE_LAST_FETCH_DATETIME_TYPE, + LAST_FETCH_DATETIME_FILE_NAME_KEY, + LAST_FETCH_DATETIME_FILE_NAME_TYPE, + LAST_FETCH_DATETIME_FROM_KEY, + LAST_FETCH_DATETIME_TO_KEY, + OBJECT_NAME_KEY, OBJECT_NAME_TYPE, + OBJECTS_KEY, OBJECTS_TYPE, + UPLOAD_FILE_NAME_KEY, + UPLOAD_FILE_NAME_TYPE) from src.util.dict_checker import DictChecker @@ -98,16 +92,24 @@ class TargetObject(): @property def is_update_last_fetch_datetime(self) -> bool: - return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY) else False + if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY): + return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] + else: + return False @property def last_fetch_datetime_file_name(self) -> str: - return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY] if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}.json' - + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY): + return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY] + else: + return f'{self.__object_info[OBJECT_NAME_KEY]}.json' @property def upload_file_name(self) -> str: - return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' + if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY): + return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) + else: + return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}' @property def datetime_column(self) -> str: @@ -126,7 +128,6 @@ class LastFetchDatetime(): self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ) - return @property @@ -135,4 +136,7 @@ class LastFetchDatetime(): @property def last_fetch_datetime_to(self) -> str: - return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FROM_KEY) else self.__execute_datetime + if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY): + return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY] + else: + return self.__execute_datetime diff --git a/ecs/crm-datafetch/src/controller.py b/ecs/crm-datafetch/src/controller.py index acbd2383..3c92277b 100644 --- a/ecs/crm-datafetch/src/controller.py +++ b/ecs/crm-datafetch/src/controller.py @@ -7,13 +7,14 @@ from src.config.objects import FetchTargetObjects from src.convert_crm_csv_data_process import convert_crm_csv_data_process from src.copy_crm_csv_data_process import copy_crm_csv_data_process from src.error.exceptions import MeDaCaCRMDataFetchException -from src.util.execute_datetime import ExecuteDateTime from src.fetch_crm_data_process import fetch_crm_data_process from src.prepare_data_fetch_process import prepare_data_fetch_process from src.set_datetime_period_process import set_datetime_period_process from src.system_var.constants import OBJECT_NAME_KEY -from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime_process +from src.upload_last_fetch_datetime_process import \ + upload_last_fetch_datetime_process from src.upload_result_data_process import upload_result_data_process +from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger @@ -63,29 +64,22 @@ def controller() -> None: def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: ExecuteDateTime, process_result: dict): + """取得対象オブジェクト情報をループし、１オブジェクトごとのデータを取得する + + Args: + fetch_target_objects (FetchTargetObjects): CRMオブジェクト情報インスタンス + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス + process_result (dict): 取得処理実行結果辞書オブジェクト + + Returns: + process_result: 取得処理実行結果辞書オブジェクト """ - 取得対象オブジェクト情報をループし、１オブジェクトごとのデータを取得する - Parameters - ---------- - fetch_target_objects : FetchTargetObjects - CRMオブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス - process_result : dict - 取得処理実行結果辞書オブジェクト - - Returns - ------- - process_result : dict - 取得処理実行結果辞書オブジェクト - - """ for object_info in fetch_target_objects: try: process_result[object_info.get(OBJECT_NAME_KEY)] = 'fail' - fetch_crm_data_per_object(object_info, execute_datetime, process_result) + fetch_crm_data_per_object(object_info, execute_datetime) process_result[object_info.get(OBJECT_NAME_KEY)] = 'success' @@ -102,28 +96,13 @@ def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: E return process_result + def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTime) -> None: - """ - オブジェクトごとにCRMのデータを取得し、取込フォルダにアップロードする - - Parameters - ---------- - object_info : dict - 取得対象オブジェクト情報 - execute_datetime : object - 実行日次取得インスタンス - - Returns - ------- - なし - - Raises - ------ - FileNotFoundException - S3上のファイルが存在しない場合 - InvalidConfigException - オブジェクト情報定義が不正だった場合 + """オブジェクトごとにCRMのデータを取得し、取込フォルダにアップロードする + Args: + object_info (dict): 取得対象オブジェクト情報 + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス """ # 1. オブジェクト処理結果の初期化 @@ -194,4 +173,4 @@ def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTi # 14. オブジェクトのアップロードが完了した旨をログに出力する logger.info(f'I-CTRL-16 [{target_object_name}] 処理正常終了') - return \ No newline at end of file + return diff --git a/ecs/crm-datafetch/src/convert_crm_csv_data_process.py b/ecs/crm-datafetch/src/convert_crm_csv_data_process.py index 1ed05c3a..fd6138b1 100644 --- a/ecs/crm-datafetch/src/convert_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/convert_crm_csv_data_process.py @@ -6,25 +6,17 @@ from src.util.logger import logger_instance as logger def convert_crm_csv_data_process(target_object: TargetObject, crm_data_response: dict): - """ - CSV変換処理 + """CSV変換処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - crm_data_response : dict - Salesforceオブジェクトデータ + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + crm_data_response (dict): Salesforceオブジェクトデータ - Returns - ------- - csv_string : str - csvデータ + Raises: + DataConvertException: データ変換が失敗した場合 - Raises - ------ - DataConvertException - データ変換が失敗した場合 + Returns: + csv_string: csvデータ """ # ① CSV変換処理の開始ログを出力する @@ -34,9 +26,8 @@ def convert_crm_csv_data_process(target_object: TargetObject, crm_data_response: try: # ② CSV変換 - CSVStringConverter(target_object, crm_data_response) - csv_string = CSVStringConverter.convert() - + csv_string_converter = CSVStringConverter(target_object, crm_data_response) + csv_string = csv_string_converter.convert() logger.debug(f'D-CONV-02 [{target_object_name}] のCSV変換処理 正常終了') diff --git a/ecs/crm-datafetch/src/converter/convert_factory.py b/ecs/crm-datafetch/src/converter/convert_factory.py index 48673f7e..bf949a6e 100644 --- a/ecs/crm-datafetch/src/converter/convert_factory.py +++ b/ecs/crm-datafetch/src/converter/convert_factory.py @@ -1,9 +1,10 @@ import re from datetime import datetime -from src.config.objects import TargetObject from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE, - CSV_TRUE_VALUE, YYYYMMDDHHMMSS, DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC) + CSV_TRUE_VALUE, + DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, + YYYYMMDDHHMMSS) class ConvertStrategyFactory: @@ -12,30 +13,30 @@ class ConvertStrategyFactory: self.__float_convert_strategy = FloatConvertStrategy() self.__boolean_convert_strategy = BooleanConvertStrategy() self.__datetime_convert_strategy = DatatimeConvertStrategy() + self.__non_convert_strategy = NonConvertStrategy() def create(self, value): - converted_value = value - if value is None: - converted_value = self.__none_value_convert_strategy.convert_value() + convert_strategy = self.__none_value_convert_strategy - # 指数表記で取得できるパターン。指数表記を整数表記に変換する。 elif type(value) == float: - converted_value = self.__float_convert_strategy.convert_value(value) + convert_strategy = self.__float_convert_strategy - # SQLの真偽値に対応するために変換する elif type(value) == bool: - converted_value = self.__boolean_convert_strategy.convert_value(value) + convert_strategy = self.__boolean_convert_strategy elif type(value) == str and re.fullmatch(DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, value): - converted_value = self.__datetime_convert_strategy.convert_value(value) + convert_strategy = self.__datetime_convert_strategy - return converted_value + else: + convert_strategy = self.__non_convert_strategy + + return convert_strategy class NoneValueConvertStrategy: - def convert_value(self) -> str: + def convert_value(self, convert_value: None) -> str: return '' @@ -53,3 +54,7 @@ class FloatConvertStrategy: def convert_value(self, convert_value: str) -> int: return int(convert_value) + +class NonConvertStrategy: + def convert_value(self, convert_value: str): + return convert_value diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index bf7faaee..9c223182 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -52,7 +52,8 @@ class CSVStringConverter: for column in columns: v = json_object[column.upper()] - converted_value = self.__convert_strategy_factory.create(v) + convert_strategy = self.__convert_strategy_factory.create(v) + converted_value = convert_strategy.convert_value(v) csv_row.append(converted_value) diff --git a/ecs/crm-datafetch/src/copy_crm_csv_data_process.py b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py index 57a3659f..ba2831ea 100644 --- a/ecs/crm-datafetch/src/copy_crm_csv_data_process.py +++ b/ecs/crm-datafetch/src/copy_crm_csv_data_process.py @@ -2,31 +2,20 @@ from src.aws.s3 import BackupBucket, DataBucket from src.config.objects import TargetObject from src.error.exceptions import FileUploadException from src.system_var.constants import UPLD_JP_NAME -from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER, - CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET) +from src.system_var.environments import CRM_IMPORT_DATA_BACKUP_FOLDER from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger def copy_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime): - """ - CSVアップロード処理 + """CSVアップロード処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① CSVデータアップロード処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/fetch_crm_data_process.py b/ecs/crm-datafetch/src/fetch_crm_data_process.py index 3a89da1e..d22efb7e 100644 --- a/ecs/crm-datafetch/src/fetch_crm_data_process.py +++ b/ecs/crm-datafetch/src/fetch_crm_data_process.py @@ -2,102 +2,91 @@ from requests.exceptions import ConnectTimeout, ReadTimeout from tenacity import retry, stop_after_attempt from tenacity.wait import wait_exponential -from src.config.objects import TargetObject, LastFetchDatetime +from src.config.objects import LastFetchDatetime, TargetObject from src.error.exceptions import DataConvertException, SalesforceAPIException from src.salesforce.salesforce_api import SalesforceApiClient from src.salesforce.soql_builder import SOQLBuilder from src.system_var.constants import FETCH_JP_NAME -from src.system_var.environments import (CRM_AUTH_TIMEOUT, - CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, - CRM_FETCH_RECORD_RETRY_INTERVAL, - CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, - CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, - CRM_FETCH_RECORD_TIMEOUT, - CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, - CRM_GET_RECORD_COUNT_RETRY_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, - CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, - CRM_GET_RECORD_COUNT_TIMEOUT) +from src.system_var.environments import ( + CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, + CRM_FETCH_RECORD_RETRY_INTERVAL, CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, + CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, CRM_FETCH_RECORD_TIMEOUT, + CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, + CRM_GET_RECORD_COUNT_RETRY_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, + CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_TIMEOUT) +from src.util.counter_object import CounterObject from src.util.logger import logger_instance as logger def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): - """ - CRMデータ取得処理 + """CRMデータ取得処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - last_fetch_datetime : LastFetchDatetime - 取得対象オブジェクト情報インスタンス + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + last_fetch_datetime (LastFetchDatetime): データ取得期間設定インスタンス - Returns - ------- - sf_object_dict : dict - Salesforceオブジェクトデータ + Raises: + SalesforceAPIException: SalseforceのAPI実行失敗が発生した場合 + DataConvertException: データ変換が失敗した場合 - Raises - ------ - SalesforceAPIException - SalseforceのAPI実行失敗が発生した場合 - DataConvertException - データ変換が失敗した場合 + Returns: + crm_data_response: Salesforceオブジェクトデータ """ # ① CRMデータ取得処理開始ログを出力する logger.info( f'I-FETCH-01 [{target_object.object_name}] のCRMからのデータ取得処理を開始します') - object_name = target_object.object_name + target_object_name = target_object.object_name - global count_contime_counter, count_readtime_counter, count_counter, data_contime_counter, data_readtime_counter, data_counter - count_contime_counter = 1 - count_readtime_counter = 1 - count_counter = 1 - data_contime_counter = 1 - data_readtime_counter = 1 - data_counter = 1 + # リトライ回数判定用のカウンタオブジェクトを生成(@retryデコレータを利用したことによるリトライ対象の関数内でのカウント変数保持不可の対策のためオブジェクト化する) + count_contime_counter = CounterObject(1) + count_readtime_counter = CounterObject(1) + count_counter = CounterObject(1) + data_contime_counter = CounterObject(1) + data_readtime_counter = CounterObject(1) + data_counter = CounterObject(1) try: # ② 取得対象オブジェクトの取得期間内のレコード件数を取得する - logger.info(f'I-FETCH-02 [{object_name}] の件数取得を開始します') + logger.info(f'I-FETCH-02 [{target_object_name}] の件数取得を開始します') soql_builder = SOQLBuilder(target_object, last_fetch_datetime) count_soql = soql_builder.create_count_soql() - record_count = fetch_record_count_retry(count_soql, object_name) + record_count = fetch_record_count_retry(count_soql, target_object_name, count_contime_counter, count_readtime_counter, count_counter) - logger.info(f'I-FETCH-03 [{object_name}] の件数：[{record_count}]') + logger.info(f'I-FETCH-03 [{target_object_name}] の件数：[{record_count}]') except Exception as e: raise SalesforceAPIException( - 'E-FETCH-01', FETCH_JP_NAME, f'[{object_name}] の件数取得に失敗しました エラー内容：[{e}]') + 'E-FETCH-01', FETCH_JP_NAME, f'[{target_object_name}] の件数取得に失敗しました エラー内容：[{e}]') try: # ③ 取得対象オブジェクトのレコードを取得する - logger.info(f'I-FETCH-04 [{object_name}] のレコード取得を開始します') + logger.info(f'I-FETCH-04 [{target_object_name}] のレコード取得を開始します') fetch_soql = soql_builder.create_fetch_soql() - record_all = fetch_sf_data_retry(fetch_soql, object_name) + record_all = fetch_sf_data_retry(fetch_soql, target_object_name, data_contime_counter, data_readtime_counter, data_counter) except Exception as e: raise SalesforceAPIException( - 'E-FETCH-02', FETCH_JP_NAME, f'[{object_name}] のレコード取得に失敗しました エラー内容：[{e}]') + 'E-FETCH-02', FETCH_JP_NAME, f'[{target_object_name}] のレコード取得に失敗しました エラー内容：[{e}]') try: # ④ 取得対象オブジェクトをJSONに変換 - logger.info(f'I-FETCH-05 [{object_name}] のレコードをJSONに変換します') + logger.info(f'I-FETCH-05 [{target_object_name}] のレコードをJSONに変換します') crm_data_response = [record for record in record_all] except Exception as e: raise DataConvertException( - 'E-FETCH-03', FETCH_JP_NAME, f'[{object_name}] のレコードのJSON変換に失敗しました エラー内容：[{e}]') + 'E-FETCH-03', FETCH_JP_NAME, f'[{target_object_name}] のレコードのJSON変換に失敗しました エラー内容：[{e}]') # ⑤ CRMデータ取得処理終了ログを出力する - logger.info(f'I-FETCH-06 [{object_name}] のレコード取得が成功しました') + logger.info(f'I-FETCH-06 [{target_object_name}] のレコード取得が成功しました') # ⑥ 次の処理へ移行する return crm_data_response @@ -107,31 +96,34 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL, min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT)) -def fetch_record_count_retry(soql: str, object_name: str): +def fetch_record_count_retry(soql: str, target_object_name: str, + count_contime_counter: CounterObject, count_readtime_counter: CounterObject, count_counter: CounterObject): try: - global count_contime_counter, count_readtime_counter, count_counter - salesforce_api_client = SalesforceApiClient() return salesforce_api_client.fetch_sf_count(soql) except ConnectTimeout as e: - if count_contime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_contime_counter += 1 - logger.warn(f'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if count_contime_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_contime_counter.increment(1) + logger.warning(f'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') raise e except ReadTimeout as e: - if count_readtime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_readtime_counter += 1 - logger.warn( - f'W-FETCH-02 [{object_name}] の件数取得処理がタイムアウトしたため、リトライします：[{CRM_GET_RECORD_COUNT_TIMEOUT}] エラー内容：[{e}]') + + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if count_readtime_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_readtime_counter.increment(1) + logger.warning( + f'W-FETCH-02 [{target_object_name}] の件数取得処理がタイムアウトしたため、リトライします：[{CRM_GET_RECORD_COUNT_TIMEOUT}] エラー内容：[{e}]') raise e except Exception as e: - if count_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: - count_counter += 1 - logger.warn( - f'W-FETCH-03 [{object_name}] の件数取得に失敗したため、リトライします エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if count_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT: + count_counter.increment(1) + logger.warning( + f'W-FETCH-03 [{target_object_name}] の件数取得に失敗したため、リトライします エラー内容：[{e}]') raise e @@ -139,29 +131,31 @@ def fetch_record_count_retry(soql: str, object_name: str): wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL, min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL), stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT)) -def fetch_sf_data_retry(soql: str, object_name: str): +def fetch_sf_data_retry(soql: str, target_object_name: str, + data_contime_counter: CounterObject, data_readtime_counter: CounterObject, data_counter: CounterObject): try: - global data_contime_counter, data_readtime_counter, data_counter - salesforce_api_client = SalesforceApiClient() return salesforce_api_client.fetch_sf_data(soql) except ConnectTimeout as e: - if data_contime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_contime_counter += 1 - logger.warn(f'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if data_contime_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_contime_counter.increment(1) + logger.warning(f'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]') raise e except ReadTimeout as e: - if data_readtime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_readtime_counter += 1 - logger.warn( - f'W-FETCH-05 [{object_name}] のレコード取得処理がタイムアウトしたため、リトライします：[{CRM_FETCH_RECORD_TIMEOUT}] エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if data_readtime_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_readtime_counter.increment(1) + logger.warning( + f'W-FETCH-05 [{target_object_name}] のレコード取得処理がタイムアウトしたため、リトライします：[{CRM_FETCH_RECORD_TIMEOUT}] エラー内容：[{e}]') raise e except Exception as e: - if data_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: - data_counter += 1 - logger.warn( - f'W-FETCH-06 [{object_name}] のレコード取得に失敗したため、リトライします エラー内容：[{e}]') + # 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う + if data_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT: + data_counter.increment(1) + logger.warning( + f'W-FETCH-06 [{target_object_name}] のレコード取得に失敗したため、リトライします エラー内容：[{e}]') raise e diff --git a/ecs/crm-datafetch/src/parser/json_parse.py b/ecs/crm-datafetch/src/parser/json_parse.py index 5e7249b9..ca1d8ed2 100644 --- a/ecs/crm-datafetch/src/parser/json_parse.py +++ b/ecs/crm-datafetch/src/parser/json_parse.py @@ -8,7 +8,7 @@ class JsonParser(): def __init__(self, json_str) -> None: self.__json_str = json_str - def json_parser(self) -> dict: + def parse(self) -> dict: for symbol in EXCLUDE_SYMBOL: # コメントアウトシンボルを含む部分を置き換える正規表現 replace_comment_regex = rf'\s(?!\"){symbol}[\s\S]*?.*' diff --git a/ecs/crm-datafetch/src/prepare_data_fetch_process.py b/ecs/crm-datafetch/src/prepare_data_fetch_process.py index 409651ad..3488c71f 100644 --- a/ecs/crm-datafetch/src/prepare_data_fetch_process.py +++ b/ecs/crm-datafetch/src/prepare_data_fetch_process.py @@ -3,36 +3,24 @@ from src.config.objects import FetchTargetObjects from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser from src.system_var.constants import PRE_JP_NAME -from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME, - OBJECT_INFO_FOLDER) +from src.system_var.environments import (CRM_CONFIG_BUCKET, + OBJECT_INFO_FILENAME, + OBJECT_INFO_FOLDER) from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger def prepare_data_fetch_process(): - """ - データ取得準備処理 + """データ取得準備処理 - Parameters - ---------- - なし - - Returns - ------- - fetch_target_objects : FetchTargetObjects - CRMオブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス - process_result : dict - 取得処理実行結果辞書オブジェクト - - Raises - ------ - FileNotFoundException - S3上のファイルが存在しない場合 - InvalidConfigException - オブジェクト情報定義が不正だった場合 + Raises: + FileNotFoundException: S3上のファイルが存在しない場合 + InvalidConfigException: オブジェクト情報定義が不正だった場合 + Returns: + fetch_target_objects : CRMオブジェクト情報インスタンス + execute_datetime : 実行日次取得インスタンス + process_result : 取得処理実行結果辞書オブジェクト """ # ① データ取得準備処理の開始ログを出力する @@ -64,7 +52,7 @@ def prepare_data_fetch_process(): logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします') json_parser = JsonParser(object_info_file_str) - object_info_file_dict = json_parser.json_parser() + object_info_file_dict = json_parser.parse() logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました') diff --git a/ecs/crm-datafetch/src/salesforce/salesforce_api.py b/ecs/crm-datafetch/src/salesforce/salesforce_api.py index 0c073b9a..4e4bb267 100644 --- a/ecs/crm-datafetch/src/salesforce/salesforce_api.py +++ b/ecs/crm-datafetch/src/salesforce/salesforce_api.py @@ -1,10 +1,9 @@ from simple_salesforce import Salesforce - from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_TIMEOUT, - CRM_FETCH_RECORD_TIMEOUT, - CRM_GET_RECORD_COUNT_TIMEOUT, CRM_USER_NAME, - CRM_USER_PASSWORD, CRM_USER_SECURITY_TOKEN) - + CRM_FETCH_RECORD_TIMEOUT, + CRM_GET_RECORD_COUNT_TIMEOUT, + CRM_USER_NAME, CRM_USER_PASSWORD, + CRM_USER_SECURITY_TOKEN) class SalesforceApiClient(): @@ -25,4 +24,5 @@ class SalesforceApiClient(): return count_res.get('records')[0].get('expr0') def fetch_sf_data(self, soql: str): - return self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + data_res = self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT) + return data_res.get('records') diff --git a/ecs/crm-datafetch/src/set_datetime_period_process.py b/ecs/crm-datafetch/src/set_datetime_period_process.py index 9795ab2e..bfebd5c2 100644 --- a/ecs/crm-datafetch/src/set_datetime_period_process.py +++ b/ecs/crm-datafetch/src/set_datetime_period_process.py @@ -1,37 +1,27 @@ from src.aws.s3 import ConfigBucket -from src.config.objects import TargetObject, LastFetchDatetime +from src.config.objects import LastFetchDatetime, TargetObject from src.error.exceptions import FileNotFoundException, InvalidConfigException from src.parser.json_parse import JsonParser from src.system_var.constants import DATE_JP_NAME -from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER +from src.system_var.environments import (CRM_CONFIG_BUCKET, + LAST_FETCH_DATE_FOLDER) from src.util.execute_datetime import ExecuteDateTime from src.util.logger import logger_instance as logger - - def set_datetime_period_process(target_object: TargetObject, execute_datetime: ExecuteDateTime): - """ - データ取得期間設定処理 + """データ取得期間設定処理 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - execute_datetime : ExecuteDateTime - 実行日次取得インスタンス + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + execute_datetime (ExecuteDateTime): 実行日次取得インスタンス - Returns - ------- - last_fetch_datetime : LastFetchDatetime - 取得対象オブジェクト情報インスタンス + Raises: + FileNotFoundException: S3上のファイルが存在しない場合 + InvalidConfigException: オブジェクト情報定義が不正だった場合 - Raises - ------ - FileNotFoundException - S3上のファイルが存在しない場合 - InvalidConfigException - オブジェクト情報定義が不正だった場合 + Returns: + last_fetch_datetime: データ取得期間設定インスタンス """ # ① データ取得期間設定処理の開始ログを出力する @@ -59,7 +49,7 @@ def set_datetime_period_process(target_object: TargetObject, execute_datetime: E logger.debug(f'D-DATE-04 前回取得日時ファイルの形式チェックを開始します') json_parser = JsonParser(last_fetch_datetime_file_str) - last_fetch_datetime_file_dict = json_parser.json_parser() + last_fetch_datetime_file_dict = json_parser.parse() last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_file_dict, execute_datetime) diff --git a/ecs/crm-datafetch/src/system_var/environments.py b/ecs/crm-datafetch/src/system_var/environments.py index 5ff41929..7dc62c1b 100644 --- a/ecs/crm-datafetch/src/system_var/environments.py +++ b/ecs/crm-datafetch/src/system_var/environments.py @@ -6,35 +6,35 @@ import src.system_var.constants as constants # ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO) # CRMへの認証処理のタイムアウト秒数 -CRM_AUTH_TIMEOUT = os.environ.get(constants.CRM_AUTH_TIMEOUT, 100) +CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)) # CRMへの認証処理の最大リトライ試行回数 -CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3) +CRM_AUTH_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3)) # CRMへの認証処理のリトライ時の初回待ち秒数 -CRM_AUTH_RETRY_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5) +CRM_AUTH_RETRY_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5)) # CRMへの認証処理のリトライ時の最小待ち秒数 -CRM_AUTH_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5) +CRM_AUTH_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5)) # CRMへの認証処理のリトライ時の最大待ち秒数 -CRM_AUTH_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50) +CRM_AUTH_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50)) # CRMのレコード件数取得処理のタイムアウト秒数 -CRM_GET_RECORD_COUNT_TIMEOUT = os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300) +CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)) # CRMのレコード件数取得処理の最大リトライ試行回数 -CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3) +CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3)) # CRMのレコード件数取得処理のリトライ時の初回待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5) +CRM_GET_RECORD_COUNT_RETRY_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5)) # CRMのレコード件数取得処理のリトライ時の最小待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, 5) +CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, 5)) # CRMのレコード件数取得処理のリトライ時の最大待ち秒数 -CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50) +CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50)) # CRMのレコード取得処理のタイムアウト秒数 -CRM_FETCH_RECORD_TIMEOUT = os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300) +CRM_FETCH_RECORD_TIMEOUT = int(os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300)) # CRMのレコード取得処理の最大リトライ試行回数 -CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3) +CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3)) # CRMのレコード取得処理のリトライ時の初回待ち秒数 -CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5) +CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最小待ち秒数 -CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5) +CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5)) # CRMのレコード取得処理のリトライ時の最大待ち秒数 -CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50) +CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50)) # environments(ECS Task Environment) # CRMのAPI実行のための認証エンドポイントのドメイン diff --git a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py index e83e58ab..e6b34c53 100644 --- a/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py +++ b/ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py @@ -1,31 +1,21 @@ import json from src.aws.s3 import ConfigBucket -from src.config.objects import TargetObject, LastFetchDatetime +from src.config.objects import LastFetchDatetime, TargetObject from src.error.exceptions import FileUploadException from src.system_var.constants import UPD_JP_NAME from src.util.logger import logger_instance as logger def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime): - """ - 前回取得日時ファイル更新 + """前回取得日時ファイル更新 - Parameters - ---------- - target_object : TargetObject - 取得対象オブジェクト情報インスタンス - last_fetch_datetime : LastFetchDatetime - 取得対象オブジェクト情報インスタンス + Args: + target_object (TargetObject): 取得対象オブジェクト情報インスタンス + last_fetch_datetime (LastFetchDatetime): データ取得期間設定インスタンス - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① 前回取得日時ファイル更新処理の開始ログを出力する diff --git a/ecs/crm-datafetch/src/upload_result_data_process.py b/ecs/crm-datafetch/src/upload_result_data_process.py index 855ec900..8248fb78 100644 --- a/ecs/crm-datafetch/src/upload_result_data_process.py +++ b/ecs/crm-datafetch/src/upload_result_data_process.py @@ -7,24 +7,14 @@ from src.util.logger import logger_instance as logger def upload_result_data_process(process_result: dict, execute_datetime: ExecuteDateTime): - """ - 取得処理実施結果アップロード処理 + """取得処理実施結果アップロード処理 - Parameters - ---------- - process_result : dict - 取得処理実行結果辞書オブジェクト - last_fetch_datetime : LastFetchDatetime - 取得対象オブジェクト情報インスタンス + Args: + process_result (dict): 取得処理実行結果辞書オブジェクト + execute_datetime (ExecuteDateTime): データ取得期間設定インスタンス - Returns - ------- - なし - - Raises - ------ - FileUploadException - S3のファイルアップロード失敗 + Raises: + FileUploadException: S3のファイルアップロード失敗 """ # ① 取得処理実施結果アップロード処理のログを出力する diff --git a/ecs/crm-datafetch/src/util/counter_object.py b/ecs/crm-datafetch/src/util/counter_object.py new file mode 100644 index 00000000..23da4f82 --- /dev/null +++ b/ecs/crm-datafetch/src/util/counter_object.py @@ -0,0 +1,14 @@ +class CounterObject: + def __init__(self, base_num=1) -> None: + self.__counter = base_num + + def describe(self) -> int: + return self.__counter + + def increment(self, num=1) -> int: + self.__counter += num + return self.__counter + + def decrement(self, num=1) -> int: + self.__counter -= num + return self.__counter diff --git a/ecs/crm-datafetch/src/util/dict_checker.py b/ecs/crm-datafetch/src/util/dict_checker.py index 9d498b5e..1f97dded 100644 --- a/ecs/crm-datafetch/src/util/dict_checker.py +++ b/ecs/crm-datafetch/src/util/dict_checker.py @@ -17,7 +17,7 @@ class DictChecker: """辞書型バリュー正規表現チェック""" return True if re.fullmatch(regex_str, self.__object_dict[check_key]) else False - def assert_key_exist(self, check_key: str, check_type: type) -> None: + def assert_key_exist(self, check_key: str) -> None: """辞書型キー存在検査""" if not self.check_key_exist(check_key): raise Exception(f'「{check_key}」キーは必須です') diff --git a/ecs/crm-datafetch/src/util/execute_datetime.py b/ecs/crm-datafetch/src/util/execute_datetime.py index 2f07736a..8a804291 100644 --- a/ecs/crm-datafetch/src/util/execute_datetime.py +++ b/ecs/crm-datafetch/src/util/execute_datetime.py @@ -1,9 +1,6 @@ from datetime import datetime -from src.system_var.constants import( - YYYYMMDDTHHMMSSTZ, - MILLISEC_FORMAT -) +from src.system_var.constants import MILLISEC_FORMAT, YYYYMMDDTHHMMSSTZ class ExecuteDateTime: diff --git a/ecs/crm-datafetch/src/util/logger.py b/ecs/crm-datafetch/src/util/logger.py index ec1df277..47fd2e45 100644 --- a/ecs/crm-datafetch/src/util/logger.py +++ b/ecs/crm-datafetch/src/util/logger.py @@ -2,7 +2,7 @@ import logging from src.system_var.environments import LOG_LEVEL -"""boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する""" +# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する for name in ["boto3", "botocore", "s3transfer", "urllib3"]: logging.getLogger(name).setLevel(logging.WARNING) diff --git a/s3/config/crm/last_fetch_datetime/Account.json b/s3/config/crm/last_fetch_datetime/Account.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Account.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/AccountShare.json b/s3/config/crm/last_fetch_datetime/AccountShare.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/AccountShare.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Account_Territory_Loader_vod__c.json b/s3/config/crm/last_fetch_datetime/Account_Territory_Loader_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Account_Territory_Loader_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Approved_Document_vod__c.json b/s3/config/crm/last_fetch_datetime/Approved_Document_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Approved_Document_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call2_Detail_vod__c.json b/s3/config/crm/last_fetch_datetime/Call2_Detail_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call2_Detail_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call2_Discussion_vod__c.json b/s3/config/crm/last_fetch_datetime/Call2_Discussion_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call2_Discussion_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call2_Key_Message_vod__c.json b/s3/config/crm/last_fetch_datetime/Call2_Key_Message_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call2_Key_Message_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call2_vod__c.json b/s3/config/crm/last_fetch_datetime/Call2_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call2_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Call_Clickstream_vod__c.json b/s3/config/crm/last_fetch_datetime/Call_Clickstream_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Call_Clickstream_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Child_Account_vod__c.json b/s3/config/crm/last_fetch_datetime/Child_Account_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Child_Account_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Clm_Presentation_Slide_vod__c.json b/s3/config/crm/last_fetch_datetime/Clm_Presentation_Slide_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Clm_Presentation_Slide_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Clm_Presentation_vod__c.json b/s3/config/crm/last_fetch_datetime/Clm_Presentation_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Clm_Presentation_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Coaching_Report_vod__c.json b/s3/config/crm/last_fetch_datetime/Coaching_Report_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Coaching_Report_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Consent_Header_vod__c.json b/s3/config/crm/last_fetch_datetime/Consent_Header_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Consent_Header_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Consent_Line_vod__c.json b/s3/config/crm/last_fetch_datetime/Consent_Line_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Consent_Line_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Consent_Type_vod__c.json b/s3/config/crm/last_fetch_datetime/Consent_Type_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Consent_Type_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Contact.json b/s3/config/crm/last_fetch_datetime/Contact.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Contact.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_Configuration_vod__c.json b/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_Configuration_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_Configuration_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_vod__c.json b/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Dynamic_Attribute_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Email_Activity_vod__c.json b/s3/config/crm/last_fetch_datetime/Email_Activity_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Email_Activity_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Event_Attendee_vod__c.json b/s3/config/crm/last_fetch_datetime/Event_Attendee_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Event_Attendee_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Group.json b/s3/config/crm/last_fetch_datetime/Group.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Group.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Key_Message_vod__c.json b/s3/config/crm/last_fetch_datetime/Key_Message_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Key_Message_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Hospital_Medical_Regimen__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Hospital_Medical_Regimen__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Hospital_Medical_Regimen__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Inquiry_Assignment__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Inquiry_Assignment__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Inquiry_Assignment__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_MR_Weekly_Report__c.json b/s3/config/crm/last_fetch_datetime/MSJ_MR_Weekly_Report__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_MR_Weekly_Report__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Medical_Event_Evaluation__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Medical_Event_Evaluation__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Medical_Event_Evaluation__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Medical_Regimen__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Medical_Regimen__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Medical_Regimen__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/MSJ_Patient__c.json b/s3/config/crm/last_fetch_datetime/MSJ_Patient__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/MSJ_Patient__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json b/s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Medical_Inquiry_vod__c.json b/s3/config/crm/last_fetch_datetime/Medical_Inquiry_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Medical_Inquiry_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Medical_Insight_vod__c.json b/s3/config/crm/last_fetch_datetime/Medical_Insight_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Medical_Insight_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Multichannel_Activity_Line_vod__c.json b/s3/config/crm/last_fetch_datetime/Multichannel_Activity_Line_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Multichannel_Activity_Line_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Multichannel_Activity_vod__c.json b/s3/config/crm/last_fetch_datetime/Multichannel_Activity_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Multichannel_Activity_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Multichannel_Consent_vod__c.json b/s3/config/crm/last_fetch_datetime/Multichannel_Consent_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Multichannel_Consent_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/My_Setup_Products_vod__c.json b/s3/config/crm/last_fetch_datetime/My_Setup_Products_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/My_Setup_Products_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/ObjectTerritory2Association.json b/s3/config/crm/last_fetch_datetime/ObjectTerritory2Association.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/ObjectTerritory2Association.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Product_Group_vod__c.json b/s3/config/crm/last_fetch_datetime/Product_Group_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Product_Group_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Product_Metrics_vod__c.json b/s3/config/crm/last_fetch_datetime/Product_Metrics_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Product_Metrics_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Product_vod__c.json b/s3/config/crm/last_fetch_datetime/Product_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Product_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Profile.json b/s3/config/crm/last_fetch_datetime/Profile.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Profile.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Question_Response_vod__c.json b/s3/config/crm/last_fetch_datetime/Question_Response_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Question_Response_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/RecordType.json b/s3/config/crm/last_fetch_datetime/RecordType.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/RecordType.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Remote_Meeting_vod__c.json b/s3/config/crm/last_fetch_datetime/Remote_Meeting_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Remote_Meeting_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Sent_Email_vod__c.json b/s3/config/crm/last_fetch_datetime/Sent_Email_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Sent_Email_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Sent_Fragment_vod__c.json b/s3/config/crm/last_fetch_datetime/Sent_Fragment_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Sent_Fragment_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Survey_Question_vod__c.json b/s3/config/crm/last_fetch_datetime/Survey_Question_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Survey_Question_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Survey_Target_vod__c.json b/s3/config/crm/last_fetch_datetime/Survey_Target_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Survey_Target_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Survey_vod__c.json b/s3/config/crm/last_fetch_datetime/Survey_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Survey_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Territory2.json b/s3/config/crm/last_fetch_datetime/Territory2.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Territory2.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/Time_Off_Territory_vod__c.json b/s3/config/crm/last_fetch_datetime/Time_Off_Territory_vod__c.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/Time_Off_Territory_vod__c.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/User.json b/s3/config/crm/last_fetch_datetime/User.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/User.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/UserRole.json b/s3/config/crm/last_fetch_datetime/UserRole.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/UserRole.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/last_fetch_datetime/UserTerritory2Association.json b/s3/config/crm/last_fetch_datetime/UserTerritory2Association.json new file mode 100644 index 00000000..7453dc17 --- /dev/null +++ b/s3/config/crm/last_fetch_datetime/UserTerritory2Association.json @@ -0,0 +1,4 @@ +{ + "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", + "last_fetch_datetime_to": "" +} diff --git a/s3/config/crm/object_info/crm_object_list_all.json b/s3/config/crm/object_info/crm_object_list_all.json index a65cf1bb..0792da37 100644 --- a/s3/config/crm/object_info/crm_object_list_all.json +++ b/s3/config/crm/object_info/crm_object_list_all.json @@ -24,7 +24,7 @@ "is_skip": false, "is_update_last_fetch_datetime": false, "last_fetch_datetime_file_name": "Territory2_ALL.json", - "upload_file_name": "CRM_Territory2_ALL_{execute_datetime}.csv" + "upload_file_name": "CRM_Territory2_ALL_{execute_datetime}" }, { "object_name": "UserTerritory2Association", @@ -41,7 +41,7 @@ "is_skip": false, "is_update_last_fetch_datetime": false, "last_fetch_datetime_file_name": "UserTerritory2Association_ALL.json", - "upload_file_name": "CRM_UserTerritory2Association_ALL_{execute_datetime}.csv" + "upload_file_name": "CRM_UserTerritory2Association_ALL_{execute_datetime}" } ] } diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index 00f7fe89..aa20d187 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -2164,7 +2164,8 @@ "IsDeleted" ], "is_skip": false, - "is_update_last_fetch_datetime": true + "is_update_last_fetch_datetime": true, + "datetime_column": "LastModifiedDate" }, { "object_name": "Contact",