nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: ソースコードレビューの追加修正

Browse Source
This commit is contained in:
Y_SAKAI 2022-07-28 22:43:32 +09:00
parent a41a40004b
commit afcca943ca
78 changed files with 491 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
ecs/crm-datafetch/src/aws/s3.py
View File

@ -1,13 +1,17 @@
import json
import boto3
from src.system_var.constants import AWS_RESOURCE_S3, S3_RESPONSE_BODY, S3_CHAR_CODE
from src.system_var.constants import (AWS_RESOURCE_S3, S3_CHAR_CODE,
S3_RESPONSE_BODY)
from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET,
CRM_IMPORT_DATA_BACKUP_FOLDER,
CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET,
LAST_FETCH_DATE_FOLDER, OBJECT_INFO_FILENAME,
OBJECT_INFO_FOLDER, PROCESS_RESULT_FOLDER,
RESPONSE_JSON_BACKUP_FOLDER)
CRM_IMPORT_DATA_BACKUP_FOLDER,
CRM_IMPORT_DATA_FOLDER,
IMPORT_DATA_BUCKET,
LAST_FETCH_DATE_FOLDER,
OBJECT_INFO_FILENAME,
OBJECT_INFO_FOLDER,
PROCESS_RESULT_FOLDER,
RESPONSE_JSON_BACKUP_FOLDER)
class S3Resource:
@ -31,16 +35,6 @@ class S3Resource:
return
#class S3ResourceNonBucket:
# def __init__(self) -> None:
# self.__s3_resource = boto3.resource(AWS_RESOURCE_S3)
#
# def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None:
# copy_source = {'Bucket': src_bucket, 'Key': src_key}
# self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key)
# return
class ConfigBucket:
__s3_resource: S3Resource = None
@ -77,11 +71,11 @@ class DataBucket:
return
def put_csv_from(self, src_bucket: str, src_key: str):
self.__s3_resource.copy(src_bucket, src_key, str(self), CRM_IMPORT_DATA_FOLDER)
dest_filename = src_key.split('/')[-1]
self.__s3_resource.copy(src_bucket, src_key, str(self), f'{CRM_IMPORT_DATA_FOLDER}/{dest_filename}')
return
class BackupBucket:
__s3_resource: S3Resource = None
@ -104,4 +98,4 @@ class BackupBucket:
def put_result_json(self, file_path: str, data: dict) -> None:
object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}'
self.__s3_resource.put_object(object_key, json.dumps(data))
return
return

27
ecs/crm-datafetch/src/backup_crm_csv_data_process.py
View File

@ -1,32 +1,21 @@
from src.aws.s3 import BackupBucket
from src.config.objects import TargetObject
from src.system_var.constants import CSVBK_JP_NAME
from src.error.exceptions import FileUploadException
from src.system_var.constants import CSVBK_JP_NAME
from src.util.execute_datetime import ExecuteDateTime
from src.util.logger import logger_instance as logger
def backup_crm_csv_data_process(target_object: TargetObject, exetute_datetime: ExecuteDateTime, csv_string: str):
"""
CSVバックアップ処理
"""CSVバックアップ処理
Parameters
----------
target_object : TargetObject
取得対象オブジェクト情報インスタンス
execute_datetime : ExecuteDateTime
実行日次取得インスタンス
csv_string : str
csvデータ
Args:
target_object (TargetObject): 取得対象オブジェクト情報インスタンス
exetute_datetime (ExecuteDateTime): 実行日次取得インスタンス
csv_string (str): csvデータ
Returns
-------
なし
Raises
------
FileUploadException
S3のファイルアップロード失敗
Raises:
FileUploadException: S3のファイルアップロード失敗
"""
# ① CSVバックアップ処理の開始ログを出力する

25
ecs/crm-datafetch/src/backup_crm_data_process.py
View File

@ -6,26 +6,15 @@ from src.util.logger import logger_instance as logger
def backup_crm_data_process(object_name: str, sf_object_dict: dict, execute_datetime: ExecuteDateTime):
"""
CRM電文データバックアップ処理
"""CRM電文データバックアップ処理
Parameters
----------
object_name : str
取得対象オブジェクト情報インスタンス
sf_object_dict : dict
Salesforceオブジェクトデータ
execute_datetime : ExecuteDateTime
実行日次取得インスタンス
Args:
object_name (str): 取得対象オブジェクト名
sf_object_dict (dict): Salesforceオブジェクトデータ
execute_datetime (ExecuteDateTime): 実行日次取得インスタンス
Returns
-------
なし
Raises
------
FileUploadException
S3のファイルアップロード失敗
Raises:
FileUploadException: S3のファイルアップロード失敗
"""
# ① CRM電文データバックアップ処理の開始ログを出力する

26
ecs/crm-datafetch/src/check_object_info_process.py
View File

@ -1,30 +1,22 @@
from src.config.objects import TargetObject
from src.error.exceptions import InvalidConfigException
from src.util.execute_datetime import ExecuteDateTime
from src.system_var.constants import CHK_JP_NAME
from src.util.execute_datetime import ExecuteDateTime
from src.util.logger import logger_instance as logger
def check_object_info_process(object_info: dict, execute_datetime: ExecuteDateTime):
"""
オブジェクト情報形式チェック処理
"""オブジェクト情報形式チェック処理
Parameters
----------
object_info : dict
取得対象オブジェクト情報
execute_datetime : ExecuteDateTime
実行日次取得インスタンス
Args:
object_info (dict): 取得対象オブジェクト情報
execute_datetime (ExecuteDateTime): 実行日次取得インスタンス
Returns
-------
target_object : TargetObject
取得対象オブジェクト情報インスタンス
Raises:
InvalidConfigException: オブジェクト情報定義が不正だった場合
Raises
------
InvalidConfigException
オブジェクト情報定義が不正だった場合
Returns:
target_object: 取得対象オブジェクト情報インスタンス
"""
# ① オブジェクト情報形式チェック処理開始ログを出力する

60
ecs/crm-datafetch/src/config/objects.py
View File

@ -1,25 +1,19 @@
from src.system_var.constants import ( DATE_PATTERN_YYYYMMDDTHHMMSSTZ,
OBJECTS_KEY,
OBJECTS_TYPE,
OBJECT_NAME_KEY,
OBJECT_NAME_TYPE,
COLUMNS_KEY,
COLUMNS_TYPE,
IS_SKIP_KEY,
IS_SKIP_TYPE,
IS_UPDATE_LAST_FETCH_DATETIME_KEY,
IS_UPDATE_LAST_FETCH_DATETIME_TYPE,
LAST_FETCH_DATETIME_FILE_NAME_KEY,
LAST_FETCH_DATETIME_FILE_NAME_TYPE,
UPLOAD_FILE_NAME_KEY,
UPLOAD_FILE_NAME_TYPE,
DATETIME_COLUMN_KEY,
DATETIME_COLUMN_TYPE,
LAST_FETCH_DATETIME_FROM_KEY,
LAST_FETCH_DATETIME_TO_KEY,
DATETIME_COLUMN_DEFAULT_VALUE
)
from src.system_var.constants import (COLUMNS_KEY, COLUMNS_TYPE,
DATE_PATTERN_YYYYMMDDTHHMMSSTZ,
DATETIME_COLUMN_DEFAULT_VALUE,
DATETIME_COLUMN_KEY,
DATETIME_COLUMN_TYPE, IS_SKIP_KEY,
IS_SKIP_TYPE,
IS_UPDATE_LAST_FETCH_DATETIME_KEY,
IS_UPDATE_LAST_FETCH_DATETIME_TYPE,
LAST_FETCH_DATETIME_FILE_NAME_KEY,
LAST_FETCH_DATETIME_FILE_NAME_TYPE,
LAST_FETCH_DATETIME_FROM_KEY,
LAST_FETCH_DATETIME_TO_KEY,
OBJECT_NAME_KEY, OBJECT_NAME_TYPE,
OBJECTS_KEY, OBJECTS_TYPE,
UPLOAD_FILE_NAME_KEY,
UPLOAD_FILE_NAME_TYPE)
from src.util.dict_checker import DictChecker
@ -98,16 +92,24 @@ class TargetObject():
@property
def is_update_last_fetch_datetime(self) -> bool:
return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY] if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY) else False
if self.__dict_checker.check_key_exist(IS_UPDATE_LAST_FETCH_DATETIME_KEY):
return self.__object_info[IS_UPDATE_LAST_FETCH_DATETIME_KEY]
else:
return False
@property
def last_fetch_datetime_file_name(self) -> str:
return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY] if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}.json'
if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FILE_NAME_KEY):
return self.__object_info[LAST_FETCH_DATETIME_FILE_NAME_KEY]
else:
return f'{self.__object_info[OBJECT_NAME_KEY]}.json'
@property
def upload_file_name(self) -> str:
return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY) else f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}'
if self.__dict_checker.check_key_exist(UPLOAD_FILE_NAME_KEY):
return self.__object_info[UPLOAD_FILE_NAME_KEY].format(execute_datetime=self.__execute_datetime)
else:
return f'{self.__object_info[OBJECT_NAME_KEY]}_{self.__execute_datetime}'
@property
def datetime_column(self) -> str:
@ -126,7 +128,6 @@ class LastFetchDatetime():
self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_FROM_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ)
if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY):
self.__dict_checker.assert_match_pattern(LAST_FETCH_DATETIME_TO_KEY, DATE_PATTERN_YYYYMMDDTHHMMSSTZ)
return
@property
@ -135,4 +136,7 @@ class LastFetchDatetime():
@property
def last_fetch_datetime_to(self) -> str:
return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY].format(execute_datetime=self.__execute_datetime) if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_FROM_KEY) else self.__execute_datetime
if self.__dict_checker.check_key_exist(LAST_FETCH_DATETIME_TO_KEY):
return self.__last_fetch_datetime_file_dict[LAST_FETCH_DATETIME_TO_KEY]
else:
return self.__execute_datetime

59
ecs/crm-datafetch/src/controller.py
View File

@ -7,13 +7,14 @@ from src.config.objects import FetchTargetObjects
from src.convert_crm_csv_data_process import convert_crm_csv_data_process
from src.copy_crm_csv_data_process import copy_crm_csv_data_process
from src.error.exceptions import MeDaCaCRMDataFetchException
from src.util.execute_datetime import ExecuteDateTime
from src.fetch_crm_data_process import fetch_crm_data_process
from src.prepare_data_fetch_process import prepare_data_fetch_process
from src.set_datetime_period_process import set_datetime_period_process
from src.system_var.constants import OBJECT_NAME_KEY
from src.upload_last_fetch_datetime_process import upload_last_fetch_datetime_process
from src.upload_last_fetch_datetime_process import \
upload_last_fetch_datetime_process
from src.upload_result_data_process import upload_result_data_process
from src.util.execute_datetime import ExecuteDateTime
from src.util.logger import logger_instance as logger
@ -63,29 +64,22 @@ def controller() -> None:
def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: ExecuteDateTime, process_result: dict):
"""取得対象オブジェクト情報をループし、１オブジェクトごとのデータを取得する
Args:
fetch_target_objects (FetchTargetObjects): CRMオブジェクト情報インスタンス
execute_datetime (ExecuteDateTime): 実行日次取得インスタンス
process_result (dict): 取得処理実行結果辞書オブジェクト
Returns:
process_result: 取得処理実行結果辞書オブジェクト
"""
取得対象オブジェクト情報をループし、１オブジェクトごとのデータを取得する
Parameters
----------
fetch_target_objects : FetchTargetObjects
CRMオブジェクト情報インスタンス
execute_datetime : ExecuteDateTime
実行日次取得インスタンス
process_result : dict
取得処理実行結果辞書オブジェクト
Returns
-------
process_result : dict
取得処理実行結果辞書オブジェクト
"""
for object_info in fetch_target_objects:
try:
process_result[object_info.get(OBJECT_NAME_KEY)] = 'fail'
fetch_crm_data_per_object(object_info, execute_datetime, process_result)
fetch_crm_data_per_object(object_info, execute_datetime)
process_result[object_info.get(OBJECT_NAME_KEY)] = 'success'
@ -102,28 +96,13 @@ def fetch_crm_data(fetch_target_objects: FetchTargetObjects, execute_datetime: E
return process_result
def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTime) -> None:
"""
オブジェクトごとにCRMのデータを取得し、取込フォルダにアップロードする
Parameters
----------
object_info : dict
取得対象オブジェクト情報
execute_datetime : object
実行日次取得インスタンス
Returns
-------
なし
Raises
------
FileNotFoundException
S3上のファイルが存在しない場合
InvalidConfigException
オブジェクト情報定義が不正だった場合
"""オブジェクトごとにCRMのデータを取得し、取込フォルダにアップロードする
Args:
object_info (dict): 取得対象オブジェクト情報
execute_datetime (ExecuteDateTime): 実行日次取得インスタンス
"""
# 1. オブジェクト処理結果の初期化
@ -194,4 +173,4 @@ def fetch_crm_data_per_object(object_info: dict, execute_datetime: ExecuteDateTi
# 14. オブジェクトのアップロードが完了した旨をログに出力する
logger.info(f'I-CTRL-16 [{target_object_name}] 処理正常終了')
return
return

29
ecs/crm-datafetch/src/convert_crm_csv_data_process.py
View File

@ -6,25 +6,17 @@ from src.util.logger import logger_instance as logger
def convert_crm_csv_data_process(target_object: TargetObject, crm_data_response: dict):
"""
CSV変換処理
"""CSV変換処理
Parameters
----------
target_object : TargetObject
取得対象オブジェクト情報インスタンス
crm_data_response : dict
Salesforceオブジェクトデータ
Args:
target_object (TargetObject): 取得対象オブジェクト情報インスタンス
crm_data_response (dict): Salesforceオブジェクトデータ
Returns
-------
csv_string : str
csvデータ
Raises:
DataConvertException: データ変換が失敗した場合
Raises
------
DataConvertException
データ変換が失敗した場合
Returns:
csv_string: csvデータ
"""
# ① CSV変換処理の開始ログを出力する
@ -34,9 +26,8 @@ def convert_crm_csv_data_process(target_object: TargetObject, crm_data_response:
try:
# ② CSV変換
CSVStringConverter(target_object, crm_data_response)
csv_string = CSVStringConverter.convert()
csv_string_converter = CSVStringConverter(target_object, crm_data_response)
csv_string = csv_string_converter.convert()
logger.debug(f'D-CONV-02 [{target_object_name}] のCSV変換処理 正常終了')

29
ecs/crm-datafetch/src/converter/convert_factory.py
View File

@ -1,9 +1,10 @@
import re
from datetime import datetime
from src.config.objects import TargetObject
from src.system_var.constants import (CRM_DATETIME_FORMAT, CSV_FALSE_VALUE,
CSV_TRUE_VALUE, YYYYMMDDHHMMSS, DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC)
CSV_TRUE_VALUE,
DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC,
YYYYMMDDHHMMSS)
class ConvertStrategyFactory:
@ -12,30 +13,30 @@ class ConvertStrategyFactory:
self.__float_convert_strategy = FloatConvertStrategy()
self.__boolean_convert_strategy = BooleanConvertStrategy()
self.__datetime_convert_strategy = DatatimeConvertStrategy()
self.__non_convert_strategy = NonConvertStrategy()
def create(self, value):
converted_value = value
if value is None:
converted_value = self.__none_value_convert_strategy.convert_value()
convert_strategy = self.__none_value_convert_strategy
# 指数表記で取得できるパターン。指数表記を整数表記に変換する。
elif type(value) == float:
converted_value = self.__float_convert_strategy.convert_value(value)
convert_strategy = self.__float_convert_strategy
# SQLの真偽値に対応するために変換する
elif type(value) == bool:
converted_value = self.__boolean_convert_strategy.convert_value(value)
convert_strategy = self.__boolean_convert_strategy
elif type(value) == str and re.fullmatch(DATE_PATTERN_YYYYMMDDHHMMSSFFF_UTC, value):
converted_value = self.__datetime_convert_strategy.convert_value(value)
convert_strategy = self.__datetime_convert_strategy
return converted_value
else:
convert_strategy = self.__non_convert_strategy
return convert_strategy
class NoneValueConvertStrategy:
def convert_value(self) -> str:
def convert_value(self, convert_value: None) -> str:
return ''
@ -53,3 +54,7 @@ class FloatConvertStrategy:
def convert_value(self, convert_value: str) -> int:
return int(convert_value)
class NonConvertStrategy:
def convert_value(self, convert_value: str):
return convert_value

3
ecs/crm-datafetch/src/converter/converter.py
View File

@ -52,7 +52,8 @@ class CSVStringConverter:
for column in columns:
v = json_object[column.upper()]
converted_value = self.__convert_strategy_factory.create(v)
convert_strategy = self.__convert_strategy_factory.create(v)
converted_value = convert_strategy.convert_value(v)
csv_row.append(converted_value)

25
ecs/crm-datafetch/src/copy_crm_csv_data_process.py
View File

@ -2,31 +2,20 @@ from src.aws.s3 import BackupBucket, DataBucket
from src.config.objects import TargetObject
from src.error.exceptions import FileUploadException
from src.system_var.constants import UPLD_JP_NAME
from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_IMPORT_DATA_BACKUP_FOLDER,
CRM_IMPORT_DATA_FOLDER, IMPORT_DATA_BUCKET)
from src.system_var.environments import CRM_IMPORT_DATA_BACKUP_FOLDER
from src.util.execute_datetime import ExecuteDateTime
from src.util.logger import logger_instance as logger
def copy_crm_csv_data_process(target_object: TargetObject, execute_datetime: ExecuteDateTime):
"""
CSVアップロード処理
"""CSVアップロード処理
Parameters
----------
target_object : TargetObject
取得対象オブジェクト情報インスタンス
execute_datetime : ExecuteDateTime
実行日次取得インスタンス
Args:
target_object (TargetObject): 取得対象オブジェクト情報インスタンス
execute_datetime (ExecuteDateTime): 実行日次取得インスタンス
Returns
-------
なし
Raises
------
FileUploadException
S3のファイルアップロード失敗
Raises:
FileUploadException: S3のファイルアップロード失敗
"""
# ① CSVデータアップロード処理の開始ログを出力する

146
ecs/crm-datafetch/src/fetch_crm_data_process.py
View File

@ -2,102 +2,91 @@ from requests.exceptions import ConnectTimeout, ReadTimeout
from tenacity import retry, stop_after_attempt
from tenacity.wait import wait_exponential
from src.config.objects import TargetObject, LastFetchDatetime
from src.config.objects import LastFetchDatetime, TargetObject
from src.error.exceptions import DataConvertException, SalesforceAPIException
from src.salesforce.salesforce_api import SalesforceApiClient
from src.salesforce.soql_builder import SOQLBuilder
from src.system_var.constants import FETCH_JP_NAME
from src.system_var.environments import (CRM_AUTH_TIMEOUT,
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT,
CRM_FETCH_RECORD_RETRY_INTERVAL,
CRM_FETCH_RECORD_RETRY_MAX_INTERVAL,
CRM_FETCH_RECORD_RETRY_MIN_INTERVAL,
CRM_FETCH_RECORD_TIMEOUT,
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT,
CRM_GET_RECORD_COUNT_RETRY_INTERVAL,
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL,
CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL,
CRM_GET_RECORD_COUNT_TIMEOUT)
from src.system_var.environments import (
CRM_AUTH_TIMEOUT, CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT,
CRM_FETCH_RECORD_RETRY_INTERVAL, CRM_FETCH_RECORD_RETRY_MAX_INTERVAL,
CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, CRM_FETCH_RECORD_TIMEOUT,
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT,
CRM_GET_RECORD_COUNT_RETRY_INTERVAL,
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL,
CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, CRM_GET_RECORD_COUNT_TIMEOUT)
from src.util.counter_object import CounterObject
from src.util.logger import logger_instance as logger
def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime):
"""
CRMデータ取得処理
"""CRMデータ取得処理
Parameters
----------
target_object : TargetObject
取得対象オブジェクト情報インスタンス
last_fetch_datetime : LastFetchDatetime
取得対象オブジェクト情報インスタンス
Args:
target_object (TargetObject): 取得対象オブジェクト情報インスタンス
last_fetch_datetime (LastFetchDatetime): データ取得期間設定インスタンス
Returns
-------
sf_object_dict : dict
Salesforceオブジェクトデータ
Raises:
SalesforceAPIException: SalseforceのAPI実行失敗が発生した場合
DataConvertException: データ変換が失敗した場合
Raises
------
SalesforceAPIException
SalseforceのAPI実行失敗が発生した場合
DataConvertException
データ変換が失敗した場合
Returns:
crm_data_response: Salesforceオブジェクトデータ
"""
# ① CRMデータ取得処理開始ログを出力する
logger.info(
f'I-FETCH-01 [{target_object.object_name}] のCRMからのデータ取得処理を開始します')
object_name = target_object.object_name
target_object_name = target_object.object_name
global count_contime_counter, count_readtime_counter, count_counter, data_contime_counter, data_readtime_counter, data_counter
count_contime_counter = 1
count_readtime_counter = 1
count_counter = 1
data_contime_counter = 1
data_readtime_counter = 1
data_counter = 1
# リトライ回数判定用のカウンタオブジェクトを生成(@retryデコレータを利用したことによるリトライ対象の関数内でのカウント変数保持不可の対策のためオブジェクト化する)
count_contime_counter = CounterObject(1)
count_readtime_counter = CounterObject(1)
count_counter = CounterObject(1)
data_contime_counter = CounterObject(1)
data_readtime_counter = CounterObject(1)
data_counter = CounterObject(1)
try:
# ② 取得対象オブジェクトの取得期間内のレコード件数を取得する
logger.info(f'I-FETCH-02 [{object_name}] の件数取得を開始します')
logger.info(f'I-FETCH-02 [{target_object_name}] の件数取得を開始します')
soql_builder = SOQLBuilder(target_object, last_fetch_datetime)
count_soql = soql_builder.create_count_soql()
record_count = fetch_record_count_retry(count_soql, object_name)
record_count = fetch_record_count_retry(count_soql, target_object_name, count_contime_counter, count_readtime_counter, count_counter)
logger.info(f'I-FETCH-03 [{object_name}] の件数：[{record_count}]')
logger.info(f'I-FETCH-03 [{target_object_name}] の件数：[{record_count}]')
except Exception as e:
raise SalesforceAPIException(
'E-FETCH-01', FETCH_JP_NAME, f'[{object_name}] の件数取得に失敗しました エラー内容：[{e}]')
'E-FETCH-01', FETCH_JP_NAME, f'[{target_object_name}] の件数取得に失敗しました エラー内容：[{e}]')
try:
# ③ 取得対象オブジェクトのレコードを取得する
logger.info(f'I-FETCH-04 [{object_name}] のレコード取得を開始します')
logger.info(f'I-FETCH-04 [{target_object_name}] のレコード取得を開始します')
fetch_soql = soql_builder.create_fetch_soql()
record_all = fetch_sf_data_retry(fetch_soql, object_name)
record_all = fetch_sf_data_retry(fetch_soql, target_object_name, data_contime_counter, data_readtime_counter, data_counter)
except Exception as e:
raise SalesforceAPIException(
'E-FETCH-02', FETCH_JP_NAME, f'[{object_name}] のレコード取得に失敗しました エラー内容：[{e}]')
'E-FETCH-02', FETCH_JP_NAME, f'[{target_object_name}] のレコード取得に失敗しました エラー内容：[{e}]')
try:
# ④ 取得対象オブジェクトをJSONに変換
logger.info(f'I-FETCH-05 [{object_name}] のレコードをJSONに変換します')
logger.info(f'I-FETCH-05 [{target_object_name}] のレコードをJSONに変換します')
crm_data_response = [record for record in record_all]
except Exception as e:
raise DataConvertException(
'E-FETCH-03', FETCH_JP_NAME, f'[{object_name}] のレコードのJSON変換に失敗しました エラー内容：[{e}]')
'E-FETCH-03', FETCH_JP_NAME, f'[{target_object_name}] のレコードのJSON変換に失敗しました エラー内容：[{e}]')
# ⑤ CRMデータ取得処理終了ログを出力する
logger.info(f'I-FETCH-06 [{object_name}] のレコード取得が成功しました')
logger.info(f'I-FETCH-06 [{target_object_name}] のレコード取得が成功しました')
# ⑥ 次の処理へ移行する
return crm_data_response
@ -107,31 +96,34 @@ def fetch_crm_data_process(target_object: TargetObject, last_fetch_datetime: Las
wait=wait_exponential(multiplier=CRM_GET_RECORD_COUNT_RETRY_INTERVAL,
min=CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, max=CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL),
stop=stop_after_attempt(CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT))
def fetch_record_count_retry(soql: str, object_name: str):
def fetch_record_count_retry(soql: str, target_object_name: str,
count_contime_counter: CounterObject, count_readtime_counter: CounterObject, count_counter: CounterObject):
try:
global count_contime_counter, count_readtime_counter, count_counter
salesforce_api_client = SalesforceApiClient()
return salesforce_api_client.fetch_sf_count(soql)
except ConnectTimeout as e:
if count_contime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT:
count_contime_counter += 1
logger.warn(f'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]')
# 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う
if count_contime_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT:
count_contime_counter.increment(1)
logger.warning(f'W-FETCH-01 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]')
raise e
except ReadTimeout as e:
if count_readtime_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT:
count_readtime_counter += 1
logger.warn(
f'W-FETCH-02 [{object_name}] の件数取得処理がタイムアウトしたため、リトライします：[{CRM_GET_RECORD_COUNT_TIMEOUT}] エラー内容：[{e}]')
# 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う
if count_readtime_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT:
count_readtime_counter.increment(1)
logger.warning(
f'W-FETCH-02 [{target_object_name}] の件数取得処理がタイムアウトしたため、リトライします：[{CRM_GET_RECORD_COUNT_TIMEOUT}] エラー内容：[{e}]')
raise e
except Exception as e:
if count_counter < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT:
count_counter += 1
logger.warn(
f'W-FETCH-03 [{object_name}] の件数取得に失敗したため、リトライします エラー内容：[{e}]')
# 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う
if count_counter.describe() < CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT:
count_counter.increment(1)
logger.warning(
f'W-FETCH-03 [{target_object_name}] の件数取得に失敗したため、リトライします エラー内容：[{e}]')
raise e
@ -139,29 +131,31 @@ def fetch_record_count_retry(soql: str, object_name: str):
wait=wait_exponential(multiplier=CRM_FETCH_RECORD_RETRY_INTERVAL,
min=CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, max=CRM_FETCH_RECORD_RETRY_MAX_INTERVAL),
stop=stop_after_attempt(CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT))
def fetch_sf_data_retry(soql: str, object_name: str):
def fetch_sf_data_retry(soql: str, target_object_name: str,
data_contime_counter: CounterObject, data_readtime_counter: CounterObject, data_counter: CounterObject):
try:
global data_contime_counter, data_readtime_counter, data_counter
salesforce_api_client = SalesforceApiClient()
return salesforce_api_client.fetch_sf_data(soql)
except ConnectTimeout as e:
if data_contime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT:
data_contime_counter += 1
logger.warn(f'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]')
# 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う
if data_contime_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT:
data_contime_counter.increment(1)
logger.warning(f'W-FETCH-04 CRMの接続処理がタイムアウトしため、リトライします：[{CRM_AUTH_TIMEOUT}] エラー内容：[{e}]')
raise e
except ReadTimeout as e:
if data_readtime_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT:
data_readtime_counter += 1
logger.warn(
f'W-FETCH-05 [{object_name}] のレコード取得処理がタイムアウトしたため、リトライします：[{CRM_FETCH_RECORD_TIMEOUT}] エラー内容：[{e}]')
# 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う
if data_readtime_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT:
data_readtime_counter.increment(1)
logger.warning(
f'W-FETCH-05 [{target_object_name}] のレコード取得処理がタイムアウトしたため、リトライします：[{CRM_FETCH_RECORD_TIMEOUT}] エラー内容：[{e}]')
raise e
except Exception as e:
if data_counter < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT:
data_counter += 1
logger.warn(
f'W-FETCH-06 [{object_name}] のレコード取得に失敗したため、リトライします エラー内容：[{e}]')
# 「リトライします」のメッセージ出力後、リトライせず例外終了になってしまうことを防ぐため、カウンタによる回数の判定を行う
if data_counter.describe() < CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT:
data_counter.increment(1)
logger.warning(
f'W-FETCH-06 [{target_object_name}] のレコード取得に失敗したため、リトライします エラー内容：[{e}]')
raise e

2
ecs/crm-datafetch/src/parser/json_parse.py
View File

@ -8,7 +8,7 @@ class JsonParser():
def __init__(self, json_str) -> None:
self.__json_str = json_str
def json_parser(self) -> dict:
def parse(self) -> dict:
for symbol in EXCLUDE_SYMBOL:
# コメントアウトシンボルを含む部分を置き換える正規表現
replace_comment_regex = rf'\s(?!\"){symbol}[\s\S]*?.*'

36
ecs/crm-datafetch/src/prepare_data_fetch_process.py
View File

@ -3,36 +3,24 @@ from src.config.objects import FetchTargetObjects
from src.error.exceptions import FileNotFoundException, InvalidConfigException
from src.parser.json_parse import JsonParser
from src.system_var.constants import PRE_JP_NAME
from src.system_var.environments import (CRM_CONFIG_BUCKET, OBJECT_INFO_FILENAME,
OBJECT_INFO_FOLDER)
from src.system_var.environments import (CRM_CONFIG_BUCKET,
OBJECT_INFO_FILENAME,
OBJECT_INFO_FOLDER)
from src.util.execute_datetime import ExecuteDateTime
from src.util.logger import logger_instance as logger
def prepare_data_fetch_process():
"""
データ取得準備処理
"""データ取得準備処理
Parameters
----------
なし
Returns
-------
fetch_target_objects : FetchTargetObjects
CRMオブジェクト情報インスタンス
execute_datetime : ExecuteDateTime
実行日次取得インスタンス
process_result : dict
取得処理実行結果辞書オブジェクト
Raises
------
FileNotFoundException
S3上のファイルが存在しない場合
InvalidConfigException
オブジェクト情報定義が不正だった場合
Raises:
FileNotFoundException: S3上のファイルが存在しない場合
InvalidConfigException: オブジェクト情報定義が不正だった場合
Returns:
fetch_target_objects : CRMオブジェクト情報インスタンス
execute_datetime : 実行日次取得インスタンス
process_result : 取得処理実行結果辞書オブジェクト
"""
# ① データ取得準備処理の開始ログを出力する
@ -64,7 +52,7 @@ def prepare_data_fetch_process():
logger.debug('D-PRE-05 CRM_取得オブジェクト情報ファイルをパースします')
json_parser = JsonParser(object_info_file_str)
object_info_file_dict = json_parser.json_parser()
object_info_file_dict = json_parser.parse()
logger.debug('D-PRE-06 CRM_取得オブジェクト情報ファイルのパースに成功しました')

12
ecs/crm-datafetch/src/salesforce/salesforce_api.py
View File

@ -1,10 +1,9 @@
from simple_salesforce import Salesforce
from src.system_var.environments import (CRM_AUTH_DOMAIN, CRM_AUTH_TIMEOUT,
CRM_FETCH_RECORD_TIMEOUT,
CRM_GET_RECORD_COUNT_TIMEOUT, CRM_USER_NAME,
CRM_USER_PASSWORD, CRM_USER_SECURITY_TOKEN)
CRM_FETCH_RECORD_TIMEOUT,
CRM_GET_RECORD_COUNT_TIMEOUT,
CRM_USER_NAME, CRM_USER_PASSWORD,
CRM_USER_SECURITY_TOKEN)
class SalesforceApiClient():
@ -25,4 +24,5 @@ class SalesforceApiClient():
return count_res.get('records')[0].get('expr0')
def fetch_sf_data(self, soql: str):
return self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT)
data_res = self.query_all(soql, conn_timeout=CRM_AUTH_TIMEOUT, read_timeout=CRM_FETCH_RECORD_TIMEOUT)
return data_res.get('records')

36
ecs/crm-datafetch/src/set_datetime_period_process.py
View File

@ -1,37 +1,27 @@
from src.aws.s3 import ConfigBucket
from src.config.objects import TargetObject, LastFetchDatetime
from src.config.objects import LastFetchDatetime, TargetObject
from src.error.exceptions import FileNotFoundException, InvalidConfigException
from src.parser.json_parse import JsonParser
from src.system_var.constants import DATE_JP_NAME
from src.system_var.environments import CRM_CONFIG_BUCKET, LAST_FETCH_DATE_FOLDER
from src.system_var.environments import (CRM_CONFIG_BUCKET,
LAST_FETCH_DATE_FOLDER)
from src.util.execute_datetime import ExecuteDateTime
from src.util.logger import logger_instance as logger
def set_datetime_period_process(target_object: TargetObject, execute_datetime: ExecuteDateTime):
"""
データ取得期間設定処理
"""データ取得期間設定処理
Parameters
----------
target_object : TargetObject
取得対象オブジェクト情報インスタンス
execute_datetime : ExecuteDateTime
実行日次取得インスタンス
Args:
target_object (TargetObject): 取得対象オブジェクト情報インスタンス
execute_datetime (ExecuteDateTime): 実行日次取得インスタンス
Returns
-------
last_fetch_datetime : LastFetchDatetime
取得対象オブジェクト情報インスタンス
Raises:
FileNotFoundException: S3上のファイルが存在しない場合
InvalidConfigException: オブジェクト情報定義が不正だった場合
Raises
------
FileNotFoundException
S3上のファイルが存在しない場合
InvalidConfigException
オブジェクト情報定義が不正だった場合
Returns:
last_fetch_datetime: データ取得期間設定インスタンス
"""
# ① データ取得期間設定処理の開始ログを出力する
@ -59,7 +49,7 @@ def set_datetime_period_process(target_object: TargetObject, execute_datetime: E
logger.debug(f'D-DATE-04 前回取得日時ファイルの形式チェックを開始します')
json_parser = JsonParser(last_fetch_datetime_file_str)
last_fetch_datetime_file_dict = json_parser.json_parser()
last_fetch_datetime_file_dict = json_parser.parse()
last_fetch_datetime = LastFetchDatetime(last_fetch_datetime_file_dict, execute_datetime)

30
ecs/crm-datafetch/src/system_var/environments.py
View File

@ -6,35 +6,35 @@ import src.system_var.constants as constants
# ログ出力レベル。DEBUG, INFO, WARNING, ERRORの4つから指定する
LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO)
# CRMへの認証処理のタイムアウト秒数
CRM_AUTH_TIMEOUT = os.environ.get(constants.CRM_AUTH_TIMEOUT, 100)
CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100))
# CRMへの認証処理の最大リトライ試行回数
CRM_AUTH_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3)
CRM_AUTH_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_AUTH_MAX_RETRY_ATTEMPT, 3))
# CRMへの認証処理のリトライ時の初回待ち秒数
CRM_AUTH_RETRY_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5)
CRM_AUTH_RETRY_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_INTERVAL, 5))
# CRMへの認証処理のリトライ時の最小待ち秒数
CRM_AUTH_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5)
CRM_AUTH_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MIN_INTERVAL, 5))
# CRMへの認証処理のリトライ時の最大待ち秒数
CRM_AUTH_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50)
CRM_AUTH_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_AUTH_RETRY_MAX_INTERVAL, 50))
# CRMのレコード件数取得処理のタイムアウト秒数
CRM_GET_RECORD_COUNT_TIMEOUT = os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300)
CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300))
# CRMのレコード件数取得処理の最大リトライ試行回数
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3)
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3))
# CRMのレコード件数取得処理のリトライ時の初回待ち秒数
CRM_GET_RECORD_COUNT_RETRY_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5)
CRM_GET_RECORD_COUNT_RETRY_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5))
# CRMのレコード件数取得処理のリトライ時の最小待ち秒数
CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, 5)
CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL, 5))
# CRMのレコード件数取得処理のリトライ時の最大待ち秒数
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50)
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50))
# CRMのレコード取得処理のタイムアウト秒数
CRM_FETCH_RECORD_TIMEOUT = os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300)
CRM_FETCH_RECORD_TIMEOUT = int(os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300))
# CRMのレコード取得処理の最大リトライ試行回数
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3)
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3))
# CRMのレコード取得処理のリトライ時の初回待ち秒数
CRM_FETCH_RECORD_RETRY_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5)
CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5))
# CRMのレコード取得処理のリトライ時の最小待ち秒数
CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5)
CRM_FETCH_RECORD_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL, 5))
# CRMのレコード取得処理のリトライ時の最大待ち秒数
CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50)
CRM_FETCH_RECORD_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL, 50))
# environments(ECS Task Environment)
# CRMのAPI実行のための認証エンドポイントのドメイン

24
ecs/crm-datafetch/src/upload_last_fetch_datetime_process.py
View File

@ -1,31 +1,21 @@
import json
from src.aws.s3 import ConfigBucket
from src.config.objects import TargetObject, LastFetchDatetime
from src.config.objects import LastFetchDatetime, TargetObject
from src.error.exceptions import FileUploadException
from src.system_var.constants import UPD_JP_NAME
from src.util.logger import logger_instance as logger
def upload_last_fetch_datetime_process(target_object: TargetObject, last_fetch_datetime: LastFetchDatetime):
"""
前回取得日時ファイル更新
"""前回取得日時ファイル更新
Parameters
----------
target_object : TargetObject
取得対象オブジェクト情報インスタンス
last_fetch_datetime : LastFetchDatetime
取得対象オブジェクト情報インスタンス
Args:
target_object (TargetObject): 取得対象オブジェクト情報インスタンス
last_fetch_datetime (LastFetchDatetime): データ取得期間設定インスタンス
Returns
-------
なし
Raises
------
FileUploadException
S3のファイルアップロード失敗
Raises:
FileUploadException: S3のファイルアップロード失敗
"""
# ① 前回取得日時ファイル更新処理の開始ログを出力する

22
ecs/crm-datafetch/src/upload_result_data_process.py
View File

@ -7,24 +7,14 @@ from src.util.logger import logger_instance as logger
def upload_result_data_process(process_result: dict, execute_datetime: ExecuteDateTime):
"""
取得処理実施結果アップロード処理
"""取得処理実施結果アップロード処理
Parameters
----------
process_result : dict
取得処理実行結果辞書オブジェクト
last_fetch_datetime : LastFetchDatetime
取得対象オブジェクト情報インスタンス
Args:
process_result (dict): 取得処理実行結果辞書オブジェクト
execute_datetime (ExecuteDateTime): データ取得期間設定インスタンス
Returns
-------
なし
Raises
------
FileUploadException
S3のファイルアップロード失敗
Raises:
FileUploadException: S3のファイルアップロード失敗
"""
# ① 取得処理実施結果アップロード処理のログを出力する

14
ecs/crm-datafetch/src/util/counter_object.py Normal file
View File

@ -0,0 +1,14 @@
class CounterObject:
def __init__(self, base_num=1) -> None:
self.__counter = base_num
def describe(self) -> int:
return self.__counter
def increment(self, num=1) -> int:
self.__counter += num
return self.__counter
def decrement(self, num=1) -> int:
self.__counter -= num
return self.__counter

2
ecs/crm-datafetch/src/util/dict_checker.py
View File

@ -17,7 +17,7 @@ class DictChecker:
"""辞書型バリュー正規表現チェック"""
return True if re.fullmatch(regex_str, self.__object_dict[check_key]) else False
def assert_key_exist(self, check_key: str, check_type: type) -> None:
def assert_key_exist(self, check_key: str) -> None:
"""辞書型キー存在検査"""
if not self.check_key_exist(check_key):
raise Exception(f'「{check_key}」キーは必須です')

5
ecs/crm-datafetch/src/util/execute_datetime.py
View File

@ -1,9 +1,6 @@
from datetime import datetime
from src.system_var.constants import(
YYYYMMDDTHHMMSSTZ,
MILLISEC_FORMAT
)
from src.system_var.constants import MILLISEC_FORMAT, YYYYMMDDTHHMMSSTZ
class ExecuteDateTime:

2
ecs/crm-datafetch/src/util/logger.py
View File

@ -2,7 +2,7 @@ import logging
from src.system_var.environments import LOG_LEVEL
"""boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する"""
# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する
for name in ["boto3", "botocore", "s3transfer", "urllib3"]:
logging.getLogger(name).setLevel(logging.WARNING)

4
s3/config/crm/last_fetch_datetime/Account.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/AccountShare.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Account_Territory_Loader_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Approved_Document_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Call2_Detail_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Call2_Discussion_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Call2_Key_Message_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Call2_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Call_Clickstream_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Child_Account_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Clm_Presentation_Slide_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Clm_Presentation_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Coaching_Report_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Consent_Header_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Consent_Line_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Consent_Type_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Contact.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Dynamic_Attribute_Configuration_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Dynamic_Attribute_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Email_Activity_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Event_Attendee_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Group.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Key_Message_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/MSJ_Hospital_Medical_Regimen__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/MSJ_Inquiry_Assignment__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/MSJ_MR_Weekly_Report__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/MSJ_Medical_Event_Evaluation__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/MSJ_Medical_Regimen__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/MSJ_Patient__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Medical_Inquiry_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Medical_Insight_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Multichannel_Activity_Line_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Multichannel_Activity_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Multichannel_Consent_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/My_Setup_Products_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/ObjectTerritory2Association.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Product_Group_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Product_Metrics_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Product_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Profile.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Question_Response_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/RecordType.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Remote_Meeting_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Sent_Email_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Sent_Fragment_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Survey_Question_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Survey_Target_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Survey_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Territory2.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/Time_Off_Territory_vod__c.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/User.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/UserRole.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/UserTerritory2Association.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/object_info/crm_object_list_all.json
View File

@ -24,7 +24,7 @@
"is_skip": false,
"is_update_last_fetch_datetime": false,
"last_fetch_datetime_file_name": "Territory2_ALL.json",
"upload_file_name": "CRM_Territory2_ALL_{execute_datetime}.csv"
"upload_file_name": "CRM_Territory2_ALL_{execute_datetime}"
},
{
"object_name": "UserTerritory2Association",
@ -41,7 +41,7 @@
"is_skip": false,
"is_update_last_fetch_datetime": false,
"last_fetch_datetime_file_name": "UserTerritory2Association_ALL.json",
"upload_file_name": "CRM_UserTerritory2Association_ALL_{execute_datetime}.csv"
"upload_file_name": "CRM_UserTerritory2Association_ALL_{execute_datetime}"
}
]
}

3
s3/config/crm/object_info/crm_object_list_diff.json
View File

@ -2164,7 +2164,8 @@
"IsDeleted"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
"is_update_last_fetch_datetime": true,
"datetime_column": "LastModifiedDate"
},
{
"object_name": "Contact",