nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #99 fix-regration-crm-datafetch-process into develop-6crm

Browse Source
This commit is contained in:
下田雅人 2022-09-07 18:26:56 +09:00
commit 8117da4766
17 changed files with 330 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ecs/crm-datafetch/Dockerfile
View File

@ -14,6 +14,6 @@ RUN \
pip uninstall -y pipenv virtualenv-clone virtualenv
COPY main.py ./
COPY src ./
COPY src ./src
CMD [ "python", "./main.py" ]

4
ecs/crm-datafetch/src/aws/s3.py
View File

@ -87,7 +87,7 @@ class BackupBucket:
def put_response_json(self, file_path: str, data: dict) -> None:
object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_path}'
self.__s3_resource.put_object(object_key, json.dumps(data))
self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False))
return
def put_csv(self, file_path: str, data: str) -> None:
@ -97,5 +97,5 @@ class BackupBucket:
def put_result_json(self, file_path: str, data: dict) -> None:
object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}'
self.__s3_resource.put_object(object_key, json.dumps(data))
self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False))
return

21
ecs/crm-datafetch/src/converter/convert_strategy.py
View File

@ -1,4 +1,6 @@
import json
import re
from collections import OrderedDict
from datetime import datetime
from dateutil.tz import gettz
@ -12,20 +14,17 @@ from src.system_var.environments import CONVERT_TZ
class ConvertStrategyFactory:
def __init__(self) -> None:
self.__none_value_convert_strategy = NoneValueConvertStrategy()
self.__float_convert_strategy = FloatConvertStrategy()
self.__boolean_convert_strategy = BooleanConvertStrategy()
self.__datetime_convert_strategy = DatetimeConvertStrategy()
self.__int_convert_strategy = IntConvertStrategy()
self.__string_convert_strategy = StringConvertStrategy()
self.__dict_convert_strategy = DictConvertStrategy()
def create(self, value):
if value is None:
convert_strategy = self.__none_value_convert_strategy
elif type(value) == float:
convert_strategy = self.__float_convert_strategy
elif type(value) == bool:
convert_strategy = self.__boolean_convert_strategy
@ -35,6 +34,8 @@ class ConvertStrategyFactory:
elif type(value) == int:
convert_strategy = self.__int_convert_strategy
elif type(value) == dict or type(value) == OrderedDict:
convert_strategy = self.__dict_convert_strategy
else:
convert_strategy = self.__string_convert_strategy
@ -60,12 +61,6 @@ class DatetimeConvertStrategy:
return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS)
class FloatConvertStrategy:
def convert_value(self, convert_value: str) -> int:
"""float型をint型に変換する処理"""
return int(convert_value)
class IntConvertStrategy:
def convert_value(self, convert_value: int):
"""int型を変換せずに返す処理"""
@ -78,3 +73,9 @@ class StringConvertStrategy:
"""string型を変換せずに返す処理"""
# ConvertStrategyFactoryにて型チェックを行っているため値を変換せずに返す
return convert_value
class DictConvertStrategy:
def convert_value(self, convert_value: dict):
"""dict型の項目を文字列に変換して返す処理"""
return json.dumps(convert_value, ensure_ascii=False)

6
ecs/crm-datafetch/src/prepare_data_fetch_process.py
View File

@ -34,9 +34,9 @@ def prepare_data_fetch_process():
try:
# ③ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する
object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}'
logger.debug(
f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]')
object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}'
logger.info(
f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]')
config_bucket = ConfigBucket()
object_info_file_str = config_bucket.get_object_info_file()

8
ecs/crm-datafetch/src/system_var/environments.py
View File

@ -9,8 +9,8 @@ LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO)
CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100))
# CRMのレコード件数取得処理のタイムアウト秒数
CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300))
# CRMのレコード件数取得処理の最大リトライ試行回数
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3))
# CRMのレコード件数取得処理の最大リトライ試行回数(処理の実施総回数)
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 4))
# CRMのレコード件数取得処理のリトライ時の初回待ち秒数
CRM_GET_RECORD_COUNT_RETRY_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5))
# CRMのレコード件数取得処理のリトライ時の最小待ち秒数
@ -19,8 +19,8 @@ CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_GET_R
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50))
# CRMのレコード取得処理のタイムアウト秒数
CRM_FETCH_RECORD_TIMEOUT = int(os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300))
# CRMのレコード取得処理の最大リトライ試行回数
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3))
# CRMのレコード取得処理の最大リトライ試行回数(処理の実施総回数)
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 4))
# CRMのレコード取得処理のリトライ時の初回待ち秒数
CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5))
# CRMのレコード取得処理のリトライ時の最小待ち秒数

145
ecs/crm-datafetch/tests/converter/test_convert_strategy.py
View File

@ -1,7 +1,9 @@
from collections import OrderedDict
from src.converter.convert_strategy import (BooleanConvertStrategy,
ConvertStrategyFactory,
DatetimeConvertStrategy,
FloatConvertStrategy,
DictConvertStrategy,
IntConvertStrategy,
NoneValueConvertStrategy,
StringConvertStrategy)
@ -29,7 +31,7 @@ class TestConvertStrategyFactory:
def test_create_float(self):
"""
Cases:
引数に指数表記を指定した場合、FloatConvertStrategyインスタンスが返ってくること
引数に指数表記を指定した場合、StringConvertStrategyインスタンスが返ってくること
Arranges:
- なし
Expects:
@ -41,7 +43,26 @@ class TestConvertStrategyFactory:
actual = sut.create(1.2345678E7)
# Expects
assert type(actual) == FloatConvertStrategy
# 変換しない
assert type(actual) == StringConvertStrategy
def test_create_float_scale(self):
"""
Cases:
引数に少数を指定した場合、StringConvertStrategyインスタンスが返ってくること
Arranges:
- なし
Expects:
- 戻り値が、期待値と一致する
"""
# Act
sut = ConvertStrategyFactory()
actual = sut.create(1.2345678)
# Expects
# 変換しない
assert type(actual) == StringConvertStrategy
def test_create_bool_true(self):
"""
@ -94,10 +115,10 @@ class TestConvertStrategyFactory:
# Expects
assert type(actual) == DatetimeConvertStrategy
def test_create_other_str(self):
def test_create_str(self):
"""
Cases:
引数にSalesforce日付型以外の文字列を指定した場合、NonConvertStrategyインスタンスが返ってくること
引数にSalesforce日付型以外の文字列を指定した場合、StringConvertStrategyインスタンスが返ってくること
Arranges:
- なし
Expects:
@ -111,10 +132,10 @@ class TestConvertStrategyFactory:
# Expects
assert type(actual) == StringConvertStrategy
def test_create_other_int(self):
def test_create_int(self):
"""
Cases:
引数に整数を指定した場合、NonConvertStrategyインスタンスが返ってくること
引数に整数を指定した場合、IntConvertStrategyインスタンスが返ってくること
Arranges:
- なし
Expects:
@ -128,6 +149,40 @@ class TestConvertStrategyFactory:
# Expects
assert type(actual) == IntConvertStrategy
def test_create_dict(self):
"""
Cases:
引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること
Arranges:
- なし
Expects:
- 戻り値が、期待値と一致する
"""
# Act
sut = ConvertStrategyFactory()
actual = sut.create({'key': 'value'})
# Expects
assert type(actual) == DictConvertStrategy
def test_create_ordered_dict_dict(self):
"""
Cases:
引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること
Arranges:
- なし
Expects:
- 戻り値が、期待値と一致する
"""
# Act
sut = ConvertStrategyFactory()
actual = sut.create(OrderedDict([('key', 'value')]))
# Expects
assert type(actual) == DictConvertStrategy
class TestNoneValueConvertStrategy:
@ -206,26 +261,6 @@ class TestDatetimeConvertStrategy:
assert actual == "2022-06-14 05:15:32"
class TestFloatConvertStrategy:
def test_convert_value(self) -> int:
"""
Cases:
引数に指数表記を指定した場合、整数で返ってくること
Arranges:
- なし
Expects:
- 戻り値が、期待値と一致する
"""
# Act
sut = FloatConvertStrategy()
actual = sut.convert_value(1.2345678E7)
# Expects
assert actual == 12345678
class TestIntConvertStrategy:
def test_convert_value(self):
@ -264,3 +299,59 @@ class TestStringConvertStrategy:
# Expects
assert actual == 'テストデータ'
class TestDictConvertStrategy:
def test_convert_value_dict(self):
"""
Cases:
引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること
Arranges:
- なし
Expects:
- 戻り値が、期待値と一致する
"""
# Act
sut = DictConvertStrategy()
actual = sut.convert_value({'テストデータキー': 'テストデータバリュー'})
# Expects
assert actual == '{"テストデータキー": "テストデータバリュー"}'
def test_convert_value_dict_in_line_break(self):
"""
Cases:
引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること(バリューに改行を含む)
Arranges:
- なし
Expects:
- 戻り値が、期待値と一致する
"""
# Act
sut = DictConvertStrategy()
actual = sut.convert_value({'テストデータキー': 'テスト\nデータ\nバリュー'})
# Expects
assert actual == '{"テストデータキー": "テスト\\nデータ\\nバリュー"}'
def test_convert_value_ordered_dict(self):
"""
Cases:
引数に整列された辞書型のデータを指定した場合、JSONの文字列が返ってくること
Arranges:
- なし
Expects:
- 戻り値が、期待値と一致する
"""
# Act
sut = DictConvertStrategy()
actual = sut.convert_value(OrderedDict(
[('テストデータキー', 'テストデータバリュー')]
))
# Expects
assert actual == '{"テストデータキー": "テストデータバリュー"}'

10
ecs/crm-datafetch/tests/converter/test_converter.py
View File

@ -71,7 +71,7 @@ class TestCSVStringConverter:
('ContactAccessLevel', 8),
('RowCause', 'テストのため2'),
('LastModifiedDate', '2022-06-02T16:30:30.000+0000'),
('LastModifiedById', 2.234567E+6),
('LastModifiedById', 2.23E+0),
('IsDeleted', True)
]),
OrderedDict([
@ -85,7 +85,7 @@ class TestCSVStringConverter:
('ContactAccessLevel', 12),
('RowCause', 'テストのため3'),
('LastModifiedDate', '2022-06-03T23:50:50.000+0000'),
('LastModifiedById', 3.234567E+6),
('LastModifiedById', 3.234567),
('IsDeleted', False)
])
]
@ -100,9 +100,9 @@ class TestCSVStringConverter:
# Expects
expected_value = '''\
"Id","AccountId","UserOrGroupId","AccountAccessLevel","OpportunityAccessLevel","CaseAccessLevel","ContactAccessLevel","RowCause","LastModifiedDate","LastModifiedById","IsDeleted"\r\n\
"TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567","0"\r\n\
"TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2234567","1"\r\n\
"TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3234567","0"\r\n\
"TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567.0","0"\r\n\
"TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2.23","1"\r\n\
"TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3.234567","0"\r\n\
'''
# expected_valueのインデントが半角スペースと認識されてしまうため、`textwrap.dedent`にて補正

110
ecs/crm-datafetch/tests/salesforce/test_salesforce.py
View File

@ -9,6 +9,8 @@ Accountオブジェクトの下記SFIDのレコードはいじらないように
変更してしまった場合は各SOQLの取得日付とデータを修正してください
"""
from typing import OrderedDict
import pytest
from requests.exceptions import ConnectTimeout, ReadTimeout
from src.config.objects import LastFetchDatetime, TargetObject
@ -33,8 +35,8 @@ class TestSalesforceApiClient:
FROM
Account
WHERE
SystemModstamp > 2022-08-04T00:00:00.000Z AND
SystemModstamp <= 2022-08-06T00:00:00.000Z
SystemModstamp > 2022-08-23T01:56:39.000Z AND
SystemModstamp <= 2022-08-24T00:00:00.000Z
"""
sut = SalesforceApiClient()
@ -63,7 +65,7 @@ class TestSalesforceApiClient:
actual = sut.fetch_sf_count(soql)
assert actual >= 0
def test_fetch_sf_count_by_soql_builder_system_modstamp_lt_from_and_to_ge(self):
def test_fetch_sf_count_by_soql_builder_system_modstamp_to_ge(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること
@ -79,8 +81,8 @@ class TestSalesforceApiClient:
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z',
'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z',
'last_fetch_datetime_from': '2022-08-23T02:38:59.000Z',
'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'Account',
@ -106,7 +108,7 @@ class TestSalesforceApiClient:
actual = sut.fetch_sf_count(soql)
assert actual == 1
def test_fetch_sf_count_by_soql_builder_system_modstamp_gt_from_and_to_lt(self):
def test_fetch_sf_count_by_soql_builder_system_modstamp_to_lt(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること
@ -122,8 +124,8 @@ class TestSalesforceApiClient:
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z',
'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z',
'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z',
'last_fetch_datetime_to': '2022-08-23T02:39:01.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'Account',
@ -160,7 +162,7 @@ class TestSalesforceApiClient:
- LastFetchDatetimeのFromに2000年1月1日を指定する
- LastFetchDatetimeのToに2100年12月31日を指定する
Expects:
取得件数が16になる
取得件数が17になる
"""
execute_datetime = ExecuteDateTime()
@ -190,7 +192,7 @@ class TestSalesforceApiClient:
sut = SalesforceApiClient()
actual = sut.fetch_sf_count(soql)
assert actual == 16
assert actual == 17
def test_fetch_sf_data_one_record(self):
"""
@ -217,8 +219,8 @@ class TestSalesforceApiClient:
FROM
Account
WHERE
SystemModstamp > 2022-08-04T00:00:00.000Z AND
SystemModstamp <= 2022-08-06T00:00:00.000Z
SystemModstamp > 2022-08-23T01:56:39.000Z AND
SystemModstamp <= 2022-08-24T00:00:00.000Z
"""
sut = SalesforceApiClient()
@ -284,7 +286,7 @@ class TestSalesforceApiClient:
actual = sut.fetch_sf_data(soql)
assert len(actual) >= 0
def test_fetch_sf_data_by_soql_builder_system_modstamp_lt_from_and_to_ge(self):
def test_fetch_sf_data_by_soql_builder_system_modstamp_to_ge(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること
@ -300,8 +302,8 @@ class TestSalesforceApiClient:
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z',
'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z',
'last_fetch_datetime_from': '2022-08-23T03:48:39.000Z',
'last_fetch_datetime_to': '2022-08-23T03:48:40.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'Account',
@ -353,7 +355,7 @@ class TestSalesforceApiClient:
assert dict(actual[0]) == expect
def test_fetch_sf_data_by_soql_builder_system_modstamp_gt_from_and_to_lt(self):
def test_fetch_sf_data_by_soql_builder_system_modstamp_to_gt(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること
@ -361,16 +363,16 @@ class TestSalesforceApiClient:
- SystemModStampのToが指定日付未満のレコードは取得できないこと
Arranges:
- SalesforceのAccountオブジェクトに、レコードを作成する
- LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定）
- LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定）
- LastFetchDatetimeのFromがSystemModstampより小さくなるように指定する（UTC指定）
- LastFetchDatetimeのToがSystemModstampより大きくなるように指定する（UTC指定）
Expects:
取得できたオブジェクト1件が期待値どおりであること
"""
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z',
'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z',
'last_fetch_datetime_from': '2022-08-23T03:42:24.000Z',
'last_fetch_datetime_to': '2022-08-23T03:42:26.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'Account',
@ -422,6 +424,69 @@ class TestSalesforceApiClient:
assert dict(actual[0]) == expect
def test_fetch_sf_data_by_soql_builder_address_item_check(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること
- できること
Arranges:
- SalesforceのAccountオブジェクトに、住所項目を持つレコードを作成する
- 住所項目を持つレコードだけが取れるよう日付を設定する
Expects:
取得できたオブジェクト件数が1件になる
住所項目(BillingAddress)が想定通りの値になっていること
"""
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z',
'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'Account',
'columns': [
'Id',
'Name',
'SystemModstamp',
'LastModifiedDate',
'BillingStreet',
'BillingCity',
'BillingState',
'BillingPostalCode',
'BillingCountry',
'BillingLatitude',
'BillingLongitude',
'BillingGeocodeAccuracy',
'BillingAddress',
]
}, execute_datetime)
soql_builder = SOQLBuilder(target_object, last_fetch_datetime)
soql = soql_builder.create_fetch_soql()
sut = SalesforceApiClient()
actual = sut.fetch_sf_data(soql)
assert len(actual) == 1
expect_address = OrderedDict([
("city", '〇〇区'),
("country", "日本"),
("geocodeAccuracy", None),
("latitude", None),
("longitude", None),
("postalCode", '999-9999'),
("state", '東京都'),
("street", '△△-✗✗'),
])
assert actual[0]['BillingAddress'] == expect_address
assert actual[0]['BillingCity'] == '〇〇区'
assert actual[0]['BillingCountry'] == '日本'
assert actual[0]['BillingGeocodeAccuracy'] is None
assert actual[0]['BillingLatitude'] is None
assert actual[0]['BillingLongitude'] is None
assert actual[0]['BillingPostalCode'] == '999-9999'
assert actual[0]['BillingState'] == '東京都'
assert actual[0]['BillingStreet'] == '△△-✗✗'
def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self):
"""
Cases:
@ -433,7 +498,7 @@ class TestSalesforceApiClient:
- LastFetchDatetimeのFromに2000年1月1日を指定する
- LastFetchDatetimeのToに2100年12月31日を指定する
Expects:
取得できたオブジェクト件数が16件になる
取得できたオブジェクト件数が17件になる
"""
execute_datetime = ExecuteDateTime()
@ -448,6 +513,7 @@ class TestSalesforceApiClient:
'Name',
'SystemModstamp',
'LastModifiedDate',
'BillingAddress',
'CustomItem1__c',
'CustomItem2__c',
'CustomItem3__c',
@ -463,7 +529,7 @@ class TestSalesforceApiClient:
sut = SalesforceApiClient()
actual = sut.fetch_sf_data(soql)
assert len(actual) == 16
assert len(actual) == 17
# 内容の確認は別のケースで行っているため省略
def test_raise_create_instance_cause_auth_failed(self, monkeypatch):

11
ecs/crm-datafetch/tests/test_backup_crm_data_process.py
View File

@ -44,7 +44,8 @@ class TestBackupCrmDataProcess:
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
('IsDeleted', False)
('IsDeleted', False),
('Name', 'テスト取引先1')
]),
OrderedDict([
('attributes', OrderedDict([('type', 'Account'),
@ -54,7 +55,8 @@ class TestBackupCrmDataProcess:
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
('IsDeleted', False)
('IsDeleted', False),
('Name', 'テスト取引先2')
]),
OrderedDict([
('attributes', OrderedDict([('type', 'Account'),
@ -64,7 +66,8 @@ class TestBackupCrmDataProcess:
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
('IsDeleted', False)
('IsDeleted', False),
('Name', 'テスト取引先3')
]),
]
@ -96,7 +99,7 @@ class TestBackupCrmDataProcess:
actual = s3_client.get_object(
Bucket=bucket_name,
Key=f'response_json/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.json')
assert actual['Body'].read().decode('utf-8') == json.dumps(response_json)
assert actual['Body'].read().decode('utf-8') == json.dumps(response_json, ensure_ascii=False)
# ログの確認
assert generate_log_message_tuple(

51
ecs/crm-datafetch/tests/test_convert_crm_csv_data_process.py
View File

@ -34,7 +34,17 @@ class TestConvertCrmCsvDataProcess:
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
('IsDeleted', True)
('IsDeleted', True),
('PersonMailingAddress', OrderedDict([
('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501
('PersonMailingCity', 'New york city'),
('PersonMailingState', 'Ohaio'),
('PersonMailingPostalCode', '999-9999'),
('PersonMailingCountry', 'US'),
('PersonMailingLatitude', 50.1234567),
('PersonMailingLongitude', 103.1234567),
('PersonMailingGeocodeAccuracy', 'Address'),
])),
]),
OrderedDict([
('attributes', OrderedDict([('type', 'Account'),
@ -42,9 +52,19 @@ class TestConvertCrmCsvDataProcess:
('Id', 'TEST002'),
('AccountNumber', 'test002'),
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('LastModifiedById', 1.23E+0),
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
('IsDeleted', False)
('IsDeleted', False),
('PersonMailingAddress', OrderedDict([
('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501
('PersonMailingCity', 'New york city'),
('PersonMailingState', 'Ohaio'),
('PersonMailingPostalCode', '999-9999'),
('PersonMailingCountry', 'US'),
('PersonMailingLatitude', 50.1234567),
('PersonMailingLongitude', 103.1234567),
('PersonMailingGeocodeAccuracy', 'Address'),
])),
]),
OrderedDict([
('attributes', OrderedDict([('type', 'Account'),
@ -52,9 +72,19 @@ class TestConvertCrmCsvDataProcess:
('Id', 'TEST003'),
('AccountNumber', 'test003'),
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('LastModifiedById', 1.234567),
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
('IsDeleted', False)
('IsDeleted', False),
('PersonMailingAddress', OrderedDict([
('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501
('PersonMailingCity', 'New york city'),
('PersonMailingState', 'Ohaio'),
('PersonMailingPostalCode', '999-9999'),
('PersonMailingCountry', 'US'),
('PersonMailingLatitude', 50.1234567),
('PersonMailingLongitude', 103.1234567),
('PersonMailingGeocodeAccuracy', 'Address'),
])),
]),
]
@ -66,7 +96,8 @@ class TestConvertCrmCsvDataProcess:
'LastModifiedDate',
'LastModifiedById',
'SystemModstamp',
'IsDeleted'
'IsDeleted',
'PersonMailingAddress'
]
}
@ -79,10 +110,10 @@ class TestConvertCrmCsvDataProcess:
# Assert
expect_csv_string = """\
"Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\
"TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\
"TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\
"TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\
"Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted","PersonMailingAddress"\r\n\
"TEST001","test001","2022-06-01 09:00:00","1234567.0","2022-06-01 09:00:00","1","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\
"TEST002","test002","2022-06-01 09:00:00","1.23","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\
"TEST003","test003","2022-06-01 09:00:00","1.234567","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\
"""
# 返り値の期待値チェック
assert isinstance(actual_csv_string, str), 'CSV文字列が返却される'

60
ecs/crm-datafetch/tests/test_fetch_crm_data_process.py
View File

@ -217,7 +217,7 @@ class TestFetchCrmDataProcess:
2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること
3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること
Arranges:
- データ件数取得処理の最大リトライ試行回数を3に設定する
- データ件数取得処理の最大リトライ試行回数を4に設定する
- timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する
- データ件数取得処理の初回に接続タイムアウト例外が発生するようにする
Expects:
@ -225,7 +225,7 @@ class TestFetchCrmDataProcess:
- データ件数取得に失敗した旨のエラーが出力されない
"""
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1)
@ -236,8 +236,8 @@ class TestFetchCrmDataProcess:
patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client:
# モック化
mock_counter_inst = mock_counter.return_value
mock_counter_inst.describe.side_effect = [1, 2, 3]
mock_counter_inst.increment.side_effect = [2, 3, 4]
mock_counter_inst.describe.side_effect = [1, 2, 3, 4]
mock_counter_inst.increment.side_effect = [2, 3, 4, 5]
mock_builder_inst = mock_soql_builder.return_value
mock_builder_inst.create_count_soql.return_value = ''
mock_builder_inst.create_fetch_soql.return_value = ''
@ -271,15 +271,15 @@ class TestFetchCrmDataProcess:
2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること
3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること
Arranges:
- データ件数取得処理の最大リトライ試行回数を3に設定する
- データ件数取得処理の最大リトライ試行回数を4に設定する
- timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する
- データ件数取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする
- データ件数取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする
Expects:
- 異常終了する
- データ件数取得に失敗した旨のエラーが出力される
"""
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1)
@ -290,13 +290,13 @@ class TestFetchCrmDataProcess:
patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client:
# モック化
mock_counter_inst = mock_counter.return_value
mock_counter_inst.describe.side_effect = [1, 2, 3]
mock_counter_inst.increment.side_effect = [2, 3, 4]
mock_counter_inst.describe.side_effect = [1, 2, 3, 4]
mock_counter_inst.increment.side_effect = [2, 3, 4, 5]
mock_builder_inst = mock_soql_builder.return_value
mock_builder_inst.create_count_soql.return_value = ''
mock_builder_inst.create_fetch_soql.return_value = ''
mock_client_inst = mock_api_client.return_value
mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception]
mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception, exception]
mock_client_inst.fetch_sf_data.return_value = common_expect
# Act
with pytest.raises(SalesforceAPIException) as e:
@ -304,16 +304,16 @@ class TestFetchCrmDataProcess:
# Assert
# 取得は3回行われる
assert mock_counter_inst.describe.call_count == 3
# 足し込みは2回のみ
assert mock_counter_inst.increment.call_count == 2
# 取得は4回行われる
assert mock_counter_inst.describe.call_count == 4
# 足し込みは3回のみ
assert mock_counter_inst.increment.call_count == 3
assert generate_log_message_tuple(
log_level=logging.WARNING,
log_message=expect_message) in caplog.record_tuples
called_log_counts = len([log for log in caplog.messages if log == expect_message])
assert called_log_counts == 2
assert called_log_counts == 3
assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples
assert e.value.error_id == 'E-FETCH-01'
@ -333,7 +333,7 @@ class TestFetchCrmDataProcess:
2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること
3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること
Arranges:
- レコード取得処理の最大リトライ試行回数を3に設定する
- レコード取得処理の最大リトライ試行回数を4に設定する
- timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する
- レコード取得処理の初回に接続タイムアウト例外が発生するようにする
Expects:
@ -341,7 +341,7 @@ class TestFetchCrmDataProcess:
- データレコード取得に失敗した旨のエラーが出力されない
"""
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1)
@ -352,8 +352,8 @@ class TestFetchCrmDataProcess:
patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client:
# モック化
mock_counter_inst = mock_counter.return_value
mock_counter_inst.describe.side_effect = [1, 2, 3]
mock_counter_inst.increment.side_effect = [2, 3, 4]
mock_counter_inst.describe.side_effect = [1, 2, 3, 4]
mock_counter_inst.increment.side_effect = [2, 3, 4, 5]
mock_builder_inst = mock_soql_builder.return_value
mock_builder_inst.create_count_soql.return_value = ''
mock_builder_inst.create_fetch_soql.return_value = ''
@ -387,15 +387,15 @@ class TestFetchCrmDataProcess:
2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること
3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること
Arranges:
- レコード取得処理の最大リトライ試行回数を3に設定する
- レコード取得処理の最大リトライ試行回数を4に設定する
- timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する
- レコード取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする
- レコード取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする
Expects:
- 異常終了する
- データレコード取得に失敗した旨のエラーが出力される
"""
monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 3)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 4)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL', 1)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL', 1)
monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_INTERVAL', 1)
@ -406,30 +406,30 @@ class TestFetchCrmDataProcess:
patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client:
# モック化
mock_counter_inst = mock_counter.return_value
mock_counter_inst.describe.side_effect = [1, 2, 3]
mock_counter_inst.increment.side_effect = [2, 3, 4]
mock_counter_inst.describe.side_effect = [1, 2, 3, 4]
mock_counter_inst.increment.side_effect = [2, 3, 4, 5]
mock_builder_inst = mock_soql_builder.return_value
mock_builder_inst.create_count_soql.return_value = ''
mock_builder_inst.create_fetch_soql.return_value = ''
mock_client_inst = mock_api_client.return_value
mock_client_inst.fetch_sf_count.return_value = 1
mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception]
mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception, exception]
# Act
with pytest.raises(SalesforceAPIException) as e:
fetch_crm_data_process(common_target_object, common_last_fetch_datetime)
# Assert
# 取得は3回行われる
assert mock_counter_inst.describe.call_count == 3
# 足し込みは2回のみ
assert mock_counter_inst.increment.call_count == 2
# 取得は4回行われる
assert mock_counter_inst.describe.call_count == 4
# 足し込みは3回のみ
assert mock_counter_inst.increment.call_count == 3
assert generate_log_message_tuple(
log_level=logging.WARNING,
log_message=expect_message) in caplog.record_tuples
called_log_counts = len([log for log in caplog.messages if log == expect_message])
assert called_log_counts == 2
assert called_log_counts == 3
assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples
assert e.value.error_id == 'E-FETCH-02'

5
ecs/crm-datafetch/tests/test_prepare_data_fetch_process.py
View File

@ -7,6 +7,9 @@ from src.config.objects import FetchTargetObjects
from src.error.exceptions import FileNotFoundException, InvalidConfigException
from src.prepare_data_fetch_process import prepare_data_fetch_process
from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ
from src.system_var.environments import (CRM_CONFIG_BUCKET,
OBJECT_INFO_FILENAME,
OBJECT_INFO_FOLDER)
from .test_utils.log_message import generate_log_message_tuple
@ -80,6 +83,8 @@ class TestPrepareDataFetchProcess:
# ログの確認
assert generate_log_message_tuple(log_message='I-PRE-01 データ取得準備処理を開始します') in caplog.record_tuples
assert generate_log_message_tuple(log_message=f'I-PRE-02 データ取得処理開始日時：{str(actual_execute_datetime)}') in caplog.record_tuples
assert generate_log_message_tuple(
log_message=f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}]') in caplog.record_tuples
assert generate_log_message_tuple(log_message='I-PRE-09 データ取得準備処理を終了します') in caplog.record_tuples
def test_call_depended_modules(self):

24
ecs/crm-datafetch/tests/test_walk_through.py
View File

@ -73,7 +73,6 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog):
object_info_list = get_object_config_list('object_info')
for object_info in object_info_list:
json_file = read_json(object_info)
json_file = to_upload_json(json_file)
upload_json(json_file, s3_client, CONFIG_BUCKET, f'{OBJECT_INFO_FOLDER}/{path.basename(object_info)}')
object_info_files.append(json_file)
last_fetch_datetime_list = get_object_config_list('last_fetch_datetime')
@ -105,7 +104,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog):
# Act
logger = logging.getLogger()
logger.setLevel(logging.INFO)
logger.setLevel(logging.DEBUG)
logger.info(f'##########################')
logger.info(f'# 差分データ取得処理:実行開始 #')
logger.info(f'##########################')
@ -118,7 +117,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog):
# ループ前のログ確認
assert 'I-CTRL-01 CRMデータ取得処理を開始します' in log_messages
assert 'I-CTRL-02 データ取得準備処理呼び出し' in log_messages
assert_prepare_process_log(log_messages, now)
assert_prepare_process_log(log_messages, now, path.basename(object_info_list[0]))
assert 'I-CTRL-03 取得対象オブジェクトのループ処理開始' in log_messages
# オブジェクト情報を取得する(diff)
object_info_list = object_info_files[0]
@ -207,9 +206,10 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog):
"""
def assert_prepare_process_log(log_messages, now):
def assert_prepare_process_log(log_messages, now, object_info_list_filename):
assert 'I-PRE-01 データ取得準備処理を開始します' in log_messages
assert f'I-PRE-02 データ取得処理開始日時：{now}' in log_messages
assert f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{object_info_list_filename}]' in log_messages
assert 'I-PRE-09 データ取得準備処理を終了します' in log_messages
@ -295,21 +295,6 @@ def upload_json(json_file, s3_client, bucket, folder):
s3_client.put_object(Bucket=bucket, Key=folder, Body=json_str)
def to_upload_json(json_file):
"""Userオブジェクトの取得できないプロパティを取り除く
TODO: Userオブジェクトの恒久対応が確定したらこのメソッドは消す
"""
for object_info in json_file['objects']:
if object_info['object_name'] != 'User':
continue
columns: list = object_info['columns']
columns.remove('LastPasswordChangeDate')
columns.remove('NumberOfFailedLogins')
columns.remove('UserPreferencesNativeEmailClient')
return json_file
def set_environment(monkeypatch):
# 環境変数を設定(CRMの認証情報は別途設定しておくこと)
monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET)
@ -324,3 +309,4 @@ def set_environment(monkeypatch):
monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER)
monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER)
monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER)
monkeypatch.setattr('src.prepare_data_fetch_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET)

3
s3/config/crm/object_info/crm_object_list_diff.json
View File

@ -1645,13 +1645,11 @@
"DelegatedApproverId",
"ManagerId",
"LastLoginDate",
"LastPasswordChangeDate",
"CreatedDate",
"CreatedById",
"LastModifiedDate",
"LastModifiedById",
"SystemModstamp",
"NumberOfFailedLogins",
"OfflineTrialExpirationDate",
"OfflinePdaTrialExpirationDate",
"UserPermissionsMarketingUser",
@ -1725,7 +1723,6 @@
"UserPreferencesSRHOverrideActivities",
"UserPreferencesNewLightningReportRunPageEnabled",
"UserPreferencesReverseOpenActivitiesView",
"UserPreferencesNativeEmailClient",
"UserPreferencesHideBrowseProductRedirectConfirmation",
"UserPreferencesHideOnlineSalesAppWelcomeMat",
"ContactId",

7
s3/config/crm/task_settings/task_settings.env
View File

@ -1,13 +1,14 @@
LOG_LEVEL=INFO
CRM_AUTH_TIMEOUT=100
CRM_GET_RECORD_COUNT_TIMEOUT=300
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=3
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=4
CRM_GET_RECORD_COUNT_RETRY_INTERVAL=5
CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL=5
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL=50
CRM_FETCH_RECORD_TIMEOUT=300
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=4
CRM_FETCH_RECORD_RETRY_INTERVAL=5
CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5
CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50
CONVERT_TZ='Asia/Tokyo'
CONVERT_TZ=Asia/Tokyo
OBJECT_INFO_FILENAME=crm_object_list_diff.json

6
s3/data/crm/settings/CRM_Event_Attendee_vod__c.txt
View File

@ -4,9 +4,9 @@ utf-8
"
CRLF
1
31
Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c
id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c
32
Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c,MSJ_Registration__c
id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c,msj_registration__c
src02.crm_event_attendee_vod__c
org02.crm_event_attendee_vod__c

6
s3/data/crm/settings/CRM_User.txt
View File

File diff suppressed because one or more lines are too long