Revert "Revert "Merge branch 'develop-6crm' of nds-tyo.git.backlog.com:/NEWDWH2021/newsdwh2021 into develop-6crm""
This reverts commit bae8447658a03ff8a0bd84948d493f332724bebb.
This commit is contained in:
shimoda.m@nds-tyo.co.jp
parent
bc9f070386
commit
6b0d978378
@ -14,6 +14,6 @@ RUN \
|
||||
pip uninstall -y pipenv virtualenv-clone virtualenv
|
||||
|
||||
COPY main.py ./
|
||||
COPY src ./
|
||||
COPY src ./src
|
||||
|
||||
CMD [ "python", "./main.py" ]
|
||||
|
||||
@ -87,7 +87,7 @@ class BackupBucket:
|
||||
|
||||
def put_response_json(self, file_path: str, data: dict) -> None:
|
||||
object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_path}'
|
||||
self.__s3_resource.put_object(object_key, json.dumps(data))
|
||||
self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False))
|
||||
return
|
||||
|
||||
def put_csv(self, file_path: str, data: str) -> None:
|
||||
@ -97,5 +97,5 @@ class BackupBucket:
|
||||
|
||||
def put_result_json(self, file_path: str, data: dict) -> None:
|
||||
object_key = f'{PROCESS_RESULT_FOLDER}/{file_path}'
|
||||
self.__s3_resource.put_object(object_key, json.dumps(data))
|
||||
self.__s3_resource.put_object(object_key, json.dumps(data, ensure_ascii=False))
|
||||
return
|
||||
|
||||
@ -1,4 +1,6 @@
|
||||
import json
|
||||
import re
|
||||
from collections import OrderedDict
|
||||
from datetime import datetime
|
||||
|
||||
from dateutil.tz import gettz
|
||||
@ -12,20 +14,17 @@ from src.system_var.environments import CONVERT_TZ
|
||||
class ConvertStrategyFactory:
|
||||
def __init__(self) -> None:
|
||||
self.__none_value_convert_strategy = NoneValueConvertStrategy()
|
||||
self.__float_convert_strategy = FloatConvertStrategy()
|
||||
self.__boolean_convert_strategy = BooleanConvertStrategy()
|
||||
self.__datetime_convert_strategy = DatetimeConvertStrategy()
|
||||
self.__int_convert_strategy = IntConvertStrategy()
|
||||
self.__string_convert_strategy = StringConvertStrategy()
|
||||
self.__dict_convert_strategy = DictConvertStrategy()
|
||||
|
||||
def create(self, value):
|
||||
|
||||
if value is None:
|
||||
convert_strategy = self.__none_value_convert_strategy
|
||||
|
||||
elif type(value) == float:
|
||||
convert_strategy = self.__float_convert_strategy
|
||||
|
||||
elif type(value) == bool:
|
||||
convert_strategy = self.__boolean_convert_strategy
|
||||
|
||||
@ -35,6 +34,8 @@ class ConvertStrategyFactory:
|
||||
elif type(value) == int:
|
||||
convert_strategy = self.__int_convert_strategy
|
||||
|
||||
elif type(value) == dict or type(value) == OrderedDict:
|
||||
convert_strategy = self.__dict_convert_strategy
|
||||
else:
|
||||
convert_strategy = self.__string_convert_strategy
|
||||
|
||||
@ -60,12 +61,6 @@ class DatetimeConvertStrategy:
|
||||
return datetime.strptime(convert_value, CRM_DATETIME_FORMAT).astimezone(gettz(CONVERT_TZ)).strftime(YYYYMMDDHHMMSS)
|
||||
|
||||
|
||||
class FloatConvertStrategy:
|
||||
def convert_value(self, convert_value: str) -> int:
|
||||
"""float型をint型に変換する処理"""
|
||||
return int(convert_value)
|
||||
|
||||
|
||||
class IntConvertStrategy:
|
||||
def convert_value(self, convert_value: int):
|
||||
"""int型を変換せずに返す処理"""
|
||||
@ -78,3 +73,9 @@ class StringConvertStrategy:
|
||||
"""string型を変換せずに返す処理"""
|
||||
# ConvertStrategyFactoryにて型チェックを行っているため値を変換せずに返す
|
||||
return convert_value
|
||||
|
||||
|
||||
class DictConvertStrategy:
|
||||
def convert_value(self, convert_value: dict):
|
||||
"""dict型の項目を文字列に変換して返す処理"""
|
||||
return json.dumps(convert_value, ensure_ascii=False)
|
||||
|
||||
@ -34,9 +34,9 @@ def prepare_data_fetch_process():
|
||||
|
||||
try:
|
||||
# ③ S3 設定ファイル保管用バケットから、CRM_取得オブジェクト情報ファイルを取得する
|
||||
object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}'
|
||||
logger.debug(
|
||||
f'D-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]')
|
||||
object_info_file_s3_path = f's3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}'
|
||||
logger.info(
|
||||
f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[{object_info_file_s3_path}]')
|
||||
|
||||
config_bucket = ConfigBucket()
|
||||
object_info_file_str = config_bucket.get_object_info_file()
|
||||
|
||||
@ -9,8 +9,8 @@ LOG_LEVEL = os.environ.get(constants.LOG_LEVEL, constants.LOG_LEVEL_INFO)
|
||||
CRM_AUTH_TIMEOUT = int(os.environ.get(constants.CRM_AUTH_TIMEOUT, 100))
|
||||
# CRMのレコード件数取得処理のタイムアウト秒数
|
||||
CRM_GET_RECORD_COUNT_TIMEOUT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_TIMEOUT, 300))
|
||||
# CRMのレコード件数取得処理の最大リトライ試行回数
|
||||
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 3))
|
||||
# CRMのレコード件数取得処理の最大リトライ試行回数(処理の実施総回数)
|
||||
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT, 4))
|
||||
# CRMのレコード件数取得処理のリトライ時の初回待ち秒数
|
||||
CRM_GET_RECORD_COUNT_RETRY_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_INTERVAL, 5))
|
||||
# CRMのレコード件数取得処理のリトライ時の最小待ち秒数
|
||||
@ -19,8 +19,8 @@ CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL = int(os.environ.get(constants.CRM_GET_R
|
||||
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL = int(os.environ.get(constants.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL, 50))
|
||||
# CRMのレコード取得処理のタイムアウト秒数
|
||||
CRM_FETCH_RECORD_TIMEOUT = int(os.environ.get(constants.CRM_FETCH_RECORD_TIMEOUT, 300))
|
||||
# CRMのレコード取得処理の最大リトライ試行回数
|
||||
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 3))
|
||||
# CRMのレコード取得処理の最大リトライ試行回数(処理の実施総回数)
|
||||
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT = int(os.environ.get(constants.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT, 4))
|
||||
# CRMのレコード取得処理のリトライ時の初回待ち秒数
|
||||
CRM_FETCH_RECORD_RETRY_INTERVAL = int(os.environ.get(constants.CRM_FETCH_RECORD_RETRY_INTERVAL, 5))
|
||||
# CRMのレコード取得処理のリトライ時の最小待ち秒数
|
||||
|
||||
@ -1,7 +1,9 @@
|
||||
from collections import OrderedDict
|
||||
|
||||
from src.converter.convert_strategy import (BooleanConvertStrategy,
|
||||
ConvertStrategyFactory,
|
||||
DatetimeConvertStrategy,
|
||||
FloatConvertStrategy,
|
||||
DictConvertStrategy,
|
||||
IntConvertStrategy,
|
||||
NoneValueConvertStrategy,
|
||||
StringConvertStrategy)
|
||||
@ -29,7 +31,7 @@ class TestConvertStrategyFactory:
|
||||
def test_create_float(self):
|
||||
"""
|
||||
Cases:
|
||||
引数に指数表記を指定した場合、FloatConvertStrategyインスタンスが返ってくること
|
||||
引数に指数表記を指定した場合、StringConvertStrategyインスタンスが返ってくること
|
||||
Arranges:
|
||||
- なし
|
||||
Expects:
|
||||
@ -41,7 +43,26 @@ class TestConvertStrategyFactory:
|
||||
actual = sut.create(1.2345678E7)
|
||||
|
||||
# Expects
|
||||
assert type(actual) == FloatConvertStrategy
|
||||
# 変換しない
|
||||
assert type(actual) == StringConvertStrategy
|
||||
|
||||
def test_create_float_scale(self):
|
||||
"""
|
||||
Cases:
|
||||
引数に少数を指定した場合、StringConvertStrategyインスタンスが返ってくること
|
||||
Arranges:
|
||||
- なし
|
||||
Expects:
|
||||
- 戻り値が、期待値と一致する
|
||||
"""
|
||||
|
||||
# Act
|
||||
sut = ConvertStrategyFactory()
|
||||
actual = sut.create(1.2345678)
|
||||
|
||||
# Expects
|
||||
# 変換しない
|
||||
assert type(actual) == StringConvertStrategy
|
||||
|
||||
def test_create_bool_true(self):
|
||||
"""
|
||||
@ -94,10 +115,10 @@ class TestConvertStrategyFactory:
|
||||
# Expects
|
||||
assert type(actual) == DatetimeConvertStrategy
|
||||
|
||||
def test_create_other_str(self):
|
||||
def test_create_str(self):
|
||||
"""
|
||||
Cases:
|
||||
引数にSalesforce日付型以外の文字列を指定した場合、NonConvertStrategyインスタンスが返ってくること
|
||||
引数にSalesforce日付型以外の文字列を指定した場合、StringConvertStrategyインスタンスが返ってくること
|
||||
Arranges:
|
||||
- なし
|
||||
Expects:
|
||||
@ -111,10 +132,10 @@ class TestConvertStrategyFactory:
|
||||
# Expects
|
||||
assert type(actual) == StringConvertStrategy
|
||||
|
||||
def test_create_other_int(self):
|
||||
def test_create_int(self):
|
||||
"""
|
||||
Cases:
|
||||
引数に整数を指定した場合、NonConvertStrategyインスタンスが返ってくること
|
||||
引数に整数を指定した場合、IntConvertStrategyインスタンスが返ってくること
|
||||
Arranges:
|
||||
- なし
|
||||
Expects:
|
||||
@ -128,6 +149,40 @@ class TestConvertStrategyFactory:
|
||||
# Expects
|
||||
assert type(actual) == IntConvertStrategy
|
||||
|
||||
def test_create_dict(self):
|
||||
"""
|
||||
Cases:
|
||||
引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること
|
||||
Arranges:
|
||||
- なし
|
||||
Expects:
|
||||
- 戻り値が、期待値と一致する
|
||||
"""
|
||||
|
||||
# Act
|
||||
sut = ConvertStrategyFactory()
|
||||
actual = sut.create({'key': 'value'})
|
||||
|
||||
# Expects
|
||||
assert type(actual) == DictConvertStrategy
|
||||
|
||||
def test_create_ordered_dict_dict(self):
|
||||
"""
|
||||
Cases:
|
||||
引数に辞書型の値を指定した場合、IntConvertStrategyインスタンスが返ってくること
|
||||
Arranges:
|
||||
- なし
|
||||
Expects:
|
||||
- 戻り値が、期待値と一致する
|
||||
"""
|
||||
|
||||
# Act
|
||||
sut = ConvertStrategyFactory()
|
||||
actual = sut.create(OrderedDict([('key', 'value')]))
|
||||
|
||||
# Expects
|
||||
assert type(actual) == DictConvertStrategy
|
||||
|
||||
|
||||
class TestNoneValueConvertStrategy:
|
||||
|
||||
@ -206,26 +261,6 @@ class TestDatetimeConvertStrategy:
|
||||
assert actual == "2022-06-14 05:15:32"
|
||||
|
||||
|
||||
class TestFloatConvertStrategy:
|
||||
|
||||
def test_convert_value(self) -> int:
|
||||
"""
|
||||
Cases:
|
||||
引数に指数表記を指定した場合、整数で返ってくること
|
||||
Arranges:
|
||||
- なし
|
||||
Expects:
|
||||
- 戻り値が、期待値と一致する
|
||||
"""
|
||||
|
||||
# Act
|
||||
sut = FloatConvertStrategy()
|
||||
actual = sut.convert_value(1.2345678E7)
|
||||
|
||||
# Expects
|
||||
assert actual == 12345678
|
||||
|
||||
|
||||
class TestIntConvertStrategy:
|
||||
|
||||
def test_convert_value(self):
|
||||
@ -264,3 +299,59 @@ class TestStringConvertStrategy:
|
||||
|
||||
# Expects
|
||||
assert actual == 'テストデータ'
|
||||
|
||||
|
||||
class TestDictConvertStrategy:
|
||||
|
||||
def test_convert_value_dict(self):
|
||||
"""
|
||||
Cases:
|
||||
引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること
|
||||
Arranges:
|
||||
- なし
|
||||
Expects:
|
||||
- 戻り値が、期待値と一致する
|
||||
"""
|
||||
|
||||
# Act
|
||||
sut = DictConvertStrategy()
|
||||
actual = sut.convert_value({'テストデータキー': 'テストデータバリュー'})
|
||||
|
||||
# Expects
|
||||
assert actual == '{"テストデータキー": "テストデータバリュー"}'
|
||||
|
||||
def test_convert_value_dict_in_line_break(self):
|
||||
"""
|
||||
Cases:
|
||||
引数に辞書型のデータを指定した場合、JSONの文字列が返ってくること(バリューに改行を含む)
|
||||
Arranges:
|
||||
- なし
|
||||
Expects:
|
||||
- 戻り値が、期待値と一致する
|
||||
"""
|
||||
|
||||
# Act
|
||||
sut = DictConvertStrategy()
|
||||
actual = sut.convert_value({'テストデータキー': 'テスト\nデータ\nバリュー'})
|
||||
|
||||
# Expects
|
||||
assert actual == '{"テストデータキー": "テスト\\nデータ\\nバリュー"}'
|
||||
|
||||
def test_convert_value_ordered_dict(self):
|
||||
"""
|
||||
Cases:
|
||||
引数に整列された辞書型のデータを指定した場合、JSONの文字列が返ってくること
|
||||
Arranges:
|
||||
- なし
|
||||
Expects:
|
||||
- 戻り値が、期待値と一致する
|
||||
"""
|
||||
|
||||
# Act
|
||||
sut = DictConvertStrategy()
|
||||
actual = sut.convert_value(OrderedDict(
|
||||
[('テストデータキー', 'テストデータバリュー')]
|
||||
))
|
||||
|
||||
# Expects
|
||||
assert actual == '{"テストデータキー": "テストデータバリュー"}'
|
||||
|
||||
@ -71,7 +71,7 @@ class TestCSVStringConverter:
|
||||
('ContactAccessLevel', 8),
|
||||
('RowCause', 'テストのため2'),
|
||||
('LastModifiedDate', '2022-06-02T16:30:30.000+0000'),
|
||||
('LastModifiedById', 2.234567E+6),
|
||||
('LastModifiedById', 2.23E+0),
|
||||
('IsDeleted', True)
|
||||
]),
|
||||
OrderedDict([
|
||||
@ -85,7 +85,7 @@ class TestCSVStringConverter:
|
||||
('ContactAccessLevel', 12),
|
||||
('RowCause', 'テストのため3'),
|
||||
('LastModifiedDate', '2022-06-03T23:50:50.000+0000'),
|
||||
('LastModifiedById', 3.234567E+6),
|
||||
('LastModifiedById', 3.234567),
|
||||
('IsDeleted', False)
|
||||
])
|
||||
]
|
||||
@ -100,9 +100,9 @@ class TestCSVStringConverter:
|
||||
# Expects
|
||||
expected_value = '''\
|
||||
"Id","AccountId","UserOrGroupId","AccountAccessLevel","OpportunityAccessLevel","CaseAccessLevel","ContactAccessLevel","RowCause","LastModifiedDate","LastModifiedById","IsDeleted"\r\n\
|
||||
"TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567","0"\r\n\
|
||||
"TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2234567","1"\r\n\
|
||||
"TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3234567","0"\r\n\
|
||||
"TEST001","test001","","1","2","3","4","テストのため1","2022-06-01 09:00:00","1234567.0","0"\r\n\
|
||||
"TEST002","test002","","5","6","7","8","テストのため2","2022-06-03 01:30:30","2.23","1"\r\n\
|
||||
"TEST003","test003","","9","10","11","12","テストのため3","2022-06-04 08:50:50","3.234567","0"\r\n\
|
||||
'''
|
||||
|
||||
# expected_valueのインデントが半角スペースと認識されてしまうため、`textwrap.dedent`にて補正
|
||||
|
||||
@ -9,6 +9,8 @@ Accountオブジェクトの下記SFIDのレコードはいじらないように
|
||||
変更してしまった場合は各SOQLの取得日付とデータを修正してください
|
||||
"""
|
||||
|
||||
from typing import OrderedDict
|
||||
|
||||
import pytest
|
||||
from requests.exceptions import ConnectTimeout, ReadTimeout
|
||||
from src.config.objects import LastFetchDatetime, TargetObject
|
||||
@ -33,8 +35,8 @@ class TestSalesforceApiClient:
|
||||
FROM
|
||||
Account
|
||||
WHERE
|
||||
SystemModstamp > 2022-08-04T00:00:00.000Z AND
|
||||
SystemModstamp <= 2022-08-06T00:00:00.000Z
|
||||
SystemModstamp > 2022-08-23T01:56:39.000Z AND
|
||||
SystemModstamp <= 2022-08-24T00:00:00.000Z
|
||||
"""
|
||||
sut = SalesforceApiClient()
|
||||
|
||||
@ -63,7 +65,7 @@ class TestSalesforceApiClient:
|
||||
actual = sut.fetch_sf_count(soql)
|
||||
assert actual >= 0
|
||||
|
||||
def test_fetch_sf_count_by_soql_builder_system_modstamp_lt_from_and_to_ge(self):
|
||||
def test_fetch_sf_count_by_soql_builder_system_modstamp_to_ge(self):
|
||||
"""
|
||||
Cases:
|
||||
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること
|
||||
@ -79,8 +81,8 @@ class TestSalesforceApiClient:
|
||||
|
||||
execute_datetime = ExecuteDateTime()
|
||||
last_fetch_datetime = LastFetchDatetime({
|
||||
'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z',
|
||||
'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z',
|
||||
'last_fetch_datetime_from': '2022-08-23T02:38:59.000Z',
|
||||
'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z',
|
||||
}, execute_datetime)
|
||||
target_object = TargetObject({
|
||||
'object_name': 'Account',
|
||||
@ -106,7 +108,7 @@ class TestSalesforceApiClient:
|
||||
actual = sut.fetch_sf_count(soql)
|
||||
assert actual == 1
|
||||
|
||||
def test_fetch_sf_count_by_soql_builder_system_modstamp_gt_from_and_to_lt(self):
|
||||
def test_fetch_sf_count_by_soql_builder_system_modstamp_to_lt(self):
|
||||
"""
|
||||
Cases:
|
||||
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトの件数が取得できること
|
||||
@ -122,8 +124,8 @@ class TestSalesforceApiClient:
|
||||
|
||||
execute_datetime = ExecuteDateTime()
|
||||
last_fetch_datetime = LastFetchDatetime({
|
||||
'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z',
|
||||
'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z',
|
||||
'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z',
|
||||
'last_fetch_datetime_to': '2022-08-23T02:39:01.000Z',
|
||||
}, execute_datetime)
|
||||
target_object = TargetObject({
|
||||
'object_name': 'Account',
|
||||
@ -160,7 +162,7 @@ class TestSalesforceApiClient:
|
||||
- LastFetchDatetimeのFromに2000年1月1日を指定する
|
||||
- LastFetchDatetimeのToに2100年12月31日を指定する
|
||||
Expects:
|
||||
取得件数が16になる
|
||||
取得件数が17になる
|
||||
"""
|
||||
|
||||
execute_datetime = ExecuteDateTime()
|
||||
@ -190,7 +192,7 @@ class TestSalesforceApiClient:
|
||||
sut = SalesforceApiClient()
|
||||
|
||||
actual = sut.fetch_sf_count(soql)
|
||||
assert actual == 16
|
||||
assert actual == 17
|
||||
|
||||
def test_fetch_sf_data_one_record(self):
|
||||
"""
|
||||
@ -217,8 +219,8 @@ class TestSalesforceApiClient:
|
||||
FROM
|
||||
Account
|
||||
WHERE
|
||||
SystemModstamp > 2022-08-04T00:00:00.000Z AND
|
||||
SystemModstamp <= 2022-08-06T00:00:00.000Z
|
||||
SystemModstamp > 2022-08-23T01:56:39.000Z AND
|
||||
SystemModstamp <= 2022-08-24T00:00:00.000Z
|
||||
"""
|
||||
sut = SalesforceApiClient()
|
||||
|
||||
@ -284,7 +286,7 @@ class TestSalesforceApiClient:
|
||||
actual = sut.fetch_sf_data(soql)
|
||||
assert len(actual) >= 0
|
||||
|
||||
def test_fetch_sf_data_by_soql_builder_system_modstamp_lt_from_and_to_ge(self):
|
||||
def test_fetch_sf_data_by_soql_builder_system_modstamp_to_ge(self):
|
||||
"""
|
||||
Cases:
|
||||
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること
|
||||
@ -300,8 +302,8 @@ class TestSalesforceApiClient:
|
||||
|
||||
execute_datetime = ExecuteDateTime()
|
||||
last_fetch_datetime = LastFetchDatetime({
|
||||
'last_fetch_datetime_from': '2022-08-05T11:14:07.000Z',
|
||||
'last_fetch_datetime_to': '2022-08-05T11:15:29.000Z',
|
||||
'last_fetch_datetime_from': '2022-08-23T03:48:39.000Z',
|
||||
'last_fetch_datetime_to': '2022-08-23T03:48:40.000Z',
|
||||
}, execute_datetime)
|
||||
target_object = TargetObject({
|
||||
'object_name': 'Account',
|
||||
@ -353,7 +355,7 @@ class TestSalesforceApiClient:
|
||||
|
||||
assert dict(actual[0]) == expect
|
||||
|
||||
def test_fetch_sf_data_by_soql_builder_system_modstamp_gt_from_and_to_lt(self):
|
||||
def test_fetch_sf_data_by_soql_builder_system_modstamp_to_gt(self):
|
||||
"""
|
||||
Cases:
|
||||
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること
|
||||
@ -361,16 +363,16 @@ class TestSalesforceApiClient:
|
||||
- SystemModStampのToが指定日付未満のレコードは取得できないこと
|
||||
Arranges:
|
||||
- SalesforceのAccountオブジェクトに、レコードを作成する
|
||||
- LastFetchDatetimeのFromがSystemModstampより大きくなるように指定する（UTC指定）
|
||||
- LastFetchDatetimeのToがSystemModstamp未満になるように指定する（UTC指定）
|
||||
- LastFetchDatetimeのFromがSystemModstampより小さくなるように指定する（UTC指定）
|
||||
- LastFetchDatetimeのToがSystemModstampより大きくなるように指定する（UTC指定）
|
||||
Expects:
|
||||
取得できたオブジェクト1件が期待値どおりであること
|
||||
"""
|
||||
|
||||
execute_datetime = ExecuteDateTime()
|
||||
last_fetch_datetime = LastFetchDatetime({
|
||||
'last_fetch_datetime_from': '2022-08-05T11:14:06.000Z',
|
||||
'last_fetch_datetime_to': '2022-08-05T11:15:28.000Z',
|
||||
'last_fetch_datetime_from': '2022-08-23T03:42:24.000Z',
|
||||
'last_fetch_datetime_to': '2022-08-23T03:42:26.000Z',
|
||||
}, execute_datetime)
|
||||
target_object = TargetObject({
|
||||
'object_name': 'Account',
|
||||
@ -422,6 +424,69 @@ class TestSalesforceApiClient:
|
||||
|
||||
assert dict(actual[0]) == expect
|
||||
|
||||
def test_fetch_sf_data_by_soql_builder_address_item_check(self):
|
||||
"""
|
||||
Cases:
|
||||
- SOQLBuilderから生成したSOQLで、Salesforceからオブジェクトが取得できること
|
||||
- できること
|
||||
Arranges:
|
||||
- SalesforceのAccountオブジェクトに、住所項目を持つレコードを作成する
|
||||
- 住所項目を持つレコードだけが取れるよう日付を設定する
|
||||
Expects:
|
||||
取得できたオブジェクト件数が1件になる
|
||||
住所項目(BillingAddress)が想定通りの値になっていること
|
||||
"""
|
||||
|
||||
execute_datetime = ExecuteDateTime()
|
||||
last_fetch_datetime = LastFetchDatetime({
|
||||
'last_fetch_datetime_from': '2022-08-23T02:38:00.000Z',
|
||||
'last_fetch_datetime_to': '2022-08-23T02:39:00.000Z',
|
||||
}, execute_datetime)
|
||||
target_object = TargetObject({
|
||||
'object_name': 'Account',
|
||||
'columns': [
|
||||
'Id',
|
||||
'Name',
|
||||
'SystemModstamp',
|
||||
'LastModifiedDate',
|
||||
'BillingStreet',
|
||||
'BillingCity',
|
||||
'BillingState',
|
||||
'BillingPostalCode',
|
||||
'BillingCountry',
|
||||
'BillingLatitude',
|
||||
'BillingLongitude',
|
||||
'BillingGeocodeAccuracy',
|
||||
'BillingAddress',
|
||||
]
|
||||
}, execute_datetime)
|
||||
soql_builder = SOQLBuilder(target_object, last_fetch_datetime)
|
||||
soql = soql_builder.create_fetch_soql()
|
||||
sut = SalesforceApiClient()
|
||||
|
||||
actual = sut.fetch_sf_data(soql)
|
||||
assert len(actual) == 1
|
||||
expect_address = OrderedDict([
|
||||
("city", '〇〇区'),
|
||||
("country", "日本"),
|
||||
("geocodeAccuracy", None),
|
||||
("latitude", None),
|
||||
("longitude", None),
|
||||
("postalCode", '999-9999'),
|
||||
("state", '東京都'),
|
||||
("street", '△△-✗✗'),
|
||||
])
|
||||
|
||||
assert actual[0]['BillingAddress'] == expect_address
|
||||
assert actual[0]['BillingCity'] == '〇〇区'
|
||||
assert actual[0]['BillingCountry'] == '日本'
|
||||
assert actual[0]['BillingGeocodeAccuracy'] is None
|
||||
assert actual[0]['BillingLatitude'] is None
|
||||
assert actual[0]['BillingLongitude'] is None
|
||||
assert actual[0]['BillingPostalCode'] == '999-9999'
|
||||
assert actual[0]['BillingState'] == '東京都'
|
||||
assert actual[0]['BillingStreet'] == '△△-✗✗'
|
||||
|
||||
def test_fetch_sf_data_by_soql_builder_system_modstamp_all_range(self):
|
||||
"""
|
||||
Cases:
|
||||
@ -433,7 +498,7 @@ class TestSalesforceApiClient:
|
||||
- LastFetchDatetimeのFromに2000年1月1日を指定する
|
||||
- LastFetchDatetimeのToに2100年12月31日を指定する
|
||||
Expects:
|
||||
取得できたオブジェクト件数が16件になる
|
||||
取得できたオブジェクト件数が17件になる
|
||||
"""
|
||||
|
||||
execute_datetime = ExecuteDateTime()
|
||||
@ -448,6 +513,7 @@ class TestSalesforceApiClient:
|
||||
'Name',
|
||||
'SystemModstamp',
|
||||
'LastModifiedDate',
|
||||
'BillingAddress',
|
||||
'CustomItem1__c',
|
||||
'CustomItem2__c',
|
||||
'CustomItem3__c',
|
||||
@ -463,7 +529,7 @@ class TestSalesforceApiClient:
|
||||
sut = SalesforceApiClient()
|
||||
|
||||
actual = sut.fetch_sf_data(soql)
|
||||
assert len(actual) == 16
|
||||
assert len(actual) == 17
|
||||
# 内容の確認は別のケースで行っているため省略
|
||||
|
||||
def test_raise_create_instance_cause_auth_failed(self, monkeypatch):
|
||||
|
||||
@ -44,7 +44,8 @@ class TestBackupCrmDataProcess:
|
||||
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
|
||||
('LastModifiedById', 1.234567E+6),
|
||||
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
|
||||
('IsDeleted', False)
|
||||
('IsDeleted', False),
|
||||
('Name', 'テスト取引先1')
|
||||
]),
|
||||
OrderedDict([
|
||||
('attributes', OrderedDict([('type', 'Account'),
|
||||
@ -54,7 +55,8 @@ class TestBackupCrmDataProcess:
|
||||
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
|
||||
('LastModifiedById', 1.234567E+6),
|
||||
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
|
||||
('IsDeleted', False)
|
||||
('IsDeleted', False),
|
||||
('Name', 'テスト取引先2')
|
||||
]),
|
||||
OrderedDict([
|
||||
('attributes', OrderedDict([('type', 'Account'),
|
||||
@ -64,7 +66,8 @@ class TestBackupCrmDataProcess:
|
||||
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
|
||||
('LastModifiedById', 1.234567E+6),
|
||||
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
|
||||
('IsDeleted', False)
|
||||
('IsDeleted', False),
|
||||
('Name', 'テスト取引先3')
|
||||
]),
|
||||
]
|
||||
|
||||
@ -96,7 +99,7 @@ class TestBackupCrmDataProcess:
|
||||
actual = s3_client.get_object(
|
||||
Bucket=bucket_name,
|
||||
Key=f'response_json/{execute_datetime.to_path()}/CRM_Account_{execute_datetime.format_date()}.json')
|
||||
assert actual['Body'].read().decode('utf-8') == json.dumps(response_json)
|
||||
assert actual['Body'].read().decode('utf-8') == json.dumps(response_json, ensure_ascii=False)
|
||||
|
||||
# ログの確認
|
||||
assert generate_log_message_tuple(
|
||||
|
||||
@ -34,7 +34,17 @@ class TestConvertCrmCsvDataProcess:
|
||||
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
|
||||
('LastModifiedById', 1.234567E+6),
|
||||
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
|
||||
('IsDeleted', True)
|
||||
('IsDeleted', True),
|
||||
('PersonMailingAddress', OrderedDict([
|
||||
('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501
|
||||
('PersonMailingCity', 'New york city'),
|
||||
('PersonMailingState', 'Ohaio'),
|
||||
('PersonMailingPostalCode', '999-9999'),
|
||||
('PersonMailingCountry', 'US'),
|
||||
('PersonMailingLatitude', 50.1234567),
|
||||
('PersonMailingLongitude', 103.1234567),
|
||||
('PersonMailingGeocodeAccuracy', 'Address'),
|
||||
])),
|
||||
]),
|
||||
OrderedDict([
|
||||
('attributes', OrderedDict([('type', 'Account'),
|
||||
@ -42,9 +52,19 @@ class TestConvertCrmCsvDataProcess:
|
||||
('Id', 'TEST002'),
|
||||
('AccountNumber', 'test002'),
|
||||
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
|
||||
('LastModifiedById', 1.234567E+6),
|
||||
('LastModifiedById', 1.23E+0),
|
||||
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
|
||||
('IsDeleted', False)
|
||||
('IsDeleted', False),
|
||||
('PersonMailingAddress', OrderedDict([
|
||||
('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501
|
||||
('PersonMailingCity', 'New york city'),
|
||||
('PersonMailingState', 'Ohaio'),
|
||||
('PersonMailingPostalCode', '999-9999'),
|
||||
('PersonMailingCountry', 'US'),
|
||||
('PersonMailingLatitude', 50.1234567),
|
||||
('PersonMailingLongitude', 103.1234567),
|
||||
('PersonMailingGeocodeAccuracy', 'Address'),
|
||||
])),
|
||||
]),
|
||||
OrderedDict([
|
||||
('attributes', OrderedDict([('type', 'Account'),
|
||||
@ -52,9 +72,19 @@ class TestConvertCrmCsvDataProcess:
|
||||
('Id', 'TEST003'),
|
||||
('AccountNumber', 'test003'),
|
||||
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
|
||||
('LastModifiedById', 1.234567E+6),
|
||||
('LastModifiedById', 1.234567),
|
||||
('SystemModstamp', '2022-06-01T00:00:00.000+0000'),
|
||||
('IsDeleted', False)
|
||||
('IsDeleted', False),
|
||||
('PersonMailingAddress', OrderedDict([
|
||||
('PersonMailingStreet', 'Lorem ipsum dolor sit amet, \nconsectetur adipiscing elit, \nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua.'), # noqa: E501
|
||||
('PersonMailingCity', 'New york city'),
|
||||
('PersonMailingState', 'Ohaio'),
|
||||
('PersonMailingPostalCode', '999-9999'),
|
||||
('PersonMailingCountry', 'US'),
|
||||
('PersonMailingLatitude', 50.1234567),
|
||||
('PersonMailingLongitude', 103.1234567),
|
||||
('PersonMailingGeocodeAccuracy', 'Address'),
|
||||
])),
|
||||
]),
|
||||
]
|
||||
|
||||
@ -66,7 +96,8 @@ class TestConvertCrmCsvDataProcess:
|
||||
'LastModifiedDate',
|
||||
'LastModifiedById',
|
||||
'SystemModstamp',
|
||||
'IsDeleted'
|
||||
'IsDeleted',
|
||||
'PersonMailingAddress'
|
||||
]
|
||||
}
|
||||
|
||||
@ -79,10 +110,10 @@ class TestConvertCrmCsvDataProcess:
|
||||
# Assert
|
||||
|
||||
expect_csv_string = """\
|
||||
"Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted"\r\n\
|
||||
"TEST001","test001","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","1"\r\n\
|
||||
"TEST002","test002","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\
|
||||
"TEST003","test003","2022-06-01 09:00:00","1234567","2022-06-01 09:00:00","0"\r\n\
|
||||
"Id","AccountNumber","LastModifiedDate","LastModifiedById","SystemModstamp","IsDeleted","PersonMailingAddress"\r\n\
|
||||
"TEST001","test001","2022-06-01 09:00:00","1234567.0","2022-06-01 09:00:00","1","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\
|
||||
"TEST002","test002","2022-06-01 09:00:00","1.23","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\
|
||||
"TEST003","test003","2022-06-01 09:00:00","1.234567","2022-06-01 09:00:00","0","{""PersonMailingStreet"": ""Lorem ipsum dolor sit amet, \\nconsectetur adipiscing elit, \\nsed do eiusmod tempor incididunt ut labore et dolore magna aliqua."", ""PersonMailingCity"": ""New york city"", ""PersonMailingState"": ""Ohaio"", ""PersonMailingPostalCode"": ""999-9999"", ""PersonMailingCountry"": ""US"", ""PersonMailingLatitude"": 50.1234567, ""PersonMailingLongitude"": 103.1234567, ""PersonMailingGeocodeAccuracy"": ""Address""}"\r\n\
|
||||
"""
|
||||
# 返り値の期待値チェック
|
||||
assert isinstance(actual_csv_string, str), 'CSV文字列が返却される'
|
||||
|
||||
@ -217,7 +217,7 @@ class TestFetchCrmDataProcess:
|
||||
2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること
|
||||
3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること
|
||||
Arranges:
|
||||
- データ件数取得処理の最大リトライ試行回数を3に設定する
|
||||
- データ件数取得処理の最大リトライ試行回数を4に設定する
|
||||
- timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する
|
||||
- データ件数取得処理の初回に接続タイムアウト例外が発生するようにする
|
||||
Expects:
|
||||
@ -225,7 +225,7 @@ class TestFetchCrmDataProcess:
|
||||
- データ件数取得に失敗した旨のエラーが出力されない
|
||||
"""
|
||||
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1)
|
||||
@ -236,8 +236,8 @@ class TestFetchCrmDataProcess:
|
||||
patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client:
|
||||
# モック化
|
||||
mock_counter_inst = mock_counter.return_value
|
||||
mock_counter_inst.describe.side_effect = [1, 2, 3]
|
||||
mock_counter_inst.increment.side_effect = [2, 3, 4]
|
||||
mock_counter_inst.describe.side_effect = [1, 2, 3, 4]
|
||||
mock_counter_inst.increment.side_effect = [2, 3, 4, 5]
|
||||
mock_builder_inst = mock_soql_builder.return_value
|
||||
mock_builder_inst.create_count_soql.return_value = ''
|
||||
mock_builder_inst.create_fetch_soql.return_value = ''
|
||||
@ -271,15 +271,15 @@ class TestFetchCrmDataProcess:
|
||||
2. データ件数取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること
|
||||
3. データ件数取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること
|
||||
Arranges:
|
||||
- データ件数取得処理の最大リトライ試行回数を3に設定する
|
||||
- データ件数取得処理の最大リトライ試行回数を4に設定する
|
||||
- timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する
|
||||
- データ件数取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする
|
||||
- データ件数取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする
|
||||
Expects:
|
||||
- 異常終了する
|
||||
- データ件数取得に失敗した旨のエラーが出力される
|
||||
"""
|
||||
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1)
|
||||
@ -290,13 +290,13 @@ class TestFetchCrmDataProcess:
|
||||
patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client:
|
||||
# モック化
|
||||
mock_counter_inst = mock_counter.return_value
|
||||
mock_counter_inst.describe.side_effect = [1, 2, 3]
|
||||
mock_counter_inst.increment.side_effect = [2, 3, 4]
|
||||
mock_counter_inst.describe.side_effect = [1, 2, 3, 4]
|
||||
mock_counter_inst.increment.side_effect = [2, 3, 4, 5]
|
||||
mock_builder_inst = mock_soql_builder.return_value
|
||||
mock_builder_inst.create_count_soql.return_value = ''
|
||||
mock_builder_inst.create_fetch_soql.return_value = ''
|
||||
mock_client_inst = mock_api_client.return_value
|
||||
mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception]
|
||||
mock_client_inst.fetch_sf_count.side_effect = [exception, exception, exception, exception]
|
||||
mock_client_inst.fetch_sf_data.return_value = common_expect
|
||||
# Act
|
||||
with pytest.raises(SalesforceAPIException) as e:
|
||||
@ -304,16 +304,16 @@ class TestFetchCrmDataProcess:
|
||||
|
||||
# Assert
|
||||
|
||||
# 取得は3回行われる
|
||||
assert mock_counter_inst.describe.call_count == 3
|
||||
# 足し込みは2回のみ
|
||||
assert mock_counter_inst.increment.call_count == 2
|
||||
# 取得は4回行われる
|
||||
assert mock_counter_inst.describe.call_count == 4
|
||||
# 足し込みは3回のみ
|
||||
assert mock_counter_inst.increment.call_count == 3
|
||||
|
||||
assert generate_log_message_tuple(
|
||||
log_level=logging.WARNING,
|
||||
log_message=expect_message) in caplog.record_tuples
|
||||
called_log_counts = len([log for log in caplog.messages if log == expect_message])
|
||||
assert called_log_counts == 2
|
||||
assert called_log_counts == 3
|
||||
assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples
|
||||
|
||||
assert e.value.error_id == 'E-FETCH-01'
|
||||
@ -333,7 +333,7 @@ class TestFetchCrmDataProcess:
|
||||
2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧し、正常終了すること
|
||||
3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧し、正常終了すること
|
||||
Arranges:
|
||||
- レコード取得処理の最大リトライ試行回数を3に設定する
|
||||
- レコード取得処理の最大リトライ試行回数を4に設定する
|
||||
- timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する
|
||||
- レコード取得処理の初回に接続タイムアウト例外が発生するようにする
|
||||
Expects:
|
||||
@ -341,7 +341,7 @@ class TestFetchCrmDataProcess:
|
||||
- データレコード取得に失敗した旨のエラーが出力されない
|
||||
"""
|
||||
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 3)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT', 4)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL', 1)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL', 1)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_GET_RECORD_COUNT_RETRY_INTERVAL', 1)
|
||||
@ -352,8 +352,8 @@ class TestFetchCrmDataProcess:
|
||||
patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client:
|
||||
# モック化
|
||||
mock_counter_inst = mock_counter.return_value
|
||||
mock_counter_inst.describe.side_effect = [1, 2, 3]
|
||||
mock_counter_inst.increment.side_effect = [2, 3, 4]
|
||||
mock_counter_inst.describe.side_effect = [1, 2, 3, 4]
|
||||
mock_counter_inst.increment.side_effect = [2, 3, 4, 5]
|
||||
mock_builder_inst = mock_soql_builder.return_value
|
||||
mock_builder_inst.create_count_soql.return_value = ''
|
||||
mock_builder_inst.create_fetch_soql.return_value = ''
|
||||
@ -387,15 +387,15 @@ class TestFetchCrmDataProcess:
|
||||
2. レコード取得処理で読み取りタイムアウト例外が発生した場合、リトライした結果復旧せず、異常終了すること
|
||||
3. レコード取得処理で予期せぬ例外が発生した場合、リトライした結果復旧せず、異常終了すること
|
||||
Arranges:
|
||||
- レコード取得処理の最大リトライ試行回数を3に設定する
|
||||
- レコード取得処理の最大リトライ試行回数を4に設定する
|
||||
- timeout_env_nameに指定されたリトライタイムアウト時間の秒数を1に設定する
|
||||
- レコード取得処理の1回目、2回目、3回目で接続タイムアウト例外が発生するようにする
|
||||
- レコード取得処理の1回目、2回目、3回目、4回目で接続タイムアウト例外が発生するようにする
|
||||
Expects:
|
||||
- 異常終了する
|
||||
- データレコード取得に失敗した旨のエラーが出力される
|
||||
"""
|
||||
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 3)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT', 4)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MAX_INTERVAL', 1)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_MIN_INTERVAL', 1)
|
||||
monkeypatch.setattr('src.fetch_crm_data_process.CRM_FETCH_RECORD_RETRY_INTERVAL', 1)
|
||||
@ -406,30 +406,30 @@ class TestFetchCrmDataProcess:
|
||||
patch('src.fetch_crm_data_process.SalesforceApiClient') as mock_api_client:
|
||||
# モック化
|
||||
mock_counter_inst = mock_counter.return_value
|
||||
mock_counter_inst.describe.side_effect = [1, 2, 3]
|
||||
mock_counter_inst.increment.side_effect = [2, 3, 4]
|
||||
mock_counter_inst.describe.side_effect = [1, 2, 3, 4]
|
||||
mock_counter_inst.increment.side_effect = [2, 3, 4, 5]
|
||||
mock_builder_inst = mock_soql_builder.return_value
|
||||
mock_builder_inst.create_count_soql.return_value = ''
|
||||
mock_builder_inst.create_fetch_soql.return_value = ''
|
||||
mock_client_inst = mock_api_client.return_value
|
||||
mock_client_inst.fetch_sf_count.return_value = 1
|
||||
mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception]
|
||||
mock_client_inst.fetch_sf_data.side_effect = [exception, exception, exception, exception]
|
||||
# Act
|
||||
with pytest.raises(SalesforceAPIException) as e:
|
||||
fetch_crm_data_process(common_target_object, common_last_fetch_datetime)
|
||||
|
||||
# Assert
|
||||
|
||||
# 取得は3回行われる
|
||||
assert mock_counter_inst.describe.call_count == 3
|
||||
# 足し込みは2回のみ
|
||||
assert mock_counter_inst.increment.call_count == 2
|
||||
# 取得は4回行われる
|
||||
assert mock_counter_inst.describe.call_count == 4
|
||||
# 足し込みは3回のみ
|
||||
assert mock_counter_inst.increment.call_count == 3
|
||||
|
||||
assert generate_log_message_tuple(
|
||||
log_level=logging.WARNING,
|
||||
log_message=expect_message) in caplog.record_tuples
|
||||
called_log_counts = len([log for log in caplog.messages if log == expect_message])
|
||||
assert called_log_counts == 2
|
||||
assert called_log_counts == 3
|
||||
assert generate_log_message_tuple(log_message='I-FETCH-06 [Account] のCRMからのデータ取得処理を終了します') not in caplog.record_tuples
|
||||
|
||||
assert e.value.error_id == 'E-FETCH-02'
|
||||
|
||||
@ -7,6 +7,9 @@ from src.config.objects import FetchTargetObjects
|
||||
from src.error.exceptions import FileNotFoundException, InvalidConfigException
|
||||
from src.prepare_data_fetch_process import prepare_data_fetch_process
|
||||
from src.system_var.constants import PRE_JP_NAME, YYYYMMDDTHHMMSSTZ
|
||||
from src.system_var.environments import (CRM_CONFIG_BUCKET,
|
||||
OBJECT_INFO_FILENAME,
|
||||
OBJECT_INFO_FOLDER)
|
||||
|
||||
from .test_utils.log_message import generate_log_message_tuple
|
||||
|
||||
@ -80,6 +83,8 @@ class TestPrepareDataFetchProcess:
|
||||
# ログの確認
|
||||
assert generate_log_message_tuple(log_message='I-PRE-01 データ取得準備処理を開始します') in caplog.record_tuples
|
||||
assert generate_log_message_tuple(log_message=f'I-PRE-02 データ取得処理開始日時：{str(actual_execute_datetime)}') in caplog.record_tuples
|
||||
assert generate_log_message_tuple(
|
||||
log_message=f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CRM_CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}]') in caplog.record_tuples
|
||||
assert generate_log_message_tuple(log_message='I-PRE-09 データ取得準備処理を終了します') in caplog.record_tuples
|
||||
|
||||
def test_call_depended_modules(self):
|
||||
|
||||
@ -73,7 +73,6 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog):
|
||||
object_info_list = get_object_config_list('object_info')
|
||||
for object_info in object_info_list:
|
||||
json_file = read_json(object_info)
|
||||
json_file = to_upload_json(json_file)
|
||||
upload_json(json_file, s3_client, CONFIG_BUCKET, f'{OBJECT_INFO_FOLDER}/{path.basename(object_info)}')
|
||||
object_info_files.append(json_file)
|
||||
last_fetch_datetime_list = get_object_config_list('last_fetch_datetime')
|
||||
@ -105,7 +104,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog):
|
||||
|
||||
# Act
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.info(f'##########################')
|
||||
logger.info(f'# 差分データ取得処理:実行開始 #')
|
||||
logger.info(f'##########################')
|
||||
@ -118,7 +117,7 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog):
|
||||
# ループ前のログ確認
|
||||
assert 'I-CTRL-01 CRMデータ取得処理を開始します' in log_messages
|
||||
assert 'I-CTRL-02 データ取得準備処理呼び出し' in log_messages
|
||||
assert_prepare_process_log(log_messages, now)
|
||||
assert_prepare_process_log(log_messages, now, path.basename(object_info_list[0]))
|
||||
assert 'I-CTRL-03 取得対象オブジェクトのループ処理開始' in log_messages
|
||||
# オブジェクト情報を取得する(diff)
|
||||
object_info_list = object_info_files[0]
|
||||
@ -207,9 +206,10 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog):
|
||||
"""
|
||||
|
||||
|
||||
def assert_prepare_process_log(log_messages, now):
|
||||
def assert_prepare_process_log(log_messages, now, object_info_list_filename):
|
||||
assert 'I-PRE-01 データ取得準備処理を開始します' in log_messages
|
||||
assert f'I-PRE-02 データ取得処理開始日時：{now}' in log_messages
|
||||
assert f'I-PRE-03 CRM_取得オブジェクト情報ファイルの取得開始します ファイルパス：[s3://{CONFIG_BUCKET}/{OBJECT_INFO_FOLDER}/{object_info_list_filename}]' in log_messages
|
||||
assert 'I-PRE-09 データ取得準備処理を終了します' in log_messages
|
||||
|
||||
|
||||
@ -295,21 +295,6 @@ def upload_json(json_file, s3_client, bucket, folder):
|
||||
s3_client.put_object(Bucket=bucket, Key=folder, Body=json_str)
|
||||
|
||||
|
||||
def to_upload_json(json_file):
|
||||
"""Userオブジェクトの取得できないプロパティを取り除く
|
||||
TODO: Userオブジェクトの恒久対応が確定したらこのメソッドは消す
|
||||
"""
|
||||
for object_info in json_file['objects']:
|
||||
if object_info['object_name'] != 'User':
|
||||
continue
|
||||
columns: list = object_info['columns']
|
||||
columns.remove('LastPasswordChangeDate')
|
||||
columns.remove('NumberOfFailedLogins')
|
||||
columns.remove('UserPreferencesNativeEmailClient')
|
||||
|
||||
return json_file
|
||||
|
||||
|
||||
def set_environment(monkeypatch):
|
||||
# 環境変数を設定(CRMの認証情報は別途設定しておくこと)
|
||||
monkeypatch.setattr('src.aws.s3.IMPORT_DATA_BUCKET', DATA_BUCKET)
|
||||
@ -324,3 +309,4 @@ def set_environment(monkeypatch):
|
||||
monkeypatch.setattr('src.aws.s3.CRM_IMPORT_DATA_BACKUP_FOLDER', BACKUP_DATA_IMPORT_FOLDER)
|
||||
monkeypatch.setattr('src.aws.s3.PROCESS_RESULT_FOLDER', BACKUP_DATA_IMPORT_FOLDER)
|
||||
monkeypatch.setattr('src.aws.s3.RESPONSE_JSON_BACKUP_FOLDER', BACKUP_RESPONSE_JSON_FOLDER)
|
||||
monkeypatch.setattr('src.prepare_data_fetch_process.CRM_CONFIG_BUCKET', CONFIG_BUCKET)
|
||||
|
||||
@ -1645,13 +1645,11 @@
|
||||
"DelegatedApproverId",
|
||||
"ManagerId",
|
||||
"LastLoginDate",
|
||||
"LastPasswordChangeDate",
|
||||
"CreatedDate",
|
||||
"CreatedById",
|
||||
"LastModifiedDate",
|
||||
"LastModifiedById",
|
||||
"SystemModstamp",
|
||||
"NumberOfFailedLogins",
|
||||
"OfflineTrialExpirationDate",
|
||||
"OfflinePdaTrialExpirationDate",
|
||||
"UserPermissionsMarketingUser",
|
||||
@ -1725,7 +1723,6 @@
|
||||
"UserPreferencesSRHOverrideActivities",
|
||||
"UserPreferencesNewLightningReportRunPageEnabled",
|
||||
"UserPreferencesReverseOpenActivitiesView",
|
||||
"UserPreferencesNativeEmailClient",
|
||||
"UserPreferencesHideBrowseProductRedirectConfirmation",
|
||||
"UserPreferencesHideOnlineSalesAppWelcomeMat",
|
||||
"ContactId",
|
||||
|
||||
@ -1,13 +1,14 @@
|
||||
LOG_LEVEL=INFO
|
||||
CRM_AUTH_TIMEOUT=100
|
||||
CRM_GET_RECORD_COUNT_TIMEOUT=300
|
||||
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=3
|
||||
CRM_GET_RECORD_COUNT_MAX_RETRY_ATTEMPT=4
|
||||
CRM_GET_RECORD_COUNT_RETRY_INTERVAL=5
|
||||
CRM_GET_RECORD_COUNT_RETRY_MIN_INTERVAL=5
|
||||
CRM_GET_RECORD_COUNT_RETRY_MAX_INTERVAL=50
|
||||
CRM_FETCH_RECORD_TIMEOUT=300
|
||||
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=3
|
||||
CRM_FETCH_RECORD_MAX_RETRY_ATTEMPT=4
|
||||
CRM_FETCH_RECORD_RETRY_INTERVAL=5
|
||||
CRM_FETCH_RECORD_RETRY_MIN_INTERVAL=5
|
||||
CRM_FETCH_RECORD_RETRY_MAX_INTERVAL=50
|
||||
CONVERT_TZ='Asia/Tokyo'
|
||||
CONVERT_TZ=Asia/Tokyo
|
||||
OBJECT_INFO_FILENAME=crm_object_list_diff.json
|
||||
@ -4,9 +4,9 @@ utf-8
|
||||
"
|
||||
CRLF
|
||||
1
|
||||
31
|
||||
Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c
|
||||
id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c
|
||||
32
|
||||
Id,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,Attendee_vod__c,User_vod__c,Medical_Event_vod__c,Attendee_Type_vod__c,Status_vod__c,Contact_vod__c,Attendee_Name_vod__c,Account_vod__c,Start_Date_vod__c,Signature_vod__c,Signature_Datetime_vod__c,MSJ_Copy_Account_Type__c,MSJ_Evaluation__c,MSJ_Hospital__c,MSJ_Role__c,Mobile_ID_vod__c,MSJ_Evaluation_Comment__c,Position_vod__c,Talk_Title_vod__c,MSJ_Attendee_Reaction__c,MSJ_Registration__c
|
||||
id,is_deleted,name,record_type_id,created_date,created_by_id,last_modified_date,last_modified_by_id,system_modstamp,may_edit,is_locked,attendee_vod__c,user_vod__c,medical_event_vod__c,attendee_type_vod__c,status_vod__c,contact_vod__c,attendee_name_vod__c,account_vod__c,start_date_vod__c,signature_vod__c,signature_datetime_vod__c,msj_copy_account_type__c,msj_evaluation__c,msj_hospital__c,msj_role__c,mobile_id_vod__c,msj_evaluation_comment__c,position_vod__c,talk_title_vod__c,msj_attendee_reaction__c,msj_registration__c
|
||||
src02.crm_event_attendee_vod__c
|
||||
org02.crm_event_attendee_vod__c
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
Loading…
x
Reference in New Issue
Block a user