nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

NEWDWH2021-1255 【実装・UT】CRMデータ取得の改修

Browse Source 
自働テスト実装（内容確認してもらいたいです）、単体テスト未実施
This commit is contained in:
nds-keisuke-matsushima 2023-10-18 10:01:22 +09:00
parent 36650b483f
commit 785ecd0f7a
4 changed files with 270 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
ecs/crm-datafetch/src/converter/converter.py
View File

@ -1,3 +1,4 @@
from collections import OrderedDict
from src.config.objects import TargetObject
from src.converter.convert_strategy import ConvertStrategyFactory
@ -25,10 +26,11 @@ class CSVStringConverter:
json_object = self.__extract_necessary_props_from(json_object)
csv_row = []
for column in columns:
v = json_object[column.upper()]
column_name = column.upper()
column_value = self.__get_column_value(json_object, column_name)
convert_strategy = self.__convert_strategy_factory.create(v)
converted_value = convert_strategy.convert_value(v)
convert_strategy = self.__convert_strategy_factory.create(column_value)
converted_value = convert_strategy.convert_value(column_value)
csv_row.append(converted_value)
@ -38,3 +40,31 @@ class CSVStringConverter:
except Exception as e:
raise Exception(
f'CSV変換に失敗しました カラム名：[{column}] 行番号: [{i}] エラー内容:[{e}]')
def __get_column_value(self, json_object: dict, column_name: str) -> str:
# 参照を辿らない通常の項目の場合、カラム名が一致するためそのまま取得
if '.' not in column_name:
return json_object[column_name]
# カラム名に`.`が含まれている場合、オブジェクトの参照を辿って終端を取得する
relationship_columns = column_name.split('.')
return self.__get_column_value_by_relationship(json_object, relationship_columns)
def __get_column_value_by_relationship(self, json_object: dict, relationship_columns: str, recurs: int = 0) -> str:
# 参照関係の終端を取得しきるまで再帰的に深掘りする
# REVIEW: 参照の終端の項目型が住所型の場合、レスポンスが辞書型になるため大抵の場合Noneになる
relationship_name = relationship_columns[recurs]
relationship_item = json_object.get(relationship_name)
# 項目が取得できなかったらNoneを返す
if relationship_item is None:
return None
# 参照が辿りきれていない場合、再帰的に深掘りする
if type(relationship_item) == dict or type(relationship_item) == OrderedDict:
# 取り回しを良くするために、辞書のキーをアッパーケースにしておく
relationship_item_upper = {k.upper(): v for k, v in relationship_item.items()}
return self.__get_column_value_by_relationship(relationship_item_upper, relationship_columns, recurs + 1)
# 終端のデータを取得
return relationship_item

53
ecs/crm-datafetch/tests/converter/test_converter.py
View File

@ -11,7 +11,8 @@ class TestCSVStringConverter:
def test_convert(self) -> str:
"""
Cases:
入力データがCSV形式の文字列で出力されること
- 入力データがCSV形式の文字列で出力されること
- 参照関係を辿った項目の終端が取得されていること
Arranges:
- オブジェクト情報の作成
- データの作成
@ -35,7 +36,10 @@ class TestCSVStringConverter:
"RowCause",
"LastModifiedDate",
"LastModifiedById",
"IsDeleted"
"IsDeleted",
"Account.Name",
"Account.attributes.type",
"Account.attributes.url"
],
"is_skip": False,
"is_update_last_fetch_datetime": False,
@ -57,7 +61,8 @@ class TestCSVStringConverter:
('RowCause', 'テストのため1'),
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('IsDeleted', False)
('IsDeleted', False),
('Account', None)
]),
OrderedDict([
('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])),
@ -71,7 +76,8 @@ class TestCSVStringConverter:
('RowCause', 'テストのため2'),
('LastModifiedDate', '2022-06-02T16:30:30.000+0000'),
('LastModifiedById', 2.23E+0),
('IsDeleted', True)
('IsDeleted', True),
('Account', None)
]),
OrderedDict([
('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])),
@ -85,7 +91,26 @@ class TestCSVStringConverter:
('RowCause', 'テストのため3'),
('LastModifiedDate', '2022-06-03T23:50:50.000+0000'),
('LastModifiedById', 3.234567),
('IsDeleted', False)
('IsDeleted', True),
('Account', None)
]),
OrderedDict([
('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])),
('Id', 'TEST004'),
('AccountId', 'test004'),
('UserOrGroupId', None),
('AccountAccessLevel', 13),
('OpportunityAccessLevel', 14),
('CaseAccessLevel', 15),
('ContactAccessLevel', 16),
('RowCause', 'テストのため4'),
('LastModifiedDate', '2022-06-03T23:50:50.000+0000'),
('LastModifiedById', 3.234567),
('IsDeleted', False),
('Account', OrderedDict([
('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/test4')])),
('Name', 'テスト取引先'),
]))
])
]
@ -99,10 +124,13 @@ class TestCSVStringConverter:
# Expects
expect = [
["Id", "AccountId", "UserOrGroupId", "AccountAccessLevel", "OpportunityAccessLevel", "CaseAccessLevel",
"ContactAccessLevel", "RowCause", "LastModifiedDate", "LastModifiedById", "IsDeleted"],
["TEST001", "test001", "", 1, 2, 3, 4, "テストのため1", "2022-06-01 09:00:00", 1234567.0, 0],
["TEST002", "test002", "", 5, 6, 7, 8, "テストのため2", "2022-06-03 01:30:30", 2.23, 1],
["TEST003", "test003", "", 9, 10, 11, 12, "テストのため3", "2022-06-04 08:50:50", 3.234567, 0]
"ContactAccessLevel", "RowCause", "LastModifiedDate", "LastModifiedById", "IsDeleted",
"Account.Name", "Account.attributes.type", "Account.attributes.url"],
["TEST001", "test001", "", 1, 2, 3, 4, "テストのため1", "2022-06-01 09:00:00", 1234567.0, 0, "", "", ""],
["TEST002", "test002", "", 5, 6, 7, 8, "テストのため2", "2022-06-03 01:30:30", 2.23, 1, "", "", ""],
["TEST003", "test003", "", 9, 10, 11, 12, "テストのため3", "2022-06-04 08:50:50", 3.234567, 1, "", "", ""],
["TEST004", "test004", "", 13, 14, 15, 16, "テストのため4", "2022-06-04 08:50:50",
3.234567, 0, "テスト取引先", "Account", "/services/data/v1.0/sobjects/Account/test4"]
]
assert actual == expect
@ -184,7 +212,12 @@ class TestCSVStringConverter:
('RowCause', 'テストのため3'),
('LastModifiedDate', '2022-06-03T23:50:50.000+0000'),
('LastModifiedById', 3.234567E+6),
('IsDeleted', False)
('IsDeleted', False),
('Account', OrderedDict([
('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/test3')])),
('Name', 'テスト取引先'),
])
),
])
]

183
ecs/crm-datafetch/tests/salesforce/test_salesforce.py
View File

@ -286,6 +286,99 @@ class TestSalesforceApiClient:
actual = sut.fetch_sf_data(soql)
assert len(actual) >= 0
def test_fetch_sf_data_relationship_object_depth_1(self):
"""
Cases:
参照関係を1回辿るSOQLを実行し、Salesforceからデータが取得できること
Arranges:
Salesforceの以下のオブジェクトに、レコードを作成する(手作業、コード上では行わない)
- RelationShipTest__c
Expects:
取得結果が期待値と一致すること
"""
soql = """SELECT
Id,
Name,
RecordTypeId,
RecordType.DeveloperName
FROM
RelationShipTest__c
ORDER BY Name ASC
"""
sut = SalesforceApiClient()
actual = sut.fetch_sf_data(soql)
assert len(actual) == 5
"""
expect = {
'Id': 'a025i00000RleEHAAZ',
'Name': 'A-0001',
'RecordTypeId': '0125i000000RUqOAAW',
'attributes': OrderedDict([('type', 'RelationshipTest__c'),
('url',
'/services/data/v57.0/sobjects/RelationshipTest__c/a025i00000RleEHAAZ')]),
'RecordType': OrderedDict([('attributes',
OrderedDict([
('type', 'RecordType'),
('url', '/services/data/v57.0/sobjects/RecordType/0125i000000RUqOAAW')])),
('DeveloperName', 'RecordTypeNormal'),
])
}
assert dict(actual[0]) == expect
"""
assert dict(actual[0])["RecordType"]["DeveloperName"] == "RecordTypeNormal"
# assert dict(actual[1])["RecordType"]["DeveloperName"] == "RecordTypeNormal"
# assert dict(actual[2])["RecordType"]["DeveloperName"] == "RecordTypeNormal"
# assert dict(actual[3])["RecordType"]["DeveloperName"] == "RecordTypeSpecial"
# assert dict(actual[4])["RecordType"]["DeveloperName"] == "RecordTypeSpecial"
def test_fetch_sf_data_relationship_object_depth_2(self):
"""
Cases:
参照関係を2回辿るSOQLを実行し、Salesforceからデータが取得できること
Arranges:
Salesforceの以下のオブジェクトに、レコードを作成する(手作業、コード上では行わない)
- RelationShipTest__c
- RelationShipTest_Child__c
Expects:
取得結果が期待値と一致すること
"""
soql = """SELECT
Id,
Name,
RelationShipTest__r.RecordType.DeveloperName
FROM
RelationShipTest_Child__c
ORDER BY Name ASC
"""
sut = SalesforceApiClient()
actual = sut.fetch_sf_data(soql)
assert len(actual) > 0
"""
expect = OrderedDict([('attributes',
OrderedDict([
('type', 'RelationshipTest_Child__c'),
('url', '/services/data/v57.0/sobjects/RelationshipTest_Child__c/a035i00000FW1qNAAT')])),
('Id', 'a035i00000FW1qNAAT'),
('Name', 'A-0001'),
('RelationshipTest__r',
OrderedDict([
('attributes', OrderedDict(
[('type', 'RelationshipTest__c'),
('url', '/services/data/v57.0/sobjects/RelationshipTest__c/a025i00000RleEHAAZ')])),
('RecordType',
OrderedDict([('attributes',
OrderedDict([('type', 'RecordType'),
('url', '/services/data/v57.0/sobjects/RecordType/0125i000000RUqOAAW')])),
('DeveloperName', 'RecordTypeNormal')]))]))])
assert dict(actual[0]) == expect
"""
assert dict(actual[0])["RelationshipTest__r"]["RecordType"]["DeveloperName"] == "RecordTypeNormal"
def test_fetch_sf_data_by_soql_builder_system_modstamp_to_ge(self):
"""
Cases:
@ -532,6 +625,96 @@ class TestSalesforceApiClient:
assert len(actual) == 17
# 内容の確認は別のケースで行っているため省略
def test_fetch_sf_data_by_soql_builder_relationship_object_depth_1(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceから参照関係を1回辿ったオブジェクト項目が取得できること
Arranges:
- Salesforceの以下のオブジェクトに、レコードを作成する(手作業、コード上では行わない)
- RelationShipTest__c
- RelationShipTest_Child__c
- LastFetchDatetimeのFromに2000年1月1日を指定する
- LastFetchDatetimeのToに2100年12月31日を指定する
Expects:
取得できたオブジェクトの1件をサンプリング確認し、レコードタイプ名(DeveloperName)が含まれている
"""
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2000-01-01T00:00:00.000Z',
'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'RelationShipTest__c',
'columns': [
'Id',
'Name',
'RecordTypeId',
'RecordType.DeveloperName'
]
}, execute_datetime)
soql_builder = SOQLBuilder(target_object, last_fetch_datetime)
soql = soql_builder.create_fetch_soql()
sut = SalesforceApiClient()
actual = sut.fetch_sf_data(soql)
assert len(actual) > 0
assert dict(actual[0])["RecordType"]["DeveloperName"] == "RecordTypeNormal"
...
def test_fetch_sf_data_by_soql_builder_relationship_object_depth_2(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceから参照関係を2回辿ったオブジェクト項目が取得できること
Arranges:
- Salesforceの以下のオブジェクトに、レコードを作成する(手作業、コード上では行わない)
- RelationShipTest__c
- RelationShipTest_Child__c
- LastFetchDatetimeのFromに2000年1月1日を指定する
- LastFetchDatetimeのToに2100年12月31日を指定する
Expects:
取得できたオブジェクトの1件をサンプリング確認し、レコードタイプ名(DeveloperName)が含まれている
"""
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2000-01-01T00:00:00.000Z',
'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'RelationShipTest_Child__c',
'columns': [
'Id',
'Name',
'RelationShipTest__r.RecordType.DeveloperName'
]
}, execute_datetime)
soql_builder = SOQLBuilder(target_object, last_fetch_datetime)
soql = soql_builder.create_fetch_soql()
sut = SalesforceApiClient()
actual = sut.fetch_sf_data(soql)
assert len(actual) > 0
""""
excepts = OrderedDict(
[('attributes', OrderedDict([
('type', 'RelationshipTest_Child__c'),
('url', '/services/data/v57.0/sobjects/RelationshipTest_Child__c/a035i00000FW1qPAAT')])),
('Id', 'a035i00000FW1qPAAT'),
('Name', 'A-0009'),
('RelationshipTest__r', OrderedDict([
('attributes', OrderedDict(
[('type', 'RelationshipTest__c'),
('url', '/services/data/v57.0/sobjects/RelationshipTest__c/a025i00000RleESAAZ')])),
('RecordType', OrderedDict([
('attributes', OrderedDict([
('type', 'RecordType'),
('url', '/services/data/v57.0/sobjects/RecordType/0125i000000RUqTAAW')])),
('DeveloperName', 'RecordTypeSpecial')]))]))])
assert f"{actual[0]}" == "aaaaa"
"""
assert dict(actual[0])["RelationshipTest__r"]["RecordType"]["DeveloperName"] == "RecordTypeSpecial"
def test_raise_create_instance_cause_auth_failed(self, monkeypatch):
"""
Cases:

17
s3/config/crm/object_info/crm_object_list_diff.json
View File

@ -37,7 +37,8 @@
"Display_Order_vod__c",
"Clm_Presentation_Name_vod__c",
"Clm_Presentation_Version_vod__c",
"Clm_Presentation_vod__c"
"Clm_Presentation_vod__c",
"Call2_vod___r.RecordTypeId"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -61,7 +62,8 @@
"Detail_Priority_vod__c",
"Mobile_ID_vod__c",
"Override_Lock_vod__c",
"Type_vod__c"
"Type_vod__c",
"Call2_vod___r.RecordTypeId"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -961,7 +963,8 @@
"Usage_Start_Time_vod__c",
"AuxillaryId_vod__c",
"ParentId_vod__c",
"Revision_vod__c"
"Revision_vod__c",
"Call2_vod___r.RecordTypeId"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -1012,7 +1015,8 @@
"EMDS_Materials__c",
"EMDS_Topic__c",
"MSJ_Visit_Purpose__c",
"MSJ_Insight_Count__c"
"MSJ_Insight_Count__c",
"Call2_vod___r.RecordTypeId"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -2783,7 +2787,8 @@
"MSJ_Therapeutic_Area_Expertise__c",
"MSJ_MAP_GAP__c",
"MSJ_Associations__c",
"MSJ_Tier_Score__c"
"MSJ_Tier_Score__c",
"Products_vod__r.MSJ_Product_Classification__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -3238,4 +3243,4 @@
"is_update_last_fetch_datetime": true
}
]
}
}