diff --git a/ecs/crm-datafetch/src/converter/converter.py b/ecs/crm-datafetch/src/converter/converter.py index 09dbf00e..ce20bb4a 100644 --- a/ecs/crm-datafetch/src/converter/converter.py +++ b/ecs/crm-datafetch/src/converter/converter.py @@ -1,3 +1,4 @@ +from collections import OrderedDict from src.config.objects import TargetObject from src.converter.convert_strategy import ConvertStrategyFactory @@ -25,10 +26,11 @@ class CSVStringConverter: json_object = self.__extract_necessary_props_from(json_object) csv_row = [] for column in columns: - v = json_object[column.upper()] + column_name = column.upper() + column_value = self.__get_column_value(json_object, column_name) - convert_strategy = self.__convert_strategy_factory.create(v) - converted_value = convert_strategy.convert_value(v) + convert_strategy = self.__convert_strategy_factory.create(column_value) + converted_value = convert_strategy.convert_value(column_value) csv_row.append(converted_value) @@ -38,3 +40,31 @@ class CSVStringConverter: except Exception as e: raise Exception( f'CSV変換に失敗しました カラム名：[{column}] 行番号: [{i}] エラー内容:[{e}]') + + def __get_column_value(self, json_object: dict, column_name: str) -> str: + # 参照を辿らない通常の項目の場合、カラム名が一致するためそのまま取得 + if '.' not in column_name: + return json_object[column_name] + + # カラム名に`.`が含まれている場合、オブジェクトの参照を辿って終端を取得する + relationship_columns = column_name.split('.') + return self.__get_column_value_by_relationship(json_object, relationship_columns) + + def __get_column_value_by_relationship(self, json_object: dict, relationship_columns: str, recurs: int = 0) -> str: + # 参照関係の終端を取得しきるまで再帰的に深掘りする + # REVIEW: 参照の終端の項目型が住所型の場合、レスポンスが辞書型になるため大抵の場合Noneになる + relationship_name = relationship_columns[recurs] + relationship_item = json_object.get(relationship_name) + + # 項目が取得できなかったらNoneを返す + if relationship_item is None: + return None + + # 参照が辿りきれていない場合、再帰的に深掘りする + if type(relationship_item) == dict or type(relationship_item) == OrderedDict: + # 取り回しを良くするために、辞書のキーをアッパーケースにしておく + relationship_item_upper = {k.upper(): v for k, v in relationship_item.items()} + return self.__get_column_value_by_relationship(relationship_item_upper, relationship_columns, recurs + 1) + + # 終端のデータを取得 + return relationship_item diff --git a/ecs/crm-datafetch/tests/converter/test_converter.py b/ecs/crm-datafetch/tests/converter/test_converter.py index 0a301f44..992be814 100644 --- a/ecs/crm-datafetch/tests/converter/test_converter.py +++ b/ecs/crm-datafetch/tests/converter/test_converter.py @@ -11,7 +11,8 @@ class TestCSVStringConverter: def test_convert(self) -> str: """ Cases: - 入力データがCSV形式の文字列で出力されること + - 入力データがCSV形式の文字列で出力されること + - 参照関係を辿った項目の終端が取得されていること Arranges: - オブジェクト情報の作成 - データの作成 @@ -35,7 +36,10 @@ class TestCSVStringConverter: "RowCause", "LastModifiedDate", "LastModifiedById", - "IsDeleted" + "IsDeleted", + "Account.Name", + "Account.attributes.type", + "Account.attributes.url" ], "is_skip": False, "is_update_last_fetch_datetime": False, @@ -57,7 +61,8 @@ class TestCSVStringConverter: ('RowCause', 'テストのため1'), ('LastModifiedDate', '2022-06-01T00:00:00.000+0000'), ('LastModifiedById', 1.234567E+6), - ('IsDeleted', False) + ('IsDeleted', False), + ('Account', None) ]), OrderedDict([ ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), @@ -71,7 +76,8 @@ class TestCSVStringConverter: ('RowCause', 'テストのため2'), ('LastModifiedDate', '2022-06-02T16:30:30.000+0000'), ('LastModifiedById', 2.23E+0), - ('IsDeleted', True) + ('IsDeleted', True), + ('Account', None) ]), OrderedDict([ ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), @@ -85,7 +91,26 @@ class TestCSVStringConverter: ('RowCause', 'テストのため3'), ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), ('LastModifiedById', 3.234567), - ('IsDeleted', False) + ('IsDeleted', True), + ('Account', None) + ]), + OrderedDict([ + ('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])), + ('Id', 'TEST004'), + ('AccountId', 'test004'), + ('UserOrGroupId', None), + ('AccountAccessLevel', 13), + ('OpportunityAccessLevel', 14), + ('CaseAccessLevel', 15), + ('ContactAccessLevel', 16), + ('RowCause', 'テストのため4'), + ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), + ('LastModifiedById', 3.234567), + ('IsDeleted', False), + ('Account', OrderedDict([ + ('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/test4')])), + ('Name', 'テスト取引先'), + ])) ]) ] @@ -99,10 +124,13 @@ class TestCSVStringConverter: # Expects expect = [ ["Id", "AccountId", "UserOrGroupId", "AccountAccessLevel", "OpportunityAccessLevel", "CaseAccessLevel", - "ContactAccessLevel", "RowCause", "LastModifiedDate", "LastModifiedById", "IsDeleted"], - ["TEST001", "test001", "", 1, 2, 3, 4, "テストのため1", "2022-06-01 09:00:00", 1234567.0, 0], - ["TEST002", "test002", "", 5, 6, 7, 8, "テストのため2", "2022-06-03 01:30:30", 2.23, 1], - ["TEST003", "test003", "", 9, 10, 11, 12, "テストのため3", "2022-06-04 08:50:50", 3.234567, 0] + "ContactAccessLevel", "RowCause", "LastModifiedDate", "LastModifiedById", "IsDeleted", + "Account.Name", "Account.attributes.type", "Account.attributes.url"], + ["TEST001", "test001", "", 1, 2, 3, 4, "テストのため1", "2022-06-01 09:00:00", 1234567.0, 0, "", "", ""], + ["TEST002", "test002", "", 5, 6, 7, 8, "テストのため2", "2022-06-03 01:30:30", 2.23, 1, "", "", ""], + ["TEST003", "test003", "", 9, 10, 11, 12, "テストのため3", "2022-06-04 08:50:50", 3.234567, 1, "", "", ""], + ["TEST004", "test004", "", 13, 14, 15, 16, "テストのため4", "2022-06-04 08:50:50", + 3.234567, 0, "テスト取引先", "Account", "/services/data/v1.0/sobjects/Account/test4"] ] assert actual == expect @@ -184,7 +212,12 @@ class TestCSVStringConverter: ('RowCause', 'テストのため3'), ('LastModifiedDate', '2022-06-03T23:50:50.000+0000'), ('LastModifiedById', 3.234567E+6), - ('IsDeleted', False) + ('IsDeleted', False), + ('Account', OrderedDict([ + ('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/test3')])), + ('Name', 'テスト取引先'), + ]) + ), ]) ] diff --git a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py index 64530071..d8df66f2 100644 --- a/ecs/crm-datafetch/tests/salesforce/test_salesforce.py +++ b/ecs/crm-datafetch/tests/salesforce/test_salesforce.py @@ -286,6 +286,99 @@ class TestSalesforceApiClient: actual = sut.fetch_sf_data(soql) assert len(actual) >= 0 + def test_fetch_sf_data_relationship_object_depth_1(self): + """ + Cases: + 参照関係を1回辿るSOQLを実行し、Salesforceからデータが取得できること + Arranges: + Salesforceの以下のオブジェクトに、レコードを作成する(手作業、コード上では行わない) + - RelationShipTest__c + Expects: + 取得結果が期待値と一致すること + """ + soql = """SELECT + Id, + Name, + RecordTypeId, + RecordType.DeveloperName + FROM + RelationShipTest__c + ORDER BY Name ASC + """ + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) == 5 + + """ + expect = { + 'Id': 'a025i00000RleEHAAZ', + 'Name': 'A-0001', + 'RecordTypeId': '0125i000000RUqOAAW', + 'attributes': OrderedDict([('type', 'RelationshipTest__c'), + ('url', + '/services/data/v57.0/sobjects/RelationshipTest__c/a025i00000RleEHAAZ')]), + 'RecordType': OrderedDict([('attributes', + OrderedDict([ + ('type', 'RecordType'), + ('url', '/services/data/v57.0/sobjects/RecordType/0125i000000RUqOAAW')])), + ('DeveloperName', 'RecordTypeNormal'), + ]) + + } + assert dict(actual[0]) == expect + """ + + assert dict(actual[0])["RecordType"]["DeveloperName"] == "RecordTypeNormal" + # assert dict(actual[1])["RecordType"]["DeveloperName"] == "RecordTypeNormal" + # assert dict(actual[2])["RecordType"]["DeveloperName"] == "RecordTypeNormal" + # assert dict(actual[3])["RecordType"]["DeveloperName"] == "RecordTypeSpecial" + # assert dict(actual[4])["RecordType"]["DeveloperName"] == "RecordTypeSpecial" + + def test_fetch_sf_data_relationship_object_depth_2(self): + """ + Cases: + 参照関係を2回辿るSOQLを実行し、Salesforceからデータが取得できること + Arranges: + Salesforceの以下のオブジェクトに、レコードを作成する(手作業、コード上では行わない) + - RelationShipTest__c + - RelationShipTest_Child__c + Expects: + 取得結果が期待値と一致すること + """ + soql = """SELECT + Id, + Name, + RelationShipTest__r.RecordType.DeveloperName + FROM + RelationShipTest_Child__c + ORDER BY Name ASC + """ + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) > 0 + """ + expect = OrderedDict([('attributes', + OrderedDict([ + ('type', 'RelationshipTest_Child__c'), + ('url', '/services/data/v57.0/sobjects/RelationshipTest_Child__c/a035i00000FW1qNAAT')])), + ('Id', 'a035i00000FW1qNAAT'), + ('Name', 'A-0001'), + ('RelationshipTest__r', + OrderedDict([ + ('attributes', OrderedDict( + [('type', 'RelationshipTest__c'), + ('url', '/services/data/v57.0/sobjects/RelationshipTest__c/a025i00000RleEHAAZ')])), + ('RecordType', + OrderedDict([('attributes', + OrderedDict([('type', 'RecordType'), + ('url', '/services/data/v57.0/sobjects/RecordType/0125i000000RUqOAAW')])), + ('DeveloperName', 'RecordTypeNormal')]))]))]) + assert dict(actual[0]) == expect + """ + assert dict(actual[0])["RelationshipTest__r"]["RecordType"]["DeveloperName"] == "RecordTypeNormal" + def test_fetch_sf_data_by_soql_builder_system_modstamp_to_ge(self): """ Cases: @@ -532,6 +625,96 @@ class TestSalesforceApiClient: assert len(actual) == 17 # 内容の確認は別のケースで行っているため省略 + def test_fetch_sf_data_by_soql_builder_relationship_object_depth_1(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceから参照関係を1回辿ったオブジェクト項目が取得できること + Arranges: + - Salesforceの以下のオブジェクトに、レコードを作成する(手作業、コード上では行わない) + - RelationShipTest__c + - RelationShipTest_Child__c + - LastFetchDatetimeのFromに2000年1月1日を指定する + - LastFetchDatetimeのToに2100年12月31日を指定する + Expects: + 取得できたオブジェクトの1件をサンプリング確認し、レコードタイプ名(DeveloperName)が含まれている + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2000-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'RelationShipTest__c', + 'columns': [ + 'Id', + 'Name', + 'RecordTypeId', + 'RecordType.DeveloperName' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_fetch_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) > 0 + assert dict(actual[0])["RecordType"]["DeveloperName"] == "RecordTypeNormal" + ... + + def test_fetch_sf_data_by_soql_builder_relationship_object_depth_2(self): + """ + Cases: + - SOQLBuilderから生成したSOQLで、Salesforceから参照関係を2回辿ったオブジェクト項目が取得できること + Arranges: + - Salesforceの以下のオブジェクトに、レコードを作成する(手作業、コード上では行わない) + - RelationShipTest__c + - RelationShipTest_Child__c + - LastFetchDatetimeのFromに2000年1月1日を指定する + - LastFetchDatetimeのToに2100年12月31日を指定する + Expects: + 取得できたオブジェクトの1件をサンプリング確認し、レコードタイプ名(DeveloperName)が含まれている + """ + + execute_datetime = ExecuteDateTime() + last_fetch_datetime = LastFetchDatetime({ + 'last_fetch_datetime_from': '2000-01-01T00:00:00.000Z', + 'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z', + }, execute_datetime) + target_object = TargetObject({ + 'object_name': 'RelationShipTest_Child__c', + 'columns': [ + 'Id', + 'Name', + 'RelationShipTest__r.RecordType.DeveloperName' + ] + }, execute_datetime) + soql_builder = SOQLBuilder(target_object, last_fetch_datetime) + soql = soql_builder.create_fetch_soql() + sut = SalesforceApiClient() + + actual = sut.fetch_sf_data(soql) + assert len(actual) > 0 + """" + excepts = OrderedDict( + [('attributes', OrderedDict([ + ('type', 'RelationshipTest_Child__c'), + ('url', '/services/data/v57.0/sobjects/RelationshipTest_Child__c/a035i00000FW1qPAAT')])), + ('Id', 'a035i00000FW1qPAAT'), + ('Name', 'A-0009'), + ('RelationshipTest__r', OrderedDict([ + ('attributes', OrderedDict( + [('type', 'RelationshipTest__c'), + ('url', '/services/data/v57.0/sobjects/RelationshipTest__c/a025i00000RleESAAZ')])), + ('RecordType', OrderedDict([ + ('attributes', OrderedDict([ + ('type', 'RecordType'), + ('url', '/services/data/v57.0/sobjects/RecordType/0125i000000RUqTAAW')])), + ('DeveloperName', 'RecordTypeSpecial')]))]))]) + assert f"{actual[0]}" == "aaaaa" + """ + assert dict(actual[0])["RelationshipTest__r"]["RecordType"]["DeveloperName"] == "RecordTypeSpecial" + def test_raise_create_instance_cause_auth_failed(self, monkeypatch): """ Cases: diff --git a/s3/config/crm/object_info/crm_object_list_diff.json b/s3/config/crm/object_info/crm_object_list_diff.json index 2840932f..eac278bf 100644 --- a/s3/config/crm/object_info/crm_object_list_diff.json +++ b/s3/config/crm/object_info/crm_object_list_diff.json @@ -37,7 +37,8 @@ "Display_Order_vod__c", "Clm_Presentation_Name_vod__c", "Clm_Presentation_Version_vod__c", - "Clm_Presentation_vod__c" + "Clm_Presentation_vod__c", + "Call2_vod___r.RecordTypeId" ], "is_skip": false, "is_update_last_fetch_datetime": true @@ -61,7 +62,8 @@ "Detail_Priority_vod__c", "Mobile_ID_vod__c", "Override_Lock_vod__c", - "Type_vod__c" + "Type_vod__c", + "Call2_vod___r.RecordTypeId" ], "is_skip": false, "is_update_last_fetch_datetime": true @@ -961,7 +963,8 @@ "Usage_Start_Time_vod__c", "AuxillaryId_vod__c", "ParentId_vod__c", - "Revision_vod__c" + "Revision_vod__c", + "Call2_vod___r.RecordTypeId" ], "is_skip": false, "is_update_last_fetch_datetime": true @@ -1012,7 +1015,8 @@ "EMDS_Materials__c", "EMDS_Topic__c", "MSJ_Visit_Purpose__c", - "MSJ_Insight_Count__c" + "MSJ_Insight_Count__c", + "Call2_vod___r.RecordTypeId" ], "is_skip": false, "is_update_last_fetch_datetime": true @@ -2783,7 +2787,8 @@ "MSJ_Therapeutic_Area_Expertise__c", "MSJ_MAP_GAP__c", "MSJ_Associations__c", - "MSJ_Tier_Score__c" + "MSJ_Tier_Score__c", + "Products_vod__r.MSJ_Product_Classification__c" ], "is_skip": false, "is_update_last_fetch_datetime": true @@ -3238,4 +3243,4 @@ "is_update_last_fetch_datetime": true } ] -} \ No newline at end of file +}