nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #294 feature-NEWDWH2021-1255 into develop-4-crm-medical

Browse Source
This commit is contained in:
朝倉 明日香 2023-10-23 09:44:57 +09:00
commit 2527593b89
7 changed files with 238 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
ecs/crm-datafetch/Pipfile
View File

@ -11,7 +11,7 @@ test = "pytest tests/"
[packages]
boto3 = "*"
simple-salesforce = "*"
simple-salesforce = "==1.12.4"
tenacity = "*"
[dev-packages]

75
ecs/crm-datafetch/Pipfile.lock generated
View File

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "e98038dbc623f05e819dd27af2e12ec2720d6e86d3b7d203aed1af789b99a38d"
"sha256": "79301fd70f05a7e9d84a96c82c8ca7d17d7c476a666f204bbde8ac46acf2c400"
},
"pipfile-spec": 6,
"requires": {
@ -26,20 +26,20 @@
},
"boto3": {
"hashes": [
"sha256:9d52a1605657aeb5b19b09cfc01d9a92f88a616a5daf5479a59656d6341ea6b3",
"sha256:ff3d0116e0ca6c096547652390025780eace3a28f6c04c9ffbf38448f1e5a87b"
"sha256:7d17f987a8b4f804e5ae509a30589736a72c6db7b0e2fb1338997128fdc9a3ec",
"sha256:8db91c0648c9dcde1cf7fb4c15cd50da1fdef573595a9b9c769a303c7531b9a6"
],
"index": "pypi",
"markers": "python_version >= '3.7'",
"version": "==1.28.65"
"version": "==1.28.67"
},
"botocore": {
"hashes": [
"sha256:90716c6f1af97e5c2f516e9a3379767ebdddcc6cbed79b026fa5038ce4e5e43e",
"sha256:f74e3da98dfcec17bc63ef58f82c643bf5bd7ec6cc11a26ede21cc4cd064917f"
"sha256:487fb6ee4a6612613da370599b1a1aca0e159dd9e94b2e8aaa8e6ad9cc546ded",
"sha256:ab3b73a2e03efa1c534a94f8db4a5cf45629a53e5478d2d154b0a3e2ffb05249"
],
"markers": "python_version >= '3.7'",
"version": "==1.31.65"
"version": "==1.31.67"
},
"certifi": {
"hashes": [
@ -353,41 +353,6 @@
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==4.9.3"
},
"more-itertools": {
"hashes": [
"sha256:626c369fa0eb37bac0291bce8259b332fd59ac792fa5497b59837309cd5b114a",
"sha256:64e0735fcfdc6f3464ea133afe8ea4483b1c5fe3a3d69852e6503b43a0b222e6"
],
"markers": "python_version >= '3.8'",
"version": "==10.1.0"
},
"pendulum": {
"hashes": [
"sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394",
"sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b",
"sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a",
"sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087",
"sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739",
"sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269",
"sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0",
"sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5",
"sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be",
"sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7",
"sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3",
"sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207",
"sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe",
"sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360",
"sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0",
"sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b",
"sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052",
"sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002",
"sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116",
"sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db",
"sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
"version": "==2.1.2"
},
"platformdirs": {
"hashes": [
"sha256:cf8ee52a3afdb965072dcc652433e0c7e3e40cf5ea1477cd4b3b1d2eb75495b3",
@ -426,14 +391,6 @@
],
"version": "==2023.3.post1"
},
"pytzdata": {
"hashes": [
"sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540",
"sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==2020.1"
},
"requests": {
"hashes": [
"sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
@ -467,11 +424,11 @@
},
"simple-salesforce": {
"hashes": [
"sha256:07029575385d04132babfd6e19c1c8068c859d616a45dab07bbf9875bdc5ab93",
"sha256:ef65f72438e3b215619f6835d3d4356e147adf3a7ece6896d239127dd6aefcd1"
"sha256:87f1ac1363987ebf9b7a66ac9e7395534ec3f5a9545e08b664cb695dc085b1ac",
"sha256:e9205f8041226c8273f020448c7fbd37999d481d25edd67f9bf86dc159cd3939"
],
"index": "pypi",
"version": "==1.12.5"
"version": "==1.12.4"
},
"six": {
"hashes": [
@ -519,20 +476,20 @@
},
"boto3": {
"hashes": [
"sha256:9d52a1605657aeb5b19b09cfc01d9a92f88a616a5daf5479a59656d6341ea6b3",
"sha256:ff3d0116e0ca6c096547652390025780eace3a28f6c04c9ffbf38448f1e5a87b"
"sha256:7d17f987a8b4f804e5ae509a30589736a72c6db7b0e2fb1338997128fdc9a3ec",
"sha256:8db91c0648c9dcde1cf7fb4c15cd50da1fdef573595a9b9c769a303c7531b9a6"
],
"index": "pypi",
"markers": "python_version >= '3.7'",
"version": "==1.28.65"
"version": "==1.28.67"
},
"botocore": {
"hashes": [
"sha256:90716c6f1af97e5c2f516e9a3379767ebdddcc6cbed79b026fa5038ce4e5e43e",
"sha256:f74e3da98dfcec17bc63ef58f82c643bf5bd7ec6cc11a26ede21cc4cd064917f"
"sha256:487fb6ee4a6612613da370599b1a1aca0e159dd9e94b2e8aaa8e6ad9cc546ded",
"sha256:ab3b73a2e03efa1c534a94f8db4a5cf45629a53e5478d2d154b0a3e2ffb05249"
],
"markers": "python_version >= '3.7'",
"version": "==1.31.65"
"version": "==1.31.67"
},
"certifi": {
"hashes": [

36
ecs/crm-datafetch/src/converter/converter.py
View File

@ -1,3 +1,4 @@
from collections import OrderedDict
from src.config.objects import TargetObject
from src.converter.convert_strategy import ConvertStrategyFactory
@ -25,10 +26,11 @@ class CSVStringConverter:
json_object = self.__extract_necessary_props_from(json_object)
csv_row = []
for column in columns:
v = json_object[column.upper()]
column_name = column.upper()
column_value = self.__get_column_value(json_object, column_name)
convert_strategy = self.__convert_strategy_factory.create(v)
converted_value = convert_strategy.convert_value(v)
convert_strategy = self.__convert_strategy_factory.create(column_value)
converted_value = convert_strategy.convert_value(column_value)
csv_row.append(converted_value)
@ -38,3 +40,31 @@ class CSVStringConverter:
except Exception as e:
raise Exception(
f'CSV変換に失敗しました カラム名：[{column}] 行番号: [{i}] エラー内容:[{e}]')
def __get_column_value(self, json_object: dict, column_name: str) -> str:
# 参照を辿らない通常の項目の場合、カラム名が一致するためそのまま取得
if '.' not in column_name:
return json_object[column_name]
# カラム名に`.`が含まれている場合、オブジェクトの参照を辿って終端を取得する
relationship_columns = column_name.split('.')
return self.__get_column_value_by_relationship(json_object, relationship_columns)
def __get_column_value_by_relationship(self, json_object: dict, relationship_columns: str, recurs: int = 0) -> str:
# 参照関係の終端を取得しきるまで再帰的に深掘りする
# REVIEW: 参照の終端の項目型が住所型の場合、レスポンスが辞書型になるため大抵の場合Noneになる
relationship_name = relationship_columns[recurs]
relationship_item = json_object.get(relationship_name)
# 項目が取得できなかったらNoneを返す
if relationship_item is None:
return None
# 参照が辿りきれていない場合、再帰的に深掘りする
if type(relationship_item) == dict or type(relationship_item) == OrderedDict:
# 取り回しを良くするために、辞書のキーをアッパーケースにしておく
relationship_item_upper = {k.upper(): v for k, v in relationship_item.items()}
return self.__get_column_value_by_relationship(relationship_item_upper, relationship_columns, recurs + 1)
# 終端のデータを取得
return relationship_item

53
ecs/crm-datafetch/tests/converter/test_converter.py
View File

@ -11,7 +11,8 @@ class TestCSVStringConverter:
def test_convert(self) -> str:
"""
Cases:
入力データがCSV形式の文字列で出力されること
- 入力データがCSV形式の文字列で出力されること
- 参照関係を辿った項目の終端が取得されていること
Arranges:
- オブジェクト情報の作成
- データの作成
@ -35,7 +36,10 @@ class TestCSVStringConverter:
"RowCause",
"LastModifiedDate",
"LastModifiedById",
"IsDeleted"
"IsDeleted",
"Account.Name",
"Account.attributes.type",
"Account.attributes.url"
],
"is_skip": False,
"is_update_last_fetch_datetime": False,
@ -57,7 +61,8 @@ class TestCSVStringConverter:
('RowCause', 'テストのため1'),
('LastModifiedDate', '2022-06-01T00:00:00.000+0000'),
('LastModifiedById', 1.234567E+6),
('IsDeleted', False)
('IsDeleted', False),
('Account', None)
]),
OrderedDict([
('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])),
@ -71,7 +76,8 @@ class TestCSVStringConverter:
('RowCause', 'テストのため2'),
('LastModifiedDate', '2022-06-02T16:30:30.000+0000'),
('LastModifiedById', 2.23E+0),
('IsDeleted', True)
('IsDeleted', True),
('Account', None)
]),
OrderedDict([
('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])),
@ -85,7 +91,26 @@ class TestCSVStringConverter:
('RowCause', 'テストのため3'),
('LastModifiedDate', '2022-06-03T23:50:50.000+0000'),
('LastModifiedById', 3.234567),
('IsDeleted', False)
('IsDeleted', True),
('Account', None)
]),
OrderedDict([
('attributes', OrderedDict([('type', 'AccountShare'), ('url', '/services/data/v1.0/sobjects/AccountShare/test1')])),
('Id', 'TEST004'),
('AccountId', 'test004'),
('UserOrGroupId', None),
('AccountAccessLevel', 13),
('OpportunityAccessLevel', 14),
('CaseAccessLevel', 15),
('ContactAccessLevel', 16),
('RowCause', 'テストのため4'),
('LastModifiedDate', '2022-06-03T23:50:50.000+0000'),
('LastModifiedById', 3.234567),
('IsDeleted', False),
('Account', OrderedDict([
('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/test4')])),
('Name', 'テスト取引先'),
]))
])
]
@ -99,10 +124,13 @@ class TestCSVStringConverter:
# Expects
expect = [
["Id", "AccountId", "UserOrGroupId", "AccountAccessLevel", "OpportunityAccessLevel", "CaseAccessLevel",
"ContactAccessLevel", "RowCause", "LastModifiedDate", "LastModifiedById", "IsDeleted"],
["TEST001", "test001", "", 1, 2, 3, 4, "テストのため1", "2022-06-01 09:00:00", 1234567.0, 0],
["TEST002", "test002", "", 5, 6, 7, 8, "テストのため2", "2022-06-03 01:30:30", 2.23, 1],
["TEST003", "test003", "", 9, 10, 11, 12, "テストのため3", "2022-06-04 08:50:50", 3.234567, 0]
"ContactAccessLevel", "RowCause", "LastModifiedDate", "LastModifiedById", "IsDeleted",
"Account.Name", "Account.attributes.type", "Account.attributes.url"],
["TEST001", "test001", "", 1, 2, 3, 4, "テストのため1", "2022-06-01 09:00:00", 1234567.0, 0, "", "", ""],
["TEST002", "test002", "", 5, 6, 7, 8, "テストのため2", "2022-06-03 01:30:30", 2.23, 1, "", "", ""],
["TEST003", "test003", "", 9, 10, 11, 12, "テストのため3", "2022-06-04 08:50:50", 3.234567, 1, "", "", ""],
["TEST004", "test004", "", 13, 14, 15, 16, "テストのため4", "2022-06-04 08:50:50",
3.234567, 0, "テスト取引先", "Account", "/services/data/v1.0/sobjects/Account/test4"]
]
assert actual == expect
@ -184,7 +212,12 @@ class TestCSVStringConverter:
('RowCause', 'テストのため3'),
('LastModifiedDate', '2022-06-03T23:50:50.000+0000'),
('LastModifiedById', 3.234567E+6),
('IsDeleted', False)
('IsDeleted', False),
('Account', OrderedDict([
('attributes', OrderedDict([('type', 'Account'), ('url', '/services/data/v1.0/sobjects/Account/test3')])),
('Name', 'テスト取引先'),
])
),
])
]

122
ecs/crm-datafetch/tests/salesforce/test_salesforce.py
View File

@ -286,6 +286,56 @@ class TestSalesforceApiClient:
actual = sut.fetch_sf_data(soql)
assert len(actual) >= 0
def test_fetch_sf_data_relationship_object_depth_1(self):
"""
Cases:
参照関係を1回辿るSOQLを実行し、Salesforceからデータが取得できること
Arranges:
Salesforceの以下のオブジェクトに、レコードを作成する(手作業、コード上では行わない)
- RelationShipTest__c
Expects:
取得結果が期待値と一致すること
"""
soql = """SELECT
Id,
Name,
RecordTypeId,
RecordType.DeveloperName
FROM
RelationShipTest__c
ORDER BY Name ASC
"""
sut = SalesforceApiClient()
actual = sut.fetch_sf_data(soql)
assert len(actual) > 0
assert dict(actual[0])["RecordType"]["DeveloperName"] == "RecordTypeNormal"
def test_fetch_sf_data_relationship_object_depth_2(self):
"""
Cases:
参照関係を2回辿るSOQLを実行し、Salesforceからデータが取得できること
Arranges:
Salesforceの以下のオブジェクトに、レコードを作成する(手作業、コード上では行わない)
- RelationShipTest__c
- RelationShipTest_Child__c
Expects:
取得結果が期待値と一致すること
"""
soql = """SELECT
Id,
Name,
RelationShipTest__r.RecordType.DeveloperName
FROM
RelationShipTest_Child__c
ORDER BY Name ASC
"""
sut = SalesforceApiClient()
actual = sut.fetch_sf_data(soql)
assert len(actual) > 0
assert dict(actual[0])["RelationshipTest__r"]["RecordType"]["DeveloperName"] == "RecordTypeNormal"
def test_fetch_sf_data_by_soql_builder_system_modstamp_to_ge(self):
"""
Cases:
@ -532,6 +582,78 @@ class TestSalesforceApiClient:
assert len(actual) == 17
# 内容の確認は別のケースで行っているため省略
def test_fetch_sf_data_by_soql_builder_relationship_object_depth_1(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceから参照関係を1回辿ったオブジェクト項目が取得できること
Arranges:
- Salesforceの以下のオブジェクトに、レコードを作成する(手作業、コード上では行わない)
- RelationShipTest__c
- RelationShipTest_Child__c
- LastFetchDatetimeのFromに2000年1月1日を指定する
- LastFetchDatetimeのToに2100年12月31日を指定する
Expects:
取得できたオブジェクトの1件をサンプリング確認し、レコードタイプ名(DeveloperName)が含まれている
"""
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2000-01-01T00:00:00.000Z',
'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'RelationShipTest__c',
'columns': [
'Id',
'Name',
'RecordTypeId',
'RecordType.DeveloperName'
]
}, execute_datetime)
soql_builder = SOQLBuilder(target_object, last_fetch_datetime)
soql = soql_builder.create_fetch_soql()
sut = SalesforceApiClient()
actual = sut.fetch_sf_data(soql)
assert len(actual) > 0
assert dict(actual[0])["RecordType"]["DeveloperName"] == "RecordTypeNormal"
...
def test_fetch_sf_data_by_soql_builder_relationship_object_depth_2(self):
"""
Cases:
- SOQLBuilderから生成したSOQLで、Salesforceから参照関係を2回辿ったオブジェクト項目が取得できること
Arranges:
- Salesforceの以下のオブジェクトに、レコードを作成する(手作業、コード上では行わない)
- RelationShipTest__c
- RelationShipTest_Child__c
- LastFetchDatetimeのFromに2000年1月1日を指定する
- LastFetchDatetimeのToに2100年12月31日を指定する
Expects:
取得できたオブジェクトの1件をサンプリング確認し、レコードタイプ名(DeveloperName)が含まれている
"""
execute_datetime = ExecuteDateTime()
last_fetch_datetime = LastFetchDatetime({
'last_fetch_datetime_from': '2000-01-01T00:00:00.000Z',
'last_fetch_datetime_to': '2100-12-31T23:59:59.000Z',
}, execute_datetime)
target_object = TargetObject({
'object_name': 'RelationShipTest_Child__c',
'columns': [
'Id',
'Name',
'RelationShipTest__r.RecordType.DeveloperName'
]
}, execute_datetime)
soql_builder = SOQLBuilder(target_object, last_fetch_datetime)
soql = soql_builder.create_fetch_soql()
sut = SalesforceApiClient()
actual = sut.fetch_sf_data(soql)
assert len(actual) > 0
assert dict(actual[0])["RelationshipTest__r"]["RecordType"]["DeveloperName"] == "RecordTypeSpecial"
def test_raise_create_instance_cause_auth_failed(self, monkeypatch):
"""
Cases:

5
s3/config/crm/last_fetch_datetime/H1Insights__H1_Insights_Configuration__mdt.json
View File

@ -1,5 +0,0 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

51
s3/config/crm/object_info/crm_object_list_diff.json
View File

@ -37,7 +37,8 @@
"Display_Order_vod__c",
"Clm_Presentation_Name_vod__c",
"Clm_Presentation_Version_vod__c",
"Clm_Presentation_vod__c"
"Clm_Presentation_vod__c",
"Call2_vod__r.RecordTypeId"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -61,7 +62,8 @@
"Detail_Priority_vod__c",
"Mobile_ID_vod__c",
"Override_Lock_vod__c",
"Type_vod__c"
"Type_vod__c",
"Call2_vod__r.RecordTypeId"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -490,7 +492,18 @@
"MSJ_Level_4B_Value__c",
"MSJ_Hospital_ID__c",
"MSJ_Hospital_Name__c",
"MSJ_Hospital__c"
"MSJ_Hospital__c",
"MSJ_Type_of_Insight__c",
"MSJ_Therapeutic_Area__c",
"MSJ_Starred_Insight__c",
"MSJ_Disclaimer__c",
"MSJ_Not_pharmacovigilance_related__c",
"MSJ_Approval_Status__c",
"MSJ_Insight_Owner_Sharing__c",
"MSJ_Description_J__c",
"MSJ_Summary_J__c",
"MSJ_Level_1J__c",
"MSJ_Level_2J__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -961,7 +974,8 @@
"Usage_Start_Time_vod__c",
"AuxillaryId_vod__c",
"ParentId_vod__c",
"Revision_vod__c"
"Revision_vod__c",
"Call_vod__r.RecordTypeId"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -1012,7 +1026,8 @@
"EMDS_Materials__c",
"EMDS_Topic__c",
"MSJ_Visit_Purpose__c",
"MSJ_Insight_Count__c"
"MSJ_Insight_Count__c",
"Call2_vod__r.RecordTypeId"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -2783,7 +2798,8 @@
"MSJ_Therapeutic_Area_Expertise__c",
"MSJ_MAP_GAP__c",
"MSJ_Associations__c",
"MSJ_Tier_Score__c"
"MSJ_Tier_Score__c",
"Products_vod__r.MSJ_Product_Classification__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -3267,27 +3283,6 @@
"is_skip": false,
"is_update_last_fetch_datetime": true
},
{
"object_name": "H1Insights__H1_Insights_Configuration__mdt",
"columns": [
"Id",
"DeveloperName",
"MasterLabel",
"Language",
"NamespacePrefix",
"Label",
"QualifiedApiName",
"SystemModstamp",
"H1Insights__API_Endpoint__c",
"H1Insights__API_Key__c",
"H1Insights__H1_Customer_ID__c",
"H1Insights__Support_Email_Address__c",
"H1Insights__Testing_H1_Person_ID__c",
"H1Insights__Testing_NPI_Number__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
},
{
"object_name": "Publication__c",
"columns": [
@ -3354,4 +3349,4 @@
"is_update_last_fetch_datetime": true
}
]
}
}