Merge pull request #357 feature-NEWDWH2021-1472 into develop

This commit is contained in:
朝倉 明日香 2024-03-21 15:11:43 +09:00
commit abe68940bf
15 changed files with 399 additions and 35 deletions

3
.gitignore vendored
View File

@ -15,3 +15,6 @@ stepfunctions/*/build
# python test
.coverage
.report/
# log
.log

20
ecs/crm-datafetch/.gitignore vendored Normal file
View File

@ -0,0 +1,20 @@
# Node.jsで実装されたLambdaの管理対象外ファイル群
package-lock.json
node_modules/
# ローカル確認用環境変数ファイル
.env
# Pythonの仮想環境ファイル
.venv
# pythonのキャッシュファイル
__pycache__/
# StepFunctionsステートメント定義変換後のフォルダ
stepfunctions/*/build
**/.vscode/settings.json
# python test
.coverage
.report/
# log
.log

View File

@ -6,6 +6,7 @@ from datetime import datetime, timezone
import boto3
import pytest
from src.controller import controller
from src.parser.json_parser import JsonParser
from src.system_var.constants import YYYYMMDDTHHMMSSTZ
@ -114,6 +115,10 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog):
logger.info(f'##########################')
# Assertion
log_messages = caplog.messages
# ログの目視確認を容易にするため、ローカルファイルに書き出す。
with open('crm_datafetch_test_walk_through_diff.log', 'w', encoding='utf8') as f:
f.write('\n'.join(log_messages))
# ループ前のログ確認
assert 'I-CTRL-01 CRMデータ取得処理を開始します' in log_messages
assert 'I-CTRL-02 データ取得準備処理呼び出し' in log_messages
@ -170,6 +175,10 @@ def test_walk_through(s3_test, s3_client, monkeypatch, caplog):
logger.info(f'##########################')
# ログ再取得
log_messages_all = caplog.messages
# ログの目視確認を容易にするため、ローカルファイルに書き出す。
with open('crm_datafetch_test_walk_through_all.log', 'w', encoding='utf8') as f:
f.write('\n'.join(log_messages_all))
object_info_list_all = object_info_files[1]
# 開始ログなどはテスト済みなのでチェックを省く
for object_info in object_info_list_all['objects']:

View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

View File

@ -104,6 +104,65 @@
"datetime_column": "LastModifiedDate",
"last_fetch_datetime_file_name": "AccountShare_ALL.json",
"upload_file_name": "CRM_AccountShare_ALL_{execute_datetime}"
},
{
"object_name": "ProcessDefinition",
"columns": [
"Id",
"Name",
"DeveloperName",
"Type",
"Description",
"TableEnumOrId",
"LockType",
"State",
"CreatedDate",
"CreatedById",
"LastModifiedDate",
"LastModifiedById",
"SystemModstamp"
],
"is_skip": false,
"is_update_last_fetch_datetime": false,
"last_fetch_datetime_file_name": "ProcessDefinition_ALL.json",
"upload_file_name": "CRM_ProcessDefinition_ALL_{execute_datetime}"
},
{
"object_name": "ProcessInstanceStep",
"columns": [
"Id",
"ProcessInstanceId",
"StepStatus",
"OriginalActorId",
"ActorId",
"Comments",
"StepNodeId",
"ElapsedTimeInDays",
"ElapsedTimeInHours",
"ElapsedTimeInMinutes",
"CreatedDate",
"CreatedById",
"SystemModstamp"
],
"is_skip": false,
"is_update_last_fetch_datetime": false,
"last_fetch_datetime_file_name": "ProcessInstanceStep_ALL.json",
"upload_file_name": "CRM_ProcessInstanceStep_ALL_{execute_datetime}"
},
{
"object_name": "ProcessNode",
"columns": [
"Id",
"Name",
"DeveloperName",
"ProcessDefinitionId",
"Description",
"SystemModstamp"
],
"is_skip": false,
"is_update_last_fetch_datetime": false,
"last_fetch_datetime_file_name": "ProcessNode_ALL.json",
"upload_file_name": "CRM_ProcessNode_ALL_{execute_datetime}"
}
]
}

View File

@ -38,6 +38,7 @@
"Clm_Presentation_Name_vod__c",
"Clm_Presentation_Version_vod__c",
"Clm_Presentation_vod__c",
"Share_Channel_vod__c",
"Call2_vod__r.RecordTypeId"
],
"is_skip": false,
@ -228,6 +229,7 @@
"MSJ_MSL_Comment_for_MR__c",
"MSJ_APD__c",
"Medical_Inquiry_vod__c",
"Suggestion_vod__c",
"MSJ_Call_Type_MSJ__c",
"MSJ_Prescription_Request__c",
"MSJ_Patient_Follow__c",
@ -275,7 +277,11 @@
"MSJ_Interaction_Duration__c",
"MSJ_SIPAGL_1A_date__c",
"MSJ_CoPromotion__c",
"Call_Channel_Formula_vod__c"
"Call_Channel_Formula_vod__c",
"Meeting_Request_vod__c",
"Phone_vod__c",
"Detail_Section_Attribute_vod__c",
"Remote_Meeting_Type_vod__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -403,11 +409,11 @@
"Description_vod__c",
"Keywords_vod__c",
"Content_Channel_vod__c",
"original_material_approved_in_veritas__c",
"keywords__c",
"trade_team__c",
"business_function__c",
"ewizard_link__c",
"business_function__c"
"keywords__c",
"original_material_approved_in_veritas__c",
"trade_team__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -493,18 +499,18 @@
"MSJ_Hospital_ID__c",
"MSJ_Hospital_Name__c",
"MSJ_Hospital__c",
"MSJ_Type_of_Insight__c",
"MSJ_Therapeutic_Area__c",
"MSJ_Starred_Insight__c",
"MSJ_Disclaimer__c",
"MSJ_Not_pharmacovigilance_related__c",
"MSJ_Approval_Status__c",
"MSJ_Insight_Owner_Sharing__c",
"MSJ_Description_J__c",
"MSJ_Summary_J__c",
"MSJ_Disclaimer__c",
"MSJ_HighlightPanel_Display_J__c",
"MSJ_Insight_Owner_Sharing__c",
"MSJ_Level_1J__c",
"MSJ_Level_2J__c",
"MSJ_HighlightPanel_Display_J__c"
"MSJ_Not_pharmacovigilance_related__c",
"MSJ_Starred_Insight__c",
"MSJ_Summary_J__c",
"MSJ_Therapeutic_Area__c",
"MSJ_Type_of_Insight__c",
"MSJ_Level_2J__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -546,7 +552,13 @@
"MSJ_Next_Week_Action_Who_MCC__c",
"MSJ_Next_Week_Action_What_MCC__c",
"MSJ_Next_Week_Action_When_MCC__c",
"MSJ_Target_Patient_Count_MCC__c"
"MSJ_Target_Patient_Count_MCC__c",
"MSJ_Activity_Results_Summary_LC__c",
"MSJ_Next_Week_Action_Where_LC__c",
"MSJ_Next_Week_Action_Who_LC__c",
"MSJ_Next_Week_Action_What_LC__c",
"MSJ_Next_Week_Action_When_LC__c",
"MSJ_Target_Patient_Count_LC__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -611,7 +623,11 @@
"Position_vod__c",
"Talk_Title_vod__c",
"MSJ_Attendee_Reaction__c",
"MSJ_Registration__c"
"MSJ_Registration__c",
"MSJ_DirectMail_Status__c",
"EM_Event_Team_Member_vod__c",
"Group_Name_vod__c",
"Role_vod__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -932,12 +948,12 @@
"MSJ_SK_10__c",
"MSJ_Specific_Action__c",
"MSJ_Training_Point__c",
"MSJ_Efforts_of_Year__c",
"MSJ_Efforts_of_Month__c",
"MSJ_Skill_Task__c",
"MSJ_Action_of_This_Month__c",
"MSJ_Achievement_of_This_Month__c",
"MSJ_Comment_from_AM__c"
"MSJ_Action_of_This_Month__c",
"MSJ_Comment_from_AM__c",
"MSJ_Efforts_of_Month__c",
"MSJ_Efforts_of_Year__c",
"MSJ_Skill_Task__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -1177,6 +1193,7 @@
"PermissionsEditPublicFilters",
"PermissionsEditPublicTemplates",
"PermissionsModifyAllData",
"PermissionsEditBillingInfo",
"PermissionsManageCases",
"PermissionsManageSolutions",
"PermissionsCustomizeApplication",
@ -1222,6 +1239,7 @@
"PermissionsViewContent",
"PermissionsManageEmailClientConfig",
"PermissionsEnableNotifications",
"PermissionsIsSsoEnabled",
"PermissionsManageDataIntegrations",
"PermissionsDistributeFromPersWksp",
"PermissionsViewDataCategories",
@ -1304,6 +1322,7 @@
"PermissionsManageSandboxes",
"PermissionsAutomaticActivityCapture",
"PermissionsImportCustomObjects",
"PermissionsSalesforceIQInbox",
"PermissionsDelegatedTwoFactor",
"PermissionsSelectFilesFromSalesforce",
"PermissionsModerateNetworkUsers",
@ -1345,9 +1364,12 @@
"PermissionsApexRestServices",
"PermissionsEnableCommunityAppLauncher",
"PermissionsGiveRecognitionBadge",
"PermissionsAllowObjectDetection",
"PermissionsSalesforceIQInternal",
"PermissionsUseMySearch",
"PermissionsLtngPromoReserved01UserPerm",
"PermissionsManageSubscriptions",
"PermissionsAllowObjectDetectionTraining",
"PermissionsManageSurveys",
"PermissionsUseAssistantDialog",
"PermissionsUseQuerySuggestions",
@ -1377,6 +1399,7 @@
"PermissionsSkipIdentityConfirmation",
"PermissionsSendCustomNotifications",
"PermissionsPackaging2Delete",
"PermissionsViewRestrictionAndScopingRules",
"PermissionsFSCComprehensiveUserAccess",
"PermissionsManageTrustMeasures",
"PermissionsViewTrustMeasures",
@ -1388,8 +1411,18 @@
"PermissionsManageExternalConnections",
"PermissionsAIViewInsightObjects",
"PermissionsAICreateInsightObjects",
"PermissionsViewMLModels",
"PermissionsNativeWebviewScrolling",
"PermissionsViewDeveloperName",
"PermissionsBypassMFAForUiLogins",
"PermissionsClientSecretRotation",
"PermissionsUpdateReportTypeReferences",
"PermissionsAccessToServiceProcess",
"PermissionsManageOrchInstsAndWorkItems",
"PermissionsCMSECEAuthoringAccess",
"PermissionsEnableIPFSUpload",
"PermissionsEnableBCTransactionPolling",
"PermissionsFSCArcGraphCommunityUser",
"Type",
"UserLicenseId",
"UserType",
@ -1398,6 +1431,7 @@
"LastModifiedDate",
"LastModifiedById",
"SystemModstamp",
"IsSsoEnabled",
"Description",
"LastViewedDate",
"LastReferencedDate"
@ -1473,7 +1507,15 @@
"MSJ_Sent_Date__c",
"MSJ_Email_Subject__c",
"MSJ_Opens__c",
"MSJ_Email_Status__c"
"MSJ_Email_Status__c",
"MSJ_Activity_Name__c",
"MSJ_Asset__c",
"MSJ_CIAM_Account__c",
"MSJ_Channel__c",
"MSJ_Content__c",
"MSJ_Registration_Time__c",
"MSJ_Related_External_Account_Data__c",
"MSJ_Tag_Id__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -1602,7 +1644,9 @@
"MSJ_MSL_Support__c",
"MSJ_Material_Requirement__c",
"MSJ_Hospital_Name_Disp__c",
"MSJ_Hospital__c"
"MSJ_Hospital__c",
"MSJ_Chatbot_Check__c",
"MSJ_File_Attached__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -1747,8 +1791,7 @@
"UserPreferencesShowStateToGuestUsers",
"UserPreferencesShowPostalCodeToGuestUsers",
"UserPreferencesShowCountryToGuestUsers",
"UserPreferencesHideInvoicesRedirectConfirmation",
"UserPreferencesHideStatementsRedirectConfirmation",
"UserPreferencesShowForecastingChangeSignals",
"UserPreferencesPathAssistantCollapsed",
"UserPreferencesCacheDiagnostics",
"UserPreferencesShowEmailToGuestUsers",
@ -1779,6 +1822,9 @@
"UserPreferencesSRHOverrideActivities",
"UserPreferencesNewLightningReportRunPageEnabled",
"UserPreferencesReverseOpenActivitiesView",
"UserPreferencesHasSentWarningEmail",
"UserPreferencesHasSentWarningEmail238",
"UserPreferencesHasSentWarningEmail240",
"UserPreferencesHideBrowseProductRedirectConfirmation",
"UserPreferencesHideOnlineSalesAppWelcomeMat",
"ContactId",
@ -1852,7 +1898,23 @@
"Last_CRMDesktop_Mac_Version_vod__c",
"Last_CRMDesktop_Windows_Sync_vod__c",
"Last_CRMDesktop_Windows_Version_vod__c",
"MSJ_Test_User__c"
"MSJ_Test_User__c",
"Last_Android_Connect_Version_vod__c",
"Last_Android_Connect_vod__c",
"Last_Android_OS_Version_vod__c",
"Last_Android_Sync_vod__c",
"Last_CRMDesktop_Mac_Conn_Ver_vod__c",
"Last_CRMDesktop_Mac_Connect_vod__c",
"Last_CRMDesktop_Windows_Conn_Ver_vod__c",
"Last_CRMDesktop_Windows_Connect_vod__c",
"MS_Graph_API_Authorized_vod__c",
"MS_Outlook_Last_Pull_Activity_vod__c",
"MS_Outlook_Last_Push_Activity_vod__c",
"Enable_MS_Outlook_Calendar_vod__c",
"Last_iPad_Updates_Applied_vod__c",
"Last_iPhone_Updates_Applied_vod__c",
"UserPreferencesHideInvoicesRedirectConfirmation",
"UserPreferencesHideStatementsRedirectConfirmation"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -2190,6 +2252,24 @@
"MSJ_Marketing_Cloud4__c",
"MSJ_Medical_Department__c",
"MSJ_Marketing_Cloud0__c",
"MSJ_CIAM_Number__c",
"MSJ_sID__c",
"MSJ_Application_Materials_Deadline__c",
"MSJ_DI_Person__c",
"MSJ_Pharmaceutical_Department_Memo__c",
"MSJ_Pharmaceutical_Examination_Date__c",
"MSJ_Pharmaceutical_Examination_Member__c",
"MSJ_Pharmaceutical_Examination_Rule_Memo__c",
"MSJ_Pharmaceutical_Examination_Rule__c",
"MSJ_Pharmacy_Director__c",
"MSJ_Visit_Regulation__c",
"MSJ_18_Digit_Salesforce_Id__c",
"MSJ_Clinical_Trial_Count__c",
"MSJ_Congresses_Count__c",
"MSJ_Deletion_H1__c",
"MSJ_External_Link_To_Full_H1_Profile__c",
"MSJ_Industry_Payments_Amount__c",
"MSJ_Publication_Count__c",
"Mobile_ID_vod__pc",
"H1Insights__H1_NPI_Value_for_Testing__pc",
"H1Insights__H1_Person_ID__pc",
@ -2327,7 +2407,8 @@
"zvod_Consent_Line_vod__c",
"zvod_Signature_Capture_vod__c",
"Double_Opt_In_vod__c",
"zvod_Consent_Activity_Tracking_vod__c"
"zvod_Consent_Activity_Tracking_vod__c",
"MSJ_Channel_Type__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -2442,7 +2523,12 @@
"MSJ_MSL_Support__c",
"MSJ_Handover_Comment__c",
"MSJ_Handover_Email__c",
"MSJ_Material_Requirement__c"
"MSJ_Material_Requirement__c",
"MSJ_Material_Used__c",
"MSJ_File_Attached__c",
"MSJ_Off_Label__c",
"MSJ_Include_Quality_Complaint__c",
"MSJ_PSC__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -2502,7 +2588,8 @@
"Bcc_vod__c",
"Audience_vod__c",
"WeChat_Template_ID_vod__c",
"Check_Consent_vod__c"
"Check_Consent_vod__c",
"Publish_Method_vod__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -2566,7 +2653,9 @@
"MSJ_Target_Call_Num__c",
"MSJ_Tech_segment__c",
"MSJ_Title__c",
"MSJ_XLK_Segment__c"
"MSJ_XLK_Segment__c",
"MSJ_Main_Treatment_Plan__c",
"MSJ_Optimal_Visiting_Hours__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -2734,7 +2823,8 @@
"MSJ_Global_Brand__c",
"MSJ_Global_Business_Unit__c",
"MSJ_Molecules__c",
"MSJ_SBU__c"
"MSJ_SBU__c",
"MSJ_Local_Business_Unit__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -3037,7 +3127,8 @@
"Activity_Tracking_Mode_vod__c",
"Email_Source_vod__c",
"Subject_vod__c",
"User_Input_Text_vod__c"
"User_Input_Text_vod__c",
"Email_Group_Id_vod__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
@ -3090,8 +3181,8 @@
"Title_vod__c",
"Suggestion_Survey_vod__c",
"Category_vod__c",
"MSJ_Reason_unformatted__c",
"MSJ_Data_Id__c",
"MSJ_Reason_unformatted__c",
"MSJ_reason_id__c"
],
"is_skip": false,
@ -3208,11 +3299,11 @@
"Mobile_ID_vod__c",
"Color_vod__c",
"Event_Canceled_vod__c",
"Followup_Activity_Type_vod__c",
"MSJ_Data_ID__c",
"MSJ_Reason_ID__c",
"MSJ_Task_External_Id__c",
"MSJ_Task_Source__c",
"Followup_Activity_Type_vod__c",
"MSJ_Visit_Type__c"
],
"is_skip": false,
@ -3265,11 +3356,11 @@
"Mobile_ID_vod__c",
"Color_vod__c",
"Event_Canceled_vod__c",
"Followup_Activity_Type_vod__c",
"MSJ_Data_ID__c",
"MSJ_Reason_ID__c",
"MSJ_Task_External_Id__c",
"MSJ_Task_Source__c",
"Followup_Activity_Type_vod__c",
"MSJ_Visit_Type__c"
],
"is_skip": false,
@ -3369,6 +3460,148 @@
],
"is_skip": false,
"is_update_last_fetch_datetime": true
},
{
"object_name": "ProcessDefinition",
"columns": [
"Id",
"Name",
"DeveloperName",
"Type",
"Description",
"TableEnumOrId",
"LockType",
"State",
"CreatedDate",
"CreatedById",
"LastModifiedDate",
"LastModifiedById",
"SystemModstamp"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
},
{
"object_name": "ProcessInstance",
"columns": [
"Id",
"ProcessDefinitionId",
"TargetObjectId",
"Status",
"CompletedDate",
"LastActorId",
"ElapsedTimeInDays",
"ElapsedTimeInHours",
"ElapsedTimeInMinutes",
"SubmittedById",
"IsDeleted",
"CreatedDate",
"CreatedById",
"LastModifiedDate",
"LastModifiedById",
"SystemModstamp"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
},
{
"object_name": "ProcessInstanceStep",
"columns": [
"Id",
"ProcessInstanceId",
"StepStatus",
"OriginalActorId",
"ActorId",
"Comments",
"StepNodeId",
"ElapsedTimeInDays",
"ElapsedTimeInHours",
"ElapsedTimeInMinutes",
"CreatedDate",
"CreatedById",
"SystemModstamp"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
},
{
"object_name": "ProcessInstanceNode",
"columns": [
"Id",
"IsDeleted",
"CreatedDate",
"CreatedById",
"LastModifiedDate",
"LastModifiedById",
"SystemModstamp",
"ProcessInstanceId",
"ProcessNodeId",
"NodeStatus",
"CompletedDate",
"LastActorId",
"ProcessNodeName",
"ElapsedTimeInDays",
"ElapsedTimeInHours",
"ElapsedTimeInMinutes"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
},
{
"object_name": "ProcessNode",
"columns": [
"Id",
"Name",
"DeveloperName",
"ProcessDefinitionId",
"Description",
"SystemModstamp"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
},
{
"object_name": "ProcessInstanceWorkitem",
"columns": [
"Id",
"ProcessInstanceId",
"OriginalActorId",
"ActorId",
"ElapsedTimeInDays",
"ElapsedTimeInHours",
"ElapsedTimeInMinutes",
"IsDeleted",
"CreatedDate",
"CreatedById",
"SystemModstamp"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
},
{
"object_name": "Directory_vod__c",
"columns": [
"Id",
"OwnerId",
"IsDeleted",
"Name",
"RecordTypeId",
"CreatedDate",
"CreatedById",
"LastModifiedDate",
"LastModifiedById",
"SystemModstamp",
"MayEdit",
"IsLocked",
"LastViewedDate",
"LastReferencedDate",
"Display_Order_vod__c",
"External_ID_vod__c",
"Level_vod__c",
"Parent_Directory_vod__c"
],
"is_skip": false,
"is_update_last_fetch_datetime": true
}
]
}