nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CRMエンハンス＆オブジェクト追加対応

Browse Source
This commit is contained in:
Asuka Asakura 2024-04-26 15:24:33 +09:00
parent 88e8b1a0b1
commit 6461e24a0f
35 changed files with 435 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
ecs/crm-datafetch/.gitignore vendored Normal file
View File

@ -0,0 +1,20 @@
# Node.jsで実装されたLambdaの管理対象外ファイル群
package-lock.json
node_modules/
# ローカル確認用環境変数ファイル
.env
# Pythonの仮想環境ファイル
.venv
# pythonのキャッシュファイル
__pycache__/
# StepFunctionsステートメント定義変換後のフォルダ
stepfunctions/*/build
**/.vscode/settings.json
# python test
.coverage
.report/
# log
.log

37
lambda/NoticeToSlack/index.js Normal file
View File

@ -0,0 +1,37 @@
"use-strict";
const request = require("request-promise");
exports.handler = (event, context, callback) => {
const attaches = event.Records.map(evt => {
return {
"fallback":`Notification from mbj-newdwh2021 AWS: ${evt.EventSubscriptionArn}`,
"pretext":`Notification from mbj-newdwh2021 AWS: ${evt.EventSubscriptionArn}`,
"color":"#0000D0",
"fields":[
{
"title": `${evt.Sns.Subject}`,
"value": `${evt.Sns.Timestamp}: ${evt.Sns.Message} (MessageId: ${evt.Sns.MessageId})`,
"short": false
}
]
};
});
request({
method: "POST",
url: process.env.webhookurl,
body: {
attachments: attaches
},
json: true
}).then((body => {
callback(null, `Request succeed. ${body}`);
})).catch((err) => {
callback("failed to request to slack.", {
result: "ERROR",
event: event,
cause: `request failed. ${err}.`
});
})
};

BIN
lambda/NoticeToSlack/mbj-newdwh2021-staging-NoticeToSlack.zip Normal file
View File

Binary file not shown.

15
lambda/NoticeToSlack/package.json Normal file
View File

@ -0,0 +1,15 @@
{
"name": "mbj-newdwh2021-staging-noticetoslack",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"request": "^2.88.2",
"request-promise": "^4.2.6"
}
}

30
lambda/PublishFromLog/index.js Normal file
View File

@ -0,0 +1,30 @@
const zlib = require("zlib");
const aws = require("aws-sdk");
const sns = new aws.SNS({
apiVersion: "2010-03-31",
region: 'ap-northeast-1'
});
exports.handler = function(input, context) {
var payload = new Buffer.from(input.awslogs.data, 'base64');
zlib.gunzip(payload, function(e, result) {
if (e) {
context.fail(e);
} else {
result = JSON.parse(result.toString('UTF-8'));
const publishMessage = {
Subject: `Detect Error(or Warning) in ${result.logGroup}`,
Message: result.logEvents.map((l) => l.message).join('\n'),
TopicArn: process.env.topicArn
};
console.log(publishMessage);
sns.publish(publishMessage, (err, data) => {
console.log(err, data);
if(err){
context.fail(err);
}
});
}
});
};

BIN
lambda/PublishFromLog/mbj-newdwh2021-staging-PublishFromLog.zip Normal file
View File

Binary file not shown.

15
lambda/PublishFromLog/package.json Normal file
View File

@ -0,0 +1,15 @@
{
"name": "mbj-newdwh2021-staging-publishfromlog",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"author": "",
"license": "ISC",
"dependencies": {
"aws-sdk": "^2.1011.0",
"zlib": "^1.0.5"
}
}

48
rds_mysql/stored_procedure/crm_data_sync.sql Normal file
View File

@ -0,0 +1,48 @@
-- A5M2で実行時に[SQL] - [スラッシュ(/)のみの行でSQLを区切る]に変えてから実行する
-- $$から始まる文字は後からREPLACEする文字を示す独自ルール
-- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない
-- 実行者の権限でストアドプロシージャを実行するために、「SQL SECURITY INVOKER」を付与している
CREATE PROCEDURE src02.crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255))
SQL SECURITY INVOKER
BEGIN
-- 例外処理
-- エラーが発生した場合に一時テーブルの削除を実施
DECLARE EXIT HANDLER FOR SQLEXCEPTION
BEGIN
GET DIAGNOSTICS CONDITION 1
@error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT;
ROLLBACK;
SIGNAL SQLSTATE '45000'
SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg;
END;
SET @error_state = NULL, @error_msg = NULL;
START TRANSACTION;
-- ①-1 Salesforce側で物理削除されたデータを検出し更新する
SET @update_end_datetime = '
UPDATE $$target_table$$ tt
SET
tt.end_datetime = CURRENT_TIMESTAMP ()
, tt.upd_user = CURRENT_USER ()
, tt.upd_date = CURRENT_TIMESTAMP ()
WHERE
tt.end_datetime = "9999-12-31 00:00:00"
AND NOT EXISTS (
SELECT
tta.Id
FROM
$$target_table_all$$ tta
WHERE
tt.Id = tta.Id
AND tt.$$target_column$$ = tta.$$target_column$$
)
';
SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table);
SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all);
SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column);
PREPARE update_end_datetime_stmt from @update_end_datetime;
EXECUTE update_end_datetime_stmt;
COMMIT;
END

95
rds_mysql/stored_procedure/crm_history.sql Normal file
View File

@ -0,0 +1,95 @@
-- A5M2で実行時に[SQL] - [スラッシュ(/)のみの行でSQLを区切る]に変えてから実行する
-- $$から始まり$$で終わる文字は後からREPLACEする文字を示す独自ルール
-- crm_historyストアドプロシージャは、同一セッション内での並列処理を実行することができない
-- 実行者の権限でストアドプロシージャを実行するために、「SQL SECURITY INVOKER」を付与している
CREATE PROCEDURE src02.crm_history(target_table VARCHAR(255), target_column VARCHAR(255))
SQL SECURITY INVOKER
BEGIN
-- 例外処理
-- エラーが発生した場合に一時テーブルの削除を実施
DECLARE EXIT HANDLER FOR SQLEXCEPTION
BEGIN
GET DIAGNOSTICS CONDITION 1
@error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT;
EXECUTE drop_tmp_table_stmt;
DEALLOCATE PREPARE drop_tmp_table_stmt;
ROLLBACK;
SIGNAL SQLSTATE '45000'
SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg;
END;
SET @error_state = NULL, @error_msg = NULL;
START TRANSACTION;
-- ②-3で利用する一時テーブル削除のSQLを準備
-- 例外処理でも利用するため先に記述
SET @drop_tmp_table = '
DROP TEMPORARY TABLE IF EXISTS $$target_table$$_make_history_tmp
';
SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table$$", target_table);
PREPARE drop_tmp_table_stmt from @drop_tmp_table;
-- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する
SET @new_history_save = '
UPDATE $$target_table$$
SET
start_datetime = $$target_column$$
, end_datetime = "9999-12-31 00:00:00"
, upd_user = CURRENT_USER()
, upd_date = CURRENT_TIMESTAMP()
WHERE
start_datetime IS NULL
AND end_datetime IS NULL
';
SET @new_history_save = REPLACE(@new_history_save, "$$target_table$$", target_table);
SET @new_history_save = REPLACE(@new_history_save, "$$target_column$$", target_column);
PREPARE new_history_save_stmt from @new_history_save;
EXECUTE new_history_save_stmt;
DEALLOCATE PREPARE new_history_save_stmt;
-- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成
SET @make_history_tmp_create = '
CREATE TEMPORARY TABLE $$target_table$$_make_history_tmp
SELECT
Id
, MIN($$target_column$$) AS min_start_datetime
, MAX(start_datetime) AS max_start_datetime
FROM
$$target_table$$
WHERE
end_datetime = "9999-12-31 00:00:00"
GROUP BY
Id
HAVING
count(Id) = 2
';
SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_table$$", target_table);
SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_column$$", target_column);
PREPARE make_history_tmp_create_stmt from @make_history_tmp_create;
EXECUTE make_history_tmp_create_stmt;
DEALLOCATE PREPARE make_history_tmp_create_stmt;
-- ②-2 「②-1」で取得した全件に更新処理を行う
SET @update_end_datetime = '
UPDATE $$target_table$$ tt
INNER JOIN $$target_table$$_make_history_tmp mht
ON tt.Id = mht.Id
AND tt.start_datetime = mht.min_start_datetime
SET
end_datetime = mht.max_start_datetime - INTERVAL 1 SECOND
, upd_user = CURRENT_USER()
, upd_date = CURRENT_TIMESTAMP()
';
SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table);
SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column);
PREPARE update_end_datetime_stmt from @update_end_datetime;
EXECUTE update_end_datetime_stmt;
DEALLOCATE PREPARE update_end_datetime_stmt;
-- ②-3 「②-1」で作成した一時テーブルを削除する
EXECUTE drop_tmp_table_stmt;
DEALLOCATE PREPARE drop_tmp_table_stmt;
COMMIT;
END

0
s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json → s3/config/crm/last_fetch_datetime/Directory_vod__c.json
View File

4
s3/config/crm/last_fetch_datetime/ProcessDefinition.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/ProcessDefinition_ALL.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/ProcessInstance.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/ProcessInstanceNode.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/ProcessInstanceStep.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/ProcessInstanceStep_ALL.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/ProcessInstanceWorkitem.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/ProcessNode.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

4
s3/config/crm/last_fetch_datetime/ProcessNode_ALL.json Normal file
View File

@ -0,0 +1,4 @@
{
"last_fetch_datetime_from": "1900-01-01T00:00:00.000Z",
"last_fetch_datetime_to": ""
}

13
s3/data/crm/settings/CRM_Directory_vod__c.txt Normal file
View File

@ -0,0 +1,13 @@
CRM
,
utf-8
"
CRLF
1
18
Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Display_Order_vod__c,External_ID_vod__c,Level_vod__c,Parent_Directory_vod__c
Id,OwnerId,IsDeleted,Name,RecordTypeId,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,MayEdit,IsLocked,LastViewedDate,LastReferencedDate,Display_Order_vod__c,External_ID_vod__c,Level_vod__c,Parent_Directory_vod__c
src02.crm_Directory_vod__c
org02.crm_Directory_vod__c

13
s3/data/crm/settings/CRM_ProcessDefinition.txt Normal file
View File

@ -0,0 +1,13 @@
CRM
,
utf-8
"
CRLF
1
13
Id,Name,DeveloperName,Type,Description,TableEnumOrId,LockType,State,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp
Id,Name,DeveloperName,Type,Description,TableEnumOrId,LockType,State,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp
src02.crm_ProcessDefinition
org02.crm_ProcessDefinition
CRM_ProcessDefinition_ex.sql

14
s3/data/crm/settings/CRM_ProcessDefinition_ALL.txt Normal file
View File

@ -0,0 +1,14 @@
CRM
,
utf-8
"
CRLF
1
13
Id,Name,DeveloperName,Type,Description,TableEnumOrId,LockType,State,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp
Id,Name,DeveloperName,Type,Description,TableEnumOrId,LockType,State,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp
src02.crm_ProcessDefinition_all
org02.crm_ProcessDefinition_all
CRM_ProcessDefinition_ALL_ex.sql
truncate_src_table:src02.crm_ProcessDefinition_all

1
s3/data/crm/settings/CRM_ProcessDefinition_ALL_ex.sql Normal file
View File

@ -0,0 +1 @@
CALL internal02.crm_data_sync('src02.crm_ProcessDefinition', 'src02.crm_ProcessDefinition_all', 'SystemModstamp');

1
s3/data/crm/settings/CRM_ProcessDefinition_ex.sql Normal file
View File

@ -0,0 +1 @@
CALL internal02.crm_history('src02.crm_ProcessDefinition', 'SystemModstamp');

13
s3/data/crm/settings/CRM_ProcessInstance.txt Normal file
View File

@ -0,0 +1,13 @@
CRM
,
utf-8
"
CRLF
1
16
Id,ProcessDefinitionId,TargetObjectId,Status,CompletedDate,LastActorId,ElapsedTimeInDays,ElapsedTimeInHours,ElapsedTimeInMinutes,SubmittedById,IsDeleted,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp
Id,ProcessDefinitionId,TargetObjectId,Status,CompletedDate,LastActorId,ElapsedTimeInDays,ElapsedTimeInHours,ElapsedTimeInMinutes,SubmittedById,IsDeleted,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp
src02.crm_ProcessInstance
org02.crm_ProcessInstance

13
s3/data/crm/settings/CRM_ProcessInstanceNode.txt Normal file
View File

@ -0,0 +1,13 @@
CRM
,
utf-8
"
CRLF
1
16
Id,IsDeleted,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,ProcessInstanceId,ProcessNodeId,NodeStatus,CompletedDate,LastActorId,ProcessNodeName,ElapsedTimeInDays,ElapsedTimeInHours,ElapsedTimeInMinutes
Id,IsDeleted,CreatedDate,CreatedById,LastModifiedDate,LastModifiedById,SystemModstamp,ProcessInstanceId,ProcessNodeId,NodeStatus,CompletedDate,LastActorId,ProcessNodeName,ElapsedTimeInDays,ElapsedTimeInHours,ElapsedTimeInMinutes
src02.crm_ProcessInstanceNode
org02.crm_ProcessInstanceNode

13
s3/data/crm/settings/CRM_ProcessInstanceStep.txt Normal file
View File

@ -0,0 +1,13 @@
CRM
,
utf-8
"
CRLF
1
13
Id,ProcessInstanceId,StepStatus,OriginalActorId,ActorId,Comments,StepNodeId,ElapsedTimeInDays,ElapsedTimeInHours,ElapsedTimeInMinutes,CreatedDate,CreatedById,SystemModstamp
Id,ProcessInstanceId,StepStatus,OriginalActorId,ActorId,Comments,StepNodeId,ElapsedTimeInDays,ElapsedTimeInHours,ElapsedTimeInMinutes,CreatedDate,CreatedById,SystemModstamp
src02.crm_ProcessInstanceStep
org02.crm_ProcessInstanceStep
CRM_ProcessInstanceStep_ex.sql

14
s3/data/crm/settings/CRM_ProcessInstanceStep_ALL.txt Normal file
View File

@ -0,0 +1,14 @@
CRM
,
utf-8
"
CRLF
1
13
Id,ProcessInstanceId,StepStatus,OriginalActorId,ActorId,Comments,StepNodeId,ElapsedTimeInDays,ElapsedTimeInHours,ElapsedTimeInMinutes,CreatedDate,CreatedById,SystemModstamp
Id,ProcessInstanceId,StepStatus,OriginalActorId,ActorId,Comments,StepNodeId,ElapsedTimeInDays,ElapsedTimeInHours,ElapsedTimeInMinutes,CreatedDate,CreatedById,SystemModstamp
src02.crm_ProcessInstanceStep_all
org02.crm_ProcessInstanceStep_all
CRM_ProcessInstanceStep_ALL_ex.sql
truncate_src_table:src02.crm_ProcessInstanceStep_all

1
s3/data/crm/settings/CRM_ProcessInstanceStep_ALL_ex.sql Normal file
View File

@ -0,0 +1 @@
CALL internal02.crm_data_sync('src02.crm_ProcessInstanceStep', 'src02.crm_ProcessInstanceStep_all', 'SystemModstamp');

1
s3/data/crm/settings/CRM_ProcessInstanceStep_ex.sql Normal file
View File

@ -0,0 +1 @@
CALL internal02.crm_history('src02.crm_ProcessInstanceStep', 'SystemModstamp');

13
s3/data/crm/settings/CRM_ProcessInstanceWorkitem.txt Normal file
View File

@ -0,0 +1,13 @@
CRM
,
utf-8
"
CRLF
1
11
Id,ProcessInstanceId,OriginalActorId,ActorId,ElapsedTimeInDays,ElapsedTimeInHours,ElapsedTimeInMinutes,IsDeleted,CreatedDate,CreatedById,SystemModstamp
Id,ProcessInstanceId,OriginalActorId,ActorId,ElapsedTimeInDays,ElapsedTimeInHours,ElapsedTimeInMinutes,IsDeleted,CreatedDate,CreatedById,SystemModstamp
src02.crm_ProcessInstanceWorkitem
org02.crm_ProcessInstanceWorkitem

13
s3/data/crm/settings/CRM_ProcessNode.txt Normal file
View File

@ -0,0 +1,13 @@
CRM
,
utf-8
"
CRLF
1
6
Id,Name,DeveloperName,ProcessDefinitionId,Description,SystemModstamp
Id,Name,DeveloperName,ProcessDefinitionId,Description,SystemModstamp
src02.crm_ProcessNode
org02.crm_ProcessNode
CRM_ProcessNode_ex.sql

14
s3/data/crm/settings/CRM_ProcessNode_ALL.txt Normal file
View File

@ -0,0 +1,14 @@
CRM
,
utf-8
"
CRLF
1
6
Id,Name,DeveloperName,ProcessDefinitionId,Description,SystemModstamp
Id,Name,DeveloperName,ProcessDefinitionId,Description,SystemModstamp
src02.crm_ProcessNode_all
org02.crm_ProcessNode_all
CRM_ProcessNode_ALL_ex.sql
truncate_src_table:src02.crm_ProcessNode_all

1
s3/data/crm/settings/CRM_ProcessNode_ALL_ex.sql Normal file
View File

@ -0,0 +1 @@
CALL internal02.crm_data_sync('src02.crm_ProcessNode', 'src02.crm_ProcessNode_all', 'SystemModstamp');

1
s3/data/crm/settings/CRM_ProcessNode_ex.sql Normal file
View File

@ -0,0 +1 @@
CALL internal02.crm_history('src02.crm_ProcessNode', 'SystemModstamp');