diff --git a/lambda/NoticeToSlack/index.js b/lambda/NoticeToSlack/index.js new file mode 100644 index 00000000..5c0a7226 --- /dev/null +++ b/lambda/NoticeToSlack/index.js @@ -0,0 +1,37 @@ +"use-strict"; + +const request = require("request-promise"); +exports.handler = (event, context, callback) => { + const attaches = event.Records.map(evt => { + return { + "fallback":`Notification from mbj-newdwh2021 AWS: ${evt.EventSubscriptionArn}`, + "pretext":`Notification from mbj-newdwh2021 AWS: ${evt.EventSubscriptionArn}`, + "color":"#0000D0", + "fields":[ + { + "title": `${evt.Sns.Subject}`, + "value": `${evt.Sns.Timestamp}: ${evt.Sns.Message} (MessageId: ${evt.Sns.MessageId})`, + "short": false + } + ] + }; + }); + request({ + method: "POST", + url: process.env.webhookurl, + body: { + attachments: attaches + + }, + json: true + }).then((body => { + callback(null, `Request succeed. ${body}`); + })).catch((err) => { + callback("failed to request to slack.", { + result: "ERROR", + event: event, + cause: `request failed. ${err}.` + }); + }) + +}; diff --git a/lambda/NoticeToSlack/mbj-newdwh2021-staging-NoticeToSlack.zip b/lambda/NoticeToSlack/mbj-newdwh2021-staging-NoticeToSlack.zip new file mode 100644 index 00000000..53f0c160 Binary files /dev/null and b/lambda/NoticeToSlack/mbj-newdwh2021-staging-NoticeToSlack.zip differ diff --git a/lambda/NoticeToSlack/package.json b/lambda/NoticeToSlack/package.json new file mode 100644 index 00000000..1495f188 --- /dev/null +++ b/lambda/NoticeToSlack/package.json @@ -0,0 +1,15 @@ +{ + "name": "mbj-newdwh2021-staging-noticetoslack", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "request": "^2.88.2", + "request-promise": "^4.2.6" + } +} diff --git a/lambda/PublishFromLog/index.js b/lambda/PublishFromLog/index.js new file mode 100644 index 00000000..f0b608f8 --- /dev/null +++ b/lambda/PublishFromLog/index.js @@ -0,0 +1,30 @@ +const zlib = require("zlib"); +const aws = require("aws-sdk"); +const sns = new aws.SNS({ + apiVersion: "2010-03-31", + region: 'ap-northeast-1' +}); + +exports.handler = function(input, context) { + var payload = new Buffer.from(input.awslogs.data, 'base64'); + zlib.gunzip(payload, function(e, result) { + if (e) { + context.fail(e); + } else { + result = JSON.parse(result.toString('UTF-8')); + const publishMessage = { + Subject: `Detect Error(or Warning) in ${result.logGroup}`, + Message: result.logEvents.map((l) => l.message).join('\n'), + TopicArn: process.env.topicArn + }; + console.log(publishMessage); + + sns.publish(publishMessage, (err, data) => { + console.log(err, data); + if(err){ + context.fail(err); + } + }); + } + }); +}; \ No newline at end of file diff --git a/lambda/PublishFromLog/mbj-newdwh2021-staging-PublishFromLog.zip b/lambda/PublishFromLog/mbj-newdwh2021-staging-PublishFromLog.zip new file mode 100644 index 00000000..6a8526bc Binary files /dev/null and b/lambda/PublishFromLog/mbj-newdwh2021-staging-PublishFromLog.zip differ diff --git a/lambda/PublishFromLog/package.json b/lambda/PublishFromLog/package.json new file mode 100644 index 00000000..1e0d12e7 --- /dev/null +++ b/lambda/PublishFromLog/package.json @@ -0,0 +1,15 @@ +{ + "name": "mbj-newdwh2021-staging-publishfromlog", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "", + "license": "ISC", + "dependencies": { + "aws-sdk": "^2.1011.0", + "zlib": "^1.0.5" + } +} diff --git a/rds_mysql/stored_procedure/crm_data_sync.sql b/rds_mysql/stored_procedure/crm_data_sync.sql new file mode 100644 index 00000000..bb3f3354 --- /dev/null +++ b/rds_mysql/stored_procedure/crm_data_sync.sql @@ -0,0 +1,48 @@ +-- A5M2で実行時に[SQL] - [スラッシュ(/)のみの行でSQLを区切る]に変えてから実行する +-- $$から始まる文字は後からREPLACEする文字を示す独自ルール +-- crm_data_syncストアドプロシージャは、同一セッション内での並列処理を実行することができない +-- 実行者の権限でストアドプロシージャを実行するために、「SQL SECURITY INVOKER」を付与している +CREATE PROCEDURE src02.crm_data_sync(target_table VARCHAR(255), target_table_all VARCHAR(255), target_column VARCHAR(255)) +SQL SECURITY INVOKER +BEGIN + -- 例外処理 + -- エラーが発生した場合に一時テーブルの削除を実施 + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + ROLLBACK; + SIGNAL SQLSTATE '45000' + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; + END; + + SET @error_state = NULL, @error_msg = NULL; + START TRANSACTION; + + -- ①-1 Salesforce側で物理削除されたデータを検出し更新する + SET @update_end_datetime = ' + UPDATE $$target_table$$ tt + SET + tt.end_datetime = CURRENT_TIMESTAMP () + , tt.upd_user = CURRENT_USER () + , tt.upd_date = CURRENT_TIMESTAMP () + WHERE + tt.end_datetime = "9999-12-31 00:00:00" + AND NOT EXISTS ( + SELECT + tta.Id + FROM + $$target_table_all$$ tta + WHERE + tt.Id = tta.Id + AND tt.$$target_column$$ = tta.$$target_column$$ + ) + '; + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table_all$$", target_table_all); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); + PREPARE update_end_datetime_stmt from @update_end_datetime; + EXECUTE update_end_datetime_stmt; + + COMMIT; +END diff --git a/rds_mysql/stored_procedure/crm_history.sql b/rds_mysql/stored_procedure/crm_history.sql new file mode 100644 index 00000000..cc94bbfb --- /dev/null +++ b/rds_mysql/stored_procedure/crm_history.sql @@ -0,0 +1,95 @@ +-- A5M2で実行時に[SQL] - [スラッシュ(/)のみの行でSQLを区切る]に変えてから実行する +-- $$から始まり$$で終わる文字は後からREPLACEする文字を示す独自ルール +-- crm_historyストアドプロシージャは、同一セッション内での並列処理を実行することができない +-- 実行者の権限でストアドプロシージャを実行するために、「SQL SECURITY INVOKER」を付与している +CREATE PROCEDURE src02.crm_history(target_table VARCHAR(255), target_column VARCHAR(255)) +SQL SECURITY INVOKER +BEGIN + -- 例外処理 + -- エラーが発生した場合に一時テーブルの削除を実施 + DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + EXECUTE drop_tmp_table_stmt; + DEALLOCATE PREPARE drop_tmp_table_stmt; + ROLLBACK; + SIGNAL SQLSTATE '45000' + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; + END; + + SET @error_state = NULL, @error_msg = NULL; + START TRANSACTION; + + -- ②-3で利用する一時テーブル削除のSQLを準備 + -- 例外処理でも利用するため先に記述 + SET @drop_tmp_table = ' + DROP TEMPORARY TABLE IF EXISTS $$target_table$$_make_history_tmp + '; + SET @drop_tmp_table = REPLACE(@drop_tmp_table, "$$target_table$$", target_table); + PREPARE drop_tmp_table_stmt from @drop_tmp_table; + + -- ①-1 Salesforce側で更新されたデータの適用開始日時と適用終了日時を設定する + SET @new_history_save = ' + UPDATE $$target_table$$ + SET + start_datetime = $$target_column$$ + , end_datetime = "9999-12-31 00:00:00" + , upd_user = CURRENT_USER() + , upd_date = CURRENT_TIMESTAMP() + WHERE + start_datetime IS NULL + AND end_datetime IS NULL + '; + SET @new_history_save = REPLACE(@new_history_save, "$$target_table$$", target_table); + SET @new_history_save = REPLACE(@new_history_save, "$$target_column$$", target_column); + PREPARE new_history_save_stmt from @new_history_save; + EXECUTE new_history_save_stmt; + DEALLOCATE PREPARE new_history_save_stmt; + + -- ②-1 Salesforce側で更新されたデータを検出用一時テーブルの作成 + SET @make_history_tmp_create = ' + CREATE TEMPORARY TABLE $$target_table$$_make_history_tmp + SELECT + Id + , MIN($$target_column$$) AS min_start_datetime + , MAX(start_datetime) AS max_start_datetime + FROM + $$target_table$$ + WHERE + end_datetime = "9999-12-31 00:00:00" + GROUP BY + Id + HAVING + count(Id) = 2 + '; + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_table$$", target_table); + SET @make_history_tmp_create = REPLACE(@make_history_tmp_create, "$$target_column$$", target_column); + PREPARE make_history_tmp_create_stmt from @make_history_tmp_create; + EXECUTE make_history_tmp_create_stmt; + DEALLOCATE PREPARE make_history_tmp_create_stmt; + + -- ②-2 「②-1」で取得した全件に更新処理を行う + SET @update_end_datetime = ' + UPDATE $$target_table$$ tt + INNER JOIN $$target_table$$_make_history_tmp mht + ON tt.Id = mht.Id + AND tt.start_datetime = mht.min_start_datetime + SET + end_datetime = mht.max_start_datetime - INTERVAL 1 SECOND + , upd_user = CURRENT_USER() + , upd_date = CURRENT_TIMESTAMP() + '; + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_table$$", target_table); + SET @update_end_datetime = REPLACE(@update_end_datetime, "$$target_column$$", target_column); + PREPARE update_end_datetime_stmt from @update_end_datetime; + EXECUTE update_end_datetime_stmt; + DEALLOCATE PREPARE update_end_datetime_stmt; + + -- ②-3 「②-1」で作成した一時テーブルを削除する + EXECUTE drop_tmp_table_stmt; + DEALLOCATE PREPARE drop_tmp_table_stmt; + + COMMIT; + +END diff --git a/s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json b/s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json deleted file mode 100644 index 7453dc17..00000000 --- a/s3/config/crm/last_fetch_datetime/Medical_Event_vod__c.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "last_fetch_datetime_from": "1900-01-01T00:00:00.000Z", - "last_fetch_datetime_to": "" -}