From 56c880be67aca73ea47d5e40004a54c41c8a54ce Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 16 Nov 2022 19:02:52 +0900 Subject: [PATCH] =?UTF-8?q?feat:=20=E7=99=BB=E9=8C=B2=E5=87=A6=E7=90=86?= =?UTF-8?q?=E4=B8=AD=E6=96=AD=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=8C?= =?UTF-8?q?=E3=81=82=E3=82=8B=E5=A0=B4=E5=90=88=E3=80=81=E5=87=A6=E7=90=86?= =?UTF-8?q?=E3=82=92=E8=B5=B7=E5=8B=95=E3=81=95=E3=81=9B=E3=81=AA=E3=81=84?= =?UTF-8?q?=E5=87=A6=E7=90=86=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/dataimport/dataimport.py | 4 +- .../r-data-import-state.json | 52 ++++++++++++++++--- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/lambda/dataimport/dataimport.py b/lambda/dataimport/dataimport.py index 0daf4194..72695078 100644 --- a/lambda/dataimport/dataimport.py +++ b/lambda/dataimport/dataimport.py @@ -61,14 +61,14 @@ def lambda_handler(event, context): if __name__ == '__main__': - # ローカル実行用 + # ローカル実行用。バケット名やファイルキーはよしなに変えてください。 lambda_handler({"Records": [{ "s3": { "bucket": { "name": "test-shimoda-bucket" }, "object": { - "key": "folder/object_key" + "key": "crm/target/CRM_Account_20221116111800.csv" } } }]}, {}) diff --git a/stepfunctions/r-data-import-state/r-data-import-state.json b/stepfunctions/r-data-import-state/r-data-import-state.json index 34d74e82..258880f1 100644 --- a/stepfunctions/r-data-import-state/r-data-import-state.json +++ b/stepfunctions/r-data-import-state/r-data-import-state.json @@ -52,7 +52,38 @@ }, "Comment": "パラメータ設定", "ResultPath": "$.params", - "Next": "data-import" + "Next": "?IsExistsFatalErrorFile" + }, + "?IsExistsFatalErrorFile": { + "Type": "Task", + "Parameters": { + "Bucket.$": "$.InputParams.BucketName", + "Key.$": "States.Format('{}/target/.detected_fatal_error_ecs', $.InputParams.DataSourceName)" + }, + "Resource": "arn:aws:states:::aws-sdk:s3:headObject", + "ResultPath": "$.result", + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "Next": "data-import", + "ResultPath": "$.result" + } + ], + "Comment": "ECSタスク起動エラーファイル存在チェック", + "Next": "PendingDataImportNotice" + }, + "PendingDataImportNotice": { + "Type": "Task", + "Resource": "arn:aws:states:::aws-sdk:sns:publish", + "Parameters": { + "TopicArn.$": "$.params.sns.TopicArn", + "Subject": "データ登録ECSタスク異常通知", + "Message.$": "States.Format('「{}/{}/target/」に登録処理中断ファイル(.detected_fatal_error_ecs)が見つかったため、データ登録の起動を中断しました。\n', $.InputParams.BucketName, $.InputParams.DataSourceName)" + }, + "End": true, + "Comment": "データ登録処理起動中断通知" }, "data-import": { "Type": "Task", @@ -67,7 +98,9 @@ "ResultPath": "$.result", "Retry": [ { - "ErrorEquals": ["States.ALL"], + "ErrorEquals": [ + "States.ALL" + ], "BackoffRate": 2, "IntervalSeconds": 3, "MaxAttempts": 3 @@ -75,7 +108,9 @@ ], "Catch": [ { - "ErrorEquals": ["States.ALL"], + "ErrorEquals": [ + "States.ALL" + ], "Next": "CreateImportPendingFile", "ResultPath": "$.result" } @@ -88,22 +123,23 @@ "Parameters": { "Body": "", "Bucket.$": "$.InputParams.BucketName", - "Key.$": "States.Format('{}.pending-import', $.InputParams.TargetKey)" + "Key.$": "States.Format('{}/target/.detected_fatal_error_ecs', $.InputParams.DataSourceName)" }, "Resource": "arn:aws:states:::aws-sdk:s3:putObject", "Next": "FailedNotice", - "Comment": "データ登録保留ファイル作成", + "Comment": "データ登録処理起動中断ファイル作成", "ResultPath": "$.result" }, "FailedNotice": { "Type": "Task", - "Resource": "arn:aws:states:::sns:publish", + "Resource": "arn:aws:states:::aws-sdk:sns:publish", "Parameters": { "TopicArn.$": "$.params.sns.TopicArn", "Subject": "データ登録ECSタスク異常通知", - "Message.$": "States.Format('【{}】{}のデータ登録が失敗したため、登録中断ファイルを作成しました。\n\n バケット名：{}\n取込ファイルパス：{}\n\n', $.InputParams.DataSourceName, $.InputParams.FileName, $.InputParams.BucketName, $.InputParams.TargetKey)" + "Message.$": "States.Format('【{}】{}のデータ登録が失敗したため、登録処理中断ファイル(.detected_fatal_error_ecs)を作成しました。\n\n バケット名：{}\n取込ファイルパス：{}\n\n', $.InputParams.DataSourceName, $.InputParams.FileName, $.InputParams.BucketName, $.InputParams.TargetKey)" }, - "Next": "Fail" + "Next": "Fail", + "Comment": "データ登録処理起動失敗通知" }, "Success": { "Type": "Succeed",