From cb6adbb45d0430de6bdff11ad093b28238b8b14d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 16 Nov 2022 11:06:23 +0900 Subject: [PATCH 1/8] =?UTF-8?q?feat:=20=E3=83=87=E3=83=BC=E3=82=BF?= =?UTF-8?q?=E7=99=BB=E9=8C=B2ECS=E3=81=AE=E3=83=AA=E3=83=88=E3=83=A9?= =?UTF-8?q?=E3=82=A4=E3=82=92=E8=A1=8C=E3=81=86=E3=81=9F=E3=82=81=E3=80=81?= =?UTF-8?q?StepFunctions=E3=82=92=E8=B5=B7=E5=8B=95=E3=81=99=E3=82=8B?= =?UTF-8?q?=E3=82=88=E3=81=86=E3=81=AB=E3=81=97=E3=81=A6=E3=81=BF=E3=81=9F?= =?UTF-8?q?=E3=82=82=E3=81=AE=20=E6=94=B9=E8=89=AF=E3=81=AE=E4=BD=99?= =?UTF-8?q?=E5=9C=B0=E6=9C=89=E3=82=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/dataimport/dataimport.py | 118 ++++++++++++++++++++++++-------- 1 file changed, 88 insertions(+), 30 deletions(-) diff --git a/lambda/dataimport/dataimport.py b/lambda/dataimport/dataimport.py index 00fe2f86..bb2df665 100644 --- a/lambda/dataimport/dataimport.py +++ b/lambda/dataimport/dataimport.py @@ -1,8 +1,11 @@ +import json import os from datetime import datetime + import boto3 # 環境変数 +SFN_STATE_MACHINE_ARN=os.environ["SFN_STATE_MACHINE_ARN"] CLUSTER_NAME = os.environ["CLUSTER_NAME"] TASK_NAME = os.environ["TASK_NAME"] CONTAINER_NAME = os.environ["CONTAINER_NAME"] @@ -14,6 +17,7 @@ MODE = os.environ["MODE"] # クラス変数 ecs_client = boto3.client('ecs') +sfn_client = boto3.client('stepfunctions') def lambda_handler(event, context): @@ -29,38 +33,92 @@ def lambda_handler(event, context): print(f'{datetime.now():%Y-%m-%d %H:%M:%S} Info I-3 ファイル名：{event_file_name}') print(f'{datetime.now():%Y-%m-%d %H:%M:%S} Info I-4 データソース名：{event_data_source_name}') - # ECSを起動する - response = ecs_client.run_task( - launchType='FARGATE', - cluster=CLUSTER_NAME, - taskDefinition=TASK_NAME, - networkConfiguration={ - "awsvpcConfiguration": { - "subnets": [ - SUBNET_ID_AP_NORTHEAST_1A, - SUBNET_ID_AP_NORTHEAST_1D, - ], - "securityGroups": [ - SECURITY_GROUP_ID_ECSALL, - SECURITY_GROUP_ID_ECSDATAIMPORT, - ], + # StepFunctionsを起動する + response = sfn_client.start_execution( + stateMachineArn=SFN_STATE_MACHINE_ARN, + input=json.dumps( + { + 'inputParams': { + 'ecs': { + 'Cluster': CLUSTER_NAME, + 'TaskDefinition': TASK_NAME, + 'network': { + # TODO: 配列もパラメータから組めるなら、StepFunctions側で組み立てる + "Subnets": [ + SUBNET_ID_AP_NORTHEAST_1A, + SUBNET_ID_AP_NORTHEAST_1D, + ], + "SecurityGroups": [ + SECURITY_GROUP_ID_ECSALL, + SECURITY_GROUP_ID_ECSDATAIMPORT, + ], + + }, + 'overrides': { + "ContainerName": CONTAINER_NAME, + "Environment": [ + # TODO: 環境変数以外でも使いたいので、外出しする + {"Name": 'BUCKET_NAME', "Value": event_bucket_name}, + {"Name": 'TARGET_KEY', "Value": event_object_key}, + {"Name": 'DATA_SOURCE_NAME', "Value": event_data_source_name}, + {"Name": 'FILE_NAME', "Value": event_file_name}, + {"Name": 'MODE', "Value": MODE}, + ], + }, + }, + } } - }, - overrides={ - "containerOverrides": [ - { - "name": CONTAINER_NAME, - "environment": [ - {"name": 'BUCKET_NAME', "value": event_bucket_name}, - {"name": 'TARGET_KEY', "value": event_object_key}, - {"name": 'DATA_SOURCE_NAME', "value": event_data_source_name}, - {"name": 'FILE_NAME', "value": event_file_name}, - {"name": 'MODE', "value": MODE}, - ], - }, - ], - }, + ) ) + # # ECSを起動する + # response = ecs_client.run_task( + # launchType='FARGATE', + # cluster=CLUSTER_NAME, + # taskDefinition=TASK_NAME, + # networkConfiguration={ + # "awsvpcConfiguration": { + # "subnets": [ + # SUBNET_ID_AP_NORTHEAST_1A, + # SUBNET_ID_AP_NORTHEAST_1D, + # ], + # "securityGroups": [ + # SECURITY_GROUP_ID_ECSALL, + # SECURITY_GROUP_ID_ECSDATAIMPORT, + # ], + # } + # }, + # overrides={ + # "containerOverrides": [ + # { + # "name": CONTAINER_NAME, + # "environment": [ + # {"name": 'BUCKET_NAME', "value": event_bucket_name}, + # {"name": 'TARGET_KEY', "value": event_object_key}, + # {"name": 'DATA_SOURCE_NAME', "value": event_data_source_name}, + # {"name": 'FILE_NAME', "value": event_file_name}, + # {"name": 'MODE', "value": MODE}, + # ], + # }, + # ], + # }, + # ) + + # TODO: コメントと設計修正 print(f'{datetime.now():%Y-%m-%d %H:%M:%S} Info I-5 ECS起動レスポンス：{str(response)}') print(f'{datetime.now():%Y-%m-%d %H:%M:%S} Info I-6 駆動処理終了') + + + +if __name__ == '__main__': + # ローカル実行用 + lambda_handler({"Records": [{ + "s3": { + "bucket": { + "name": "test-shimoda-bucket" + }, + "object": { + "key": "folder/object_key" + } + } + }]}, {}) From deac56ef5096064b1bd85b8c143dff7a6efa64ac Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 16 Nov 2022 11:06:23 +0900 Subject: [PATCH 2/8] =?UTF-8?q?feat:=20Sfn=E3=81=AB=E6=B8=A1=E3=81=99?= =?UTF-8?q?=E3=83=91=E3=83=A9=E3=83=A1=E3=83=BC=E3=82=BF=E3=82=92=E6=9C=80?= =?UTF-8?q?=E9=81=A9=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/dataimport/dataimport.py | 78 ++++++--------------------------- 1 file changed, 14 insertions(+), 64 deletions(-) diff --git a/lambda/dataimport/dataimport.py b/lambda/dataimport/dataimport.py index bb2df665..0daf4194 100644 --- a/lambda/dataimport/dataimport.py +++ b/lambda/dataimport/dataimport.py @@ -16,7 +16,6 @@ SECURITY_GROUP_ID_ECSDATAIMPORT = os.environ["SECURITY_GROUP_ID_ECSDATAIMPORT"] MODE = os.environ["MODE"] # クラス変数 -ecs_client = boto3.client('ecs') sfn_client = boto3.client('stepfunctions') @@ -38,73 +37,24 @@ def lambda_handler(event, context): stateMachineArn=SFN_STATE_MACHINE_ARN, input=json.dumps( { - 'inputParams': { - 'ecs': { - 'Cluster': CLUSTER_NAME, - 'TaskDefinition': TASK_NAME, - 'network': { - # TODO: 配列もパラメータから組めるなら、StepFunctions側で組み立てる - "Subnets": [ - SUBNET_ID_AP_NORTHEAST_1A, - SUBNET_ID_AP_NORTHEAST_1D, - ], - "SecurityGroups": [ - SECURITY_GROUP_ID_ECSALL, - SECURITY_GROUP_ID_ECSDATAIMPORT, - ], - - }, - 'overrides': { - "ContainerName": CONTAINER_NAME, - "Environment": [ - # TODO: 環境変数以外でも使いたいので、外出しする - {"Name": 'BUCKET_NAME', "Value": event_bucket_name}, - {"Name": 'TARGET_KEY', "Value": event_object_key}, - {"Name": 'DATA_SOURCE_NAME', "Value": event_data_source_name}, - {"Name": 'FILE_NAME', "Value": event_file_name}, - {"Name": 'MODE', "Value": MODE}, - ], - }, - }, + 'InputParams': { + 'ClusterName': CLUSTER_NAME, + 'TaskName': TASK_NAME, + 'SubNetIdApNorthEast1A': SUBNET_ID_AP_NORTHEAST_1A, + 'SubNetIdApNorthEast1D': SUBNET_ID_AP_NORTHEAST_1D, + 'SecurityGroupIdECSAll': SECURITY_GROUP_ID_ECSALL, + 'SecurityGroupIdECSDataImport': SECURITY_GROUP_ID_ECSDATAIMPORT, + 'ContainerName': CONTAINER_NAME, + 'BucketName': event_bucket_name, + 'TargetKey': event_object_key, + 'DataSourceName': event_data_source_name, + 'FileName': event_file_name, + 'Mode': MODE } } ) ) - # # ECSを起動する - # response = ecs_client.run_task( - # launchType='FARGATE', - # cluster=CLUSTER_NAME, - # taskDefinition=TASK_NAME, - # networkConfiguration={ - # "awsvpcConfiguration": { - # "subnets": [ - # SUBNET_ID_AP_NORTHEAST_1A, - # SUBNET_ID_AP_NORTHEAST_1D, - # ], - # "securityGroups": [ - # SECURITY_GROUP_ID_ECSALL, - # SECURITY_GROUP_ID_ECSDATAIMPORT, - # ], - # } - # }, - # overrides={ - # "containerOverrides": [ - # { - # "name": CONTAINER_NAME, - # "environment": [ - # {"name": 'BUCKET_NAME', "value": event_bucket_name}, - # {"name": 'TARGET_KEY', "value": event_object_key}, - # {"name": 'DATA_SOURCE_NAME', "value": event_data_source_name}, - # {"name": 'FILE_NAME', "value": event_file_name}, - # {"name": 'MODE', "value": MODE}, - # ], - # }, - # ], - # }, - # ) - - # TODO: コメントと設計修正 - print(f'{datetime.now():%Y-%m-%d %H:%M:%S} Info I-5 ECS起動レスポンス：{str(response)}') + print(f'{datetime.now():%Y-%m-%d %H:%M:%S} Info I-5 StepFunctions起動レスポンス：{str(response)}') print(f'{datetime.now():%Y-%m-%d %H:%M:%S} Info I-6 駆動処理終了') From dc48dd8bf10a1e3f062b78ae28de8cece0dad6d3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 16 Nov 2022 11:52:07 +0900 Subject: [PATCH 3/8] =?UTF-8?q?feat:=20=E3=82=B9=E3=83=86=E3=83=BC?= =?UTF-8?q?=E3=83=88=E3=83=9E=E3=82=B7=E3=83=B3=E3=83=86=E3=83=B3=E3=83=97?= =?UTF-8?q?=E3=83=AC=E3=83=BC=E3=83=88=E3=81=A8=E7=BD=AE=E3=81=8D=E6=8F=9B?= =?UTF-8?q?=E3=81=88=E8=A8=AD=E5=AE=9A=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- stepfunctions/TOOLS/convert_config.yaml | 15 +++ .../r-data-import-state.json | 123 ++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 stepfunctions/r-data-import-state/r-data-import-state.json diff --git a/stepfunctions/TOOLS/convert_config.yaml b/stepfunctions/TOOLS/convert_config.yaml index 2b6487ea..f393460f 100644 --- a/stepfunctions/TOOLS/convert_config.yaml +++ b/stepfunctions/TOOLS/convert_config.yaml @@ -62,3 +62,18 @@ config: SG_ECS_ALL: *PRD_SG_ECS_ALL # セキュリティグループ(ecs-crm-datafetch) SG_CRM_DATAFETCH: *PRD_SG_CRM_DATAFETCH + +config: + r-data-import-state: + # ステージング環境 + staging: + # AWSアカウントID + AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID + # 東京リージョン + REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1 + # 本番環境 + product: + # AWSアカウントID + AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID + # 東京リージョン + REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1 diff --git a/stepfunctions/r-data-import-state/r-data-import-state.json b/stepfunctions/r-data-import-state/r-data-import-state.json new file mode 100644 index 00000000..41de6967 --- /dev/null +++ b/stepfunctions/r-data-import-state/r-data-import-state.json @@ -0,0 +1,123 @@ +{ + "Comment": "MeDaCa データ取込ECSタスク起動 ステートマシン", + "StartAt": "params", + "States": { + "params": { + "Type": "Pass", + "Parameters": { + "sns": { + "TopicArn": "arn:aws:sns:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:nds-notice-#{ENV_NAME}" + }, + "ecs": { + "LaunchType": "FARGATE", + "Cluster.$": "$.InputParams.ClusterName", + "TaskDefinition.$": "$.InputParams.TaskName", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "Subnets.$": "States.Array($.InputParams.SubNetIdApNorthEast1A, $.InputParams.SubNetIdApNorthEast1D)", + "SecurityGroups.$": "States.Array($.InputParams.SecurityGroupIdECSAll, $.InputParams.SecurityGroupIdECSDataImport)", + "AssignPublicIp": "DISABLED" + } + }, + "Overrides": { + "ContainerOverrides": [ + { + "Name.$": "$.InputParams.ContainerName", + "Environment": [ + { + "Name": "BUCKET_NAME", + "Value.$": "$.InputParams.BucketName" + }, + { + "Name": "TARGET_KEY", + "Value.$": "$.InputParams.TargetKey" + }, + { + "Name": "DATA_SOURCE_NAME", + "Value.$": "$.InputParams.DataSourceName" + }, + { + "Name": "FILE_NAME", + "Value.$": "$.InputParams.FileName" + }, + { + "Name": "MODE", + "Value.$": "$.InputParams.Mode" + } + ] + } + ] + } + } + }, + "Comment": "パラメータ設定", + "ResultPath": "$.params", + "Next": "data-import" + }, + "data-import": { + "Type": "Task", + "Resource": "arn:aws:states:::ecs:runTask.sync", + "Parameters": { + "LaunchType.$": "$.params.ecs.LaunchType", + "Cluster.$": "$.params.ecs.Cluster", + "TaskDefinition.$": "$.params.ecs.TaskDefinition", + "NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration", + "Overrides.$": "$.params.ecs.Overrides" + }, + "ResultPath": "$.result", + "Retry": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "BackoffRate": 2, + "IntervalSeconds": 3, + "MaxAttempts": 3 + } + ], + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "Next": "CreateImportPendingFile", + "ResultPath": "$.result" + } + ], + "Next": "Success", + "Comment": "データ登録処理" + }, + "CreateImportPendingFile": { + "Type": "Task", + "Parameters": { + "Body": "", + "Bucket.$": "$.InputParams.BucketName", + "Key.$": "States.Format('{}.pending-import', $.InputParams.TargetKey)" + }, + "Resource": "arn:aws:states:::aws-sdk:s3:putObject", + "Next": "FailedNotice", + "Comment": "データ登録保留ファイル作成", + "ResultPath": "$.result" + }, + "FailedNotice": { + "Type": "Task", + "Resource": "arn:aws:states:::sns:publish", + "Parameters": { + "TopicArn.$": "$.params.sns.TopicArn", + "Subject": "データ登録ECSタスク異常通知", + "Message.$": "States.Format('({}){}のデータ登録が失敗したため、登録中断ファイルを作成しました。\n\n バケット名：{}\n取込ファイルパス：{}\n\n', $.InputParams.DataSourceName, $.InputParams.FileName, $.InputParams.BucketName, $.InputParams.TargetKey)" + }, + "Next": "Fail" + }, + "Success": { + "Type": "Succeed", + "Comment": "正常終了" + }, + "Fail": { + "Type": "Fail", + "Comment": "異常終了", + "Error": "StatusError", + "Cause": "StepFunctions ErrorEnd" + } + } + } \ No newline at end of file From 715027b4d0cb25e64de74e4285819378a8fa8030 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 16 Nov 2022 11:56:51 +0900 Subject: [PATCH 4/8] =?UTF-8?q?feat:=20=E3=81=A1=E3=82=87=E3=81=A3?= =?UTF-8?q?=E3=81=A8=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- stepfunctions/r-data-import-state/r-data-import-state.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stepfunctions/r-data-import-state/r-data-import-state.json b/stepfunctions/r-data-import-state/r-data-import-state.json index 41de6967..1b8f18c6 100644 --- a/stepfunctions/r-data-import-state/r-data-import-state.json +++ b/stepfunctions/r-data-import-state/r-data-import-state.json @@ -105,7 +105,7 @@ "Parameters": { "TopicArn.$": "$.params.sns.TopicArn", "Subject": "データ登録ECSタスク異常通知", - "Message.$": "States.Format('({}){}のデータ登録が失敗したため、登録中断ファイルを作成しました。\n\n バケット名：{}\n取込ファイルパス：{}\n\n', $.InputParams.DataSourceName, $.InputParams.FileName, $.InputParams.BucketName, $.InputParams.TargetKey)" + "Message.$": "States.Format('【{}】{}のデータ登録が失敗したため、登録中断ファイルを作成しました。\n\n バケット名：{}\n取込ファイルパス：{}\n\n', $.InputParams.DataSourceName, $.InputParams.FileName, $.InputParams.BucketName, $.InputParams.TargetKey)" }, "Next": "Fail" }, From 0da7efa1ed7fe08554f51c6d881f932de35d5450 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 16 Nov 2022 17:57:29 +0900 Subject: [PATCH 5/8] =?UTF-8?q?style:=20=E3=83=95=E3=82=A9=E3=83=BC?= =?UTF-8?q?=E3=83=9E=E3=83=83=E3=83=86=E3=82=A3=E3=83=B3=E3=82=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../r-data-import-state.json | 226 +++++++++--------- 1 file changed, 111 insertions(+), 115 deletions(-) diff --git a/stepfunctions/r-data-import-state/r-data-import-state.json b/stepfunctions/r-data-import-state/r-data-import-state.json index 1b8f18c6..34d74e82 100644 --- a/stepfunctions/r-data-import-state/r-data-import-state.json +++ b/stepfunctions/r-data-import-state/r-data-import-state.json @@ -1,123 +1,119 @@ { - "Comment": "MeDaCa データ取込ECSタスク起動 ステートマシン", - "StartAt": "params", - "States": { - "params": { - "Type": "Pass", - "Parameters": { - "sns": { - "TopicArn": "arn:aws:sns:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:nds-notice-#{ENV_NAME}" - }, - "ecs": { - "LaunchType": "FARGATE", - "Cluster.$": "$.InputParams.ClusterName", - "TaskDefinition.$": "$.InputParams.TaskName", - "NetworkConfiguration": { - "AwsvpcConfiguration": { - "Subnets.$": "States.Array($.InputParams.SubNetIdApNorthEast1A, $.InputParams.SubNetIdApNorthEast1D)", - "SecurityGroups.$": "States.Array($.InputParams.SecurityGroupIdECSAll, $.InputParams.SecurityGroupIdECSDataImport)", - "AssignPublicIp": "DISABLED" - } - }, - "Overrides": { - "ContainerOverrides": [ - { - "Name.$": "$.InputParams.ContainerName", - "Environment": [ - { - "Name": "BUCKET_NAME", - "Value.$": "$.InputParams.BucketName" - }, - { - "Name": "TARGET_KEY", - "Value.$": "$.InputParams.TargetKey" - }, - { - "Name": "DATA_SOURCE_NAME", - "Value.$": "$.InputParams.DataSourceName" - }, - { - "Name": "FILE_NAME", - "Value.$": "$.InputParams.FileName" - }, - { - "Name": "MODE", - "Value.$": "$.InputParams.Mode" - } - ] - } - ] + "Comment": "MeDaCa データ取込ECSタスク起動 ステートマシン", + "StartAt": "params", + "States": { + "params": { + "Type": "Pass", + "Parameters": { + "sns": { + "TopicArn": "arn:aws:sns:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:nds-notice-#{ENV_NAME}" + }, + "ecs": { + "LaunchType": "FARGATE", + "Cluster.$": "$.InputParams.ClusterName", + "TaskDefinition.$": "$.InputParams.TaskName", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "Subnets.$": "States.Array($.InputParams.SubNetIdApNorthEast1A, $.InputParams.SubNetIdApNorthEast1D)", + "SecurityGroups.$": "States.Array($.InputParams.SecurityGroupIdECSAll, $.InputParams.SecurityGroupIdECSDataImport)", + "AssignPublicIp": "DISABLED" } + }, + "Overrides": { + "ContainerOverrides": [ + { + "Name.$": "$.InputParams.ContainerName", + "Environment": [ + { + "Name": "BUCKET_NAME", + "Value.$": "$.InputParams.BucketName" + }, + { + "Name": "TARGET_KEY", + "Value.$": "$.InputParams.TargetKey" + }, + { + "Name": "DATA_SOURCE_NAME", + "Value.$": "$.InputParams.DataSourceName" + }, + { + "Name": "FILE_NAME", + "Value.$": "$.InputParams.FileName" + }, + { + "Name": "MODE", + "Value.$": "$.InputParams.Mode" + } + ] + } + ] } - }, - "Comment": "パラメータ設定", - "ResultPath": "$.params", - "Next": "data-import" + } }, - "data-import": { - "Type": "Task", - "Resource": "arn:aws:states:::ecs:runTask.sync", - "Parameters": { - "LaunchType.$": "$.params.ecs.LaunchType", - "Cluster.$": "$.params.ecs.Cluster", - "TaskDefinition.$": "$.params.ecs.TaskDefinition", - "NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration", - "Overrides.$": "$.params.ecs.Overrides" - }, - "ResultPath": "$.result", - "Retry": [ - { - "ErrorEquals": [ - "States.ALL" - ], - "BackoffRate": 2, - "IntervalSeconds": 3, - "MaxAttempts": 3 - } - ], - "Catch": [ - { - "ErrorEquals": [ - "States.ALL" - ], - "Next": "CreateImportPendingFile", - "ResultPath": "$.result" - } - ], - "Next": "Success", - "Comment": "データ登録処理" + "Comment": "パラメータ設定", + "ResultPath": "$.params", + "Next": "data-import" + }, + "data-import": { + "Type": "Task", + "Resource": "arn:aws:states:::ecs:runTask.sync", + "Parameters": { + "LaunchType.$": "$.params.ecs.LaunchType", + "Cluster.$": "$.params.ecs.Cluster", + "TaskDefinition.$": "$.params.ecs.TaskDefinition", + "NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration", + "Overrides.$": "$.params.ecs.Overrides" }, - "CreateImportPendingFile": { - "Type": "Task", - "Parameters": { - "Body": "", - "Bucket.$": "$.InputParams.BucketName", - "Key.$": "States.Format('{}.pending-import', $.InputParams.TargetKey)" - }, - "Resource": "arn:aws:states:::aws-sdk:s3:putObject", - "Next": "FailedNotice", - "Comment": "データ登録保留ファイル作成", - "ResultPath": "$.result" + "ResultPath": "$.result", + "Retry": [ + { + "ErrorEquals": ["States.ALL"], + "BackoffRate": 2, + "IntervalSeconds": 3, + "MaxAttempts": 3 + } + ], + "Catch": [ + { + "ErrorEquals": ["States.ALL"], + "Next": "CreateImportPendingFile", + "ResultPath": "$.result" + } + ], + "Next": "Success", + "Comment": "データ登録処理" + }, + "CreateImportPendingFile": { + "Type": "Task", + "Parameters": { + "Body": "", + "Bucket.$": "$.InputParams.BucketName", + "Key.$": "States.Format('{}.pending-import', $.InputParams.TargetKey)" }, - "FailedNotice": { - "Type": "Task", - "Resource": "arn:aws:states:::sns:publish", - "Parameters": { - "TopicArn.$": "$.params.sns.TopicArn", - "Subject": "データ登録ECSタスク異常通知", - "Message.$": "States.Format('【{}】{}のデータ登録が失敗したため、登録中断ファイルを作成しました。\n\n バケット名：{}\n取込ファイルパス：{}\n\n', $.InputParams.DataSourceName, $.InputParams.FileName, $.InputParams.BucketName, $.InputParams.TargetKey)" - }, - "Next": "Fail" + "Resource": "arn:aws:states:::aws-sdk:s3:putObject", + "Next": "FailedNotice", + "Comment": "データ登録保留ファイル作成", + "ResultPath": "$.result" + }, + "FailedNotice": { + "Type": "Task", + "Resource": "arn:aws:states:::sns:publish", + "Parameters": { + "TopicArn.$": "$.params.sns.TopicArn", + "Subject": "データ登録ECSタスク異常通知", + "Message.$": "States.Format('【{}】{}のデータ登録が失敗したため、登録中断ファイルを作成しました。\n\n バケット名：{}\n取込ファイルパス：{}\n\n', $.InputParams.DataSourceName, $.InputParams.FileName, $.InputParams.BucketName, $.InputParams.TargetKey)" }, - "Success": { - "Type": "Succeed", - "Comment": "正常終了" - }, - "Fail": { - "Type": "Fail", - "Comment": "異常終了", - "Error": "StatusError", - "Cause": "StepFunctions ErrorEnd" - } + "Next": "Fail" + }, + "Success": { + "Type": "Succeed", + "Comment": "正常終了" + }, + "Fail": { + "Type": "Fail", + "Comment": "異常終了", + "Error": "StatusError", + "Cause": "StepFunctions ErrorEnd" } - } \ No newline at end of file + } +} From 56c880be67aca73ea47d5e40004a54c41c8a54ce Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 16 Nov 2022 19:02:52 +0900 Subject: [PATCH 6/8] =?UTF-8?q?feat:=20=E7=99=BB=E9=8C=B2=E5=87=A6?= =?UTF-8?q?=E7=90=86=E4=B8=AD=E6=96=AD=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E3=81=8C=E3=81=82=E3=82=8B=E5=A0=B4=E5=90=88=E3=80=81=E5=87=A6?= =?UTF-8?q?=E7=90=86=E3=82=92=E8=B5=B7=E5=8B=95=E3=81=95=E3=81=9B=E3=81=AA?= =?UTF-8?q?=E3=81=84=E5=87=A6=E7=90=86=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/dataimport/dataimport.py | 4 +- .../r-data-import-state.json | 52 ++++++++++++++++--- 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/lambda/dataimport/dataimport.py b/lambda/dataimport/dataimport.py index 0daf4194..72695078 100644 --- a/lambda/dataimport/dataimport.py +++ b/lambda/dataimport/dataimport.py @@ -61,14 +61,14 @@ def lambda_handler(event, context): if __name__ == '__main__': - # ローカル実行用 + # ローカル実行用。バケット名やファイルキーはよしなに変えてください。 lambda_handler({"Records": [{ "s3": { "bucket": { "name": "test-shimoda-bucket" }, "object": { - "key": "folder/object_key" + "key": "crm/target/CRM_Account_20221116111800.csv" } } }]}, {}) diff --git a/stepfunctions/r-data-import-state/r-data-import-state.json b/stepfunctions/r-data-import-state/r-data-import-state.json index 34d74e82..258880f1 100644 --- a/stepfunctions/r-data-import-state/r-data-import-state.json +++ b/stepfunctions/r-data-import-state/r-data-import-state.json @@ -52,7 +52,38 @@ }, "Comment": "パラメータ設定", "ResultPath": "$.params", - "Next": "data-import" + "Next": "?IsExistsFatalErrorFile" + }, + "?IsExistsFatalErrorFile": { + "Type": "Task", + "Parameters": { + "Bucket.$": "$.InputParams.BucketName", + "Key.$": "States.Format('{}/target/.detected_fatal_error_ecs', $.InputParams.DataSourceName)" + }, + "Resource": "arn:aws:states:::aws-sdk:s3:headObject", + "ResultPath": "$.result", + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "Next": "data-import", + "ResultPath": "$.result" + } + ], + "Comment": "ECSタスク起動エラーファイル存在チェック", + "Next": "PendingDataImportNotice" + }, + "PendingDataImportNotice": { + "Type": "Task", + "Resource": "arn:aws:states:::aws-sdk:sns:publish", + "Parameters": { + "TopicArn.$": "$.params.sns.TopicArn", + "Subject": "データ登録ECSタスク異常通知", + "Message.$": "States.Format('「{}/{}/target/」に登録処理中断ファイル(.detected_fatal_error_ecs)が見つかったため、データ登録の起動を中断しました。\n', $.InputParams.BucketName, $.InputParams.DataSourceName)" + }, + "End": true, + "Comment": "データ登録処理起動中断通知" }, "data-import": { "Type": "Task", @@ -67,7 +98,9 @@ "ResultPath": "$.result", "Retry": [ { - "ErrorEquals": ["States.ALL"], + "ErrorEquals": [ + "States.ALL" + ], "BackoffRate": 2, "IntervalSeconds": 3, "MaxAttempts": 3 @@ -75,7 +108,9 @@ ], "Catch": [ { - "ErrorEquals": ["States.ALL"], + "ErrorEquals": [ + "States.ALL" + ], "Next": "CreateImportPendingFile", "ResultPath": "$.result" } @@ -88,22 +123,23 @@ "Parameters": { "Body": "", "Bucket.$": "$.InputParams.BucketName", - "Key.$": "States.Format('{}.pending-import', $.InputParams.TargetKey)" + "Key.$": "States.Format('{}/target/.detected_fatal_error_ecs', $.InputParams.DataSourceName)" }, "Resource": "arn:aws:states:::aws-sdk:s3:putObject", "Next": "FailedNotice", - "Comment": "データ登録保留ファイル作成", + "Comment": "データ登録処理起動中断ファイル作成", "ResultPath": "$.result" }, "FailedNotice": { "Type": "Task", - "Resource": "arn:aws:states:::sns:publish", + "Resource": "arn:aws:states:::aws-sdk:sns:publish", "Parameters": { "TopicArn.$": "$.params.sns.TopicArn", "Subject": "データ登録ECSタスク異常通知", - "Message.$": "States.Format('【{}】{}のデータ登録が失敗したため、登録中断ファイルを作成しました。\n\n バケット名：{}\n取込ファイルパス：{}\n\n', $.InputParams.DataSourceName, $.InputParams.FileName, $.InputParams.BucketName, $.InputParams.TargetKey)" + "Message.$": "States.Format('【{}】{}のデータ登録が失敗したため、登録処理中断ファイル(.detected_fatal_error_ecs)を作成しました。\n\n バケット名：{}\n取込ファイルパス：{}\n\n', $.InputParams.DataSourceName, $.InputParams.FileName, $.InputParams.BucketName, $.InputParams.TargetKey)" }, - "Next": "Fail" + "Next": "Fail", + "Comment": "データ登録処理起動失敗通知" }, "Success": { "Type": "Succeed", From 50219627ed272d3b5fb68b8d2d1488bf733e2d22 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 18 Nov 2022 08:45:46 +0900 Subject: [PATCH 7/8] =?UTF-8?q?feat:=20=E3=83=87=E3=83=90=E3=83=83?= =?UTF-8?q?=E3=82=B0=E7=94=A8=E3=82=B3=E3=83=BC=E3=83=89=E3=82=92=E3=82=B3?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=A2=E3=82=A6=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- lambda/dataimport/dataimport.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/lambda/dataimport/dataimport.py b/lambda/dataimport/dataimport.py index 72695078..b9a57494 100644 --- a/lambda/dataimport/dataimport.py +++ b/lambda/dataimport/dataimport.py @@ -60,15 +60,17 @@ def lambda_handler(event, context): -if __name__ == '__main__': - # ローカル実行用。バケット名やファイルキーはよしなに変えてください。 - lambda_handler({"Records": [{ - "s3": { - "bucket": { - "name": "test-shimoda-bucket" - }, - "object": { - "key": "crm/target/CRM_Account_20221116111800.csv" - } - } - }]}, {}) +# ローカルでのデバック実行用。 +# 利用する際にコメントを外してください +# if __name__ == '__main__': +# lambda_handler({"Records": [{ +# # バケット名やファイルキーはよしなに変えてください。 +# "s3": { +# "bucket": { +# "name": "test-shimoda-bucket" +# }, +# "object": { +# "key": "crm/target/CRM_Account_20221116111800.csv" +# } +# } +# }]}, {}) From 777a6f188abc7357e541dbe5b69fecfecb87cdb3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 18 Nov 2022 12:42:14 +0900 Subject: [PATCH 8/8] =?UTF-8?q?refactor:=20=E5=87=A6=E7=90=86=E5=90=8D?= =?UTF-8?q?=E3=82=92=E7=9B=AE=E7=9A=84=E3=81=AB=E5=90=88=E3=82=8F=E3=81=9B?= =?UTF-8?q?=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- stepfunctions/r-data-import-state/r-data-import-state.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stepfunctions/r-data-import-state/r-data-import-state.json b/stepfunctions/r-data-import-state/r-data-import-state.json index 258880f1..67d11016 100644 --- a/stepfunctions/r-data-import-state/r-data-import-state.json +++ b/stepfunctions/r-data-import-state/r-data-import-state.json @@ -52,9 +52,9 @@ }, "Comment": "パラメータ設定", "ResultPath": "$.params", - "Next": "?IsExistsFatalErrorFile" + "Next": "CheckExclusive" }, - "?IsExistsFatalErrorFile": { + "CheckExclusive": { "Type": "Task", "Parameters": { "Bucket.$": "$.InputParams.BucketName",