From 4d2bffcf3bc68ac80a17cb3412522675f184f29d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 28 May 2025 17:08:02 +0900 Subject: [PATCH 01/11] =?UTF-8?q?feat:=20DCF=E5=89=8A=E9=99=A4=E6=96=B0?= =?UTF-8?q?=E8=A6=8F=20=E9=96=8B=E5=A7=8B=E3=83=AD=E3=82=B0=E3=82=92?= =?UTF-8?q?=E5=87=BA=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/dcf_inst_merge_io.py | 1 + stepfunctions/TOOLS/convert_config.yaml | 34 +++++ ...-jskult-batch-dcf-inst-merge-io-state.json | 122 ++++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json diff --git a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py index ad311d42..82c7cfbe 100644 --- a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py +++ b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py @@ -22,6 +22,7 @@ class DcfInstMergeIO(JskultBatchEntrypoint): super().__init__() def execute(self): + logger.info("DCF削除新規マスタ作成処理を開始します。") jskult_hdke_tbl_manager = JskultHdkeTblManager() jskult_batch_run_manager = JskultBatchRunManager( environment.BATCH_EXECUTION_ID) diff --git a/stepfunctions/TOOLS/convert_config.yaml b/stepfunctions/TOOLS/convert_config.yaml index 560a6594..98700734 100644 --- a/stepfunctions/TOOLS/convert_config.yaml +++ b/stepfunctions/TOOLS/convert_config.yaml @@ -32,6 +32,8 @@ resource: - &STG_SG_JSKULT_TRANSFER_RECEIVE_FILE "sg-08d43e8e118178d39" # セキュリティグループ(ecs-jskult-batch-ultmarc-io) - &STG_SG_JSKULT_ULTMARC_IO "sg-014caf29e738d106a" + # セキュリティグループ(ecs-jskult-batch-ultmarc-io) + - &STG_SG_DCF_INST_MERGE_IO "sg-0b5d1639a83d28f66" # 本番環境 product: # サブネット(PrivateSubnet1) @@ -56,6 +58,9 @@ resource: # セキュリティグループ(ecs-jskult-batch-ultmarc-io) # TODO: 本番リリース時にIDを正式版にする - &PRD_SG_JSKULT_ULTMARC_IO "sg-xxxxxxxxxxxxx" + # セキュリティグループ(ecs-jskult-batch-ultmarc-io) + # TODO: 本番リリース時にIDを正式版にする + - &PRD_SG_DCF_INST_MERGE_IO "sg-xxxxxxxxxxxxx" config: # CRMデータ取得 r-crm-datafetch-state: @@ -281,3 +286,32 @@ config: SG_ECS_ALL: *PRD_SG_ECS_ALL # セキュリティグループ(ecs-jskult-batch-ultmarc-io) SG_JSKULT_ULTMARC_IO: *PRD_SG_JSKULT_ULTMARC_IO + r-jskult-batch-dcf-inst-merge-io-state: + # ステージング環境 + staging: + # AWSアカウントID + AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID + # 東京リージョン + REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1 + # サブネット(PrivateSubnet1) + SUBNET_PRI_1A: *STG_SUBNET_PRI_1A + # サブネット(PrivateSubnet2) + SUBNET_PRI_1D: *STG_SUBNET_PRI_1D + # セキュリティグループ(ecs-all) + SG_ECS_ALL: *STG_SG_ECS_ALL + # セキュリティグループ(ecs-jskult-batch-ultmarc-io) + SG_DCF_INST_MERGE_IO: *STG_SG_DCF_INST_MERGE_IO + # 本番環境 + product: + # AWSアカウントID + AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID + # 東京リージョン + REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1 + # サブネット(PrivateSubnet1) + SUBNET_PRI_1A: *PRD_SUBNET_PRI_1A + # サブネット(PrivateSubnet2) + SUBNET_PRI_1D: *PRD_SUBNET_PRI_1D + # セキュリティグループ(ecs-all) + SG_ECS_ALL: *PRD_SG_ECS_ALL + # セキュリティグループ(ecs-jskult-batch-ultmarc-io) + SG_DCF_INST_MERGE_IO: *PRD_SG_DCF_INST_MERGE_IO diff --git a/stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json b/stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json new file mode 100644 index 00000000..4497f85c --- /dev/null +++ b/stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json @@ -0,0 +1,122 @@ +{ + "Comment": "実消化&アルトマーク DCF施設削除新規マスタ作成ステートマシン", + "StartAt": "params", + "States": { + "params": { + "Comment": "パラメータ設定", + "Type": "Pass", + "Parameters": { + "ecs": { + "LaunchType": "FARGATE", + "Cluster": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:cluster/mbj-newdwh2021-#{ENV_NAME}-jskult-batch-dcf-inst-merge-io-ecs", + "TaskDefinition": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-#{ENV_NAME}-task-jskult-batch-dcf-inst-merge-io", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "Subnets": [ + "#{SUBNET_PRI_1A}", + "#{SUBNET_PRI_1D}" + ], + "SecurityGroups": [ + "#{SG_ECS_ALL}", + "#{SG_DCF_INST_MERGE_IO}" + ], + "AssignPublicIp": "DISABLED" + } + }, + "Overrides": { + "ContainerOverrides": [ + { + "Name": "mbj-newdwh2021-#{ENV_NAME}-container-jskult-batch-dcf-inst-merge-io", + "Environment": [ + { + "Name": " BATCH_EXECUTION_ID", + "Value.$": "$$.Execution.Id" + }, + { + "Name": "MAX_RUN_COUNT", + "Value.$": "$.maxRunCount" + } + ] + } + ] + } + } + }, + "ResultPath": "$.params", + "Next": "exec-ecs-task" + }, + "exec-ecs-task": { + "Type": "Task", + "Resource": "arn:aws:states:::ecs:runTask.sync", + "Parameters": { + "LaunchType.$": "$.params.ecs.LaunchType", + "Cluster.$": "$.params.ecs.Cluster", + "TaskDefinition.$": "$.params.ecs.TaskDefinition", + "NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration", + "Overrides.$": "$.params.ecs.Overrides" + }, + "ResultPath": "$.result", + "Retry": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "BackoffRate": 2, + "IntervalSeconds": 3, + "MaxAttempts": 3 + } + ], + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "Next": "ErrorEnd", + "ResultPath": "$.result" + } + ], + "Next": "scan-jskult-batch-run-manage", + "Comment": "ECSタスク起動" + }, + "scan-jskult-batch-run-manage": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "mbj-newdwh2021-#{ENV_NAME}-jskult-batch-run-manage", + "Key": { + "execution_id": { + "S.$": "$$.Execution.Id" + } + } + }, + "Next": "Choice", + "ResultPath": "$.scan" + }, + "Choice": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.scan.Item.batch_run_status.S", + "StringEquals": "retry", + "Next": "wait-for-retry" + } + ], + "Default": "NormalEnd" + }, + "wait-for-retry": { + "Type": "Wait", + "SecondsPath": "$.retryIntervalSecond", + "Next": "exec-ecs-task" + }, + "NormalEnd": { + "Comment": "正常終了", + "Type": "Succeed" + }, + "ErrorEnd": { + "Comment": "異常終了", + "Type": "Fail", + "Error": "StatesError", + "Cause": "StepFunctions ErrorEnd" + } + } +} \ No newline at end of file From a0ffea108644f3c33a60c636861b7ec0bbb4037f Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 28 May 2025 17:14:35 +0900 Subject: [PATCH 02/11] =?UTF-8?q?feat:=20DCF=E5=89=8A=E9=99=A4=E6=96=B0?= =?UTF-8?q?=E8=A6=8F=E3=83=9E=E3=82=B9=E3=82=BF=E4=BD=9C=E6=88=90=E3=81=AE?= =?UTF-8?q?=E3=82=B9=E3=83=86=E3=83=BC=E3=83=88=E3=83=9E=E3=82=B7=E3=83=B3?= =?UTF-8?q?=E5=AE=9A=E7=BE=A9=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../r-jskult-batch-dcf-inst-merge-io-state.json | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json b/stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json index 4497f85c..b4d3e751 100644 --- a/stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json +++ b/stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json @@ -95,13 +95,20 @@ "Choice": { "Type": "Choice", "Choices": [ + { + "Variable": "$.scan.Item", + "IsPresent": false, + "Next": "NormalEnd", + "Comment": "バッチ実行管理テーブルにデータが存在しない場合" + }, { "Variable": "$.scan.Item.batch_run_status.S", "StringEquals": "retry", - "Next": "wait-for-retry" + "Next": "wait-for-retry", + "Comment": "バッチ実行管理テーブルのスターテスがリトライの場合" } ], - "Default": "NormalEnd" + "Default": "ErrorEnd" }, "wait-for-retry": { "Type": "Wait", From 75072eb024b763107a511c46cb480eb331a3b9aa Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 28 May 2025 19:00:14 +0900 Subject: [PATCH 03/11] =?UTF-8?q?feat:=20DCF=E5=89=8A=E9=99=A4=E6=96=B0?= =?UTF-8?q?=E8=A6=8F=E3=83=9E=E3=82=B9=E3=82=BF=E4=BD=9C=E6=88=90=E3=81=AE?= =?UTF-8?q?=E3=82=B9=E3=83=86=E3=83=BC=E3=83=88=E3=83=9E=E3=82=B7=E3=83=B3?= =?UTF-8?q?=E5=AE=9A=E7=BE=A9=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../r-jskult-batch-dcf-inst-merge-io-state.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json b/stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json index b4d3e751..73816f70 100644 --- a/stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json +++ b/stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json @@ -29,7 +29,7 @@ "Name": "mbj-newdwh2021-#{ENV_NAME}-container-jskult-batch-dcf-inst-merge-io", "Environment": [ { - "Name": " BATCH_EXECUTION_ID", + "Name": "BATCH_EXECUTION_ID", "Value.$": "$$.Execution.Id" }, { @@ -98,7 +98,7 @@ { "Variable": "$.scan.Item", "IsPresent": false, - "Next": "NormalEnd", + "Next": "ErrorEnd", "Comment": "バッチ実行管理テーブルにデータが存在しない場合" }, { @@ -108,7 +108,7 @@ "Comment": "バッチ実行管理テーブルのスターテスがリトライの場合" } ], - "Default": "ErrorEnd" + "Default": "NormalEnd" }, "wait-for-retry": { "Type": "Wait", From e5222796490f1c57360061dda1088eddc56d31f6 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 28 May 2025 19:24:06 +0900 Subject: [PATCH 04/11] =?UTF-8?q?feat:=20=E3=83=A1=E3=83=AB=E3=82=AF?= =?UTF-8?q?=E6=96=BD=E8=A8=AD=E3=83=9E=E3=82=B9=E3=82=BF=E4=BD=9C=E6=88=90?= =?UTF-8?q?=E3=82=B9=E3=83=86=E3=83=BC=E3=83=88=E3=83=9E=E3=82=B7=E3=83=B3?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- stepfunctions/TOOLS/convert_config.yaml | 34 +++++ .../r-jskult-batch-mst-inst-all-state.json | 129 ++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 stepfunctions/r-jskult-batch-mst-inst-all-state/r-jskult-batch-mst-inst-all-state.json diff --git a/stepfunctions/TOOLS/convert_config.yaml b/stepfunctions/TOOLS/convert_config.yaml index 98700734..9c893fa5 100644 --- a/stepfunctions/TOOLS/convert_config.yaml +++ b/stepfunctions/TOOLS/convert_config.yaml @@ -34,6 +34,8 @@ resource: - &STG_SG_JSKULT_ULTMARC_IO "sg-014caf29e738d106a" # セキュリティグループ(ecs-jskult-batch-ultmarc-io) - &STG_SG_DCF_INST_MERGE_IO "sg-0b5d1639a83d28f66" + # セキュリティグループ(ecs-jskult-batch-mst-inst-all) + - &STG_SG_MST_INST_ALL "sg-0483fd75c76fa5808" # 本番環境 product: # サブネット(PrivateSubnet1) @@ -61,6 +63,9 @@ resource: # セキュリティグループ(ecs-jskult-batch-ultmarc-io) # TODO: 本番リリース時にIDを正式版にする - &PRD_SG_DCF_INST_MERGE_IO "sg-xxxxxxxxxxxxx" + # セキュリティグループ(ecs-jskult-batch-mst-inst-all) + # TODO: 本番リリース時にIDを正式版にする + - &PRD_SG_MST_INST_ALL "sg-xxxxxxxxxxxxx" config: # CRMデータ取得 r-crm-datafetch-state: @@ -315,3 +320,32 @@ config: SG_ECS_ALL: *PRD_SG_ECS_ALL # セキュリティグループ(ecs-jskult-batch-ultmarc-io) SG_DCF_INST_MERGE_IO: *PRD_SG_DCF_INST_MERGE_IO +r-jskult-batch-mst-inst-all-state: + # ステージング環境 + staging: + # AWSアカウントID + AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID + # 東京リージョン + REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1 + # サブネット(PrivateSubnet1) + SUBNET_PRI_1A: *STG_SUBNET_PRI_1A + # サブネット(PrivateSubnet2) + SUBNET_PRI_1D: *STG_SUBNET_PRI_1D + # セキュリティグループ(ecs-all) + SG_ECS_ALL: *STG_SG_ECS_ALL + # セキュリティグループ(ecs-jskult-batch-ultmarc-io) + SG_MST_INST_ALL: *STG_SG_MST_INST_ALL + # 本番環境 + product: + # AWSアカウントID + AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID + # 東京リージョン + REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1 + # サブネット(PrivateSubnet1) + SUBNET_PRI_1A: *PRD_SUBNET_PRI_1A + # サブネット(PrivateSubnet2) + SUBNET_PRI_1D: *PRD_SUBNET_PRI_1D + # セキュリティグループ(ecs-all) + SG_ECS_ALL: *PRD_SG_ECS_ALL + # セキュリティグループ(ecs-jskult-batch-ultmarc-io) + SG_MST_INST_ALL: *PRD_SG_MST_INST_ALL \ No newline at end of file diff --git a/stepfunctions/r-jskult-batch-mst-inst-all-state/r-jskult-batch-mst-inst-all-state.json b/stepfunctions/r-jskult-batch-mst-inst-all-state/r-jskult-batch-mst-inst-all-state.json new file mode 100644 index 00000000..54086560 --- /dev/null +++ b/stepfunctions/r-jskult-batch-mst-inst-all-state/r-jskult-batch-mst-inst-all-state.json @@ -0,0 +1,129 @@ +{ + "Comment": "実消化&アルトマーク メルク施設マスタステートマシン", + "StartAt": "params", + "States": { + "params": { + "Comment": "パラメータ設定", + "Type": "Pass", + "Parameters": { + "ecs": { + "LaunchType": "FARGATE", + "Cluster": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:cluster/mbj-newdwh2021-#{ENV_NAME}-jskult-batch-mst-inst-all-ecs", + "TaskDefinition": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-#{ENV_NAME}-task-jskult-batch-mst-inst-all", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "Subnets": [ + "#{SUBNET_PRI_1A}", + "#{SUBNET_PRI_1D}" + ], + "SecurityGroups": [ + "#{SG_ECS_ALL}", + "#{SG_MST_INST_ALL}" + ], + "AssignPublicIp": "DISABLED" + } + }, + "Overrides": { + "ContainerOverrides": [ + { + "Name": "mbj-newdwh2021-#{ENV_NAME}-container-jskult-batch-mst-inst-all", + "Environment": [ + { + "Name": "BATCH_EXECUTION_ID", + "Value.$": "$$.Execution.Id" + }, + { + "Name": "MAX_RUN_COUNT", + "Value.$": "$.maxRunCount" + } + ] + } + ] + } + } + }, + "ResultPath": "$.params", + "Next": "exec-ecs-task" + }, + "exec-ecs-task": { + "Type": "Task", + "Resource": "arn:aws:states:::ecs:runTask.sync", + "Parameters": { + "LaunchType.$": "$.params.ecs.LaunchType", + "Cluster.$": "$.params.ecs.Cluster", + "TaskDefinition.$": "$.params.ecs.TaskDefinition", + "NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration", + "Overrides.$": "$.params.ecs.Overrides" + }, + "ResultPath": "$.result", + "Retry": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "BackoffRate": 2, + "IntervalSeconds": 3, + "MaxAttempts": 3 + } + ], + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "Next": "ErrorEnd", + "ResultPath": "$.result" + } + ], + "Next": "scan-jskult-batch-run-manage", + "Comment": "ECSタスク起動" + }, + "scan-jskult-batch-run-manage": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "mbj-newdwh2021-#{ENV_NAME}-jskult-batch-run-manage", + "Key": { + "execution_id": { + "S.$": "$$.Execution.Id" + } + } + }, + "Next": "Choice", + "ResultPath": "$.scan" + }, + "Choice": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.scan.Item", + "IsPresent": false, + "Next": "ErrorEnd", + "Comment": "バッチ実行管理テーブルにデータが存在しない場合" + }, + { + "Variable": "$.scan.Item.batch_run_status.S", + "StringEquals": "retry", + "Next": "wait-for-retry", + "Comment": "バッチ実行管理テーブルのスターテスがリトライの場合" + } + ], + "Default": "NormalEnd" + }, + "wait-for-retry": { + "Type": "Wait", + "SecondsPath": "$.retryIntervalSecond", + "Next": "exec-ecs-task" + }, + "NormalEnd": { + "Comment": "正常終了", + "Type": "Succeed" + }, + "ErrorEnd": { + "Comment": "異常終了", + "Type": "Fail", + "Error": "StatesError", + "Cause": "StepFunctions ErrorEnd" + } + } +} \ No newline at end of file From eda21bdc9424d3fe1d2562240032fd12818b4321 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 28 May 2025 19:24:27 +0900 Subject: [PATCH 05/11] =?UTF-8?q?feat:=20=E5=90=84=E3=82=BF=E3=82=B9?= =?UTF-8?q?=E3=82=AF=E8=A8=AD=E5=AE=9A=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E3=82=92=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../task_settings/archive_jsk_data_task_settings.env | 11 +++++++++++ .../task_settings/mst_inst_all_task_settings.env | 9 +++++++++ .../trn_result_data_bio_lot_task_settings.env | 9 +++++++++ .../update_business_day_task_settings.env | 11 +++++++++++ 4 files changed, 40 insertions(+) create mode 100644 s3/config/jskult/task_settings/archive_jsk_data_task_settings.env create mode 100644 s3/config/jskult/task_settings/mst_inst_all_task_settings.env create mode 100644 s3/config/jskult/task_settings/trn_result_data_bio_lot_task_settings.env create mode 100644 s3/config/jskult/task_settings/update_business_day_task_settings.env diff --git a/s3/config/jskult/task_settings/archive_jsk_data_task_settings.env b/s3/config/jskult/task_settings/archive_jsk_data_task_settings.env new file mode 100644 index 00000000..1a8b2051 --- /dev/null +++ b/s3/config/jskult/task_settings/archive_jsk_data_task_settings.env @@ -0,0 +1,11 @@ +# task environment file. +LOG_LEVEL=INFO +ARCHIVE_TRN_RESULT_FOLDER=/jsk/trn_result_data +ARCHIVE_TRN_RESULT_BIO_FOLDER=/jsk/trn_result_data_bio +ARCHIVE_TRN_RESULT_BIO_LOT_FOLDER=/jsk/trn_result_data_bio_lot +ARCHIVE_TRN_RESULT_INVENTORY_FOLDER=/jsk/trn_recive_inventry +DB_CONNECTION_MAX_RETRY_ATTEMPT=4 +DB_CONNECTION_RETRY_INTERVAL_INIT=5 +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=5 +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=50 + diff --git a/s3/config/jskult/task_settings/mst_inst_all_task_settings.env b/s3/config/jskult/task_settings/mst_inst_all_task_settings.env new file mode 100644 index 00000000..85e70ad1 --- /dev/null +++ b/s3/config/jskult/task_settings/mst_inst_all_task_settings.env @@ -0,0 +1,9 @@ +# task environment file. +LOG_LEVEL=INFO +ENTRYPOINT_MODULE_NAME=jskult-batch-mst-inst-all +TRANSFER_RESULT_FOLDER=transfer_result +TRANSFER_RESULT_FILE_NAME=transfer_result.json +DB_CONNECTION_MAX_RETRY_ATTEMPT=4 +DB_CONNECTION_RETRY_INTERVAL_INIT=5 +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=5 +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=50 diff --git a/s3/config/jskult/task_settings/trn_result_data_bio_lot_task_settings.env b/s3/config/jskult/task_settings/trn_result_data_bio_lot_task_settings.env new file mode 100644 index 00000000..f5adb7ab --- /dev/null +++ b/s3/config/jskult/task_settings/trn_result_data_bio_lot_task_settings.env @@ -0,0 +1,9 @@ +# task environment file. +LOG_LEVEL=INFO +PROCESS_NAME=jskult-batch-trn-result-data-bio-lot +TRANSFER_RESULT_FOLDER=transfer_result +TRANSFER_RESULT_FILE_NAME=transfer_result.json +DB_CONNECTION_MAX_RETRY_ATTEMPT=4 +DB_CONNECTION_RETRY_INTERVAL_INIT=5 +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=5 +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=50 diff --git a/s3/config/jskult/task_settings/update_business_day_task_settings.env b/s3/config/jskult/task_settings/update_business_day_task_settings.env new file mode 100644 index 00000000..b5217d60 --- /dev/null +++ b/s3/config/jskult/task_settings/update_business_day_task_settings.env @@ -0,0 +1,11 @@ +# task environment file. +LOG_LEVEL=INFO +PROCESS_NAME=jskult-batch-update-business-day +JSKULT_CONFIG_CALENDAR_FOLDER=jskult/calendar +JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME=jsk_archive_run_day.txt +JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER=jskult/expected_data_list +JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME=jsk_expected_data_list.json +DB_CONNECTION_MAX_RETRY_ATTEMPT=4 +DB_CONNECTION_RETRY_INTERVAL_INIT=5 +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=5 +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=50 From 9c4848aab77edbfd5c6b05c53c79fbc9b343e9af Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 28 May 2025 19:24:47 +0900 Subject: [PATCH 06/11] =?UTF-8?q?feat:=20config=E3=81=AE=E8=AA=A4=E3=82=8A?= =?UTF-8?q?=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- stepfunctions/TOOLS/convert_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stepfunctions/TOOLS/convert_config.yaml b/stepfunctions/TOOLS/convert_config.yaml index 9c893fa5..6533829e 100644 --- a/stepfunctions/TOOLS/convert_config.yaml +++ b/stepfunctions/TOOLS/convert_config.yaml @@ -320,7 +320,7 @@ config: SG_ECS_ALL: *PRD_SG_ECS_ALL # セキュリティグループ(ecs-jskult-batch-ultmarc-io) SG_DCF_INST_MERGE_IO: *PRD_SG_DCF_INST_MERGE_IO -r-jskult-batch-mst-inst-all-state: + r-jskult-batch-mst-inst-all-state: # ステージング環境 staging: # AWSアカウントID From 25a37e45526b2771ef3b10d8bff0ae474e3dce8e Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 28 May 2025 19:40:03 +0900 Subject: [PATCH 07/11] =?UTF-8?q?feat:=20=E7=94=9F=E7=89=A9=E7=94=B1?= =?UTF-8?q?=E6=9D=A5=E3=83=AD=E3=83=83=E3=83=88=E5=88=86=E8=A7=A3=E3=81=AE?= =?UTF-8?q?=E3=82=B9=E3=83=86=E3=83=BC=E3=83=88=E3=83=9E=E3=82=B7=E3=83=B3?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- stepfunctions/TOOLS/convert_config.yaml | 36 ++++- ...t-batch-trn-result-data-bio-lot-state.json | 129 ++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 stepfunctions/r-jskult-batch-trn-result-data-bio-lot-state/r-jskult-batch-trn-result-data-bio-lot-state.json diff --git a/stepfunctions/TOOLS/convert_config.yaml b/stepfunctions/TOOLS/convert_config.yaml index 6533829e..d22087bd 100644 --- a/stepfunctions/TOOLS/convert_config.yaml +++ b/stepfunctions/TOOLS/convert_config.yaml @@ -36,6 +36,8 @@ resource: - &STG_SG_DCF_INST_MERGE_IO "sg-0b5d1639a83d28f66" # セキュリティグループ(ecs-jskult-batch-mst-inst-all) - &STG_SG_MST_INST_ALL "sg-0483fd75c76fa5808" + # セキュリティグループ(ecs-jskult-batch-trn-result-data-bio-lot) + - &STG_SG_TRN_RESULT_DATA_BIO_LOT "sg-07ca545e440dd83b7" # 本番環境 product: # サブネット(PrivateSubnet1) @@ -66,6 +68,9 @@ resource: # セキュリティグループ(ecs-jskult-batch-mst-inst-all) # TODO: 本番リリース時にIDを正式版にする - &PRD_SG_MST_INST_ALL "sg-xxxxxxxxxxxxx" + # セキュリティグループ(ecs-jskult-batch-trn-result-data-bio-lot) + # TODO: 本番リリース時にIDを正式版にする + - &PRD_SG_TRN_RESULT_DATA_BIO_LOT "sg-xxxxxxxxxxxxx" config: # CRMデータ取得 r-crm-datafetch-state: @@ -348,4 +353,33 @@ config: # セキュリティグループ(ecs-all) SG_ECS_ALL: *PRD_SG_ECS_ALL # セキュリティグループ(ecs-jskult-batch-ultmarc-io) - SG_MST_INST_ALL: *PRD_SG_MST_INST_ALL \ No newline at end of file + SG_MST_INST_ALL: *PRD_SG_MST_INST_ALL + r-jskult-batch-trn-result-data-bio-lot-state: + # ステージング環境 + staging: + # AWSアカウントID + AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID + # 東京リージョン + REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1 + # サブネット(PrivateSubnet1) + SUBNET_PRI_1A: *STG_SUBNET_PRI_1A + # サブネット(PrivateSubnet2) + SUBNET_PRI_1D: *STG_SUBNET_PRI_1D + # セキュリティグループ(ecs-all) + SG_ECS_ALL: *STG_SG_ECS_ALL + # セキュリティグループ(ecs-jskult-batch-ultmarc-io) + SG_TRN_RESULT_DATA_BIO_LOT: *STG_SG_TRN_RESULT_DATA_BIO_LOT + # 本番環境 + product: + # AWSアカウントID + AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID + # 東京リージョン + REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1 + # サブネット(PrivateSubnet1) + SUBNET_PRI_1A: *PRD_SUBNET_PRI_1A + # サブネット(PrivateSubnet2) + SUBNET_PRI_1D: *PRD_SUBNET_PRI_1D + # セキュリティグループ(ecs-all) + SG_ECS_ALL: *PRD_SG_ECS_ALL + # セキュリティグループ(ecs-jskult-batch-ultmarc-io) + SG_TRN_RESULT_DATA_BIO_LOT: *PRD_SG_TRN_RESULT_DATA_BIO_LOT \ No newline at end of file diff --git a/stepfunctions/r-jskult-batch-trn-result-data-bio-lot-state/r-jskult-batch-trn-result-data-bio-lot-state.json b/stepfunctions/r-jskult-batch-trn-result-data-bio-lot-state/r-jskult-batch-trn-result-data-bio-lot-state.json new file mode 100644 index 00000000..4720e9e4 --- /dev/null +++ b/stepfunctions/r-jskult-batch-trn-result-data-bio-lot-state/r-jskult-batch-trn-result-data-bio-lot-state.json @@ -0,0 +1,129 @@ +{ + "Comment": "実消化&アルトマーク 生物由来ロット分解ステートマシン", + "StartAt": "params", + "States": { + "params": { + "Comment": "パラメータ設定", + "Type": "Pass", + "Parameters": { + "ecs": { + "LaunchType": "FARGATE", + "Cluster": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:cluster/mbj-newdwh2021-#{ENV_NAME}-jskult-batch-trn-result-data-bio-lot-ecs", + "TaskDefinition": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-#{ENV_NAME}-task-jskult-batch-trn-result-data-bio-lot", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "Subnets": [ + "#{SUBNET_PRI_1A}", + "#{SUBNET_PRI_1D}" + ], + "SecurityGroups": [ + "#{SG_ECS_ALL}", + "#{SG_TRN_RESULT_DATA_BIO_LOT}" + ], + "AssignPublicIp": "DISABLED" + } + }, + "Overrides": { + "ContainerOverrides": [ + { + "Name": "mbj-newdwh2021-#{ENV_NAME}-container-jskult-batch-trn-result-data-bio-lot", + "Environment": [ + { + "Name": "BATCH_EXECUTION_ID", + "Value.$": "$$.Execution.Id" + }, + { + "Name": "MAX_RUN_COUNT", + "Value.$": "$.maxRunCount" + } + ] + } + ] + } + } + }, + "ResultPath": "$.params", + "Next": "exec-ecs-task" + }, + "exec-ecs-task": { + "Type": "Task", + "Resource": "arn:aws:states:::ecs:runTask.sync", + "Parameters": { + "LaunchType.$": "$.params.ecs.LaunchType", + "Cluster.$": "$.params.ecs.Cluster", + "TaskDefinition.$": "$.params.ecs.TaskDefinition", + "NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration", + "Overrides.$": "$.params.ecs.Overrides" + }, + "ResultPath": "$.result", + "Retry": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "BackoffRate": 2, + "IntervalSeconds": 3, + "MaxAttempts": 3 + } + ], + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "Next": "ErrorEnd", + "ResultPath": "$.result" + } + ], + "Next": "scan-jskult-batch-run-manage", + "Comment": "ECSタスク起動" + }, + "scan-jskult-batch-run-manage": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "mbj-newdwh2021-#{ENV_NAME}-jskult-batch-run-manage", + "Key": { + "execution_id": { + "S.$": "$$.Execution.Id" + } + } + }, + "Next": "Choice", + "ResultPath": "$.scan" + }, + "Choice": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.scan.Item", + "IsPresent": false, + "Next": "ErrorEnd", + "Comment": "バッチ実行管理テーブルにデータが存在しない場合" + }, + { + "Variable": "$.scan.Item.batch_run_status.S", + "StringEquals": "retry", + "Next": "wait-for-retry", + "Comment": "バッチ実行管理テーブルのスターテスがリトライの場合" + } + ], + "Default": "NormalEnd" + }, + "wait-for-retry": { + "Type": "Wait", + "SecondsPath": "$.retryIntervalSecond", + "Next": "exec-ecs-task" + }, + "NormalEnd": { + "Comment": "正常終了", + "Type": "Succeed" + }, + "ErrorEnd": { + "Comment": "異常終了", + "Type": "Fail", + "Error": "StatesError", + "Cause": "StepFunctions ErrorEnd" + } + } +} \ No newline at end of file From d45f39850ec62ed66abdb571bc2d92e8344353c3 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 28 May 2025 19:51:21 +0900 Subject: [PATCH 08/11] =?UTF-8?q?feat:=20=E6=97=A5=E4=BB=98=E3=83=86?= =?UTF-8?q?=E3=83=BC=E3=83=96=E3=83=AB=E6=9B=B4=E6=96=B0=E3=81=AE=E3=82=B9?= =?UTF-8?q?=E3=83=86=E3=83=BC=E3=83=88=E3=83=9E=E3=82=B7=E3=83=B3=E3=82=92?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- stepfunctions/TOOLS/convert_config.yaml | 35 ++++- ...skult-batch-update-business-day-state.json | 129 ++++++++++++++++++ 2 files changed, 163 insertions(+), 1 deletion(-) create mode 100644 stepfunctions/r-jskult-batch-update-business-day-state/r-jskult-batch-update-business-day-state.json diff --git a/stepfunctions/TOOLS/convert_config.yaml b/stepfunctions/TOOLS/convert_config.yaml index d22087bd..27659dcf 100644 --- a/stepfunctions/TOOLS/convert_config.yaml +++ b/stepfunctions/TOOLS/convert_config.yaml @@ -38,6 +38,8 @@ resource: - &STG_SG_MST_INST_ALL "sg-0483fd75c76fa5808" # セキュリティグループ(ecs-jskult-batch-trn-result-data-bio-lot) - &STG_SG_TRN_RESULT_DATA_BIO_LOT "sg-07ca545e440dd83b7" + # TODO: 本番リリース時にIDを正式版にする + - &STG_SG_UPDATE_BIS_DAY "sg-0b58046abfa017e0e" # 本番環境 product: # サブネット(PrivateSubnet1) @@ -71,6 +73,8 @@ resource: # セキュリティグループ(ecs-jskult-batch-trn-result-data-bio-lot) # TODO: 本番リリース時にIDを正式版にする - &PRD_SG_TRN_RESULT_DATA_BIO_LOT "sg-xxxxxxxxxxxxx" + # TODO: 本番リリース時にIDを正式版にする + - &PRD_SG_UPDATE_BIS_DAY "sg-xxxxxxxxxxxxx" config: # CRMデータ取得 r-crm-datafetch-state: @@ -382,4 +386,33 @@ config: # セキュリティグループ(ecs-all) SG_ECS_ALL: *PRD_SG_ECS_ALL # セキュリティグループ(ecs-jskult-batch-ultmarc-io) - SG_TRN_RESULT_DATA_BIO_LOT: *PRD_SG_TRN_RESULT_DATA_BIO_LOT \ No newline at end of file + SG_TRN_RESULT_DATA_BIO_LOT: *PRD_SG_TRN_RESULT_DATA_BIO_LOT + r-jskult-batch-update-business-day-state: + # ステージング環境 + staging: + # AWSアカウントID + AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID + # 東京リージョン + REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1 + # サブネット(PrivateSubnet1) + SUBNET_PRI_1A: *STG_SUBNET_PRI_1A + # サブネット(PrivateSubnet2) + SUBNET_PRI_1D: *STG_SUBNET_PRI_1D + # セキュリティグループ(ecs-all) + SG_ECS_ALL: *STG_SG_ECS_ALL + # セキュリティグループ(ecs-jskult-batch-ultmarc-io) + SG_UPDATE_BIS_DAY: *STG_SG_UPDATE_BIS_DAY + # 本番環境 + product: + # AWSアカウントID + AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID + # 東京リージョン + REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1 + # サブネット(PrivateSubnet1) + SUBNET_PRI_1A: *PRD_SUBNET_PRI_1A + # サブネット(PrivateSubnet2) + SUBNET_PRI_1D: *PRD_SUBNET_PRI_1D + # セキュリティグループ(ecs-all) + SG_ECS_ALL: *PRD_SG_ECS_ALL + # セキュリティグループ(ecs-jskult-batch-ultmarc-io) + SG_UPDATE_BIS_DAY: *PRD_SG_UPDATE_BIS_DAY \ No newline at end of file diff --git a/stepfunctions/r-jskult-batch-update-business-day-state/r-jskult-batch-update-business-day-state.json b/stepfunctions/r-jskult-batch-update-business-day-state/r-jskult-batch-update-business-day-state.json new file mode 100644 index 00000000..b6cd2a25 --- /dev/null +++ b/stepfunctions/r-jskult-batch-update-business-day-state/r-jskult-batch-update-business-day-state.json @@ -0,0 +1,129 @@ +{ + "Comment": "実消化&アルトマーク 日付テーブル更新ステートマシン", + "StartAt": "params", + "States": { + "params": { + "Comment": "パラメータ設定", + "Type": "Pass", + "Parameters": { + "ecs": { + "LaunchType": "FARGATE", + "Cluster": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:cluster/mbj-newdwh2021-#{ENV_NAME}-jskult-batch-update-business-day-ecs", + "TaskDefinition": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-#{ENV_NAME}-task-jskult-batch-update-business-day", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "Subnets": [ + "#{SUBNET_PRI_1A}", + "#{SUBNET_PRI_1D}" + ], + "SecurityGroups": [ + "#{SG_ECS_ALL}", + "#{SG_UPDATE_BIS_DAY}" + ], + "AssignPublicIp": "DISABLED" + } + }, + "Overrides": { + "ContainerOverrides": [ + { + "Name": "mbj-newdwh2021-#{ENV_NAME}-container-jskult-batch-update-business-day", + "Environment": [ + { + "Name": "BATCH_EXECUTION_ID", + "Value.$": "$$.Execution.Id" + }, + { + "Name": "MAX_RUN_COUNT", + "Value.$": "$.maxRunCount" + } + ] + } + ] + } + } + }, + "ResultPath": "$.params", + "Next": "exec-ecs-task" + }, + "exec-ecs-task": { + "Type": "Task", + "Resource": "arn:aws:states:::ecs:runTask.sync", + "Parameters": { + "LaunchType.$": "$.params.ecs.LaunchType", + "Cluster.$": "$.params.ecs.Cluster", + "TaskDefinition.$": "$.params.ecs.TaskDefinition", + "NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration", + "Overrides.$": "$.params.ecs.Overrides" + }, + "ResultPath": "$.result", + "Retry": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "BackoffRate": 2, + "IntervalSeconds": 3, + "MaxAttempts": 3 + } + ], + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "Next": "ErrorEnd", + "ResultPath": "$.result" + } + ], + "Next": "scan-jskult-batch-run-manage", + "Comment": "ECSタスク起動" + }, + "scan-jskult-batch-run-manage": { + "Type": "Task", + "Resource": "arn:aws:states:::dynamodb:getItem", + "Parameters": { + "TableName": "mbj-newdwh2021-#{ENV_NAME}-jskult-batch-run-manage", + "Key": { + "execution_id": { + "S.$": "$$.Execution.Id" + } + } + }, + "Next": "Choice", + "ResultPath": "$.scan" + }, + "Choice": { + "Type": "Choice", + "Choices": [ + { + "Variable": "$.scan.Item", + "IsPresent": false, + "Next": "ErrorEnd", + "Comment": "バッチ実行管理テーブルにデータが存在しない場合" + }, + { + "Variable": "$.scan.Item.batch_run_status.S", + "StringEquals": "retry", + "Next": "wait-for-retry", + "Comment": "バッチ実行管理テーブルのスターテスがリトライの場合" + } + ], + "Default": "NormalEnd" + }, + "wait-for-retry": { + "Type": "Wait", + "SecondsPath": "$.retryIntervalSecond", + "Next": "exec-ecs-task" + }, + "NormalEnd": { + "Comment": "正常終了", + "Type": "Succeed" + }, + "ErrorEnd": { + "Comment": "異常終了", + "Type": "Fail", + "Error": "StatesError", + "Cause": "StepFunctions ErrorEnd" + } + } +} \ No newline at end of file From 288c73b50f20193c772eaefecc150d1cf7350158 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 28 May 2025 20:02:57 +0900 Subject: [PATCH 09/11] =?UTF-8?q?feat:=20=E3=82=A2=E3=83=BC=E3=82=AB?= =?UTF-8?q?=E3=82=A4=E3=83=96=E5=8F=96=E5=BE=97=E3=81=AE=E3=82=B9=E3=83=86?= =?UTF-8?q?=E3=83=BC=E3=83=88=E3=83=9E=E3=82=B7=E3=83=B3=E3=82=92=E8=BF=BD?= =?UTF-8?q?=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- stepfunctions/TOOLS/convert_config.yaml | 40 ++++++++++- ...r-jskult-batch-archive-jsk-data-state.json | 72 +++++++++++++++++++ 2 files changed, 110 insertions(+), 2 deletions(-) create mode 100644 stepfunctions/r-jskult-batch-archive-jsk-data-state/r-jskult-batch-archive-jsk-data-state.json diff --git a/stepfunctions/TOOLS/convert_config.yaml b/stepfunctions/TOOLS/convert_config.yaml index 27659dcf..d68153e1 100644 --- a/stepfunctions/TOOLS/convert_config.yaml +++ b/stepfunctions/TOOLS/convert_config.yaml @@ -38,8 +38,11 @@ resource: - &STG_SG_MST_INST_ALL "sg-0483fd75c76fa5808" # セキュリティグループ(ecs-jskult-batch-trn-result-data-bio-lot) - &STG_SG_TRN_RESULT_DATA_BIO_LOT "sg-07ca545e440dd83b7" - # TODO: 本番リリース時にIDを正式版にする + # セキュリティグループ(ecs-jskult-batch-update-business-day) - &STG_SG_UPDATE_BIS_DAY "sg-0b58046abfa017e0e" + # セキュリティグループ(ecs-jskult-batch-archive-jsk-data) + - &STG_SG_JSK_ARCHIVE "sg-0bbdd7d4ebb5ba222" + # 本番環境 product: # サブネット(PrivateSubnet1) @@ -73,8 +76,12 @@ resource: # セキュリティグループ(ecs-jskult-batch-trn-result-data-bio-lot) # TODO: 本番リリース時にIDを正式版にする - &PRD_SG_TRN_RESULT_DATA_BIO_LOT "sg-xxxxxxxxxxxxx" + # セキュリティグループ(ecs-jskult-batch-update-business-day) # TODO: 本番リリース時にIDを正式版にする - &PRD_SG_UPDATE_BIS_DAY "sg-xxxxxxxxxxxxx" + # セキュリティグループ(ecs-jskult-batch-archive-jsk-data) + # TODO: 本番リリース時にIDを正式版にする + - &PRD_SG_JSK_ARCHIVE "sg-xxxxxxxxxxxxx" config: # CRMデータ取得 r-crm-datafetch-state: @@ -415,4 +422,33 @@ config: # セキュリティグループ(ecs-all) SG_ECS_ALL: *PRD_SG_ECS_ALL # セキュリティグループ(ecs-jskult-batch-ultmarc-io) - SG_UPDATE_BIS_DAY: *PRD_SG_UPDATE_BIS_DAY \ No newline at end of file + SG_UPDATE_BIS_DAY: *PRD_SG_UPDATE_BIS_DAY + r-jskult-batch-archive-jsk-data-state: + # ステージング環境 + staging: + # AWSアカウントID + AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID + # 東京リージョン + REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1 + # サブネット(PrivateSubnet1) + SUBNET_PRI_1A: *STG_SUBNET_PRI_1A + # サブネット(PrivateSubnet2) + SUBNET_PRI_1D: *STG_SUBNET_PRI_1D + # セキュリティグループ(ecs-all) + SG_ECS_ALL: *STG_SG_ECS_ALL + # セキュリティグループ(ecs-jskult-batch-ultmarc-io) + SG_JSK_ARCHIVE: *STG_SG_JSK_ARCHIVE + # 本番環境 + product: + # AWSアカウントID + AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID + # 東京リージョン + REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1 + # サブネット(PrivateSubnet1) + SUBNET_PRI_1A: *PRD_SUBNET_PRI_1A + # サブネット(PrivateSubnet2) + SUBNET_PRI_1D: *PRD_SUBNET_PRI_1D + # セキュリティグループ(ecs-all) + SG_ECS_ALL: *PRD_SG_ECS_ALL + # セキュリティグループ(ecs-jskult-batch-ultmarc-io) + SG_JSK_ARCHIVE: *PRD_SG_JSK_ARCHIVE diff --git a/stepfunctions/r-jskult-batch-archive-jsk-data-state/r-jskult-batch-archive-jsk-data-state.json b/stepfunctions/r-jskult-batch-archive-jsk-data-state/r-jskult-batch-archive-jsk-data-state.json new file mode 100644 index 00000000..3ff8d13b --- /dev/null +++ b/stepfunctions/r-jskult-batch-archive-jsk-data-state/r-jskult-batch-archive-jsk-data-state.json @@ -0,0 +1,72 @@ +{ + "Comment": "MeDaCA 実消化&アルトマーク 実消化過去データアーカイブ処理ステートマシン", + "StartAt": "params", + "States": { + "params": { + "Comment": "パラメータ設定", + "Type": "Pass", + "Parameters": { + "sns": { + "TopicArn": "arn:aws:sns:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:nds-notice-#{ENV_NAME}" + }, + "ecs": { + "Cluster": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:cluster/mbj-newdwh2021-#{ENV_NAME}-jskult-batch-archive-jsk-data-ecs", + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "Subnets": [ + "#{SUBNET_PRI_1A}", + "#{SUBNET_PRI_1D}" + ], + "SecurityGroups": [ + "#{SG_ECS_ALL}", + "#{SG_JSK_ARCHIVE}" + ], + "AssignPublicIp": "DISABLED" + } + } + } + }, + "ResultPath": "$.params", + "Next": "exec-ecs-task" + }, + "exec-ecs-task": { + "Comment": "ECSタスク起動", + "Type": "Task", + "Resource": "arn:aws:states:::ecs:runTask.sync", + "Parameters": { + "Cluster.$": "$.params.ecs.Cluster", + "LaunchType.$": "$.params.ecs.LaunchType", + "TaskDefinition": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-#{ENV_NAME}-task-jskult-batch-archive-jsk-data", + "NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration" + }, + "Retry": [ + { + "ErrorEquals": ["States.ALL"], + "BackoffRate": 2, + "IntervalSeconds": 5, + "MaxAttempts": 3 + } + ], + "Catch": [ + { + "ErrorEquals": ["States.ALL"], + "ResultPath": "$.result", + "Next": "ErrorEnd" + } + ], + "ResultPath": "$.result", + "Next": "NormalEnd" + }, + "NormalEnd": { + "Comment": "正常終了", + "Type": "Succeed" + }, + "ErrorEnd": { + "Comment": "異常終了", + "Type": "Fail", + "Error": "StatesError", + "Cause": "StepFunctions ErrorEnd" + } + } +} From 1324408a97969a39d6a2374e47e48ffa6115a72d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 28 May 2025 20:04:44 +0900 Subject: [PATCH 10/11] =?UTF-8?q?feat:=20MAX=5FRUN=5FCOUNT=E3=81=AF?= =?UTF-8?q?=E6=96=87=E5=AD=97=E5=88=97=E3=81=A7=E5=8F=97=E3=81=91=E5=8F=96?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch/src/system_var/environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/jskult-batch/src/system_var/environment.py b/ecs/jskult-batch/src/system_var/environment.py index 4e220ba0..bd1c9c24 100644 --- a/ecs/jskult-batch/src/system_var/environment.py +++ b/ecs/jskult-batch/src/system_var/environment.py @@ -10,7 +10,7 @@ DB_SCHEMA = os.environ['DB_SCHEMA'] # AWS JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] BATCH_EXECUTION_ID = os.environ['BATCH_EXECUTION_ID'] -MAX_RUN_COUNT = os.environ['MAX_RUN_COUNT'] +MAX_RUN_COUNT = int(os.environ['MAX_RUN_COUNT']) TRANSFER_RESULT_FOLDER = os.environ['TRANSFER_RESULT_FOLDER'] TRANSFER_RESULT_FILE_NAME = os.environ['TRANSFER_RESULT_FILE_NAME'] DCF_INST_MERGE_SEND_FILE_NAME = os.environ['DCF_INST_MERGE_SEND_FILE_NAME'] From c0a0b2ffdae66237b6e54da691e26da0f736b7fe Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 29 May 2025 14:38:23 +0900 Subject: [PATCH 11/11] =?UTF-8?q?feat:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../r-jskult-batch-archive-jsk-data-state.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stepfunctions/r-jskult-batch-archive-jsk-data-state/r-jskult-batch-archive-jsk-data-state.json b/stepfunctions/r-jskult-batch-archive-jsk-data-state/r-jskult-batch-archive-jsk-data-state.json index 3ff8d13b..44ef2300 100644 --- a/stepfunctions/r-jskult-batch-archive-jsk-data-state/r-jskult-batch-archive-jsk-data-state.json +++ b/stepfunctions/r-jskult-batch-archive-jsk-data-state/r-jskult-batch-archive-jsk-data-state.json @@ -1,5 +1,5 @@ { - "Comment": "MeDaCA 実消化&アルトマーク 実消化過去データアーカイブ処理ステートマシン", + "Comment": "実消化_過去データアーカイブ処理ステートマシン", "StartAt": "params", "States": { "params": {