nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop' into feature-NEWDWH2021-1864

Browse Source
This commit is contained in:
mori.k 2025-05-29 15:52:21 +09:00
commit 45ccc48b97
36 changed files with 2855 additions and 291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
ecs/jskult-batch-ultmarc-io/src/batch/ultmarc/import_ultmarc_process.py
View File

@ -1,6 +1,7 @@
"""アルトマークデータ処理"""
import json
from urllib.parse import unquote
from src.aws.s3 import ConfigBucket, UltmarcBucket
from src.batch.ultmarc.datfile import DatFile
@ -34,7 +35,9 @@ def exec_import():
# ファイルの件数は必ず1件になる
dat_file_info = dat_file_list[0]
dat_file_name = dat_file_info['filename']
if environment.IMPORT_FILE_KEY != dat_file_name:
# ファイル名が日本語のとき、URLエンコードされるているため、デコードする
decoded_import_file_key = unquote(environment.IMPORT_FILE_KEY, encoding='utf-8', errors='replace')
if decoded_import_file_key != dat_file_name:
raise BatchOperationException(f'取込対象のファイルが見つからないため、異常終了 ファイル名:{environment.IMPORT_FILE_KEY}')
# 0Byteの場合、
if dat_file_info['size'] == 0:

4
ecs/jskult-batch-ultmarc-io/src/error/exceptions.py
View File

@ -8,3 +8,7 @@ class DBException(MeDaCaException):
class BatchOperationException(MeDaCaException):
pass
class MaxRunCountReachedException(MeDaCaException):
pass

30
ecs/jskult-batch-ultmarc-io/src/main.py
View File

@ -4,6 +4,7 @@ from src.batch.common.batch_context import BatchContext
from src.batch.ultmarc import import_ultmarc_process, output_dcf_dsf_data
from src.error.exceptions import BatchOperationException
from src.logging.get_logger import get_logger
from src.manager.jskult_batch_status_manager import JskultBatchStatusManager
from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager
from src.system_var import constants
@ -18,13 +19,22 @@ def exec():
logger.info('アルトマーク取込/データ出力：開始')
hdke_tbl_manager = JskultHdkeTblManager()
batch_status_manager = JskultBatchStatusManager(
constants.PROCESS_NAME_ULTMARC_IO,
constants.PROCESS_TYPE_DATA_IMPORT,
0, # 最大起動回数は使用しないため、0をセット
0, # 受信ファイル数は使用しないため、0をセット
)
# バッチステータスを処理開始に変更
batch_status_manager.set_process_status(constants.PROCESS_STATUS_START)
try:
if not hdke_tbl_manager.can_run_process():
logger.error('日次バッチ処理中またはdump取得が正常終了していないため、日次バッチ処理を終了します。')
return constants.BATCH_EXIT_CODE_SUCCESS
except BatchOperationException as e:
logger.exception(f'日付テーブルチェック処理エラー（異常終了）{e}')
# TODO: バッチステータス管理テーブルにエラーを登録
# バッチステータスをエラーに変更
batch_status_manager.set_process_status(constants.PROCESS_STATUS_ERROR)
return constants.BATCH_EXIT_CODE_SUCCESS
_, _, syor_date = hdke_tbl_manager.get_batch_statuses()
@ -32,7 +42,8 @@ def exec():
# バッチ共通設定に処理日を追加
batch_context.syor_date = syor_date
# TODO: バッチステータス管理テーブルに処理中を登録
# バッチステータスを処理中を変更
batch_status_manager.set_process_status(constants.PROCESS_STATUS_DOING)
try:
logger.info('アルトマーク取込：起動')
@ -40,19 +51,22 @@ def exec():
logger.info('アルトマーク取込：終了')
except BatchOperationException as e:
logger.exception(f'アルトマーク取込処理エラー（異常終了）{e}')
# TODO: バッチステータス管理テーブルにエラーを登録
# バッチステータスをエラーに変更
batch_status_manager.set_process_status(constants.PROCESS_STATUS_ERROR)
return constants.BATCH_EXIT_CODE_SUCCESS
try:
logger.info('V実消化用施設データ作成処理：起動')
logger.info('実消化用DCF/DSFデータ作成処理：起動')
output_dcf_dsf_data.exec()
logger.info('V実消化用施設データ作成処理：終了')
logger.info('実消化用DCF/DSFデータ作成処理：終了')
except BatchOperationException as e:
logger.exception(f'V実消化用施設データ作成処理エラー（異常終了）{e}')
# TODO: バッチステータス管理テーブルにエラーを登録
logger.exception(f'実消化用施設DCF/DSF作成処理エラー（異常終了）{e}')
# バッチステータスをエラーに変更
batch_status_manager.set_process_status(constants.PROCESS_STATUS_ERROR)
return constants.BATCH_EXIT_CODE_SUCCESS
# TODO: バッチステータス管理テーブルに処理済を登録
# バッチステータスを処理済に変更
batch_status_manager.set_process_status(constants.PROCESS_STATUS_DONE)
logger.info('アルトマーク取込/データ出力：終了')
return constants.BATCH_EXIT_CODE_SUCCESS

450
ecs/jskult-batch-ultmarc-io/src/manager/jskult_batch_status_manager.py Normal file
View File

@ -0,0 +1,450 @@
from src.db.database import Database
from src.error.exceptions import (BatchOperationException,
MaxRunCountReachedException)
from src.system_var import constants
class JskultBatchStatusManager:
"""実消化&アルトマーク_バッチステータス管理テーブルを管理するクラス"""
def __init__(self, process_name: str, process_type: str, max_run_count: int, receive_file_count: int):
"""コンストラクタ
Args:
process_name (str): 処理名
process_type (str): 管理区分
max_run_count (int): 最大起動回数
receive_file_count (int): 受信ファイル数
"""
self._process_name: str = process_name
self._process_type: str = process_type
self._max_run_count: str = max_run_count
self._receive_file_count: str = receive_file_count
# DB接続モジュールを初期化
self._db = Database.get_instance()
# 処理ステータスの登録および更新を行う
def set_process_status(self, process_status: str):
"""
処理ステータスの登録および更新を行う
Args:
process_status (str): 処理ステータス
Raises:
BatchOperationException: DB操作の何らかのエラー
"""
try:
self._db.connect()
self._db.begin()
self._db.to_jst()
self._db.execute(
"""
CALL
internal07.upsert_jskult_batch_status_manage(
:process_name,
:process_type,
:process_status,
NULL,
NULL
);""",
{
'process_name': self._process_name,
'process_type': self._process_type,
'process_status': process_status,
}
)
self._db.commit()
except Exception as e:
# 例外発生時はrollback
self._db.rollback()
raise BatchOperationException(e)
finally:
self._db.disconnect()
def can_run_post_process(self) -> bool:
"""
後続処理を実行してよいか判定する
Raises:
BatchOperationException: DB操作の何らかのエラー
MaxRunCountReachedException: 最大起動回数到達時の例外
Returns:
bool 後続処理を実行してよい場合はTrue
"""
try:
self._db.connect()
# 自身（後続処理 or 日付テーブル更新）のレコードを取得する
record = self._db.execute_select(
"""
SELECT
process_name,
process_date
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = :process_name
AND process_date = src07.get_syor_date();
""",
{'process_name': self._process_name}
)
record_count = len(record)
if record_count == 0:
raise BatchOperationException("レコードの取得が出来ませんでした。")
# 起動回数のインクリメント
self._increment_run_count()
# データ取込が完了していた場合
if self._is_done_data_import():
return True
# 最大起動回数に到達していない場合
if not self._is_max_run_count_reached():
return False
# 最大起動回数に到達していた場合
# 最大起動回数フラグを立てて例外を送出する
self._activate_max_run_count_flg()
raise MaxRunCountReachedException("最大起動回数に到達しました")
except MaxRunCountReachedException as e:
raise e
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
def can_run_business_day_update(self) -> bool:
"""
日付テーブル更新を実行してよいか判定する
Raises:
BatchOperationException: DB操作の何らかのエラー
MaxRunCountReachedException: 最大起動回数到達時の例外
Returns:
bool: 日付テーブル更新を実行してよい場合はTrue
"""
try:
# 自身（後続処理 or 日付テーブル更新）のレコードを取得する
self._db.connect()
record = self._db.execute_select(
"""
SELECT
process_name,
process_date
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = :process_name
AND process_date = src07.get_syor_date();
""",
{'process_name': self._process_name}
)
record_count = len(record)
if record_count == 0:
raise BatchOperationException("レコードの取得が出来ませんでした。")
# 起動回数のインクリメント
self._increment_run_count()
# 後続処理が完了していた場合
if self._is_done_post_process():
return True
# 最大起動回数に到達していない場合
if not self._is_max_run_count_reached():
return False
# 最大起動回数フラグを立てる
self._activate_max_run_count_flg()
# 最大起動回数に到達した場合にメッセージをスロー
raise MaxRunCountReachedException("最大起動回数に到達しました")
except MaxRunCountReachedException as e:
raise e
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
def is_done_ultmarc_import(self) -> bool:
"""アルトマークデータ連携があったかを確認する
Raises:
BatchOperationException: DB操作の何らかのエラー
Returns:
bool アルトマークデータ連携があった場合はTrue
"""
try:
self._db.connect()
record = self._db.execute_select(
"""
SELECT
process_name,
process_date
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = :process_name
AND process_date = src07.get_syor_date()
AND process_status = :process_status;
""",
{
'process_name': constants.PROCESS_NAME_ULTMARC_IO,
'process_status': constants.PROCESS_STATUS_DONE
}
)
record_count = len(record)
return record_count != 0
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
def _increment_run_count(self):
"""起動回数をインクリメントする
Raises:
BatchOperationException: DB操作の何らかのエラー
"""
try:
self._db.connect()
self._db.begin()
self._db.to_jst()
# 自身（後続処理 or 日付テーブル更新）のレコードを取得する
record = self._db.execute_select(
"""
SELECT
total_run_count
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = :process_name
AND process_date = src07.get_syor_date();
""",
{'process_name': self._process_name}
)
if len(record) == 0:
raise BatchOperationException("レコードの取得が出来ませんでした。")
total_run_count = record[0]['total_run_count']
self._db.execute(
"""
CALL
internal07.upsert_jskult_batch_status_manage(
:process_name,
:process_type,
NULL,
:total_run_count,
NULL
)
""",
{
'process_name': self._process_name,
'process_type': self._process_type,
'total_run_count': total_run_count + 1,
}
)
self._db.commit()
except Exception as e:
# 例外発生時はrollback
self._db.rollback()
raise BatchOperationException(e)
finally:
self._db.disconnect()
def _is_done_data_import(self) -> bool:
"""データ取込処理が完了しているかを判定する
Raises:
BatchOperationException: DB操作の何らかのエラー
MaxRunCountReachedException: 最大起動回数到達時の例外
Returns:
bool 後続処理を実行してよい場合はTrue
"""
try:
self._db.connect()
# 自身（後続処理 or 日付テーブル更新）のレコードを取得する
record = self._db.execute_select(
"""
SELECT
process_name,
process_date
FROM
internal07.jskult_batch_status_manage
WHERE
process_type = :process_type
AND process_status = :process_status
AND process_date = src07.get_syor_date();
""",
{
'process_type': constants.PROCESS_TYPE_DATA_IMPORT,
'process_status': constants.PROCESS_STATUS_DONE
}
)
record_count = len(record)
# データ取込数とデータ登録の件数が一致しているか確認
return self._receive_file_count == record_count
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
def _is_done_post_process(self) -> bool:
"""後続処理のすべての処理が完了しているかを判定する
Returns:
bool: 後続処理がすべて完了していたらTrue
Raises:
BatchOperationException: DB操作の何らかのエラー
"""
# 生物由来データロット分解のチェック
if not self._is_done_process(constants.PROCESS_NAME_TRN_RESULT_DATA_BIO_LOT):
return False
# メルク施設マスタ作成のチェック
if not self._is_done_process(constants.PROCESS_NAME_MST_INST_ALL):
return False
# DCF削除新規マスタ作成のチェック
if not self._is_done_process(constants.PROCESS_NAME_DCF_INST_MERGE_IO):
return False
# 全ての後続処理が完了している場合Trueを返す
return True
# データ取込処理が完了しているかを判定する
def _is_done_process(self, process_name: str) -> bool:
"""指定された処理名の処理が完了しているかを判定する
Args:
process_name (str): 処理名
Returns:
bool: 処理名に一致する処理が完了していたらTrue
Raises:
BatchOperationException: DB操作の何らかのエラー
"""
try:
self._db.connect()
record = self._db.execute_select(
"""
SELECT
process_name,
process_date
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = :process_name
AND process_status = :process_status
AND process_date = src07.get_syor_date();
""",
{
'process_name': process_name,
'process_status': constants.PROCESS_STATUS_DONE
}
)
record_count = len(record)
return record_count != 0
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
def _is_max_run_count_reached(self) -> bool:
"""起動回数が最大回数に到達しているか判定する
Returns:
bool: 最大起動回数に到達していたらTrue
Raises:
BatchOperationException: DB操作の何らかのエラー
"""
try:
self._db.connect()
record = self._db.execute_select(
"""
SELECT
total_run_count
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = :process_name
AND
process_date = src07.get_syor_date();
""",
{'process_name': self._process_name}
)
total_run_count = record[0]['total_run_count']
# 取得した起動回数とフィールド変数の最大起動回数が一致を確認
return total_run_count == self._max_run_count
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
def _activate_max_run_count_flg(self):
"""最大起動回数フラグにフラグを立てる
Raises:
BatchOperationException: DB操作の何らかのエラー
"""
try:
self._db.connect()
self._db.begin()
self._db.to_jst()
self._db.execute(
"""
CALL
internal07.upsert_jskult_batch_status_manage(
:process_name,
:process_type,
NULL,
NULL,
:max_run_count_flag);
""",
{
'process_name': self._process_name,
'process_type': self._process_type,
'max_run_count_flag': constants.MAX_RUN_COUNT_FLAG_ON
}
)
self._db.commit()
except Exception as e:
self._db.rollback()
raise BatchOperationException(e)
finally:
self._db.disconnect()

19
ecs/jskult-batch-ultmarc-io/src/system_var/constants.py
View File

@ -9,3 +9,22 @@ BATCH_ACTF_BATCH_START = '1'
DUMP_STATUS_KBN_UNPROCESSED = '0'
# dump取得状態区分：dump取得正常終了
DUMP_STATUS_KBN_COMPLETE = '2'
# バッチステータス管理:
# 処理名:
PROCESS_NAME_ULTMARC_IO = 'jskult-batch-ultmarc-io'
# 管理区分:
# データ取込
PROCESS_TYPE_DATA_IMPORT = 'data_import'
# 処理ステータス:
# 処理開始
PROCESS_STATUS_START = 'start'
# 処理待
PROCESS_STATUS_WAITING = 'waiting'
# 処理中
PROCESS_STATUS_DOING = 'doing'
# 処理済
PROCESS_STATUS_DONE = 'done'
# エラー
PROCESS_STATUS_ERROR = 'error'

4
ecs/jskult-batch/.vscode/recommended_settings.json vendored
View File

@ -3,7 +3,7 @@
"editor.defaultFormatter": null,
"editor.formatOnSave": true,
"editor.codeActionsOnSave": {
"source.organizeImports": true
"source.organizeImports": "explicit"
}
},
// 自身の環境に合わせて変えてください
@ -23,7 +23,7 @@
"--ignore=F541"
],
"python.testing.pytestArgs": [
"tests/batch/"
"tests/"
],
"python.testing.unittestEnabled": false,

6
ecs/jskult-batch/Pipfile
View File

@ -4,10 +4,8 @@ verify_ssl = true
name = "pypi"
[scripts]
"test:ultmarc" = "pytest tests/batch/ultmarc/"
"test:ultmarc:cov" = "pytest --cov=src/batch/ultmarc/ --cov-branch --cov-report=term-missing tests/batch/ultmarc/"
"test:vjsk" = "pytest tests/batch/vjsk/"
"test:vjsk:cov" = "pytest --cov=src/batch/vjsk/ --cov-branch --cov-report=term-missing tests/batch/vjsk/"
"test" = "pytest tests -vvv"
"test:cov" = "pytest --cov=src/manager/ --cov=src/batch/common --cov-branch --cov-report=term-missing tests/"
[packages]
boto3 = "*"

23
ecs/jskult-batch/src/batch/common/jskult_batch_entrypoint_factory.py
View File

@ -1,10 +1,11 @@
from src.batch.update_business_day import UpdateBusinessDay
from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint
from src.batch.trn_result_data_bio_lot import TrnResultDataBioLot
from src.batch.mst_inst import MstInst
from src.batch.dcf_inst_merge_io import DcfInstMergeIO
from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint
from src.batch.mst_inst_all import MstInstAll
from src.batch.trn_result_data_bio_lot import TrnResultDataBioLot
from src.batch.update_business_day import UpdateBusinessDay
from src.error.exceptions import BatchOperationException
from src.logging.get_logger import get_logger
from src.system_var import constants
logger = get_logger("後続処理/日付更新処理振り分け")
@ -16,15 +17,15 @@ class JskultBatchEntrypointFactory:
def create(self) -> JskultBatchEntrypoint:
if self._entrypoint_module_name == "jskult-batch-trn-result-data-bio-lot":
if self._entrypoint_module_name == constants.PROCESS_NAME_TRN_RESULT_DATA_BIO_LOT:
return TrnResultDataBioLot()
if self._entrypoint_module_name == "jskult-batch-mst-inst":
return MstInst()
if self._entrypoint_module_name == "jskult-batch-dcf-inst-merge-io":
if self._entrypoint_module_name == constants.PROCESS_NAME_MST_INST_ALL:
return MstInstAll()
if self._entrypoint_module_name == constants.PROCESS_NAME_DCF_INST_MERGE_IO:
return DcfInstMergeIO()
if self._entrypoint_module_name == "jskult-batch-update-business-day":
if self._entrypoint_module_name == constants.PROCESS_NAME_UPDATE_BUSINESS_DAY:
return UpdateBusinessDay()
logger.error(
f"一致するエントリーポイント識別子ではありませんでした。エントリーポイント識別子:{self._entrypoint_module_name}")
raise ValueError()
raise BatchOperationException()

1
ecs/jskult-batch/src/batch/dcf_inst_merge_io.py
View File

@ -22,6 +22,7 @@ class DcfInstMergeIO(JskultBatchEntrypoint):
super().__init__()
def execute(self):
logger.info("DCF削除新規マスタ作成処理を開始します。")
jskult_hdke_tbl_manager = JskultHdkeTblManager()
jskult_batch_run_manager = JskultBatchRunManager(
environment.BATCH_EXECUTION_ID)

1
ecs/jskult-batch/src/batch/jskult_batch_entrypoint.py
View File

@ -1,4 +1,5 @@
import abc
# 実消化&アルトマークの後続処理/日付更新実行クラスの基底クラス

2
ecs/jskult-batch/src/batch/mst_inst.py → ecs/jskult-batch/src/batch/mst_inst_all.py
View File

@ -1,7 +1,7 @@
from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint
class MstInst(JskultBatchEntrypoint):
class MstInstAll(JskultBatchEntrypoint):
def __init__(self):
super().__init__()

39
ecs/jskult-batch/src/manager/jskult_batch_run_manager.py
View File

@ -1,39 +1,44 @@
import os
from datetime import datetime, timezone
from datetime import UTC, datetime, timedelta
import boto3
TABLE_NAME = 'mbj-newdwh2021-staging-jskult-batch-run-manage'
dynamodb = boto3.client('dynamodb')
class JskultBatchRunManager:
def __init__(self, execution_id: str):
"""実消化&アルトマーク_バッチ実行管理テーブル(DynamoDB)の操作クラス"""
def __init__(self, table_name: str, execution_id: str):
self._table_name = table_name
self._execution_id: str = execution_id
self._dynamodb = boto3.client('dynamodb')
# バッチ処理ステータスをsuccessで登録および更新を行う
def batch_success(self):
"""バッチ処理ステータスをsuccessで登録および更新を行う"""
self._put_dynamodb_record('success')
# バッチ処理ステータスをfailedで登録および更新を行う
def batch_failed(self):
"""バッチ処理ステータスをfailedで登録および更新を行う"""
self._put_dynamodb_record('failed')
# バッチ処理ステータスをretryで登録および更新を行う
def batch_retry(self):
"""バッチ処理ステータスをretryで登録および更新を行う"""
self._put_dynamodb_record('retry')
def _put_dynamodb_record(self, record: str):
# バッチ実行管理テーブルの登録、更新（upsert)
now = int(datetime.now(timezone.utc).timestamp() * 1000)
def _put_dynamodb_record(self, batch_run_status: str):
"""バッチ実行管理テーブルの登録、更新（upsert)
Args:
batch_run_status (str): バッチ処理ステータス
"""
# レコードの有効期限を現在時刻+24hのタイムスタンプで生成
now = datetime.now(UTC)
later = now + timedelta(hours=24)
timestamp_24h = int(later.timestamp())
options = {
'TableName': TABLE_NAME,
'TableName': self._table_name,
'Item': {
'execution_id': {'S': self._execution_id},
'status': {'S': record},
'createdAt': {'N': str(now)},
'batch_run_status': {'S': batch_run_status},
'record_expiration_time': {'N': str(timestamp_24h)},
},
}
dynamodb.put_item(**options)
self._dynamodb.put_item(**options)

420
ecs/jskult-batch/src/manager/jskult_batch_status_manager.py
View File

@ -1,77 +1,103 @@
import os
from src.error.exceptions import MaxRunCountReachedException
from src.db.database import Database
# 実消化&アルトマーク_バッチステータス管理テーブルを管理するクラス
from src.error.exceptions import (BatchOperationException,
MaxRunCountReachedException)
from src.system_var import constants
class JskultBatchStatusManager:
def __init__(self, process_name: str, process_type: str, max_run_count_flg: int, receive_file_count: int):
"""実消化&アルトマーク_バッチステータス管理テーブルを管理するクラス"""
def __init__(self, process_name: str, process_type: str, max_run_count: int, receive_file_count: int):
"""コンストラクタ
Args:
process_name (str): 処理名
process_type (str): 管理区分
max_run_count (int): 最大起動回数
receive_file_count (int): 受信ファイル数
"""
# 処理名
self._process_name: str = process_name
# 管理区分
self._process_type: str = process_type
# 最大起動回数
self._max_run_count_flg: str = max_run_count_flg
# 受信ファイル数
self._max_run_count: str = max_run_count
self._receive_file_count: str = receive_file_count
# DB接続モジュール(バッチ)のget_instanceを呼び出し設定
# DB接続モジュールを初期化
self._db = Database.get_instance()
# 処理ステータスの登録および更新を行う
def set_process_status(self, process_status: str):
"""
処理ステータスの登録および更新を行う
Args:
process_status (str): 処理ステータス
Raises:
BatchOperationException: DB操作の何らかのエラー
"""
try:
# DB接続開始
self._db.connect()
self._db.begin()
self._db.to_jst()
self._db.execute(
f"""
"""
CALL
internal07.upsert_jskult_batch_status_manage(
{self._process_name},
{self._process_type},
{process_status},
:process_name,
:process_type,
:process_status,
NULL,
NULL
);"""
);""",
{
'process_name': self._process_name,
'process_type': self._process_type,
'process_status': process_status,
}
)
self._db.commit()
except Exception as e:
# Exceptionをcatchした場合はrollback
print(self._db)
# 例外発生時はrollback
self._db.rollback()
raise e
raise BatchOperationException(e)
finally:
self._db.disconnect()
# 後続処理を実行してよいか判定する
def can_run_post_process(self):
def can_run_post_process(self) -> bool:
"""
後続処理を実行してよいか判定する
# SELECTの結果からレコード数を取得
Raises:
BatchOperationException: DB操作の何らかのエラー
MaxRunCountReachedException: 最大起動回数到達時の例外
Returns:
bool 後続処理を実行してよい場合はTrue
"""
try:
self._db.connect()
# 自身（後続処理 or 日付テーブル更新）のレコードを取得する
record = self._db.execute_select(
f"""
"""
SELECT
COUNT(*)
process_name,
process_date
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = {self._process_name}
AND
process_date = src07.get_syor_date();
"""
process_name = :process_name
AND process_date = src07.get_syor_date();
""",
{'process_name': self._process_name}
)
record_count = record[0]['count']
record_count = len(record)
if record_count == 0:
raise ValueError("レコードの取得が出来ませんでした。")
raise BatchOperationException("レコードの取得が出来ませんでした。")
# 起動回数のインクリメント
self._increment_run_count()
@ -81,35 +107,53 @@ class JskultBatchStatusManager:
return True
# 最大起動回数に到達していない場合
if not self._is_max_run_count_reached:
if not self._is_max_run_count_reached():
return False
# 最大起動回数フラグを立てる
# 最大起動回数に到達していた場合
# 最大起動回数フラグを立てて例外を送出する
self._activate_max_run_count_flg()
raise MaxRunCountReachedException("最大起動回数に到達しました")
return True
except MaxRunCountReachedException as e:
raise e
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
# 日付テーブル更新を実行してよいか判定する
def can_run_business_day_update(self):
def can_run_business_day_update(self) -> bool:
"""
日付テーブル更新を実行してよいか判定する
# SELECTの結果からレコード数を取得
Raises:
BatchOperationException: DB操作の何らかのエラー
MaxRunCountReachedException: 最大起動回数到達時の例外
Returns:
bool: 日付テーブル更新を実行してよい場合はTrue
"""
try:
# 自身（後続処理 or 日付テーブル更新）のレコードを取得する
self._db.connect()
record = self._db.execute_select(
f"""
"""
SELECT
COUNT(*)
process_name,
process_date
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = {self._process_name}
AND
process_date = src07.get_syor_date();
"""
process_name = :process_name
AND process_date = src07.get_syor_date();
""",
{'process_name': self._process_name}
)
record_count = record[0]['count']
record_count = len(record)
if record_count == 0:
raise ValueError("レコードの取得が出来ませんでした。")
raise BatchOperationException("レコードの取得が出来ませんでした。")
# 起動回数のインクリメント
self._increment_run_count()
@ -128,112 +172,166 @@ class JskultBatchStatusManager:
# 最大起動回数に到達した場合にメッセージをスロー
raise MaxRunCountReachedException("最大起動回数に到達しました")
# アルトマークデータ連携があったかを確認する
except MaxRunCountReachedException as e:
raise e
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
def is_done_ultmarc_import(self):
def is_done_ultmarc_import(self) -> bool:
"""アルトマークデータ連携があったかを確認する
# SELECTの結果からレコード数を取得
record_count = self._db.execute_select(
f"""
Raises:
BatchOperationException: DB操作の何らかのエラー
Returns:
bool アルトマークデータ連携があった場合はTrue
"""
try:
self._db.connect()
record = self._db.execute_select(
"""
SELECT
COUNT(*)
process_name,
process_date
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = 'jskult-batch-ultmarc-io'
AND
process_date = src07.get_syor_date();
"""
process_name = :process_name
AND process_date = src07.get_syor_date()
AND process_status = :process_status;
""",
{
'process_name': constants.PROCESS_NAME_ULTMARC_IO,
'process_status': constants.PROCESS_STATUS_DONE
}
)
# アルトマークデータ連携が無かった場合
if record_count.pop() == 0:
return False
return True
record_count = len(record)
return record_count != 0
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
# 起動回数をインクリメントする
def _increment_run_count(self):
"""起動回数をインクリメントする
Raises:
BatchOperationException: DB操作の何らかのエラー
"""
try:
# DB接続開始
self._db.connect()
self._db.begin()
self._db.to_jst()
# SELECTの結果からレコードを取得
# 自身（後続処理 or 日付テーブル更新）のレコードを取得する
record = self._db.execute_select(
f"""
"""
SELECT
*
total_run_count
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = {self._process_name}
AND
process_date = src07.get_syor_date();
"""
process_name = :process_name
AND process_date = src07.get_syor_date();
""",
{'process_name': self._process_name}
)
run_count += record[0]['run_count']
if len(record) == 0:
raise BatchOperationException("レコードの取得が出来ませんでした。")
total_run_count = record[0]['total_run_count']
self._db.execute(
f"""
CALL
upsert_jskult_batch_status_manage(
{self._process_name},
{self._process_type},
NULL,
{run_count},
NULL);
"""
CALL
internal07.upsert_jskult_batch_status_manage(
:process_name,
:process_type,
NULL,
:total_run_count,
NULL
)
""",
{
'process_name': self._process_name,
'process_type': self._process_type,
'total_run_count': total_run_count + 1,
}
)
self._db.commit()
except Exception as e:
# Exceptionをcatchした場合はrollbakc
# 例外発生時はrollback
self._db.rollback()
raise e
raise BatchOperationException(e)
finally:
self._db.disconnect()
# データ取込処理が完了しているかを判定する
def _is_done_data_import(self):
def _is_done_data_import(self) -> bool:
"""データ取込処理が完了しているかを判定する
# SELECTの結果からレコード数を取得
Raises:
BatchOperationException: DB操作の何らかのエラー
MaxRunCountReachedException: 最大起動回数到達時の例外
Returns:
bool 後続処理を実行してよい場合はTrue
"""
try:
self._db.connect()
# 自身（後続処理 or 日付テーブル更新）のレコードを取得する
record = self._db.execute_select(
f"""
"""
SELECT
COUNT(*)
process_name,
process_date
FROM
internal07.jskult_batch_status_manage
WHERE
process_type = {self._process_type}
AND
process_status = 'done'
AND
process_date = src07.get_syor_date();
"""
process_type = :process_type
AND process_status = :process_status
AND process_date = src07.get_syor_date();
""",
{
'process_type': constants.PROCESS_TYPE_DATA_IMPORT,
'process_status': constants.PROCESS_STATUS_DONE
}
)
record_count = record[0]['count']
record_count = len(record)
# データ取込数が一致している場合
if (self._receive_file_count == record_count):
return True
# データ取込数とデータ登録の件数が一致しているか確認
return self._receive_file_count == record_count
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
return False
def _is_done_post_process(self) -> bool:
"""後続処理のすべての処理が完了しているかを判定する
# 後続処理のすべての処理が完了しているかを判定する
def _is_done_post_process(self):
Returns:
bool: 後続処理がすべて完了していたらTrue
Raises:
BatchOperationException: DB操作の何らかのエラー
"""
# 生物由来データロット分解のチェック
if not self._is_done_process("jskult-batch-trn-result-data-bio-lot"):
if not self._is_done_process(constants.PROCESS_NAME_TRN_RESULT_DATA_BIO_LOT):
return False
# メルク施設マスタ作成のチェック
if not self._is_done_process("jskult-batch-mst-inst"):
if not self._is_done_process(constants.PROCESS_NAME_MST_INST_ALL):
return False
# DCF削除新規マスタ作成のチェック
if not self._is_done_process("jskult-batch-dcf-inst-merge-io"):
if not self._is_done_process(constants.PROCESS_NAME_DCF_INST_MERGE_IO):
return False
# 全ての後続処理が完了している場合Trueを返す
@ -241,80 +339,112 @@ class JskultBatchStatusManager:
# データ取込処理が完了しているかを判定する
def _is_done_process(self):
def _is_done_process(self, process_name: str) -> bool:
"""指定された処理名の処理が完了しているかを判定する
# SELECTの結果からレコード数を取得
Args:
process_name (str): 処理名
Returns:
bool: 処理名に一致する処理が完了していたらTrue
Raises:
BatchOperationException: DB操作の何らかのエラー
"""
try:
self._db.connect()
record = self._db.execute_select(
f"""
"""
SELECT
*
process_name,
process_date
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = {self._process_name}
AND
process_status = 'done'
AND
process_date = src07.get_syor_date();
"""
process_name = :process_name
AND process_status = :process_status
AND process_date = src07.get_syor_date();
""",
{
'process_name': process_name,
'process_status': constants.PROCESS_STATUS_DONE
}
)
record_count = record[0]['count']
record_count = len(record)
if (record_count == 0):
return False
return record_count != 0
return True
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
# 起動回数が最大回数に到達しているか判定する
def _is_max_run_count_reached(self) -> bool:
"""起動回数が最大回数に到達しているか判定する
def _is_max_run_count_reached(self):
Returns:
bool: 最大起動回数に到達していたらTrue
# SELECTの結果からレコードを取得
Raises:
BatchOperationException: DB操作の何らかのエラー
"""
try:
self._db.connect()
record = self._db.execute_select(
f"""
"""
SELECT
*
total_run_count
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = {self._process_name}
process_name = :process_name
AND
process_date = src07.get_syor_date();
"""
""",
{'process_name': self._process_name}
)
run_count = record[0]['run_count']
total_run_count = record[0]['total_run_count']
# 取得した起動回数とフィールド変数の最大起動回数が一致するか確認
if run_count == self._max_run_count_flg:
return True
return False
# 取得した起動回数とフィールド変数の最大起動回数が一致を確認
return total_run_count == self._max_run_count
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
def _activate_max_run_count_flg(self):
"""最大起動回数フラグにフラグを立てる
Raises:
BatchOperationException: DB操作の何らかのエラー
"""
try:
# DB接続開始
self._db.connect()
self._db.begin()
self._db.to_jst()
# 最大起動回数フラグにフラグを立てる
self._db.execute(
f"""
CALL
upsert_jskult_batch_status_manage(
{self._process_name},
{self._process_type},
NULL,
NULL,
1);
"""
CALL
internal07.upsert_jskult_batch_status_manage(
:process_name,
:process_type,
NULL,
NULL,
:max_run_count_flag);
""",
{
'process_name': self._process_name,
'process_type': self._process_type,
'max_run_count_flag': constants.MAX_RUN_COUNT_FLAG_ON
}
)
self._db.commit()
except Exception as e:
self._db.rollback()
raise e
raise BatchOperationException(e)
finally:
self._db.disconnect()

3
ecs/jskult-batch/src/manager/jskult_hdke_tbl_manager.py
View File

@ -108,10 +108,11 @@ class JskultHdkeTblManager:
try:
# 日次バッチ処置中フラグ、dump処理状態区分を取得
batch_processing_flag, dump_status_kbn, _ = self.get_batch_statuses()
except DBException as e:
except Exception as e:
raise BatchOperationException(e)
finally:
self._db.disconnect()
# 日次バッチ処理中ではない場合、後続の処理は行わない
if batch_processing_flag != constants.BATCH_ACTF_BATCH_START:
return False

36
ecs/jskult-batch/src/system_var/constants.py
View File

@ -10,6 +10,42 @@ DUMP_STATUS_KBN_UNPROCESSED = '0'
# dump取得状態区分：dump取得正常終了
DUMP_STATUS_KBN_COMPLETE = '2'
# バッチステータス管理
# 処理名:
# アルトマーク取込
PROCESS_NAME_ULTMARC_IO = 'jskult-batch-ultmarc-io'
# 生物由来ロット分解
PROCESS_NAME_TRN_RESULT_DATA_BIO_LOT = 'jskult-batch-trn-result-data-bio-lot'
# メルク施設マスタ作成
PROCESS_NAME_MST_INST_ALL = 'jskult-batch-mst-inst-all'
# DCF削除新規マスタ作成
PROCESS_NAME_DCF_INST_MERGE_IO = 'jskult-batch-dcf-inst-merge-io'
# 日付テーブル更新
PROCESS_NAME_UPDATE_BUSINESS_DAY = 'jskult-batch-update-business-day'
# 管理区分:
# データ取込
PROCESS_TYPE_DATA_IMPORT = 'data_import'
# 後続処理
PROCESS_TYPE_POST_PROCESS = 'post_process'
# 日付テーブル更新
PROCESS_TYPE_UPDATE_BUSINESS_DAY = 'update_business_day'
# 処理ステータス:
# 処理開始
PROCESS_STATUS_START = 'start'
# 処理待
PROCESS_STATUS_WAITING = 'waiting'
# 処理中
PROCESS_STATUS_DOING = 'doing'
# 処理済
PROCESS_STATUS_DONE = 'done'
# エラー
PROCESS_STATUS_ERROR = 'error'
# 最大起動回数フラグ: ON
MAX_RUN_COUNT_FLAG_ON = 1
# カレンダーファイルのコメントシンボル
CALENDAR_COMMENT_SYMBOL = '#'

5
ecs/jskult-batch/src/system_var/environment.py
View File

@ -10,7 +10,7 @@ DB_SCHEMA = os.environ['DB_SCHEMA']
# AWS
JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET']
BATCH_EXECUTION_ID = os.environ['BATCH_EXECUTION_ID']
MAX_RUN_COUNT = os.environ['MAX_RUN_COUNT']
MAX_RUN_COUNT = int(os.environ['MAX_RUN_COUNT'])
TRANSFER_RESULT_FOLDER = os.environ['TRANSFER_RESULT_FOLDER']
TRANSFER_RESULT_FILE_NAME = os.environ['TRANSFER_RESULT_FILE_NAME']
DCF_INST_MERGE_SEND_FILE_NAME = os.environ['DCF_INST_MERGE_SEND_FILE_NAME']
@ -20,6 +20,9 @@ JSK_IO_BUCKET = os.environ['JSK_IO_BUCKET']
JSK_BACKUP_FOLDER = os.environ['JSK_BACKUP_FOLDER']
JSK_DATA_SEND_FOLDER = os.environ['JSK_DATA_SEND_FOLDER']
# AWS
BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get('BATCH_MANAGE_DYNAMODB_TABLE_NAME')
# 初期値がある環境変数
LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO')
DB_CONNECTION_MAX_RETRY_ATTEMPT = int(

0
ecs/jskult-batch/tests/__init__.py Normal file
View File

0
ecs/jskult-batch/tests/batch/__init__.py Normal file
View File

0
ecs/jskult-batch/tests/batch/common/__init__.py Normal file
View File

56
ecs/jskult-batch/tests/batch/common/test_jskult_batch_entrypoint_factory.py Normal file
View File

@ -0,0 +1,56 @@
import pytest
from src.batch.common.jskult_batch_entrypoint_factory import \
JskultBatchEntrypointFactory
from src.batch.dcf_inst_merge_io import DcfInstMergeIO
from src.batch.mst_inst_all import MstInstAll
from src.batch.trn_result_data_bio_lot import TrnResultDataBioLot
from src.batch.update_business_day import UpdateBusinessDay
from src.error.exceptions import BatchOperationException
class TestJskultBatchEntrypointFactory:
def test_create_trn_result_data_bio_lot(self):
# Arrange
# Act
sut = JskultBatchEntrypointFactory(
'jskult-batch-trn-result-data-bio-lot')
actual = sut.create()
# Assert
assert isinstance(actual, TrnResultDataBioLot)
def test_create_mst_inst_all(self):
# Arrange
# Act
sut = JskultBatchEntrypointFactory(
'jskult-batch-mst-inst-all')
actual = sut.create()
# Assert
assert isinstance(actual, MstInstAll)
def test_create_dcf_inst_merge_io(self):
# Arrange
# Act
sut = JskultBatchEntrypointFactory(
'jskult-batch-dcf-inst-merge-io')
actual = sut.create()
# Assert
assert isinstance(actual, DcfInstMergeIO)
def test_create_update_business_day(self):
# Arrange
# Act
sut = JskultBatchEntrypointFactory(
'jskult-batch-update-business-day')
actual = sut.create()
# Assert
assert isinstance(actual, UpdateBusinessDay)
def test_create_raise_exception(self):
# Arrange
# Act
sut = JskultBatchEntrypointFactory('unknown')
with pytest.raises(BatchOperationException):
sut.create()

0
ecs/jskult-batch/tests/manager/__init__.py Normal file
View File

85
ecs/jskult-batch/tests/manager/test_jskult_batch_run_manager.py Normal file
View File

@ -0,0 +1,85 @@
import boto3
import pytest
from src.manager.jskult_batch_run_manager import JskultBatchRunManager
from src.system_var.environment import BATCH_MANAGE_DYNAMODB_TABLE_NAME
UNITTEST_EXECUTION_ID = 'unittest'
class TestJskultBatchRunManager:
@pytest.fixture(scope='function', autouse=True)
def dynamodb_client(self):
dynamodb = boto3.client('dynamodb')
yield dynamodb
dynamodb.delete_item(
TableName=BATCH_MANAGE_DYNAMODB_TABLE_NAME,
Key={
'execution_id': {'S': UNITTEST_EXECUTION_ID}
}
)
def test_batch_success(self, dynamodb_client):
"""バッチ実行管理テーブルに成功のステータスが書き込まれること
Args:
dynamodb_client (boto3.Client): DynamoDB クライアント
"""
# Arrange
# Act
sut = JskultBatchRunManager(BATCH_MANAGE_DYNAMODB_TABLE_NAME, UNITTEST_EXECUTION_ID)
sut.batch_success()
# Assert
options = {
'TableName': BATCH_MANAGE_DYNAMODB_TABLE_NAME,
'Key': {
'execution_id': {'S': UNITTEST_EXECUTION_ID},
},
}
actual = dynamodb_client.get_item(**options)
assert actual['Item']['batch_run_status']['S'] == 'success'
def test_batch_failed(self, dynamodb_client):
"""バッチ実行管理テーブルに失敗のステータスが書き込まれること
Args:
dynamodb_client (boto3.Client): DynamoDB クライアント
"""
# Arrange
# Act
sut = JskultBatchRunManager(BATCH_MANAGE_DYNAMODB_TABLE_NAME, UNITTEST_EXECUTION_ID)
sut.batch_failed()
# Assert
options = {
'TableName': BATCH_MANAGE_DYNAMODB_TABLE_NAME,
'Key': {
'execution_id': {'S': UNITTEST_EXECUTION_ID},
},
}
actual = dynamodb_client.get_item(**options)
assert actual['Item']['batch_run_status']['S'] == 'failed'
def test_batch_retry(self, dynamodb_client):
"""バッチ実行管理テーブルにリトライのステータスが書き込まれること
Args:
dynamodb_client (boto3.Client): DynamoDB クライアント
"""
# Arrange
# Act
sut = JskultBatchRunManager(BATCH_MANAGE_DYNAMODB_TABLE_NAME, UNITTEST_EXECUTION_ID)
sut.batch_retry()
# Assert
options = {
'TableName': BATCH_MANAGE_DYNAMODB_TABLE_NAME,
'Key': {
'execution_id': {'S': UNITTEST_EXECUTION_ID},
},
}
actual = dynamodb_client.get_item(**options)
assert actual['Item']['batch_run_status']['S'] == 'retry'

762
ecs/jskult-batch/tests/manager/test_jskult_batch_status_manager.py Normal file
View File

@ -0,0 +1,762 @@
import datetime
import pytest
from src.db.database import Database
from src.error.exceptions import (BatchOperationException,
MaxRunCountReachedException)
from src.manager.jskult_batch_status_manager import JskultBatchStatusManager
class TestJskultBatchStatusManager:
@pytest.fixture(scope='function', autouse=True)
def backup_hdke_tbl_record(self):
"""
テスト実行前にテーブルのバックアップを取得する。
テスト実行後にバックアップから復元する。
"""
# Setup
db = Database.get_instance()
db.connect()
# ステータス管理テーブル
backup_status_manage_records = db.execute_select(
'SELECT * FROM internal07.jskult_batch_status_manage')
db.execute('DELETE FROM internal07.jskult_batch_status_manage')
# 日付テーブル
backup_hdke_tbl = db.execute_select('SELECT * FROM src07.hdke_tbl')
db.execute('DELETE FROM src07.hdke_tbl')
yield
# Teardown
# ステータス管理テーブルを復元
db.execute('DELETE FROM internal07.jskult_batch_status_manage')
if len(backup_status_manage_records) != 0:
db.execute("""
INSERT INTO internal07.jskult_batch_status_manage
(process_name,process_date,process_type,process_status,total_run_count,max_run_count_flg,ins_user,ins_date,upd_user,upd_date)
VALUES
(:process_name,:process_date,:process_type,:process_status,:total_run_count,:max_run_count_flg,:ins_user,:ins_date,:upd_user,:upd_date)
""", backup_status_manage_records)
# 日付テーブルを復元
if len(backup_hdke_tbl) != 0:
db.execute('DELETE FROM src07.hdke_tbl')
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
(:syor_date, :bch_actf, :dump_sts_kbn, CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""", {
'syor_date': backup_hdke_tbl[0]['syor_date'],
'bch_actf': backup_hdke_tbl[0]['bch_actf'],
'dump_sts_kbn': backup_hdke_tbl[0]['dump_sts_kbn'],
})
db.disconnect()
def test_set_process_status_record_not_exists(self):
"""
ステータス管理テーブルのレコードが0件のとき、INSERTされること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# Act
sut = JskultBatchStatusManager(
'unittest_process_name',
'unittest_process_type',
3,
33
)
sut.set_process_status('start')
# Assert
actual = db.execute_select("""
SELECT
process_name,
process_date,
process_type,
process_status
FROM
internal07.jskult_batch_status_manage
""")[0]
assert actual['process_name'] == 'unittest_process_name'
assert datetime.datetime.strftime(
actual['process_date'], '%Y/%m/%d') == '2025/05/30'
assert actual['process_type'] == 'unittest_process_type'
assert actual['process_status'] == 'start'
def test_set_process_status_record_exists(self):
"""
ステータス管理テーブルのレコードが1件のとき、UPDATEされること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録
db.execute("""
INSERT INTO internal07.jskult_batch_status_manage
(process_name, process_date, process_type,
process_status, total_run_count, max_run_count_flg)
VALUES
('unittest_process_name', '2025-05-30','unittest_process_type',
'retry', '1', '0')
""")
# Act
sut = JskultBatchStatusManager(
'unittest_process_name',
'unittest_process_type',
3,
33
)
sut.set_process_status('doing')
# Assert
actual = db.execute_select("""
SELECT
process_name,
process_date,
process_type,
process_status
FROM
internal07.jskult_batch_status_manage
""")[0]
assert actual['process_name'] == 'unittest_process_name'
assert datetime.datetime.strftime(
actual['process_date'], '%Y/%m/%d') == '2025/05/30'
assert actual['process_type'] == 'unittest_process_type'
assert actual['process_status'] == 'doing'
def test_set_process_status_raise_exception(self):
"""
ステータス管理テーブルのレコードが0件のとき、INSERTされること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# Act
# 処理名と管理区分をNoneで初期化
sut = JskultBatchStatusManager(
None,
None,
3,
33
)
# SQL実行時にNOT NULL制約に引っかかる
with pytest.raises(BatchOperationException):
sut.set_process_status(None)
def test_can_run_post_process_success(self):
"""
完了しているデータ取り込み件数と受信ファイル数が一致している場合、trueが返却されること
起動回数が+1されていること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録
db.execute("""
INSERT INTO internal07.jskult_batch_status_manage
(process_name, process_date, process_type,
process_status, total_run_count, max_run_count_flg)
VALUES
('unittest_process_name', '2025-05-30', 'unittest_process_type',
'start', '0', '0'),
('data_import_process_1', '2025-05-30','data_import',
'done', '0', '0'),
('data_import_process_2', '2025-05-30','data_import',
'done', '0', '0'),
('data_import_process_3', '2025-05-30','data_import',
'done', '0', '0')
""")
# Act
sut = JskultBatchStatusManager(
'unittest_process_name',
'unittest_process_type',
3,
3
)
actual = sut.can_run_post_process()
# Assert
assert actual is True
actual_record = db.execute_select("""
SELECT
total_run_count
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = 'unittest_process_name'
AND process_date = '2025-05-30'
""")[0]
assert actual_record['total_run_count'] == 1
def test_can_run_post_process_failed(self):
"""
完了しているデータ取り込み件数と受信ファイル数が一致していない場合、falseが返却されること
起動回数が+1されていること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録
db.execute("""
INSERT INTO internal07.jskult_batch_status_manage
(process_name, process_date, process_type,
process_status, total_run_count, max_run_count_flg)
VALUES
('unittest_process_name', '2025-05-30', 'unittest_process_type',
'start', '0', '0'),
('data_import_process_1', '2025-05-30','data_import',
'done', '0', '0'),
('data_import_process_2', '2025-05-30','data_import',
'done', '0', '0'),
('data_import_process_3', '2025-05-30','data_import',
'done', '0', '0')
""")
# Act
sut = JskultBatchStatusManager(
'unittest_process_name',
'unittest_process_type',
3,
4
)
actual = sut.can_run_post_process()
# Assert
assert actual is False
actual_record = db.execute_select("""
SELECT
total_run_count
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = 'unittest_process_name'
AND process_date = '2025-05-30'
""")[0]
assert actual_record['total_run_count'] == 1
def test_can_run_post_process_raise_operation_exception(self):
"""
自分自身のレコードが存在しない場合、例外が送出されること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録しない
# Act
sut = JskultBatchStatusManager(
'unittest_process_name',
'unittest_process_type',
3,
4
)
with pytest.raises(BatchOperationException):
sut.can_run_post_process()
def test_can_run_post_process_raise_max_run_count_reached_exception(self):
"""
完了しているデータ取り込み件数と受信ファイル数が一致していないかつ、最大起動回数に到達している場合、例外が送出されること
最大起動回数フラグが1になっていること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録
db.execute("""
INSERT INTO internal07.jskult_batch_status_manage
(process_name, process_date, process_type,
process_status, total_run_count, max_run_count_flg)
VALUES
('unittest_process_name', '2025-05-30', 'unittest_process_type',
'start', '2', '0'),
('data_import_process_1', '2025-05-30','data_import',
'done', '0', '0'),
('data_import_process_2', '2025-05-30','data_import',
'done', '0', '0'),
('data_import_process_3', '2025-05-30','data_import',
'done', '0', '0')
""")
# Act
sut = JskultBatchStatusManager(
'unittest_process_name',
'unittest_process_type',
3,
4
)
with pytest.raises(MaxRunCountReachedException):
sut.can_run_post_process()
# Assert
actual_record = db.execute_select("""
SELECT
max_run_count_flg
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = 'unittest_process_name'
AND process_date = '2025-05-30'
""")[0]
assert actual_record['max_run_count_flg'] == 1
def test_can_run_business_day_update_success(self):
"""
後続処理3件が完了している場合、trueが返却されること
起動回数が+1されていること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録
db.execute("""
INSERT INTO internal07.jskult_batch_status_manage
(process_name, process_date, process_type,
process_status, total_run_count, max_run_count_flg)
VALUES
('jskult-batch-business-day-update', '2025-05-30', 'update_business_day',
'start', '0', '0'),
('jskult-batch-trn-result-data-bio-lot', '2025-05-30','post_process',
'done', '1', '0'),
('jskult-batch-dcf-inst-merge-io', '2025-05-30','post_process',
'done', '1', '0'),
('jskult-batch-mst-inst-all', '2025-05-30','post_process',
'done', '1', '0')
""")
# Act
sut = JskultBatchStatusManager(
'jskult-batch-business-day-update',
'business_day_update',
3,
3
)
actual = sut.can_run_business_day_update()
# Assert
assert actual is True
actual_record = db.execute_select("""
SELECT
total_run_count
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = 'jskult-batch-business-day-update'
AND process_date = '2025-05-30'
""")[0]
assert actual_record['total_run_count'] == 1
def test_can_run_business_day_update_failed_because_bio_lot_doing(self):
"""
後続処理3件のうち、生物由来ロット分解が終了していない場合、falseが返却されること
起動回数が+1されていること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録
db.execute("""
INSERT INTO internal07.jskult_batch_status_manage
(process_name, process_date, process_type,
process_status, total_run_count, max_run_count_flg)
VALUES
('jskult-batch-business-day-update', '2025-05-30', 'update_business_day',
'start', '0', '0'),
('jskult-batch-trn-result-data-bio-lot', '2025-05-30','post_process',
'doing', '1', '0'),
('jskult-batch-dcf-inst-merge-io', '2025-05-30','post_process',
'done', '1', '0'),
('jskult-batch-mst-inst-all', '2025-05-30','post_process',
'done', '1', '0')
""")
# Act
sut = JskultBatchStatusManager(
'jskult-batch-business-day-update',
'update_business_day',
3,
3
)
actual = sut.can_run_business_day_update()
# Assert
assert actual is False
actual_record = db.execute_select("""
SELECT
total_run_count
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = 'jskult-batch-business-day-update'
AND process_date = '2025-05-30'
""")[0]
assert actual_record['total_run_count'] == 1
def test_can_run_business_day_update_failed_because_dcf_inst_merge_io_doing(self):
"""
後続処理3件のうち、DCF削除新規マスタ作成が終了していない場合、falseが返却されること
起動回数が+1されていること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録
db.execute("""
INSERT INTO internal07.jskult_batch_status_manage
(process_name, process_date, process_type,
process_status, total_run_count, max_run_count_flg)
VALUES
('jskult-batch-business-day-update', '2025-05-30', 'update_business_day',
'start', '0', '0'),
('jskult-batch-trn-result-data-bio-lot', '2025-05-30','post_process',
'done', '1', '0'),
('jskult-batch-dcf-inst-merge-io', '2025-05-30','post_process',
'doing', '1', '0'),
('jskult-batch-mst-inst-all', '2025-05-30','post_process',
'done', '1', '0')
""")
# Act
sut = JskultBatchStatusManager(
'jskult-batch-business-day-update',
'update_business_day',
3,
3
)
actual = sut.can_run_business_day_update()
# Assert
assert actual is False
actual_record = db.execute_select("""
SELECT
total_run_count
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = 'jskult-batch-business-day-update'
AND process_date = '2025-05-30'
""")[0]
assert actual_record['total_run_count'] == 1
def test_can_run_business_day_update_failed_because_mst_inst_all_doing(self):
"""
後続処理3件のうち、メルク施設マスタ作成が終了していない場合、falseが返却されること
起動回数が+1されていること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録
db.execute("""
INSERT INTO internal07.jskult_batch_status_manage
(process_name, process_date, process_type,
process_status, total_run_count, max_run_count_flg)
VALUES
('jskult-batch-business-day-update', '2025-05-30', 'update_business_day',
'start', '0', '0'),
('jskult-batch-trn-result-data-bio-lot', '2025-05-30','post_process',
'done', '1', '0'),
('jskult-batch-dcf-inst-merge-io', '2025-05-30','post_process',
'done', '1', '0'),
('jskult-batch-mst-inst-all', '2025-05-30','post_process',
'doing', '1', '0')
""")
# Act
sut = JskultBatchStatusManager(
'jskult-batch-business-day-update',
'update_business_day',
3,
3
)
actual = sut.can_run_business_day_update()
# Assert
assert actual is False
actual_record = db.execute_select("""
SELECT
total_run_count
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = 'jskult-batch-business-day-update'
AND process_date = '2025-05-30'
""")[0]
assert actual_record['total_run_count'] == 1
def test_can_run_business_day_update_raise_operation_exception(self):
"""
自分自身のレコードが存在しない場合、例外が送出されること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録しない
# Act
sut = JskultBatchStatusManager(
'unittest_process_name',
'unittest_process_type',
3,
4
)
with pytest.raises(BatchOperationException):
sut.can_run_business_day_update()
def test_can_run_business_day_update_raise_max_run_count_reached_exception(self):
"""
後続処理3件のいずれかが終了していないかつ、最大起動回数に到達している場合、例外が送出されること
最大起動回数フラグが1になっていること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録
db.execute("""
INSERT INTO internal07.jskult_batch_status_manage
(process_name, process_date, process_type,
process_status, total_run_count, max_run_count_flg)
VALUES
('jskult-batch-business-day-update', '2025-05-30', 'update_business_day',
'start', '2', '0'),
('jskult-batch-trn-result-data-bio-lot', '2025-05-30','post_process',
'doing', '1', '0'),
('jskult-batch-dcf-inst-merge-io', '2025-05-30','post_process',
'done', '1', '0'),
('jskult-batch-mst-inst-all', '2025-05-30','post_process',
'done', '1', '0')
""")
# Act
sut = JskultBatchStatusManager(
'jskult-batch-business-day-update',
'update_business_day',
3,
4
)
with pytest.raises(MaxRunCountReachedException):
sut.can_run_business_day_update()
# Assert
actual_record = db.execute_select("""
SELECT
max_run_count_flg
FROM
internal07.jskult_batch_status_manage
WHERE
process_name = 'jskult-batch-business-day-update'
AND process_date = '2025-05-30'
""")[0]
assert actual_record['max_run_count_flg'] == 1
def test_is_done_ultmarc_import_success(self):
"""
アルトマーク取込が完了している場合、trueが返却されること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録
db.execute("""
INSERT INTO internal07.jskult_batch_status_manage
(process_name, process_date, process_type,
process_status, total_run_count, max_run_count_flg)
VALUES
('unittest_process_name', '2025-05-30', 'unittest_process_type',
'start', '0', '0'),
('jskult-batch-ultmarc-io', '2025-05-30','data_import',
'done', '0', '0')
""")
# Act
sut = JskultBatchStatusManager(
'unittest_process_name',
'unittest_process_type',
3,
3
)
actual = sut.is_done_ultmarc_import()
# Assert
assert actual is True
def test_is_done_ultmarc_import_failed(self):
"""
アルトマーク取込が完了していない場合、falseが返却されること
"""
# Arrange
# 日付テーブルを登録
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn,
creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# バッチステータス管理テーブルを登録
db.execute("""
INSERT INTO internal07.jskult_batch_status_manage
(process_name, process_date, process_type,
process_status, total_run_count, max_run_count_flg)
VALUES
('unittest_process_name', '2025-05-30', 'unittest_process_type',
'start', '0', '0'),
('jskult-batch-ultmarc-io', '2025-05-30','data_import',
'start', '0', '0')
""")
# Act
sut = JskultBatchStatusManager(
'unittest_process_name',
'unittest_process_type',
3,
3
)
actual = sut.is_done_ultmarc_import()
# Assert
assert actual is False

197
ecs/jskult-batch/tests/manager/test_jskult_hdke_tbl_manager.py Normal file
View File

@ -0,0 +1,197 @@
import pytest
from src.db.database import Database
from src.error.exceptions import BatchOperationException
from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager
class TestJskultHdkeTblManager:
@pytest.fixture(scope='function', autouse=True)
def backup_hdke_tbl_record(self):
"""
テスト実行前にテーブルのバックアップを取得する。
テスト実行後にバックアップから復元する。
"""
# Setup
db = Database.get_instance()
db.connect()
backup_hdke_tbl = db.execute_select('SELECT * FROM src07.hdke_tbl')
db.execute('DELETE FROM src07.hdke_tbl')
yield
# Teardown
if len(backup_hdke_tbl) != 0:
db.execute('DELETE FROM src07.hdke_tbl')
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn, creater, create_date, updater, update_date)
VALUES
(:syor_date, :bch_actf, :dump_sts_kbn, CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""", {
'syor_date': backup_hdke_tbl[0]['syor_date'],
'bch_actf': backup_hdke_tbl[0]['bch_actf'],
'dump_sts_kbn': backup_hdke_tbl[0]['dump_sts_kbn'],
})
db.disconnect()
def test_get_batch_statuses(self):
"""
日付テーブルから以下を取得できること
- 日次バッチ処理中フラグ
- dump取得状況区分
- 処理日(yyyy/mm/dd)
"""
# Arrange
# 日付テーブルを作成
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn, creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# Act
sut = JskultHdkeTblManager()
bch_actf, dump_sts_kbn, syor_date = sut.get_batch_statuses()
# Assert
assert bch_actf == '1'
assert dump_sts_kbn == '2'
assert syor_date == '2025/05/30'
def test_get_batch_statuses_raise_exception(self):
"""
日付テーブルのレコードがない場合、例外が送出されること
"""
# Arrange
# 日付テーブルを作成しない
# Act
sut = JskultHdkeTblManager()
with pytest.raises(BatchOperationException):
sut.get_batch_statuses()
def test_update_batch_process_start(self):
"""
日付テーブルの日次バッチ処理中フラグを1(処理中)に更新できること
"""
# Arrange
# 日付テーブルを作成
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn, creater, create_date, updater, update_date)
VALUES
('20250530', '0', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# Act
sut = JskultHdkeTblManager()
sut.update_batch_process_start()
# Assert
actual_record = db.execute_select('SELECT * FROM src07.hdke_tbl')[0]
assert actual_record['bch_actf'] == '1'
assert actual_record['dump_sts_kbn'] == '2'
def test_update_batch_process_complete(self):
"""
日付テーブルを以下の通り更新できること
- 日次バッチ処理中フラグを0(未処理)に更新
- dump取得状態区分を0(未処理)に更新
"""
# Arrange
# 日付テーブルを作成
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn, creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# Act
sut = JskultHdkeTblManager()
sut.update_batch_process_complete()
# Assert
actual_record = db.execute_select('SELECT * FROM src07.hdke_tbl')[0]
assert actual_record['bch_actf'] == '0'
assert actual_record['dump_sts_kbn'] == '0'
def test_can_run_process_true(self):
"""
以下の条件の時、Trueが返却されること
- 日次バッチ処理中フラグが1(処理中)
- dump取得状態区分が2(正常終了)
"""
# Arrange
# 日付テーブルを作成
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn, creater, create_date, updater, update_date)
VALUES
('20250530', '1', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# Act
sut = JskultHdkeTblManager()
actual = sut.can_run_process()
# Assert
assert actual is True
def test_can_run_process_false_because_bch_actf_off(self):
"""
日次バッチ処理中フラグが1ではないため、Falseが返却されること
"""
# Arrange
# 日付テーブルを作成
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn, creater, create_date, updater, update_date)
VALUES
('20250530', '0', '2', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# Act
sut = JskultHdkeTblManager()
actual = sut.can_run_process()
# Assert
assert actual is False
def test_can_run_process_false_because_dump_sts_kbn_not_complete(self):
"""
dump取得状態区分が2ではないため、Falseが返却されること
"""
# Arrange
# 日付テーブルを作成
db = Database.get_instance()
db.connect()
db.execute("""
INSERT INTO src07.hdke_tbl
(syor_date, bch_actf, dump_sts_kbn, creater, create_date, updater, update_date)
VALUES
('20250530', '1', '0', CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL)
""")
# Act
sut = JskultHdkeTblManager()
actual = sut.can_run_process()
# Assert
assert actual is False
def test_can_run_process_raise_exception(self):
"""
日付テーブルのレコードがない場合、例外が送出されること
"""
# Arrange
# 日付テーブルを作成しない
# Act
sut = JskultHdkeTblManager()
with pytest.raises(BatchOperationException):
sut.can_run_process()

17
rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql
View File

@ -42,10 +42,10 @@ SET @upsert_statement =
CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL
)
ON DUPLICATE KEY UPDATE
process_type = CASE WHEN ? is NULL THEN process_type ELSE ? END,
process_status = CASE WHEN ? is NULL THEN process_status ELSE ? END,
total_run_count = CASE WHEN ? is NULL THEN total_run_count ELSE ? END,
max_run_count_flg = CASE WHEN ? is NULL THEN max_run_count_flg ELSE ? END,
process_type = IFNULL(?, process_type),
process_status = IFNULL(?, process_status),
total_run_count = IFNULL(?, total_run_count),
max_run_count_flg = IFNULL(?, max_run_count_flg),
upd_user = CURRENT_USER(),
upd_date = CURRENT_TIMESTAMP();';
@ -70,14 +70,13 @@ EXECUTE stmt USING
@max_run_count_flg,
-- UPDATE用管理区分
@process_type, @process_type,
@process_type,
-- UPDATE用処理ステータス
@process_status, @process_status,
@process_status,
-- UPDATE用起動回数
@total_run_count, @total_run_count,
@total_run_count,
-- UPDATE用最大起動回数フラグ
@max_run_count_flg, @max_run_count_flg;
@max_run_count_flg;
DEALLOCATE PREPARE stmt;
END;

11
s3/config/jskult/task_settings/archive_jsk_data_task_settings.env Normal file
View File

@ -0,0 +1,11 @@
# task environment file.
LOG_LEVEL=INFO
ARCHIVE_TRN_RESULT_FOLDER=/jsk/trn_result_data
ARCHIVE_TRN_RESULT_BIO_FOLDER=/jsk/trn_result_data_bio
ARCHIVE_TRN_RESULT_BIO_LOT_FOLDER=/jsk/trn_result_data_bio_lot
ARCHIVE_TRN_RESULT_INVENTORY_FOLDER=/jsk/trn_recive_inventry
DB_CONNECTION_MAX_RETRY_ATTEMPT=4
DB_CONNECTION_RETRY_INTERVAL_INIT=5
DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=5
DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=50

9
s3/config/jskult/task_settings/mst_inst_all_task_settings.env Normal file
View File

@ -0,0 +1,9 @@
# task environment file.
LOG_LEVEL=INFO
ENTRYPOINT_MODULE_NAME=jskult-batch-mst-inst-all
TRANSFER_RESULT_FOLDER=transfer_result
TRANSFER_RESULT_FILE_NAME=transfer_result.json
DB_CONNECTION_MAX_RETRY_ATTEMPT=4
DB_CONNECTION_RETRY_INTERVAL_INIT=5
DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=5
DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=50

9
s3/config/jskult/task_settings/trn_result_data_bio_lot_task_settings.env Normal file
View File

@ -0,0 +1,9 @@
# task environment file.
LOG_LEVEL=INFO
PROCESS_NAME=jskult-batch-trn-result-data-bio-lot
TRANSFER_RESULT_FOLDER=transfer_result
TRANSFER_RESULT_FILE_NAME=transfer_result.json
DB_CONNECTION_MAX_RETRY_ATTEMPT=4
DB_CONNECTION_RETRY_INTERVAL_INIT=5
DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=5
DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=50

11
s3/config/jskult/task_settings/update_business_day_task_settings.env Normal file
View File

@ -0,0 +1,11 @@
# task environment file.
LOG_LEVEL=INFO
PROCESS_NAME=jskult-batch-update-business-day
JSKULT_CONFIG_CALENDAR_FOLDER=jskult/calendar
JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME=jsk_archive_run_day.txt
JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER=jskult/expected_data_list
JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME=jsk_expected_data_list.json
DB_CONNECTION_MAX_RETRY_ATTEMPT=4
DB_CONNECTION_RETRY_INTERVAL_INIT=5
DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=5
DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=50

171
stepfunctions/TOOLS/convert_config.yaml
View File

@ -32,6 +32,17 @@ resource:
- &STG_SG_JSKULT_TRANSFER_RECEIVE_FILE "sg-08d43e8e118178d39"
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
- &STG_SG_JSKULT_ULTMARC_IO "sg-014caf29e738d106a"
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
- &STG_SG_DCF_INST_MERGE_IO "sg-0b5d1639a83d28f66"
# セキュリティグループ(ecs-jskult-batch-mst-inst-all)
- &STG_SG_MST_INST_ALL "sg-0483fd75c76fa5808"
# セキュリティグループ(ecs-jskult-batch-trn-result-data-bio-lot)
- &STG_SG_TRN_RESULT_DATA_BIO_LOT "sg-07ca545e440dd83b7"
# セキュリティグループ(ecs-jskult-batch-update-business-day)
- &STG_SG_UPDATE_BIS_DAY "sg-0b58046abfa017e0e"
# セキュリティグループ(ecs-jskult-batch-archive-jsk-data)
- &STG_SG_JSK_ARCHIVE "sg-0bbdd7d4ebb5ba222"
# 本番環境
product:
# サブネット(PrivateSubnet1)
@ -56,6 +67,21 @@ resource:
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
# TODO: 本番リリース時にIDを正式版にする
- &PRD_SG_JSKULT_ULTMARC_IO "sg-xxxxxxxxxxxxx"
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
# TODO: 本番リリース時にIDを正式版にする
- &PRD_SG_DCF_INST_MERGE_IO "sg-xxxxxxxxxxxxx"
# セキュリティグループ(ecs-jskult-batch-mst-inst-all)
# TODO: 本番リリース時にIDを正式版にする
- &PRD_SG_MST_INST_ALL "sg-xxxxxxxxxxxxx"
# セキュリティグループ(ecs-jskult-batch-trn-result-data-bio-lot)
# TODO: 本番リリース時にIDを正式版にする
- &PRD_SG_TRN_RESULT_DATA_BIO_LOT "sg-xxxxxxxxxxxxx"
# セキュリティグループ(ecs-jskult-batch-update-business-day)
# TODO: 本番リリース時にIDを正式版にする
- &PRD_SG_UPDATE_BIS_DAY "sg-xxxxxxxxxxxxx"
# セキュリティグループ(ecs-jskult-batch-archive-jsk-data)
# TODO: 本番リリース時にIDを正式版にする
- &PRD_SG_JSK_ARCHIVE "sg-xxxxxxxxxxxxx"
config:
# CRMデータ取得
r-crm-datafetch-state:
@ -281,3 +307,148 @@ config:
SG_ECS_ALL: *PRD_SG_ECS_ALL
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
SG_JSKULT_ULTMARC_IO: *PRD_SG_JSKULT_ULTMARC_IO
r-jskult-batch-dcf-inst-merge-io-state:
# ステージング環境
staging:
# AWSアカウントID
AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID
# 東京リージョン
REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1
# サブネット(PrivateSubnet1)
SUBNET_PRI_1A: *STG_SUBNET_PRI_1A
# サブネット(PrivateSubnet2)
SUBNET_PRI_1D: *STG_SUBNET_PRI_1D
# セキュリティグループ(ecs-all)
SG_ECS_ALL: *STG_SG_ECS_ALL
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
SG_DCF_INST_MERGE_IO: *STG_SG_DCF_INST_MERGE_IO
# 本番環境
product:
# AWSアカウントID
AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID
# 東京リージョン
REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1
# サブネット(PrivateSubnet1)
SUBNET_PRI_1A: *PRD_SUBNET_PRI_1A
# サブネット(PrivateSubnet2)
SUBNET_PRI_1D: *PRD_SUBNET_PRI_1D
# セキュリティグループ(ecs-all)
SG_ECS_ALL: *PRD_SG_ECS_ALL
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
SG_DCF_INST_MERGE_IO: *PRD_SG_DCF_INST_MERGE_IO
r-jskult-batch-mst-inst-all-state:
# ステージング環境
staging:
# AWSアカウントID
AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID
# 東京リージョン
REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1
# サブネット(PrivateSubnet1)
SUBNET_PRI_1A: *STG_SUBNET_PRI_1A
# サブネット(PrivateSubnet2)
SUBNET_PRI_1D: *STG_SUBNET_PRI_1D
# セキュリティグループ(ecs-all)
SG_ECS_ALL: *STG_SG_ECS_ALL
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
SG_MST_INST_ALL: *STG_SG_MST_INST_ALL
# 本番環境
product:
# AWSアカウントID
AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID
# 東京リージョン
REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1
# サブネット(PrivateSubnet1)
SUBNET_PRI_1A: *PRD_SUBNET_PRI_1A
# サブネット(PrivateSubnet2)
SUBNET_PRI_1D: *PRD_SUBNET_PRI_1D
# セキュリティグループ(ecs-all)
SG_ECS_ALL: *PRD_SG_ECS_ALL
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
SG_MST_INST_ALL: *PRD_SG_MST_INST_ALL
r-jskult-batch-trn-result-data-bio-lot-state:
# ステージング環境
staging:
# AWSアカウントID
AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID
# 東京リージョン
REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1
# サブネット(PrivateSubnet1)
SUBNET_PRI_1A: *STG_SUBNET_PRI_1A
# サブネット(PrivateSubnet2)
SUBNET_PRI_1D: *STG_SUBNET_PRI_1D
# セキュリティグループ(ecs-all)
SG_ECS_ALL: *STG_SG_ECS_ALL
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
SG_TRN_RESULT_DATA_BIO_LOT: *STG_SG_TRN_RESULT_DATA_BIO_LOT
# 本番環境
product:
# AWSアカウントID
AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID
# 東京リージョン
REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1
# サブネット(PrivateSubnet1)
SUBNET_PRI_1A: *PRD_SUBNET_PRI_1A
# サブネット(PrivateSubnet2)
SUBNET_PRI_1D: *PRD_SUBNET_PRI_1D
# セキュリティグループ(ecs-all)
SG_ECS_ALL: *PRD_SG_ECS_ALL
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
SG_TRN_RESULT_DATA_BIO_LOT: *PRD_SG_TRN_RESULT_DATA_BIO_LOT
r-jskult-batch-update-business-day-state:
# ステージング環境
staging:
# AWSアカウントID
AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID
# 東京リージョン
REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1
# サブネット(PrivateSubnet1)
SUBNET_PRI_1A: *STG_SUBNET_PRI_1A
# サブネット(PrivateSubnet2)
SUBNET_PRI_1D: *STG_SUBNET_PRI_1D
# セキュリティグループ(ecs-all)
SG_ECS_ALL: *STG_SG_ECS_ALL
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
SG_UPDATE_BIS_DAY: *STG_SG_UPDATE_BIS_DAY
# 本番環境
product:
# AWSアカウントID
AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID
# 東京リージョン
REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1
# サブネット(PrivateSubnet1)
SUBNET_PRI_1A: *PRD_SUBNET_PRI_1A
# サブネット(PrivateSubnet2)
SUBNET_PRI_1D: *PRD_SUBNET_PRI_1D
# セキュリティグループ(ecs-all)
SG_ECS_ALL: *PRD_SG_ECS_ALL
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
SG_UPDATE_BIS_DAY: *PRD_SG_UPDATE_BIS_DAY
r-jskult-batch-archive-jsk-data-state:
# ステージング環境
staging:
# AWSアカウントID
AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID
# 東京リージョン
REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1
# サブネット(PrivateSubnet1)
SUBNET_PRI_1A: *STG_SUBNET_PRI_1A
# サブネット(PrivateSubnet2)
SUBNET_PRI_1D: *STG_SUBNET_PRI_1D
# セキュリティグループ(ecs-all)
SG_ECS_ALL: *STG_SG_ECS_ALL
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
SG_JSK_ARCHIVE: *STG_SG_JSK_ARCHIVE
# 本番環境
product:
# AWSアカウントID
AWS_ACCOUNT_ID: *AWS_ACCOUNT_ID
# 東京リージョン
REGION_AP_NORTHEAST_1: *REGION_AP_NORTHEAST_1
# サブネット(PrivateSubnet1)
SUBNET_PRI_1A: *PRD_SUBNET_PRI_1A
# サブネット(PrivateSubnet2)
SUBNET_PRI_1D: *PRD_SUBNET_PRI_1D
# セキュリティグループ(ecs-all)
SG_ECS_ALL: *PRD_SG_ECS_ALL
# セキュリティグループ(ecs-jskult-batch-ultmarc-io)
SG_JSK_ARCHIVE: *PRD_SG_JSK_ARCHIVE

72
stepfunctions/r-jskult-batch-archive-jsk-data-state/r-jskult-batch-archive-jsk-data-state.json Normal file
View File

@ -0,0 +1,72 @@
{
"Comment": "実消化_過去データアーカイブ処理ステートマシン",
"StartAt": "params",
"States": {
"params": {
"Comment": "パラメータ設定",
"Type": "Pass",
"Parameters": {
"sns": {
"TopicArn": "arn:aws:sns:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:nds-notice-#{ENV_NAME}"
},
"ecs": {
"Cluster": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:cluster/mbj-newdwh2021-#{ENV_NAME}-jskult-batch-archive-jsk-data-ecs",
"LaunchType": "FARGATE",
"NetworkConfiguration": {
"AwsvpcConfiguration": {
"Subnets": [
"#{SUBNET_PRI_1A}",
"#{SUBNET_PRI_1D}"
],
"SecurityGroups": [
"#{SG_ECS_ALL}",
"#{SG_JSK_ARCHIVE}"
],
"AssignPublicIp": "DISABLED"
}
}
}
},
"ResultPath": "$.params",
"Next": "exec-ecs-task"
},
"exec-ecs-task": {
"Comment": "ECSタスク起動",
"Type": "Task",
"Resource": "arn:aws:states:::ecs:runTask.sync",
"Parameters": {
"Cluster.$": "$.params.ecs.Cluster",
"LaunchType.$": "$.params.ecs.LaunchType",
"TaskDefinition": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-#{ENV_NAME}-task-jskult-batch-archive-jsk-data",
"NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration"
},
"Retry": [
{
"ErrorEquals": ["States.ALL"],
"BackoffRate": 2,
"IntervalSeconds": 5,
"MaxAttempts": 3
}
],
"Catch": [
{
"ErrorEquals": ["States.ALL"],
"ResultPath": "$.result",
"Next": "ErrorEnd"
}
],
"ResultPath": "$.result",
"Next": "NormalEnd"
},
"NormalEnd": {
"Comment": "正常終了",
"Type": "Succeed"
},
"ErrorEnd": {
"Comment": "異常終了",
"Type": "Fail",
"Error": "StatesError",
"Cause": "StepFunctions ErrorEnd"
}
}
}

129
stepfunctions/r-jskult-batch-dcf-inst-merge-io-state/r-jskult-batch-dcf-inst-merge-io-state.json Normal file
View File

@ -0,0 +1,129 @@
{
"Comment": "実消化&アルトマーク DCF施設削除新規マスタ作成ステートマシン",
"StartAt": "params",
"States": {
"params": {
"Comment": "パラメータ設定",
"Type": "Pass",
"Parameters": {
"ecs": {
"LaunchType": "FARGATE",
"Cluster": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:cluster/mbj-newdwh2021-#{ENV_NAME}-jskult-batch-dcf-inst-merge-io-ecs",
"TaskDefinition": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-#{ENV_NAME}-task-jskult-batch-dcf-inst-merge-io",
"NetworkConfiguration": {
"AwsvpcConfiguration": {
"Subnets": [
"#{SUBNET_PRI_1A}",
"#{SUBNET_PRI_1D}"
],
"SecurityGroups": [
"#{SG_ECS_ALL}",
"#{SG_DCF_INST_MERGE_IO}"
],
"AssignPublicIp": "DISABLED"
}
},
"Overrides": {
"ContainerOverrides": [
{
"Name": "mbj-newdwh2021-#{ENV_NAME}-container-jskult-batch-dcf-inst-merge-io",
"Environment": [
{
"Name": "BATCH_EXECUTION_ID",
"Value.$": "$$.Execution.Id"
},
{
"Name": "MAX_RUN_COUNT",
"Value.$": "$.maxRunCount"
}
]
}
]
}
}
},
"ResultPath": "$.params",
"Next": "exec-ecs-task"
},
"exec-ecs-task": {
"Type": "Task",
"Resource": "arn:aws:states:::ecs:runTask.sync",
"Parameters": {
"LaunchType.$": "$.params.ecs.LaunchType",
"Cluster.$": "$.params.ecs.Cluster",
"TaskDefinition.$": "$.params.ecs.TaskDefinition",
"NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration",
"Overrides.$": "$.params.ecs.Overrides"
},
"ResultPath": "$.result",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"BackoffRate": 2,
"IntervalSeconds": 3,
"MaxAttempts": 3
}
],
"Catch": [
{
"ErrorEquals": [
"States.ALL"
],
"Next": "ErrorEnd",
"ResultPath": "$.result"
}
],
"Next": "scan-jskult-batch-run-manage",
"Comment": "ECSタスク起動"
},
"scan-jskult-batch-run-manage": {
"Type": "Task",
"Resource": "arn:aws:states:::dynamodb:getItem",
"Parameters": {
"TableName": "mbj-newdwh2021-#{ENV_NAME}-jskult-batch-run-manage",
"Key": {
"execution_id": {
"S.$": "$$.Execution.Id"
}
}
},
"Next": "Choice",
"ResultPath": "$.scan"
},
"Choice": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.scan.Item",
"IsPresent": false,
"Next": "ErrorEnd",
"Comment": "バッチ実行管理テーブルにデータが存在しない場合"
},
{
"Variable": "$.scan.Item.batch_run_status.S",
"StringEquals": "retry",
"Next": "wait-for-retry",
"Comment": "バッチ実行管理テーブルのスターテスがリトライの場合"
}
],
"Default": "NormalEnd"
},
"wait-for-retry": {
"Type": "Wait",
"SecondsPath": "$.retryIntervalSecond",
"Next": "exec-ecs-task"
},
"NormalEnd": {
"Comment": "正常終了",
"Type": "Succeed"
},
"ErrorEnd": {
"Comment": "異常終了",
"Type": "Fail",
"Error": "StatesError",
"Cause": "StepFunctions ErrorEnd"
}
}
}

129
stepfunctions/r-jskult-batch-mst-inst-all-state/r-jskult-batch-mst-inst-all-state.json Normal file
View File

@ -0,0 +1,129 @@
{
"Comment": "実消化&アルトマーク メルク施設マスタステートマシン",
"StartAt": "params",
"States": {
"params": {
"Comment": "パラメータ設定",
"Type": "Pass",
"Parameters": {
"ecs": {
"LaunchType": "FARGATE",
"Cluster": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:cluster/mbj-newdwh2021-#{ENV_NAME}-jskult-batch-mst-inst-all-ecs",
"TaskDefinition": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-#{ENV_NAME}-task-jskult-batch-mst-inst-all",
"NetworkConfiguration": {
"AwsvpcConfiguration": {
"Subnets": [
"#{SUBNET_PRI_1A}",
"#{SUBNET_PRI_1D}"
],
"SecurityGroups": [
"#{SG_ECS_ALL}",
"#{SG_MST_INST_ALL}"
],
"AssignPublicIp": "DISABLED"
}
},
"Overrides": {
"ContainerOverrides": [
{
"Name": "mbj-newdwh2021-#{ENV_NAME}-container-jskult-batch-mst-inst-all",
"Environment": [
{
"Name": "BATCH_EXECUTION_ID",
"Value.$": "$$.Execution.Id"
},
{
"Name": "MAX_RUN_COUNT",
"Value.$": "$.maxRunCount"
}
]
}
]
}
}
},
"ResultPath": "$.params",
"Next": "exec-ecs-task"
},
"exec-ecs-task": {
"Type": "Task",
"Resource": "arn:aws:states:::ecs:runTask.sync",
"Parameters": {
"LaunchType.$": "$.params.ecs.LaunchType",
"Cluster.$": "$.params.ecs.Cluster",
"TaskDefinition.$": "$.params.ecs.TaskDefinition",
"NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration",
"Overrides.$": "$.params.ecs.Overrides"
},
"ResultPath": "$.result",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"BackoffRate": 2,
"IntervalSeconds": 3,
"MaxAttempts": 3
}
],
"Catch": [
{
"ErrorEquals": [
"States.ALL"
],
"Next": "ErrorEnd",
"ResultPath": "$.result"
}
],
"Next": "scan-jskult-batch-run-manage",
"Comment": "ECSタスク起動"
},
"scan-jskult-batch-run-manage": {
"Type": "Task",
"Resource": "arn:aws:states:::dynamodb:getItem",
"Parameters": {
"TableName": "mbj-newdwh2021-#{ENV_NAME}-jskult-batch-run-manage",
"Key": {
"execution_id": {
"S.$": "$$.Execution.Id"
}
}
},
"Next": "Choice",
"ResultPath": "$.scan"
},
"Choice": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.scan.Item",
"IsPresent": false,
"Next": "ErrorEnd",
"Comment": "バッチ実行管理テーブルにデータが存在しない場合"
},
{
"Variable": "$.scan.Item.batch_run_status.S",
"StringEquals": "retry",
"Next": "wait-for-retry",
"Comment": "バッチ実行管理テーブルのスターテスがリトライの場合"
}
],
"Default": "NormalEnd"
},
"wait-for-retry": {
"Type": "Wait",
"SecondsPath": "$.retryIntervalSecond",
"Next": "exec-ecs-task"
},
"NormalEnd": {
"Comment": "正常終了",
"Type": "Succeed"
},
"ErrorEnd": {
"Comment": "異常終了",
"Type": "Fail",
"Error": "StatesError",
"Cause": "StepFunctions ErrorEnd"
}
}
}

129
stepfunctions/r-jskult-batch-trn-result-data-bio-lot-state/r-jskult-batch-trn-result-data-bio-lot-state.json Normal file
View File

@ -0,0 +1,129 @@
{
"Comment": "実消化&アルトマーク 生物由来ロット分解ステートマシン",
"StartAt": "params",
"States": {
"params": {
"Comment": "パラメータ設定",
"Type": "Pass",
"Parameters": {
"ecs": {
"LaunchType": "FARGATE",
"Cluster": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:cluster/mbj-newdwh2021-#{ENV_NAME}-jskult-batch-trn-result-data-bio-lot-ecs",
"TaskDefinition": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-#{ENV_NAME}-task-jskult-batch-trn-result-data-bio-lot",
"NetworkConfiguration": {
"AwsvpcConfiguration": {
"Subnets": [
"#{SUBNET_PRI_1A}",
"#{SUBNET_PRI_1D}"
],
"SecurityGroups": [
"#{SG_ECS_ALL}",
"#{SG_TRN_RESULT_DATA_BIO_LOT}"
],
"AssignPublicIp": "DISABLED"
}
},
"Overrides": {
"ContainerOverrides": [
{
"Name": "mbj-newdwh2021-#{ENV_NAME}-container-jskult-batch-trn-result-data-bio-lot",
"Environment": [
{
"Name": "BATCH_EXECUTION_ID",
"Value.$": "$$.Execution.Id"
},
{
"Name": "MAX_RUN_COUNT",
"Value.$": "$.maxRunCount"
}
]
}
]
}
}
},
"ResultPath": "$.params",
"Next": "exec-ecs-task"
},
"exec-ecs-task": {
"Type": "Task",
"Resource": "arn:aws:states:::ecs:runTask.sync",
"Parameters": {
"LaunchType.$": "$.params.ecs.LaunchType",
"Cluster.$": "$.params.ecs.Cluster",
"TaskDefinition.$": "$.params.ecs.TaskDefinition",
"NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration",
"Overrides.$": "$.params.ecs.Overrides"
},
"ResultPath": "$.result",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"BackoffRate": 2,
"IntervalSeconds": 3,
"MaxAttempts": 3
}
],
"Catch": [
{
"ErrorEquals": [
"States.ALL"
],
"Next": "ErrorEnd",
"ResultPath": "$.result"
}
],
"Next": "scan-jskult-batch-run-manage",
"Comment": "ECSタスク起動"
},
"scan-jskult-batch-run-manage": {
"Type": "Task",
"Resource": "arn:aws:states:::dynamodb:getItem",
"Parameters": {
"TableName": "mbj-newdwh2021-#{ENV_NAME}-jskult-batch-run-manage",
"Key": {
"execution_id": {
"S.$": "$$.Execution.Id"
}
}
},
"Next": "Choice",
"ResultPath": "$.scan"
},
"Choice": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.scan.Item",
"IsPresent": false,
"Next": "ErrorEnd",
"Comment": "バッチ実行管理テーブルにデータが存在しない場合"
},
{
"Variable": "$.scan.Item.batch_run_status.S",
"StringEquals": "retry",
"Next": "wait-for-retry",
"Comment": "バッチ実行管理テーブルのスターテスがリトライの場合"
}
],
"Default": "NormalEnd"
},
"wait-for-retry": {
"Type": "Wait",
"SecondsPath": "$.retryIntervalSecond",
"Next": "exec-ecs-task"
},
"NormalEnd": {
"Comment": "正常終了",
"Type": "Succeed"
},
"ErrorEnd": {
"Comment": "異常終了",
"Type": "Fail",
"Error": "StatesError",
"Cause": "StepFunctions ErrorEnd"
}
}
}

129
stepfunctions/r-jskult-batch-update-business-day-state/r-jskult-batch-update-business-day-state.json Normal file
View File

@ -0,0 +1,129 @@
{
"Comment": "実消化&アルトマーク 日付テーブル更新ステートマシン",
"StartAt": "params",
"States": {
"params": {
"Comment": "パラメータ設定",
"Type": "Pass",
"Parameters": {
"ecs": {
"LaunchType": "FARGATE",
"Cluster": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:cluster/mbj-newdwh2021-#{ENV_NAME}-jskult-batch-update-business-day-ecs",
"TaskDefinition": "arn:aws:ecs:#{REGION_AP_NORTHEAST_1}:#{AWS_ACCOUNT_ID}:task-definition/mbj-newdwh2021-#{ENV_NAME}-task-jskult-batch-update-business-day",
"NetworkConfiguration": {
"AwsvpcConfiguration": {
"Subnets": [
"#{SUBNET_PRI_1A}",
"#{SUBNET_PRI_1D}"
],
"SecurityGroups": [
"#{SG_ECS_ALL}",
"#{SG_UPDATE_BIS_DAY}"
],
"AssignPublicIp": "DISABLED"
}
},
"Overrides": {
"ContainerOverrides": [
{
"Name": "mbj-newdwh2021-#{ENV_NAME}-container-jskult-batch-update-business-day",
"Environment": [
{
"Name": "BATCH_EXECUTION_ID",
"Value.$": "$$.Execution.Id"
},
{
"Name": "MAX_RUN_COUNT",
"Value.$": "$.maxRunCount"
}
]
}
]
}
}
},
"ResultPath": "$.params",
"Next": "exec-ecs-task"
},
"exec-ecs-task": {
"Type": "Task",
"Resource": "arn:aws:states:::ecs:runTask.sync",
"Parameters": {
"LaunchType.$": "$.params.ecs.LaunchType",
"Cluster.$": "$.params.ecs.Cluster",
"TaskDefinition.$": "$.params.ecs.TaskDefinition",
"NetworkConfiguration.$": "$.params.ecs.NetworkConfiguration",
"Overrides.$": "$.params.ecs.Overrides"
},
"ResultPath": "$.result",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"BackoffRate": 2,
"IntervalSeconds": 3,
"MaxAttempts": 3
}
],
"Catch": [
{
"ErrorEquals": [
"States.ALL"
],
"Next": "ErrorEnd",
"ResultPath": "$.result"
}
],
"Next": "scan-jskult-batch-run-manage",
"Comment": "ECSタスク起動"
},
"scan-jskult-batch-run-manage": {
"Type": "Task",
"Resource": "arn:aws:states:::dynamodb:getItem",
"Parameters": {
"TableName": "mbj-newdwh2021-#{ENV_NAME}-jskult-batch-run-manage",
"Key": {
"execution_id": {
"S.$": "$$.Execution.Id"
}
}
},
"Next": "Choice",
"ResultPath": "$.scan"
},
"Choice": {
"Type": "Choice",
"Choices": [
{
"Variable": "$.scan.Item",
"IsPresent": false,
"Next": "ErrorEnd",
"Comment": "バッチ実行管理テーブルにデータが存在しない場合"
},
{
"Variable": "$.scan.Item.batch_run_status.S",
"StringEquals": "retry",
"Next": "wait-for-retry",
"Comment": "バッチ実行管理テーブルのスターテスがリトライの場合"
}
],
"Default": "NormalEnd"
},
"wait-for-retry": {
"Type": "Wait",
"SecondsPath": "$.retryIntervalSecond",
"Next": "exec-ecs-task"
},
"NormalEnd": {
"Comment": "正常終了",
"Type": "Succeed"
},
"ErrorEnd": {
"Comment": "異常終了",
"Type": "Fail",
"Error": "StatesError",
"Cause": "StepFunctions ErrorEnd"
}
}
}