nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: jskult-batch内の各機能事に使う環境変数を切り分けられるようにクラス化 生物由来の部分追加

Browse Source
This commit is contained in:
mori.k 2025-05-30 13:18:21 +09:00
parent 397a8a0198
commit 7e778a62f9
6 changed files with 125 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
ecs/jskult-batch/src/batch/dcf_inst_merge_io.py
View File

@ -4,15 +4,18 @@ import os.path as path
import tempfile
from src.aws.s3 import JskSendBucket, JskTransferListBucket
from src.batch.environment.dcf_inst_merge_environment import \
DCFInstMergeEnvironment
from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint
from src.db.database import Database
from src.error.exceptions import (BatchOperationException,
EnvironmentVariableNotSetException,
MaxRunCountReachedException)
from src.logging.get_logger import get_logger
from src.manager.jskult_batch_run_manager import JskultBatchRunManager
from src.manager.jskult_batch_status_manager import JskultBatchStatusManager
from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager
from src.system_var import environment
from src.system_var import constants
logger = get_logger('DCF削除新規マスタ作成')
@ -20,12 +23,22 @@ logger = get_logger('DCF削除新規マスタ作成')
class DcfInstMergeIO(JskultBatchEntrypoint):
def __init__(self):
super().__init__()
# 環境変数をimport
self.environment = DCFInstMergeEnvironment()
def execute(self):
logger.info("DCF削除新規マスタ作成処理を開始します。")
# 必須の環境変数が設定されていない場合、エラーにする
try:
self.environment.validate()
except EnvironmentVariableNotSetException as e:
logger.exception(e)
return
jskult_hdke_tbl_manager = JskultHdkeTblManager()
jskult_batch_run_manager = JskultBatchRunManager(
environment.BATCH_EXECUTION_ID)
self.environment.BATCH_EXECUTION_ID)
if not jskult_hdke_tbl_manager.can_run_process():
logger.error(
'日次バッチ処理中またはdump取得が正常終了していないため、DCF削除新規マスタ作成を終了します。')
@ -54,20 +67,21 @@ class DcfInstMergeIO(JskultBatchEntrypoint):
transfer_list['jsk_transfer_list']) + len(transfer_list['ult_transfer_list'])
jskult_batch_status_manager = JskultBatchStatusManager(
environment.PROCESS_NAME,
# TODO チケットNEWDWH2021-1847の実装で作成した定数に置き換え
'post_process',
environment.MAX_RUN_COUNT,
self.environment.PROCESS_NAME,
constants.PROCESS_TYPE_POST_PROCESS,
self.environment.MAX_RUN_COUNT,
receive_file_count
)
try:
jskult_batch_status_manager.set_process_status("start")
jskult_batch_status_manager.set_process_status(
constants.PROCESS_STATUS_START)
try:
if not jskult_batch_status_manager.can_run_post_process():
# 後続処理の起動条件を満たしていない場合
# 処理ステータスを「処理待」に設定
jskult_batch_status_manager.set_process_status("waiting")
jskult_batch_status_manager.set_process_status(
constants.PROCESS_STATUS_WAITING)
# バッチ実行管理テーブルに「retry」で登録
jskult_batch_run_manager.batch_retry()
@ -76,7 +90,8 @@ class DcfInstMergeIO(JskultBatchEntrypoint):
except MaxRunCountReachedException:
logger.info('最大起動回数に到達したため、DCF削除新規マスタ作成処理を実行します。')
jskult_batch_status_manager.set_process_status("doing")
jskult_batch_status_manager.set_process_status(
constants.PROCESS_STATUS_DOING)
# アルトマーク取込が実行されていた場合にDCF施設削除新規マスタの作成処理を実行
if jskult_batch_status_manager.is_done_ultmarc_import():
@ -90,17 +105,18 @@ class DcfInstMergeIO(JskultBatchEntrypoint):
# CSV出力
dcf_inst_merge_all_records = self._select_dcf_inst_merge_all()
file_path = self._make_csv_data(
environment.DCF_INST_MERGE_SEND_FILE_NAME,
self.environment.DCF_INST_MERGE_SEND_FILE_NAME,
dcf_inst_merge_all_records)
# CSVをS3にアップロード
self._upload_dcf_inst_merge_csv_file(
file_path, process_date, environment.DCF_INST_MERGE_SEND_FILE_NAME)
file_path, process_date, self.environment.DCF_INST_MERGE_SEND_FILE_NAME)
# 処理が全て正常終了した際に、バッチ実行管理テーブルに「success」で登録
logger.info("DCF削除新規マスタ作成処理を正常終了します。")
jskult_batch_run_manager.batch_success()
jskult_batch_status_manager.set_process_status("done")
jskult_batch_status_manager.set_process_status(
constants.PROCESS_STATUS_DONE)
return
@ -108,7 +124,8 @@ class DcfInstMergeIO(JskultBatchEntrypoint):
# 何らかのエラーが発生した際に、バッチ実行管理テーブルに「failed」で登録
logger.exception(f'予期せぬエラーが発生したため、DCF削除新規マスタ作成処理を終了します。{e}')
jskult_batch_run_manager.batch_failed()
jskult_batch_status_manager.set_process_status("failed")
jskult_batch_status_manager.set_process_status(
constants.PROCESS_STATUS_ERROR)
def _select_dcf_inst_merge_all(self) -> tuple[bool, list[dict]]:
try:
@ -272,7 +289,7 @@ class DcfInstMergeIO(JskultBatchEntrypoint):
'CREATER', 'CREATE_DATE', 'UPDATER', 'UPDATE_DATE']
with open(csv_file_path, mode='w', encoding='UTF-8') as csv_file:
# ヘッダ行書き込み（くくり文字をつけない為にwriterowではなく、writeを使用しています）
csv_file.write(f"{','.join(head_str)}\n")
csv_file.write(f"{','.join(head_str)}\r\n")
# UTF-8、CRLF、価囲いありで書き込む
writer = csv.writer(csv_file, delimiter=',', lineterminator='\r\n',
quotechar='"', doublequote=True, quoting=csv.QUOTE_ALL,

0
ecs/jskult-batch/test.py → ecs/jskult-batch/src/batch/environment/__init__.py
View File

48
ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py Normal file
View File

@ -0,0 +1,48 @@
from src.batch.environment.jskult_batch_environment import \
JskultBatchEnvironment
from src.system_var import environment
class DCFInstMergeEnvironment(JskultBatchEnvironment):
"""実消化&アルトマークのバッチ処理で使用する環境変数を管理するクラス"""
def __init__(self):
self.JSK_IO_BUCKET = environment.JSK_IO_BUCKET
self.JSKULT_BACKUP_BUCKET = environment.JSKULT_BACKUP_BUCKET
self.BATCH_MANAGE_DYNAMODB_TABLE_NAME = environment.BATCH_MANAGE_DYNAMODB_TABLE_NAME
self.BATCH_EXECUTION_ID = environment.BATCH_EXECUTION_ID
self.MAX_RUN_COUNT = environment.MAX_RUN_COUNT
self.PROCESS_NAME = environment.PROCESS_NAME
self.JSK_DATA_SEND_FOLDER = environment.JSK_DATA_SEND_FOLDER
self.JSK_BACKUP_FOLDER = environment.JSK_BACKUP_FOLDER
self.TRANSFER_RESULT_FOLDER = environment.TRANSFER_RESULT_FOLDER
self.TRANSFER_RESULT_FILE_NAME = environment.TRANSFER_RESULT_FILE_NAME
self.DCF_INST_MERGE_SEND_FILE_NAME = environment.DCF_INST_MERGE_SEND_FILE_NAME
def validate(self):
"""
必須の環境変数が設定されているかどうか検査する。
DB関連の環境変数は対象外とする。
Raises:
EnvironmentVariableNotSetException: 環境変数の設定ミス
"""
super()._assert_variable_not_empty(self.JSK_IO_BUCKET, 'JSK_IO_BUCKET')
super()._assert_variable_not_empty(
self.JSKULT_BACKUP_BUCKET, 'JSKULT_BACKUP_BUCKET')
super()._assert_variable_not_empty(self.BATCH_MANAGE_DYNAMODB_TABLE_NAME,
'BATCH_MANAGE_DYNAMODB_TABLE_NAME')
super()._assert_variable_not_empty(self.BATCH_EXECUTION_ID, 'BATCH_EXECUTION_ID')
super()._assert_variable_is_int(self.MAX_RUN_COUNT, 'MAX_RUN_COUNT')
# MAX_RUN_COUNTは数値として扱うため、検査後に変換
self.MAX_RUN_COUNT = int(self.MAX_RUN_COUNT)
super()._assert_variable_not_empty(self.PROCESS_NAME, 'PROCESS_NAME')
super()._assert_variable_not_empty(
self.JSK_DATA_SEND_FOLDER, 'JSK_DATA_SEND_FOLDER')
super()._assert_variable_not_empty(self.JSK_BACKUP_FOLDER, 'JSK_BACKUP_FOLDER')
super()._assert_variable_not_empty(
self.TRANSFER_RESULT_FOLDER, 'TRANSFER_RESULT_FOLDER')
super()._assert_variable_not_empty(
self.TRANSFER_RESULT_FILE_NAME, 'TRANSFER_RESULT_FILE_NAME')
super()._assert_variable_not_empty(self.DCF_INST_MERGE_SEND_FILE_NAME,
'DCF_INST_MERGE_SEND_FILE_NAME')

26
ecs/jskult-batch/src/batch/environment/jskult_batch_environment.py Normal file
View File

@ -0,0 +1,26 @@
import abc
from src.error.exceptions import EnvironmentVariableNotSetException
class JskultBatchEnvironment(metaclass=abc.ABCMeta):
"""実消化&アルトマークのバッチ処理で使用する環境変数を管理するクラス"""
@abc.abstractmethod
def validate(self):
"""
必須の環境変数が設定されているかどうか検査する。
DB関連の環境変数は対象外とする。
"""
pass
def _assert_variable_not_empty(self, variable: str, variable_name: str):
if variable is None:
raise EnvironmentVariableNotSetException(
f'環境変数[{variable_name}]が設定されていません。')
def _assert_variable_is_int(self, variable: str, variable_name: str):
try:
int(variable)
except ValueError:
raise EnvironmentVariableNotSetException(
f'環境変数[{variable_name}]が整数ではありません。')

4
ecs/jskult-batch/src/error/exceptions.py
View File

@ -12,3 +12,7 @@ class BatchOperationException(MeDaCaException):
class MaxRunCountReachedException(MeDaCaException):
pass
class EnvironmentVariableNotSetException(MeDaCaException):
pass

29
ecs/jskult-batch/src/system_var/environment.py
View File

@ -7,21 +7,24 @@ DB_USERNAME = os.environ['DB_USERNAME']
DB_PASSWORD = os.environ['DB_PASSWORD']
DB_SCHEMA = os.environ['DB_SCHEMA']
# AWS
JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET']
BATCH_EXECUTION_ID = os.environ['BATCH_EXECUTION_ID']
MAX_RUN_COUNT = int(os.environ['MAX_RUN_COUNT'])
TRANSFER_RESULT_FOLDER = os.environ['TRANSFER_RESULT_FOLDER']
TRANSFER_RESULT_FILE_NAME = os.environ['TRANSFER_RESULT_FILE_NAME']
DCF_INST_MERGE_SEND_FILE_NAME = os.environ['DCF_INST_MERGE_SEND_FILE_NAME']
PROCESS_NAME = os.environ['PROCESS_NAME']
JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET']
JSK_IO_BUCKET = os.environ['JSK_IO_BUCKET']
JSK_BACKUP_FOLDER = os.environ['JSK_BACKUP_FOLDER']
JSK_DATA_SEND_FOLDER = os.environ['JSK_DATA_SEND_FOLDER']
# 処理名
PROCESS_NAME = os.environ.get('PROCESS_NAME', None)
# AWS
BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get('BATCH_MANAGE_DYNAMODB_TABLE_NAME')
JSKULT_CONFIG_BUCKET = os.environ.get('JSKULT_CONFIG_BUCKET', None)
BATCH_EXECUTION_ID = os.environ.get('BATCH_EXECUTION_ID', None)
MAX_RUN_COUNT = os.environ.get('MAX_RUN_COUNT', None)
TRANSFER_RESULT_FOLDER = os.environ.get('TRANSFER_RESULT_FOLDER', None)
TRANSFER_RESULT_FILE_NAME = os.environ.get('TRANSFER_RESULT_FILE_NAME', None)
DCF_INST_MERGE_SEND_FILE_NAME = os.environ.get(
'DCF_INST_MERGE_SEND_FILE_NAME', None)
JSKULT_BACKUP_BUCKET = os.environ.get('JSKULT_BACKUP_BUCKET', None)
JSK_IO_BUCKET = os.environ.get('JSK_IO_BUCKET', None)
JSK_BACKUP_FOLDER = os.environ.get('JSK_BACKUP_FOLDER', None)
JSK_DATA_SEND_FOLDER = os.environ.get('JSK_DATA_SEND_FOLDER', None)
BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get(
'BATCH_MANAGE_DYNAMODB_TABLE_NAME', None)
# 初期値がある環境変数
LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO')