diff --git a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py index 82c7cfbe..499cfa98 100644 --- a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py +++ b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py @@ -4,15 +4,18 @@ import os.path as path import tempfile from src.aws.s3 import JskSendBucket, JskTransferListBucket +from src.batch.environment.dcf_inst_merge_environment import \ + DCFInstMergeEnvironment from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint from src.db.database import Database from src.error.exceptions import (BatchOperationException, + EnvironmentVariableNotSetException, MaxRunCountReachedException) from src.logging.get_logger import get_logger from src.manager.jskult_batch_run_manager import JskultBatchRunManager from src.manager.jskult_batch_status_manager import JskultBatchStatusManager from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager -from src.system_var import environment +from src.system_var import constants logger = get_logger('DCF削除新規マスタ作成') @@ -20,12 +23,22 @@ logger = get_logger('DCF削除新規マスタ作成') class DcfInstMergeIO(JskultBatchEntrypoint): def __init__(self): super().__init__() + # 環境変数をimport + self.environment = DCFInstMergeEnvironment() def execute(self): logger.info("DCF削除新規マスタ作成処理を開始します。") + + # 必須の環境変数が設定されていない場合、エラーにする + try: + self.environment.validate() + except EnvironmentVariableNotSetException as e: + logger.exception(e) + return + jskult_hdke_tbl_manager = JskultHdkeTblManager() jskult_batch_run_manager = JskultBatchRunManager( - environment.BATCH_EXECUTION_ID) + self.environment.BATCH_EXECUTION_ID) if not jskult_hdke_tbl_manager.can_run_process(): logger.error( '日次バッチ処理中またはdump取得が正常終了していないため、DCF削除新規マスタ作成を終了します。') @@ -54,20 +67,21 @@ class DcfInstMergeIO(JskultBatchEntrypoint): transfer_list['jsk_transfer_list']) + len(transfer_list['ult_transfer_list']) jskult_batch_status_manager = JskultBatchStatusManager( - environment.PROCESS_NAME, - # TODO チケットNEWDWH2021-1847の実装で作成した定数に置き換え - 'post_process', - environment.MAX_RUN_COUNT, + self.environment.PROCESS_NAME, + constants.PROCESS_TYPE_POST_PROCESS, + self.environment.MAX_RUN_COUNT, receive_file_count ) try: - jskult_batch_status_manager.set_process_status("start") + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_START) try: if not jskult_batch_status_manager.can_run_post_process(): # 後続処理の起動条件を満たしていない場合 # 処理ステータスを「処理待」に設定 - jskult_batch_status_manager.set_process_status("waiting") + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_WAITING) # バッチ実行管理テーブルに「retry」で登録 jskult_batch_run_manager.batch_retry() @@ -76,7 +90,8 @@ class DcfInstMergeIO(JskultBatchEntrypoint): except MaxRunCountReachedException: logger.info('最大起動回数に到達したため、DCF削除新規マスタ作成処理を実行します。') - jskult_batch_status_manager.set_process_status("doing") + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_DOING) # アルトマーク取込が実行されていた場合にDCF施設削除新規マスタの作成処理を実行 if jskult_batch_status_manager.is_done_ultmarc_import(): @@ -90,17 +105,18 @@ class DcfInstMergeIO(JskultBatchEntrypoint): # CSV出力 dcf_inst_merge_all_records = self._select_dcf_inst_merge_all() file_path = self._make_csv_data( - environment.DCF_INST_MERGE_SEND_FILE_NAME, + self.environment.DCF_INST_MERGE_SEND_FILE_NAME, dcf_inst_merge_all_records) # CSVをS3にアップロード self._upload_dcf_inst_merge_csv_file( - file_path, process_date, environment.DCF_INST_MERGE_SEND_FILE_NAME) + file_path, process_date, self.environment.DCF_INST_MERGE_SEND_FILE_NAME) # 処理が全て正常終了した際に、バッチ実行管理テーブルに「success」で登録 logger.info("DCF削除新規マスタ作成処理を正常終了します。") jskult_batch_run_manager.batch_success() - jskult_batch_status_manager.set_process_status("done") + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_DONE) return @@ -108,7 +124,8 @@ class DcfInstMergeIO(JskultBatchEntrypoint): # 何らかのエラーが発生した際に、バッチ実行管理テーブルに「failed」で登録 logger.exception(f'予期せぬエラーが発生したため、DCF削除新規マスタ作成処理を終了します。{e}') jskult_batch_run_manager.batch_failed() - jskult_batch_status_manager.set_process_status("failed") + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_ERROR) def _select_dcf_inst_merge_all(self) -> tuple[bool, list[dict]]: try: @@ -272,7 +289,7 @@ class DcfInstMergeIO(JskultBatchEntrypoint): 'CREATER', 'CREATE_DATE', 'UPDATER', 'UPDATE_DATE'] with open(csv_file_path, mode='w', encoding='UTF-8') as csv_file: # ヘッダ行書き込み（くくり文字をつけない為にwriterowではなく、writeを使用しています） - csv_file.write(f"{','.join(head_str)}\n") + csv_file.write(f"{','.join(head_str)}\r\n") # UTF-8、CRLF、価囲いありで書き込む writer = csv.writer(csv_file, delimiter=',', lineterminator='\r\n', quotechar='"', doublequote=True, quoting=csv.QUOTE_ALL, diff --git a/ecs/jskult-batch/test.py b/ecs/jskult-batch/src/batch/environment/__init__.py similarity index 100% rename from ecs/jskult-batch/test.py rename to ecs/jskult-batch/src/batch/environment/__init__.py diff --git a/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py b/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py new file mode 100644 index 00000000..3da34501 --- /dev/null +++ b/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py @@ -0,0 +1,48 @@ +from src.batch.environment.jskult_batch_environment import \ + JskultBatchEnvironment +from src.system_var import environment + + +class DCFInstMergeEnvironment(JskultBatchEnvironment): + """実消化&アルトマークのバッチ処理で使用する環境変数を管理するクラス""" + + def __init__(self): + self.JSK_IO_BUCKET = environment.JSK_IO_BUCKET + self.JSKULT_BACKUP_BUCKET = environment.JSKULT_BACKUP_BUCKET + self.BATCH_MANAGE_DYNAMODB_TABLE_NAME = environment.BATCH_MANAGE_DYNAMODB_TABLE_NAME + self.BATCH_EXECUTION_ID = environment.BATCH_EXECUTION_ID + self.MAX_RUN_COUNT = environment.MAX_RUN_COUNT + self.PROCESS_NAME = environment.PROCESS_NAME + self.JSK_DATA_SEND_FOLDER = environment.JSK_DATA_SEND_FOLDER + self.JSK_BACKUP_FOLDER = environment.JSK_BACKUP_FOLDER + self.TRANSFER_RESULT_FOLDER = environment.TRANSFER_RESULT_FOLDER + self.TRANSFER_RESULT_FILE_NAME = environment.TRANSFER_RESULT_FILE_NAME + self.DCF_INST_MERGE_SEND_FILE_NAME = environment.DCF_INST_MERGE_SEND_FILE_NAME + + def validate(self): + """ + 必須の環境変数が設定されているかどうか検査する。 + DB関連の環境変数は対象外とする。 + + Raises: + EnvironmentVariableNotSetException: 環境変数の設定ミス + """ + super()._assert_variable_not_empty(self.JSK_IO_BUCKET, 'JSK_IO_BUCKET') + super()._assert_variable_not_empty( + self.JSKULT_BACKUP_BUCKET, 'JSKULT_BACKUP_BUCKET') + super()._assert_variable_not_empty(self.BATCH_MANAGE_DYNAMODB_TABLE_NAME, + 'BATCH_MANAGE_DYNAMODB_TABLE_NAME') + super()._assert_variable_not_empty(self.BATCH_EXECUTION_ID, 'BATCH_EXECUTION_ID') + super()._assert_variable_is_int(self.MAX_RUN_COUNT, 'MAX_RUN_COUNT') + # MAX_RUN_COUNTは数値として扱うため、検査後に変換 + self.MAX_RUN_COUNT = int(self.MAX_RUN_COUNT) + super()._assert_variable_not_empty(self.PROCESS_NAME, 'PROCESS_NAME') + super()._assert_variable_not_empty( + self.JSK_DATA_SEND_FOLDER, 'JSK_DATA_SEND_FOLDER') + super()._assert_variable_not_empty(self.JSK_BACKUP_FOLDER, 'JSK_BACKUP_FOLDER') + super()._assert_variable_not_empty( + self.TRANSFER_RESULT_FOLDER, 'TRANSFER_RESULT_FOLDER') + super()._assert_variable_not_empty( + self.TRANSFER_RESULT_FILE_NAME, 'TRANSFER_RESULT_FILE_NAME') + super()._assert_variable_not_empty(self.DCF_INST_MERGE_SEND_FILE_NAME, + 'DCF_INST_MERGE_SEND_FILE_NAME') diff --git a/ecs/jskult-batch/src/batch/environment/jskult_batch_environment.py b/ecs/jskult-batch/src/batch/environment/jskult_batch_environment.py new file mode 100644 index 00000000..e5be1d41 --- /dev/null +++ b/ecs/jskult-batch/src/batch/environment/jskult_batch_environment.py @@ -0,0 +1,26 @@ +import abc + +from src.error.exceptions import EnvironmentVariableNotSetException + + +class JskultBatchEnvironment(metaclass=abc.ABCMeta): + """実消化&アルトマークのバッチ処理で使用する環境変数を管理するクラス""" + @abc.abstractmethod + def validate(self): + """ + 必須の環境変数が設定されているかどうか検査する。 + DB関連の環境変数は対象外とする。 + """ + pass + + def _assert_variable_not_empty(self, variable: str, variable_name: str): + if variable is None: + raise EnvironmentVariableNotSetException( + f'環境変数[{variable_name}]が設定されていません。') + + def _assert_variable_is_int(self, variable: str, variable_name: str): + try: + int(variable) + except ValueError: + raise EnvironmentVariableNotSetException( + f'環境変数[{variable_name}]が整数ではありません。') diff --git a/ecs/jskult-batch/src/error/exceptions.py b/ecs/jskult-batch/src/error/exceptions.py index aa5f9be6..a6db7ed7 100644 --- a/ecs/jskult-batch/src/error/exceptions.py +++ b/ecs/jskult-batch/src/error/exceptions.py @@ -12,3 +12,7 @@ class BatchOperationException(MeDaCaException): class MaxRunCountReachedException(MeDaCaException): pass + + +class EnvironmentVariableNotSetException(MeDaCaException): + pass diff --git a/ecs/jskult-batch/src/system_var/environment.py b/ecs/jskult-batch/src/system_var/environment.py index d44805d4..492c9661 100644 --- a/ecs/jskult-batch/src/system_var/environment.py +++ b/ecs/jskult-batch/src/system_var/environment.py @@ -7,21 +7,24 @@ DB_USERNAME = os.environ['DB_USERNAME'] DB_PASSWORD = os.environ['DB_PASSWORD'] DB_SCHEMA = os.environ['DB_SCHEMA'] -# AWS -JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] -BATCH_EXECUTION_ID = os.environ['BATCH_EXECUTION_ID'] -MAX_RUN_COUNT = int(os.environ['MAX_RUN_COUNT']) -TRANSFER_RESULT_FOLDER = os.environ['TRANSFER_RESULT_FOLDER'] -TRANSFER_RESULT_FILE_NAME = os.environ['TRANSFER_RESULT_FILE_NAME'] -DCF_INST_MERGE_SEND_FILE_NAME = os.environ['DCF_INST_MERGE_SEND_FILE_NAME'] -PROCESS_NAME = os.environ['PROCESS_NAME'] -JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] -JSK_IO_BUCKET = os.environ['JSK_IO_BUCKET'] -JSK_BACKUP_FOLDER = os.environ['JSK_BACKUP_FOLDER'] -JSK_DATA_SEND_FOLDER = os.environ['JSK_DATA_SEND_FOLDER'] +# 処理名 +PROCESS_NAME = os.environ.get('PROCESS_NAME', None) # AWS -BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get('BATCH_MANAGE_DYNAMODB_TABLE_NAME') +JSKULT_CONFIG_BUCKET = os.environ.get('JSKULT_CONFIG_BUCKET', None) +BATCH_EXECUTION_ID = os.environ.get('BATCH_EXECUTION_ID', None) +MAX_RUN_COUNT = os.environ.get('MAX_RUN_COUNT', None) +TRANSFER_RESULT_FOLDER = os.environ.get('TRANSFER_RESULT_FOLDER', None) +TRANSFER_RESULT_FILE_NAME = os.environ.get('TRANSFER_RESULT_FILE_NAME', None) +DCF_INST_MERGE_SEND_FILE_NAME = os.environ.get( + 'DCF_INST_MERGE_SEND_FILE_NAME', None) +JSKULT_BACKUP_BUCKET = os.environ.get('JSKULT_BACKUP_BUCKET', None) +JSK_IO_BUCKET = os.environ.get('JSK_IO_BUCKET', None) +JSK_BACKUP_FOLDER = os.environ.get('JSK_BACKUP_FOLDER', None) +JSK_DATA_SEND_FOLDER = os.environ.get('JSK_DATA_SEND_FOLDER', None) +BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get( + 'BATCH_MANAGE_DYNAMODB_TABLE_NAME', None) + # 初期値がある環境変数 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO')