diff --git a/ecs/jskult-batch/src/aws/s3.py b/ecs/jskult-batch/src/aws/s3.py index 6e5755be..27701f7b 100644 --- a/ecs/jskult-batch/src/aws/s3.py +++ b/ecs/jskult-batch/src/aws/s3.py @@ -53,46 +53,31 @@ class S3Bucket(): class ConfigBucket(S3Bucket): - # TODO 日付更新処理で内容の修正を行う _bucket_name = environment.JSKULT_CONFIG_BUCKET - def download_holiday_list(self): + def download_jsk_archive_run_day_list(self): # 一時ファイルとして保存する temporary_dir = tempfile.mkdtemp() temporary_file_path = path.join( - temporary_dir, environment.JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME) - holiday_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME}' + temporary_dir, environment.JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME) + jsk_archive_run_day_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME}' with open(temporary_file_path, mode='wb') as f: self._s3_client.download_file( - self._bucket_name, holiday_list_key, f) + self._bucket_name, jsk_archive_run_day_list_key, f) f.seek(0) return temporary_file_path - - def download_wholesaler_stock_input_day_list(self): - # 一時ファイルとして保存する + + def download_jsk_expected_data_list(self): temporary_dir = tempfile.mkdtemp() temporary_file_path = path.join( - temporary_dir, environment.JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME) - wholesaler_stock_input_day_list_key = \ - f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME}' - + temporary_dir, environment.JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME) + jsk_expected_data_list_key = f'{environment.JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER}/{environment.JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME}' with open(temporary_file_path, mode='wb') as f: self._s3_client.download_file( - self._bucket_name, wholesaler_stock_input_day_list_key, f) + self._bucket_name, jsk_expected_data_list_key, f) f.seek(0) return temporary_file_path - def download_ultmarc_hex_convert_config(self): - # 一時ファイルとして保存する - temporary_dir = tempfile.mkdtemp() - temporary_file_path = path.join( - temporary_dir, environment.JSKULT_ULTMARC_HEX_CONVERT_CONFIG_FILE_NAME) - hex_convert_config_key = f'{environment.JSKULT_CONFIG_CONVERT_FOLDER}/{environment.JSKULT_ULTMARC_HEX_CONVERT_CONFIG_FILE_NAME}' - with open(temporary_file_path, mode='wb') as f: - self._s3_client.download_file( - self._bucket_name, hex_convert_config_key, f) - f.seek(0) - return temporary_file_path class JskUltBackupBucket(S3Bucket): @@ -137,4 +122,4 @@ class JskSendBucket(S3Bucket): dat_key = f'{self._send_folder}/{dat_file_key}' backup_key = f'{jskult_backup_bucket._folder}/{self._send_folder}/{datetime_key}/{dat_file_key.replace(f"{self._send_folder}/", "")}' self._s3_client.copy(self._bucket_name, dat_key, - jskult_backup_bucket._bucket_name, backup_key) + jskult_backup_bucket._bucket_name, backup_key) \ No newline at end of file diff --git a/ecs/jskult-batch/src/batch/common/calendar_file.py b/ecs/jskult-batch/src/batch/common/calendar_file.py new file mode 100644 index 00000000..b456f03c --- /dev/null +++ b/ecs/jskult-batch/src/batch/common/calendar_file.py @@ -0,0 +1,32 @@ +from src.system_var import constants + + +class CalendarFile: + """カレンダーファイル""" + + __calendar_file_lines: list[str] + + def __init__(self, calendar_file_path): + with open(calendar_file_path) as f: + self.__calendar_file_lines: list[str] = f.readlines() + + def compare_date(self, date_str: str) -> bool: + """与えられた日付がカレンダーファイル内に含まれているかどうか + カレンダーファイル内の日付はyyyy/mm/ddで書かれている前提 + コメント（#）が含まれている行は無視される + + Args: + date_str (str): yyyy/mm/dd文字列 + + Returns: + bool: 含まれていればTrue + """ + for calendar_date in self.__calendar_file_lines: + # コメント行が含まれている場合はスキップ + if constants.CALENDAR_COMMENT_SYMBOL in calendar_date: + continue + + if date_str in calendar_date: + return True + + return False diff --git a/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py b/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py new file mode 100644 index 00000000..cc826e93 --- /dev/null +++ b/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py @@ -0,0 +1,59 @@ +from src.batch.environment.jskult_batch_environment import \ + JskultBatchEnvironment +from src.system_var import environment + + +class UpdateBusinessDayEnvironment(JskultBatchEnvironment): + """実消化&アルトマークのバッチ処理で使用する環境変数を管理するクラス""" + + def __init__(self): + self.PROCESS_NAME = environment.PROCESS_NAME + self.JSKULT_CONFIG_BUCKET = environment.JSKULT_CONFIG_BUCKET + self.JSKULT_BACKUP_BUCKET = environment.JSKULT_BACKUP_BUCKET + self.BATCH_MANAGE_DYNAMODB_TABLE_NAME = environment.BATCH_MANAGE_DYNAMODB_TABLE_NAME + self.ARCHIVE_STATEMACHINE_ARN = environment.ARCHIVE_STATEMACHINE_ARN + self.BATCH_EXECUTION_ID = environment.BATCH_EXECUTION_ID + self.MAX_RUN_COUNT = environment.MAX_RUN_COUNT + self.JSKULT_CONFIG_CALENDAR_FOLDER = environment.JSKULT_CONFIG_CALENDAR_FOLDER + self.JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME = environment.JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME + self.JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER = environment.JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER + self.JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME = environment.JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME + def validate(self): + """ + 必須の環境変数が設定されているかどうか検査する。 + DB関連の環境変数は対象外とする。 + + Raises: + EnvironmentVariableNotSetException: 環境変数の設定ミス + """ + + super()._assert_variable_not_empty( + self.PROCESS_NAME, 'PROCESS_NAME') + super()._assert_variable_not_empty( + self.JSKULT_BACKUP_BUCKET, 'JSKULT_BACKUP_BUCKET') + super()._assert_variable_not_empty( + self.JSKULT_CONFIG_BUCKET, 'JSKULT_CONFIG_BUCKET') + super()._assert_variable_not_empty( + self.BATCH_MANAGE_DYNAMODB_TABLE_NAME, 'BATCH_MANAGE_DYNAMODB_TABLE_NAME') + super()._assert_variable_not_empty( + self.ARCHIVE_STATEMACHINE_ARN, 'ARCHIVE_STATEMACHINE_ARN') + super()._assert_variable_not_empty( + self.BATCH_EXECUTION_ID, 'BATCH_EXECUTION_ID') + super()._assert_variable_is_int( + self.MAX_RUN_COUNT, 'MAX_RUN_COUNT') + # MAX_RUN_COUNTは数値として扱うため、検査後に変換 + self.MAX_RUN_COUNT = int(self.MAX_RUN_COUNT) + + super()._assert_variable_not_empty( + self.JSKULT_CONFIG_CALENDAR_FOLDER, + 'JSKULT_CONFIG_CALENDAR_FOLDER') + super()._assert_variable_not_empty( + self.JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME, + 'JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME') + super()._assert_variable_not_empty( + self.JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER, + 'JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER') + super()._assert_variable_not_empty( + self.JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME, + 'JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME') + diff --git a/ecs/jskult-batch/src/batch/update_business_day.py b/ecs/jskult-batch/src/batch/update_business_day.py index 57e9372f..963c17f4 100644 --- a/ecs/jskult-batch/src/batch/update_business_day.py +++ b/ecs/jskult-batch/src/batch/update_business_day.py @@ -1,10 +1,213 @@ +import json +from datetime import datetime + +import boto3 + +from src.aws.s3 import ConfigBucket, JskTransferListBucket +from src.batch.common.calendar_file import CalendarFile +from src.batch.environment.update_business_day_environment import \ + UpdateBusinessDayEnvironment from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint +from src.error.exceptions import (EnvironmentVariableNotSetException, + MaxRunCountReachedException) +from src.logging.get_logger import get_logger +from src.manager.jskult_batch_run_manager import JskultBatchRunManager +from src.manager.jskult_batch_status_manager import JskultBatchStatusManager +from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager +from src.system_var import constants + +logger = get_logger('日付テーブル更新') class UpdateBusinessDay(JskultBatchEntrypoint): def __init__(self): super().__init__() + self.environment = UpdateBusinessDayEnvironment() + + # 必須の環境変数が設定されていない場合、エラーにする + try: + self.environment.validate() + except EnvironmentVariableNotSetException as e: + logger.exception(e) + return def execute(self): - # TODO: ここで日付更新処理を実行する - pass + """日付テーブル更新""" + logger.info('I-1 処理開始: 実消化＆アルトマーク_日付テーブル更新') + + jskult_hdke_tbl_manager = JskultHdkeTblManager() + + jskult_batch_status_manager = JskultBatchStatusManager( + self.environment.PROCESS_NAME, + constants.PROCESS_NAME_UPDATE_BUSINESS_DAY, + self.environment.MAX_RUN_COUNT, + # 日付テーブル更新では連携ファイル数を参照しないため、0を指定する。 + 0 + ) + + jskult_batch_run_manager = JskultBatchRunManager( + self.environment.BATCH_MANAGE_DYNAMODB_TABLE_NAME, + self.environment.BATCH_EXECUTION_ID) + + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_START) + + if not jskult_hdke_tbl_manager.can_run_process(): + logger.error( + '日次バッチ処理中でない、もしくはdump取得が正常終了していないため、日次バッチ処理を異常終了します。') + # バッチ実行管理テーブルをfailedで登録 + jskult_batch_run_manager.batch_failed() + + # バッチステータス管理テーブルをエラーに更新 + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_ERROR) + return + + try: + if not jskult_batch_status_manager.can_run_post_process(): + # 後続処理の起動条件を満たしていない場合 + # 処理ステータスを「処理待」に設定 + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_WAITING) + + # バッチ実行管理テーブルに「retry」で登録 + jskult_batch_run_manager.batch_retry() + + return + + except MaxRunCountReachedException: + logger.error('後続処理の実行が完了していないため、日付テーブル更新処理を異常終了します。') + + jskult_batch_run_manager.batch_failed() + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_ERROR) + return + try: + # 日付テーブルの処理年月日を取得する + _, _, syor_date = jskult_hdke_tbl_manager.get_batch_statuses() + + if self._is_archive_run_day(syor_date): + logger.info('I-7 [NOTICE]実消化データアーカイブ取得処理を実行します。') + + # 実消化データアーカイブ取得処理を起動する + sfn_client = boto3.client('stepfunctions') + + # StepFunctionsを起動する + sfn_client.start_execution( + stateMachineArn=self.environment.ARCHIVE_STATEMACHINE_ARN + ) + + dt = datetime.strptime(syor_date, "%Y/%m/%d") + # 日付テーブルの処理年月日が月曜日の場合 + if dt.weekday() == constants.WEEKDAY_MONDAY: + + if not jskult_batch_status_manager.is_done_ultmarc_import(): + logger.info( + 'I-3 [NOTICE]アルトマーク取込稼働日でしたが、アルトマーク取込が行われませんでした。') + + # 実消化転送データ一覧を取得する + try: + transfer_list_bucket = JskTransferListBucket() + + transfer_list_file_path = transfer_list_bucket.download_transfer_result_file( + syor_date) + except Exception as e: + logger.exception(f'転送ファイル一覧の取得に失敗しました。 {e}') + # バッチ実行管理テーブルをfailedで登録 + jskult_batch_run_manager.batch_failed() + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_ERROR) + return + + with open(transfer_list_file_path) as f: + transfer_data_list = json.load(f) + + jsk_transfer_data_list = transfer_data_list['jsk_transfer_list'] + + # 実消化&アルトマーク_実消化受領予定データリストを取得する + try: + config_bucket = ConfigBucket() + jsk_expected_data_path = config_bucket.download_jsk_expected_data_list() + except Exception as e: + logger.exception(f'転送ファイル一覧の取得に失敗しました。 {e}') + # バッチ実行管理テーブルをfailedで登録 + jskult_batch_run_manager.batch_failed() + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_ERROR) + return + + with open(jsk_expected_data_path) as f: + jsk_expected_data_list = json.load(f) + + jsk_transfer_expected_data_list = jsk_expected_data_list['jsk_transfer_list'] + + if len(jsk_transfer_data_list) != 0: + # 不足ファイルがあった場合ログ出力 + self._check_missing_received_files( + jsk_transfer_expected_data_list, jsk_transfer_data_list) + # 受領予定にないファイルがあった場合ログ出力 + self._check_extra_received_files( + jsk_transfer_expected_data_list, jsk_transfer_data_list) + + jskult_hdke_tbl_manager.update_batch_process_complete() + # 処理ステータスを「処理済」に設定 + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_DONE) + + # 日付テーブルに対しバッチ処理中ステータスとダンプ取得状態区分を0に、処理日を+1日に更新 + jskult_batch_run_manager.batch_success() + + logger.info('I-4 [NOTICE]処理終了: 実消化＆アルトマーク_日付テーブル更新') + except Exception as e: + # 何らかのエラーが発生した際に、バッチ実行管理テーブルに「failed」で登録 + jskult_batch_run_manager.batch_failed() + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_ERROR) + logger.error(f'E-1 異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') + + def _is_archive_run_day(self, syor_date: str) -> bool: + """アーカイブ取得日カレンダーを取得し、処理日がアーカイブ起動日かを確認 + + Returns: + bool + + """ + archive_calendar_file_path = ConfigBucket().download_jsk_archive_run_day_list() + + # 設定ファイル「実消化データアーカイブ起動日カレンダーファイル」の定義内容を取得する + target_days = CalendarFile(archive_calendar_file_path) + + # 処理日付が、設定ファイル「実消化データアーカイブ起動日カレンダーファイル」の定義に該当するかを判定する + return target_days.compare_date(syor_date) + + def _check_missing_received_files(self, jsk_transfer_expected_data_list: list, jsk_transfer_data_list: list): + """jsk_transfer_data_listと比較し、結果不足ファイルがあった場合ログ出力 + """ + missing_files = [ + jsk_transfer_expected_data + for jsk_transfer_expected_data in jsk_transfer_expected_data_list + if not any( + jsk_transfer_data.startswith( + jsk_transfer_expected_data) + for jsk_transfer_data in jsk_transfer_data_list + ) + ] + if missing_files != 0: + logger.info( + f'I-5 [NOTICE]日次連携受領ファイルに不足がありました。ファイル名：{missing_files}') + + def _check_extra_received_files(self, jsk_transfer_expected_data_list: list, jsk_transfer_data_list: list): + """jsk_transfer_data_listと比較し、受領予定にないファイルがあった場合ログ出力 + """ + unexpected_data = [ + jsk_transfer_data + for jsk_transfer_data in jsk_transfer_data_list + if not any( + jsk_transfer_data.startswith( + jsk_transfer_expected_data) + for jsk_transfer_expected_data in jsk_transfer_expected_data_list + ) + ] + if len(unexpected_data) != 0: + logger.info( + f'I-6 [NOTICE]受領ファイルに日次連携ファイルでないファイルがありました。ファイル名：{unexpected_data}') diff --git a/ecs/jskult-batch/src/system_var/environment.py b/ecs/jskult-batch/src/system_var/environment.py index 492c9661..fbb334f3 100644 --- a/ecs/jskult-batch/src/system_var/environment.py +++ b/ecs/jskult-batch/src/system_var/environment.py @@ -12,18 +12,28 @@ PROCESS_NAME = os.environ.get('PROCESS_NAME', None) # AWS JSKULT_CONFIG_BUCKET = os.environ.get('JSKULT_CONFIG_BUCKET', None) +JSKULT_BACKUP_BUCKET = os.environ.get('JSKULT_BACKUP_BUCKET', None) +BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get( + 'BATCH_MANAGE_DYNAMODB_TABLE_NAME', None) +ARCHIVE_STATEMACHINE_ARN = os.environ.get('ARCHIVE_STATEMACHINE_ARN', None) BATCH_EXECUTION_ID = os.environ.get('BATCH_EXECUTION_ID', None) MAX_RUN_COUNT = os.environ.get('MAX_RUN_COUNT', None) TRANSFER_RESULT_FOLDER = os.environ.get('TRANSFER_RESULT_FOLDER', None) TRANSFER_RESULT_FILE_NAME = os.environ.get('TRANSFER_RESULT_FILE_NAME', None) -DCF_INST_MERGE_SEND_FILE_NAME = os.environ.get( - 'DCF_INST_MERGE_SEND_FILE_NAME', None) -JSKULT_BACKUP_BUCKET = os.environ.get('JSKULT_BACKUP_BUCKET', None) +JSKULT_CONFIG_CALENDAR_FOLDER = os.environ.get( + 'JSKULT_CONFIG_CALENDAR_FOLDER', None +) +JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME = os.environ.get( + 'JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME', None +) +JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER = os.environ.get( + 'JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER', None +) +JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME = os.environ.get( + 'JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME', None +) JSK_IO_BUCKET = os.environ.get('JSK_IO_BUCKET', None) -JSK_BACKUP_FOLDER = os.environ.get('JSK_BACKUP_FOLDER', None) JSK_DATA_SEND_FOLDER = os.environ.get('JSK_DATA_SEND_FOLDER', None) -BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get( - 'BATCH_MANAGE_DYNAMODB_TABLE_NAME', None) # 初期値がある環境変数 diff --git a/s3/config/jskult/calendar/jsk_archive_run_day.txt b/s3/config/jskult/calendar/jsk_archive_run_day.txt new file mode 100644 index 00000000..cc8eee46 --- /dev/null +++ b/s3/config/jskult/calendar/jsk_archive_run_day.txt @@ -0,0 +1 @@ +2026/01/09 \ No newline at end of file diff --git a/s3/config/jskult/expected_data_list/jsk_expected_data_list.json b/s3/config/jskult/expected_data_list/jsk_expected_data_list.json new file mode 100644 index 00000000..ff6a05ed --- /dev/null +++ b/s3/config/jskult/expected_data_list/jsk_expected_data_list.json @@ -0,0 +1,37 @@ +{ + "jsk_transfer_list": [ + "TRN_RESULT_DATA", + "TRN_RESULT_DATA_BIO", + "TRN_Recive_Inventry", + "INST_IM_PLN", + "MR_YR_PLN_INST_KY", + "MST_WHLSLR", + "MST_SALES_PLACE", + "WHLSLR_LVL4", + "WHLSLR_LVL3", + "WHLSLR_LVL2", + "WHLSLR_LVL1", + "MST_COMPANY", + "MST_COMPANY_GRP", + "MST_DEAL_DIV", + "MST_PROD_PKG", + "PROD_PRICE", + "IM_PLN_PROD", + "MST_ASSN_PROD_GRP", + "PROD_ASSN_TEAM", + "MST_ATC_PHARM_PROD_GRP", + "CUSTOMER_LOTNO_ALL", + "MST_INST", + "ATC_PHARM", + "MST_WAREHOUSE", + "EMP", + "ORG_LVL4", + "ORG_LVL3", + "ORG_LVL2", + "ORG_LVL1", + "MST_JIS_PREF", + "MST_JIS_CITY", + "MST_INST_ASSN", + "MST_GENERAL" + ] +}