From 2783da332836bf183e4ac08b0a4ea210bbfe4c2d Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Tue, 3 Jun 2025 10:00:49 +0900 Subject: [PATCH 01/11] =?UTF-8?q?=E6=97=A5=E4=BB=98=E3=83=86=E3=83=BC?= =?UTF-8?q?=E3=83=96=E3=83=AB=E6=9B=B4=E6=96=B0=20first=20commit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../update_business_day_environment.py | 35 +++++++++++++++++++ .../src/batch/update_business_day.py | 22 ++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 ecs/jskult-batch/src/batch/environment/update_business_day_environment.py diff --git a/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py b/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py new file mode 100644 index 00000000..ff0ca12e --- /dev/null +++ b/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py @@ -0,0 +1,35 @@ +from src.batch.environment.jskult_batch_environment import \ + JskultBatchEnvironment +from src.system_var import environment + + +class TrnResultDataBioLotEnvironment(JskultBatchEnvironment): + """実消化&アルトマークのバッチ処理で使用する環境変数を管理するクラス""" + + def __init__(self): + self.JSKULT_BACKUP_BUCKET = environment.JSKULT_BACKUP_BUCKET + self.BATCH_MANAGE_DYNAMODB_TABLE_NAME = environment.BATCH_MANAGE_DYNAMODB_TABLE_NAME + self.BATCH_EXECUTION_ID = environment.BATCH_EXECUTION_ID + self.MAX_RUN_COUNT = environment.MAX_RUN_COUNT + self.PROCESS_NAME = environment.PROCESS_NAME + self.TRANSFER_RESULT_FOLDER = environment.TRANSFER_RESULT_FOLDER + self.TRANSFER_RESULT_FILE_NAME = environment.TRANSFER_RESULT_FILE_NAME + + def validate(self): + """ + 必須の環境変数が設定されているかどうか検査する。 + DB関連の環境変数は対象外とする。 + + Raises: + EnvironmentVariableNotSetException: 環境変数の設定ミス + """ + super()._assert_variable_not_empty(self.JSKULT_BACKUP_BUCKET, 'JSKULT_BACKUP_BUCKET') + super()._assert_variable_not_empty(self.BATCH_MANAGE_DYNAMODB_TABLE_NAME, 'BATCH_MANAGE_DYNAMODB_TABLE_NAME') + super()._assert_variable_not_empty(self.BATCH_EXECUTION_ID, 'BATCH_EXECUTION_ID') + super()._assert_variable_is_int(self.MAX_RUN_COUNT, 'MAX_RUN_COUNT') + # MAX_RUN_COUNTは数値として扱うため、検査後に変換 + self.MAX_RUN_COUNT = int(self.MAX_RUN_COUNT) + super()._assert_variable_not_empty(self.PROCESS_NAME, 'PROCESS_NAME') + super()._assert_variable_not_empty(self.TRANSFER_RESULT_FOLDER, 'TRANSFER_RESULT_FOLDER') + super()._assert_variable_not_empty(self.TRANSFER_RESULT_FILE_NAME, 'TRANSFER_RESULT_FILE_NAME') + diff --git a/ecs/jskult-batch/src/batch/update_business_day.py b/ecs/jskult-batch/src/batch/update_business_day.py index 57e9372f..c689cc48 100644 --- a/ecs/jskult-batch/src/batch/update_business_day.py +++ b/ecs/jskult-batch/src/batch/update_business_day.py @@ -1,9 +1,31 @@ from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint +from src.aws.s3 import JskTransferListBucket +from src.batch.environment.trn_result_data_bio_lot_environment import \ + UpdateBusinessDayEnvironment +from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint +from src.manager.jskult_batch_run_manager import JskultBatchRunManager +from src.manager.jskult_batch_status_manager import JskultBatchStatusManager +from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager +from src.db.database import Database +from src.error.exceptions import (BatchOperationException, + EnvironmentVariableNotSetException, + MaxRunCountReachedException) +from src.logging.get_logger import get_logger +from src.system_var import constants +logger = get_logger('日付テーブル更新') class UpdateBusinessDay(JskultBatchEntrypoint): def __init__(self): super().__init__() + self.environment = UpdateBusinessDayEnvironment() + + # 必須の環境変数が設定されていない場合、エラーにする + try: + self.environment.validate() + except EnvironmentVariableNotSetException as e: + logger.exception(e) + return def execute(self): # TODO: ここで日付更新処理を実行する From e8253f27da73e1b648f7ee7e2f8783259ee8fac7 Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Tue, 3 Jun 2025 17:46:41 +0900 Subject: [PATCH 02/11] =?UTF-8?q?=E4=B8=8D=E8=B6=B3=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=A4=E3=83=AB=E3=81=AE=E6=AF=94=E8=BC=83=E3=81=A8=E5=8F=97?= =?UTF-8?q?=E9=A0=98=E4=BA=88=E5=AE=9A=E3=81=AB=E3=81=AA=E3=81=84=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=AE=E6=AF=94=E8=BC=83=E4=BB=A5?= =?UTF-8?q?=E5=A4=96=E3=81=AE=E9=83=A8=E5=88=86=E3=81=AE=E5=AE=9F=E8=A3=85?= =?UTF-8?q?=E3=81=A8=E3=82=A2=E3=83=BC=E3=82=AB=E3=82=A4=E3=83=96=E9=96=A2?= =?UTF-8?q?=E9=80=A3=E3=81=AE=E5=8F=96=E3=82=8A=E8=BE=BC=E3=81=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch/src/aws/s3.py | 72 +++++++++++ .../src/batch/archive_jsk_data.py | 75 +++++++++++ .../update_business_day_environment.py | 20 ++- .../src/batch/update_business_day.py | 121 ++++++++++++++++-- ecs/jskult-batch/src/error/exceptions.py | 3 + .../src/manager/jskult_archive_manager.py | 112 ++++++++++++++++ .../src/system_var/environment.py | 4 + 7 files changed, 392 insertions(+), 15 deletions(-) create mode 100644 ecs/jskult-batch/src/batch/archive_jsk_data.py create mode 100644 ecs/jskult-batch/src/manager/jskult_archive_manager.py diff --git a/ecs/jskult-batch/src/aws/s3.py b/ecs/jskult-batch/src/aws/s3.py index 6e5755be..b261f871 100644 --- a/ecs/jskult-batch/src/aws/s3.py +++ b/ecs/jskult-batch/src/aws/s3.py @@ -1,5 +1,9 @@ import os.path as path import tempfile +import shutil +import os +import os.path as path +import gzip import boto3 from src.system_var import environment @@ -138,3 +142,71 @@ class JskSendBucket(S3Bucket): backup_key = f'{jskult_backup_bucket._folder}/{self._send_folder}/{datetime_key}/{dat_file_key.replace(f"{self._send_folder}/", "")}' self._s3_client.copy(self._bucket_name, dat_key, jskult_backup_bucket._bucket_name, backup_key) + +class JskIOBucket(S3Bucket): + _bucket_name = environment.JSK_IO_BUCKET + _recv_folder = environment.JSK_RECEIVE_FOLDER + + _s3_file_list = None + + def get_file_list(self): + self._s3_file_list = self._s3_client.list_objects( + self._bucket_name, self._recv_folder) + return self._s3_file_list + + def download_data_file(self, data_filename: str): + temporary_dir = tempfile.mkdtemp() + temporary_file_path = path.join( + temporary_dir, f'{data_filename.replace(f"{self._recv_folder}/", "")}') + with open(temporary_file_path, mode='wb') as f: + self._s3_client.download_file(self._bucket_name, data_filename, f) + f.seek(0) + return temporary_file_path + + def unzip_data_file(self, filename: str): + temp_dir = os.path.dirname(filename) + decompress_filename = os.path.basename(filename).replace('.gz', '') + decompress_file_path = os.path.join(temp_dir, decompress_filename) + with gzip.open(filename, 'rb') as gz: + with open(decompress_file_path, 'wb') as decompressed_file: + shutil.copyfileobj(gz, decompressed_file) + + ret = [decompress_file_path] + return ret + + def transfer_file_to_import(self, target_file: dict): + data_import_bucket = DataImportBucket() + transfer_from_file_path = target_file.get("filename") + transfer_to_filename = transfer_from_file_path.replace( + f"{self._recv_folder}/", "") + data_import_key = f'{data_import_bucket._folder}/{transfer_to_filename}' + self._s3_client.copy(self._bucket_name, transfer_from_file_path, + data_import_bucket._bucket_name, data_import_key) + + def backup_file(self, target_file: dict, datetime_key: str): + jsk_backup_bucket = JskBackupBucket() + backup_from_file_path = target_file.get("filename") + backup_to_filename = backup_from_file_path.replace( + f"{self._recv_folder}/", "") + backup_key = f'{jsk_backup_bucket._folder}/{datetime_key}/{backup_to_filename}' + self._s3_client.copy(self._bucket_name, backup_from_file_path, + jsk_backup_bucket._bucket_name, backup_key) + + def delete_file(self, target_file: dict): + delete_path = target_file.get("filename") + self._s3_client.delete_file( + self._bucket_name, delete_path) +class JskultArchiveBucket(S3Bucket): + _bucket_name = environment.JSKULT_ARCHIVE_BUCKET + + def upload_archive_zip_file(self, archive_zip: str, archive_zip_path: str, send_folder: str): + # S3バケットにファイルを移動 + archive_zip_name = f'{send_folder}/{archive_zip}' + s3_client = S3Client() + s3_client.upload_file( + archive_zip_path, self._bucket_name, archive_zip_name) + return f"{self._bucket_name}/{archive_zip_name}" + +class DataImportBucket(S3Bucket): + _bucket_name = environment.DATA_IMPORT_BUCKET + _folder = environment.DATA_IMPORT_FOLDER \ No newline at end of file diff --git a/ecs/jskult-batch/src/batch/archive_jsk_data.py b/ecs/jskult-batch/src/batch/archive_jsk_data.py new file mode 100644 index 00000000..34b5397b --- /dev/null +++ b/ecs/jskult-batch/src/batch/archive_jsk_data.py @@ -0,0 +1,75 @@ +import csv +import os.path as path +import tempfile +import zipfile +from datetime import timedelta + +from src.aws.s3 import JskultArchiveBucket +from src.manager.jskult_archive_manager import JskultArchiveManager +from src.logging.get_logger import get_logger + +logger = get_logger("実消化_過去データアーカイブ処理") + + +def exec(): + """実消化_過去データアーカイブ処理""" + try: + logger.info("処理開始:実消化_過去データアーカイブ処理") + jskult_archive_manager = JskultArchiveManager() + # アーカイブ管理テーブルから対象テーブル、条件項目、条件年月、実行間隔（月）、前回条件年月、保存先を取得 + jskult_archive_manage_data_list = jskult_archive_manager.get_archive_manage() + + # 取得したレコード分繰り返す + for jskult_archive_manage_data in jskult_archive_manage_data_list: + # 対象テーブルで条件項目が条件年月以前のデータを取得 + archive_data = jskult_archive_manager.get_archive_data( + jskult_archive_manage_data["target_table"], jskult_archive_manage_data["filter_column"], jskult_archive_manage_data["filter_date"]) + # 取得データが0件の場合、スキップする + if not archive_data: + logger.info( + f"アーカイブ対象データがありませんでした。対象テーブル：{jskult_archive_manage_data['target_table']} 条件年月：{jskult_archive_manage_data['filter_date']}") + continue + + # 一時フォルダ作成 + with tempfile.TemporaryDirectory() as temporary_dir: + # 取得したデータをCSVに出力 + day_after_prev_filter_date = jskult_archive_manage_data["prev_filter_date"] + timedelta( + days=1) + file_name = f'{jskult_archive_manage_data["target_table"]}_{day_after_prev_filter_date.strftime('%Y%m%d')}_{jskult_archive_manage_data["filter_date"].strftime('%Y%m%d')}' + csv_file_path = path.join(temporary_dir, f"{file_name}.csv") + headers = archive_data[0].keys() + with open(csv_file_path, 'w', newline='') as file: + writer = csv.DictWriter( + file, fieldnames=headers, quoting=csv.QUOTE_ALL) + writer.writeheader() + writer.writerows(archive_data) + logger.info(f"CSVファイル作成に成功しました。{file_name}.csv") + + # 作成したCSVをzip形式に圧縮 + zip_file_path = path.join(temporary_dir, f"{file_name}.zip") + with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + zipf.write(csv_file_path) + logger.info(f"zip形式への圧縮に成功しました。{file_name}.zip") + + # 圧縮したCSVを保存先へアップロード + archive_bucket = JskultArchiveBucket() + upload_file_path = archive_bucket.upload_archive_zip_file( + f"{file_name}.zip", zip_file_path, jskult_archive_manage_data["archive_storage"]) + logger.info(f"{upload_file_path}へのアップロードに成功しました。") + + # アーカイブしたデータをDBから削除 + jskult_archive_manager.delete_archive_data( + jskult_archive_manage_data["target_table"], + jskult_archive_manage_data["filter_column"], + jskult_archive_manage_data["filter_date"]) + logger.info( + f"アーカイブしたデータのDBから削除に成功しました。対象テーブル：{jskult_archive_manage_data['target_table']} 条件年月：{jskult_archive_manage_data['filter_date']}") + + # 次回に向けてアーカイブ管理テーブルを更新する + jskult_archive_manager.update_archive_manage( + jskult_archive_manage_data["target_table"]) + logger.info( + f"アーカイブ管理テーブルの更新に成功しました。対象テーブル：{jskult_archive_manage_data['target_table']}") + logger.info("処理終了:実消化_過去データアーカイブ処理") + except Exception as e: + logger.exception(f"異常終了:実消化_過去データアーカイブ処理 {e}") diff --git a/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py b/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py index ff0ca12e..1056f83f 100644 --- a/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py +++ b/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py @@ -3,7 +3,7 @@ from src.batch.environment.jskult_batch_environment import \ from src.system_var import environment -class TrnResultDataBioLotEnvironment(JskultBatchEnvironment): +class UpdateBusinessDayEnvironment(JskultBatchEnvironment): """実消化&アルトマークのバッチ処理で使用する環境変数を管理するクラス""" def __init__(self): @@ -14,6 +14,9 @@ class TrnResultDataBioLotEnvironment(JskultBatchEnvironment): self.PROCESS_NAME = environment.PROCESS_NAME self.TRANSFER_RESULT_FOLDER = environment.TRANSFER_RESULT_FOLDER self.TRANSFER_RESULT_FILE_NAME = environment.TRANSFER_RESULT_FILE_NAME + self.JSK_RECEIVE_FOLDER = environment.JSK_RECEIVE_FOLDER + self.DATA_IMPORT_BUCKET = environment.DATA_IMPORT_BUCKET + self.DATA_IMPORT_FOLDER = environment.DATA_IMPORT_FOLDER def validate(self): """ @@ -23,13 +26,18 @@ class TrnResultDataBioLotEnvironment(JskultBatchEnvironment): Raises: EnvironmentVariableNotSetException: 環境変数の設定ミス """ - super()._assert_variable_not_empty(self.JSKULT_BACKUP_BUCKET, 'JSKULT_BACKUP_BUCKET') - super()._assert_variable_not_empty(self.BATCH_MANAGE_DYNAMODB_TABLE_NAME, 'BATCH_MANAGE_DYNAMODB_TABLE_NAME') + super()._assert_variable_not_empty( + self.JSKULT_BACKUP_BUCKET, 'JSKULT_BACKUP_BUCKET') + super()._assert_variable_not_empty( + self.BATCH_MANAGE_DYNAMODB_TABLE_NAME, 'BATCH_MANAGE_DYNAMODB_TABLE_NAME') super()._assert_variable_not_empty(self.BATCH_EXECUTION_ID, 'BATCH_EXECUTION_ID') super()._assert_variable_is_int(self.MAX_RUN_COUNT, 'MAX_RUN_COUNT') # MAX_RUN_COUNTは数値として扱うため、検査後に変換 self.MAX_RUN_COUNT = int(self.MAX_RUN_COUNT) super()._assert_variable_not_empty(self.PROCESS_NAME, 'PROCESS_NAME') - super()._assert_variable_not_empty(self.TRANSFER_RESULT_FOLDER, 'TRANSFER_RESULT_FOLDER') - super()._assert_variable_not_empty(self.TRANSFER_RESULT_FILE_NAME, 'TRANSFER_RESULT_FILE_NAME') - + super()._assert_variable_not_empty( + self.TRANSFER_RESULT_FOLDER, 'TRANSFER_RESULT_FOLDER') + super()._assert_variable_not_empty( + self.TRANSFER_RESULT_FILE_NAME, 'TRANSFER_RESULT_FILE_NAME') + super()._assert_variable_not_empty(self.DATA_IMPORT_BUCKET, 'DATA_IMPORT_BUCKET') + super()._assert_variable_not_empty(self.DATA_IMPORT_FOLDER, 'DATA_IMPORT_FOLDER') diff --git a/ecs/jskult-batch/src/batch/update_business_day.py b/ecs/jskult-batch/src/batch/update_business_day.py index c689cc48..deafcde2 100644 --- a/ecs/jskult-batch/src/batch/update_business_day.py +++ b/ecs/jskult-batch/src/batch/update_business_day.py @@ -1,20 +1,23 @@ +import json +from datetime import datetime from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint -from src.aws.s3 import JskTransferListBucket -from src.batch.environment.trn_result_data_bio_lot_environment import \ - UpdateBusinessDayEnvironment -from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint +from src.aws.s3 import (JskTransferListBucket, JskIOBucket) +from src.batch.environment.update_business_day_environment import \ + UpdateBusinessDayEnvironment +from src.batch import archive_jsk_data from src.manager.jskult_batch_run_manager import JskultBatchRunManager from src.manager.jskult_batch_status_manager import JskultBatchStatusManager from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager -from src.db.database import Database -from src.error.exceptions import (BatchOperationException, +from src.manager.jskult_archive_manager import JskultArchiveManager +from src.error.exceptions import (NotReadyException, EnvironmentVariableNotSetException, - MaxRunCountReachedException) + ) from src.logging.get_logger import get_logger from src.system_var import constants logger = get_logger('日付テーブル更新') + class UpdateBusinessDay(JskultBatchEntrypoint): def __init__(self): super().__init__() @@ -28,5 +31,105 @@ class UpdateBusinessDay(JskultBatchEntrypoint): return def execute(self): - # TODO: ここで日付更新処理を実行する - pass + """日付テーブル更新""" + logger.info('I-1 処理開始: 実消化＆アルトマーク_日付テーブル更新') + + jskult_hdke_tbl_manager = JskultHdkeTblManager() + jskult_archive_manager = JskultArchiveManager( + ) + + jskult_batch_status_manager = JskultBatchStatusManager( + self.environment.PROCESS_NAME, + # TODO チケットNEWDWH2021-1847の実装で作成した定数に置き換え + 'update_business_day', + self.environment.MAX_RUN_COUNT, + 0 + ) + + jskult_batch_run_manager = JskultBatchRunManager( + self.environment.BATCH_MANAGE_DYNAMODB_TABLE_NAME, + self.environment.BATCH_EXECUTION_ID) + + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_START) + + if not jskult_hdke_tbl_manager.can_run_process(): + logger.error( + '日次バッチ処理中でない、もしくはdump取得が正常終了していないため、日次バッチ処理を異常終了します。') + # バッチ実行管理テーブルをfailedで登録 + jskult_batch_run_manager.batch_failed() + raise NotReadyException() + + if not jskult_batch_status_manager.can_run_business_day_update(): + # 後続処理の起動条件を満たしていない場合 + # 処理ステータスを「処理待」に設定 + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_WAITING) + + # バッチ実行管理テーブルに「retry」で登録 + jskult_batch_run_manager.batch_retry() + + logger.info('処理待：実消化＆アルトマーク_日付テーブル更新') + + return + + _, _, archive_date, _, _, _ = jskult_archive_manager.get_archive_manage() + + _, _, process_date = jskult_hdke_tbl_manager.get_batch_statuses() + + try: + if archive_date == process_date: + logger.info('[NOTICE]実消化データアーカイブ取得処理を実行します。') + # 処理ステータスを「処理中」に設定 + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_DOING) + + jskult_archive_manager.get_archive_data() + archive_jsk_data.exec() + + dt = datetime.strptime(process_date, "%Y/%m/%d") + # 日付テーブルの処理年月日が月曜日の場合 + if dt.weekday() == 0: + if not JskultBatchStatusManager.is_done_ultmarc_import(): + logger.info('[NOTICE]アルトマーク取込稼働日でしたが、アルトマーク取込が行われませんでした。') + + # 転送ファイル一覧を取得 + try: + transfer_list_bucket = JskTransferListBucket() + transfer_list_file_path = transfer_list_bucket.download_transfer_result_file( + process_date) + except Exception as e: + logger.exception(f'転送ファイル一覧の取得に失敗しました。 {e}') + # バッチ実行管理テーブルをfailedで登録 + jskult_batch_run_manager.batch_failed() + + with open(transfer_list_file_path) as f: + transfer_list = json.load(f) + + try: + jsk_io_bucket = JskIOBucket() + jsk_receive_file_list: str = jsk_io_bucket.get_file_list() + except Exception as e: + logger.exception(f'実消化データリスト取得に失敗しました。{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + if transfer_list is not None: + # TODO jsk_receive_file_listと比較して結果不足ファイルがあった場合(I-5)ログ出力 + + # TODO jsk_receive_file_listと比較して受領予定にないファイルがあった場合(I-6)ログ出力 + pass + + JskultHdkeTblManager.update_batch_process_complete() + # 処理ステータスを「処理済」に設定 + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_DONE) + + JskultBatchRunManager.batch_success() + + logger.info('[NOTICE]処理終了: 実消化＆アルトマーク_日付テーブル更新') + except Exception as e: + # 何らかのエラーが発生した際に、バッチ実行管理テーブルに「failed」で登録 + jskult_batch_run_manager.batch_failed() + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_ERROR) + logger.exception(f'異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') diff --git a/ecs/jskult-batch/src/error/exceptions.py b/ecs/jskult-batch/src/error/exceptions.py index a6db7ed7..b4673b28 100644 --- a/ecs/jskult-batch/src/error/exceptions.py +++ b/ecs/jskult-batch/src/error/exceptions.py @@ -16,3 +16,6 @@ class MaxRunCountReachedException(MeDaCaException): class EnvironmentVariableNotSetException(MeDaCaException): pass + +class NotReadyException(MeDaCaException): + pass diff --git a/ecs/jskult-batch/src/manager/jskult_archive_manager.py b/ecs/jskult-batch/src/manager/jskult_archive_manager.py new file mode 100644 index 00000000..9bb25341 --- /dev/null +++ b/ecs/jskult-batch/src/manager/jskult_archive_manager.py @@ -0,0 +1,112 @@ +from src.db.database import Database +from src.logging.get_logger import get_logger + +logger = get_logger("アーカイブ管理テーブル操作") + + +class JskultArchiveManager: + _db: Database = None + + def __init__(self): + self._db = Database.get_instance() + + def get_archive_manage(self): + """対象テーブル、対象項目、対象年月、実行間隔（月）、前回対象年月、保存先を取得""" + try: + logger.info("処理開始：get_archive_manage") + sql = """ + select + target_table + , filter_column + , filter_date + , run_interval_months + , prev_filter_date + , archive_storage + from + internal07.jskult_archive_manage; + """ + self._db.connect() + jskult_archive_manage_data = self._db.execute_select(sql) + logger.info("処理終了：get_archive_manage") + return jskult_archive_manage_data + except Exception as e: + logger.info("異常終了：get_archive_manage") + raise + finally: + self._db.disconnect() + + def get_archive_data(self,target_table:str, filter_column:str, filter_date:str): + """アーカイブするデータを取得""" + try: + logger.info("処理開始：get_archive_data") + sql = f""" + select + * + from + src07.{target_table} + where + str_to_date({filter_column},'%Y%m%d') <= :filter_date; + """ + self._db.connect() + parameter_dict = {'filter_date' : filter_date} + target_table_data = self._db.execute_select(sql, parameter_dict) + logger.info("処理終了：get_archive_data") + return target_table_data + except Exception as e: + logger.info("異常終了：get_archive_data") + raise + finally: + self._db.disconnect() + + def delete_archive_data(self, target_table:str, filter_column:str, filter_date:str): + """アーカイブしたデータを削除""" + try: + logger.info("処理開始：delete_archive_data") + sql = f""" + delete from + src07.{target_table} + where + str_to_date({filter_column},'%Y%m%d') <= :filter_date; + """ + self._db.connect() + self._db.begin() + parameter_dict = {'filter_date' : filter_date} + self._db.execute(sql, parameter_dict) + self._db.commit() + logger.info("処理終了：delete_archive_data") + return + except Exception as e: + self._db.rollback() + logger.info("異常終了：delete_archive_data") + raise + finally: + self._db.disconnect() + + def update_archive_manage(self, target_table:str): + """アーカイブ管理テーブルの指定した対象テーブルのレコードを更新する""" + try: + logger.info("処理開始：update_archive_manage") + sql = f""" + update internal07.jskult_archive_manage + set + prev_filter_date = filter_date + , filter_date = LAST_DAY( + DATE_ADD(filter_date, INTERVAL run_interval_months MONTH) + ) + , upd_user = CURRENT_USER() + , upd_date = NOW() + where + target_table = '{target_table}'; + """ + self._db.connect() + self._db.begin() + self._db.execute(sql) + self._db.commit() + logger.info("処理終了：update_archive_manage") + return + except Exception as e: + self._db.rollback() + logger.info("異常終了：update_archive_manage") + raise + finally: + self._db.disconnect() diff --git a/ecs/jskult-batch/src/system_var/environment.py b/ecs/jskult-batch/src/system_var/environment.py index 492c9661..afe7c8d9 100644 --- a/ecs/jskult-batch/src/system_var/environment.py +++ b/ecs/jskult-batch/src/system_var/environment.py @@ -22,8 +22,12 @@ JSKULT_BACKUP_BUCKET = os.environ.get('JSKULT_BACKUP_BUCKET', None) JSK_IO_BUCKET = os.environ.get('JSK_IO_BUCKET', None) JSK_BACKUP_FOLDER = os.environ.get('JSK_BACKUP_FOLDER', None) JSK_DATA_SEND_FOLDER = os.environ.get('JSK_DATA_SEND_FOLDER', None) +JSK_RECEIVE_FOLDER = os.environ.get('JSK_RECEIVE_FOLDER', None) BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get( 'BATCH_MANAGE_DYNAMODB_TABLE_NAME', None) +DATA_IMPORT_BUCKET = os.environ.get('DATA_IMPORT_BUCKET', None) +DATA_IMPORT_FOLDER = os.environ.get('DATA_IMPORT_FOLDER', None) +JSKULT_ARCHIVE_BUCKET = os.environ.get('JSKULT_ARCHIVE_BUCKET', None) # 初期値がある環境変数 From 3607229dee492b82b2340fcfdbbf5887984cf7d2 Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Tue, 3 Jun 2025 18:37:08 +0900 Subject: [PATCH 03/11] =?UTF-8?q?=E4=B8=8D=E8=A6=81=E9=83=A8=E5=88=86?= =?UTF-8?q?=E3=81=AE=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch/src/aws/s3.py | 74 +----------- .../src/batch/archive_jsk_data.py | 75 ------------ .../environment/dcf_inst_merge_environment.py | 6 - .../src/batch/update_business_day.py | 18 +-- ecs/jskult-batch/src/error/exceptions.py | 3 - .../src/manager/jskult_archive_manager.py | 112 ------------------ .../src/system_var/environment.py | 4 - 7 files changed, 6 insertions(+), 286 deletions(-) delete mode 100644 ecs/jskult-batch/src/batch/archive_jsk_data.py delete mode 100644 ecs/jskult-batch/src/manager/jskult_archive_manager.py diff --git a/ecs/jskult-batch/src/aws/s3.py b/ecs/jskult-batch/src/aws/s3.py index b261f871..b37b38d2 100644 --- a/ecs/jskult-batch/src/aws/s3.py +++ b/ecs/jskult-batch/src/aws/s3.py @@ -1,9 +1,5 @@ import os.path as path import tempfile -import shutil -import os -import os.path as path -import gzip import boto3 from src.system_var import environment @@ -141,72 +137,4 @@ class JskSendBucket(S3Bucket): dat_key = f'{self._send_folder}/{dat_file_key}' backup_key = f'{jskult_backup_bucket._folder}/{self._send_folder}/{datetime_key}/{dat_file_key.replace(f"{self._send_folder}/", "")}' self._s3_client.copy(self._bucket_name, dat_key, - jskult_backup_bucket._bucket_name, backup_key) - -class JskIOBucket(S3Bucket): - _bucket_name = environment.JSK_IO_BUCKET - _recv_folder = environment.JSK_RECEIVE_FOLDER - - _s3_file_list = None - - def get_file_list(self): - self._s3_file_list = self._s3_client.list_objects( - self._bucket_name, self._recv_folder) - return self._s3_file_list - - def download_data_file(self, data_filename: str): - temporary_dir = tempfile.mkdtemp() - temporary_file_path = path.join( - temporary_dir, f'{data_filename.replace(f"{self._recv_folder}/", "")}') - with open(temporary_file_path, mode='wb') as f: - self._s3_client.download_file(self._bucket_name, data_filename, f) - f.seek(0) - return temporary_file_path - - def unzip_data_file(self, filename: str): - temp_dir = os.path.dirname(filename) - decompress_filename = os.path.basename(filename).replace('.gz', '') - decompress_file_path = os.path.join(temp_dir, decompress_filename) - with gzip.open(filename, 'rb') as gz: - with open(decompress_file_path, 'wb') as decompressed_file: - shutil.copyfileobj(gz, decompressed_file) - - ret = [decompress_file_path] - return ret - - def transfer_file_to_import(self, target_file: dict): - data_import_bucket = DataImportBucket() - transfer_from_file_path = target_file.get("filename") - transfer_to_filename = transfer_from_file_path.replace( - f"{self._recv_folder}/", "") - data_import_key = f'{data_import_bucket._folder}/{transfer_to_filename}' - self._s3_client.copy(self._bucket_name, transfer_from_file_path, - data_import_bucket._bucket_name, data_import_key) - - def backup_file(self, target_file: dict, datetime_key: str): - jsk_backup_bucket = JskBackupBucket() - backup_from_file_path = target_file.get("filename") - backup_to_filename = backup_from_file_path.replace( - f"{self._recv_folder}/", "") - backup_key = f'{jsk_backup_bucket._folder}/{datetime_key}/{backup_to_filename}' - self._s3_client.copy(self._bucket_name, backup_from_file_path, - jsk_backup_bucket._bucket_name, backup_key) - - def delete_file(self, target_file: dict): - delete_path = target_file.get("filename") - self._s3_client.delete_file( - self._bucket_name, delete_path) -class JskultArchiveBucket(S3Bucket): - _bucket_name = environment.JSKULT_ARCHIVE_BUCKET - - def upload_archive_zip_file(self, archive_zip: str, archive_zip_path: str, send_folder: str): - # S3バケットにファイルを移動 - archive_zip_name = f'{send_folder}/{archive_zip}' - s3_client = S3Client() - s3_client.upload_file( - archive_zip_path, self._bucket_name, archive_zip_name) - return f"{self._bucket_name}/{archive_zip_name}" - -class DataImportBucket(S3Bucket): - _bucket_name = environment.DATA_IMPORT_BUCKET - _folder = environment.DATA_IMPORT_FOLDER \ No newline at end of file + jskult_backup_bucket._bucket_name, backup_key) \ No newline at end of file diff --git a/ecs/jskult-batch/src/batch/archive_jsk_data.py b/ecs/jskult-batch/src/batch/archive_jsk_data.py deleted file mode 100644 index 34b5397b..00000000 --- a/ecs/jskult-batch/src/batch/archive_jsk_data.py +++ /dev/null @@ -1,75 +0,0 @@ -import csv -import os.path as path -import tempfile -import zipfile -from datetime import timedelta - -from src.aws.s3 import JskultArchiveBucket -from src.manager.jskult_archive_manager import JskultArchiveManager -from src.logging.get_logger import get_logger - -logger = get_logger("実消化_過去データアーカイブ処理") - - -def exec(): - """実消化_過去データアーカイブ処理""" - try: - logger.info("処理開始:実消化_過去データアーカイブ処理") - jskult_archive_manager = JskultArchiveManager() - # アーカイブ管理テーブルから対象テーブル、条件項目、条件年月、実行間隔（月）、前回条件年月、保存先を取得 - jskult_archive_manage_data_list = jskult_archive_manager.get_archive_manage() - - # 取得したレコード分繰り返す - for jskult_archive_manage_data in jskult_archive_manage_data_list: - # 対象テーブルで条件項目が条件年月以前のデータを取得 - archive_data = jskult_archive_manager.get_archive_data( - jskult_archive_manage_data["target_table"], jskult_archive_manage_data["filter_column"], jskult_archive_manage_data["filter_date"]) - # 取得データが0件の場合、スキップする - if not archive_data: - logger.info( - f"アーカイブ対象データがありませんでした。対象テーブル：{jskult_archive_manage_data['target_table']} 条件年月：{jskult_archive_manage_data['filter_date']}") - continue - - # 一時フォルダ作成 - with tempfile.TemporaryDirectory() as temporary_dir: - # 取得したデータをCSVに出力 - day_after_prev_filter_date = jskult_archive_manage_data["prev_filter_date"] + timedelta( - days=1) - file_name = f'{jskult_archive_manage_data["target_table"]}_{day_after_prev_filter_date.strftime('%Y%m%d')}_{jskult_archive_manage_data["filter_date"].strftime('%Y%m%d')}' - csv_file_path = path.join(temporary_dir, f"{file_name}.csv") - headers = archive_data[0].keys() - with open(csv_file_path, 'w', newline='') as file: - writer = csv.DictWriter( - file, fieldnames=headers, quoting=csv.QUOTE_ALL) - writer.writeheader() - writer.writerows(archive_data) - logger.info(f"CSVファイル作成に成功しました。{file_name}.csv") - - # 作成したCSVをzip形式に圧縮 - zip_file_path = path.join(temporary_dir, f"{file_name}.zip") - with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) as zipf: - zipf.write(csv_file_path) - logger.info(f"zip形式への圧縮に成功しました。{file_name}.zip") - - # 圧縮したCSVを保存先へアップロード - archive_bucket = JskultArchiveBucket() - upload_file_path = archive_bucket.upload_archive_zip_file( - f"{file_name}.zip", zip_file_path, jskult_archive_manage_data["archive_storage"]) - logger.info(f"{upload_file_path}へのアップロードに成功しました。") - - # アーカイブしたデータをDBから削除 - jskult_archive_manager.delete_archive_data( - jskult_archive_manage_data["target_table"], - jskult_archive_manage_data["filter_column"], - jskult_archive_manage_data["filter_date"]) - logger.info( - f"アーカイブしたデータのDBから削除に成功しました。対象テーブル：{jskult_archive_manage_data['target_table']} 条件年月：{jskult_archive_manage_data['filter_date']}") - - # 次回に向けてアーカイブ管理テーブルを更新する - jskult_archive_manager.update_archive_manage( - jskult_archive_manage_data["target_table"]) - logger.info( - f"アーカイブ管理テーブルの更新に成功しました。対象テーブル：{jskult_archive_manage_data['target_table']}") - logger.info("処理終了:実消化_過去データアーカイブ処理") - except Exception as e: - logger.exception(f"異常終了:実消化_過去データアーカイブ処理 {e}") diff --git a/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py b/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py index 3da34501..5688fa84 100644 --- a/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py +++ b/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py @@ -15,8 +15,6 @@ class DCFInstMergeEnvironment(JskultBatchEnvironment): self.PROCESS_NAME = environment.PROCESS_NAME self.JSK_DATA_SEND_FOLDER = environment.JSK_DATA_SEND_FOLDER self.JSK_BACKUP_FOLDER = environment.JSK_BACKUP_FOLDER - self.TRANSFER_RESULT_FOLDER = environment.TRANSFER_RESULT_FOLDER - self.TRANSFER_RESULT_FILE_NAME = environment.TRANSFER_RESULT_FILE_NAME self.DCF_INST_MERGE_SEND_FILE_NAME = environment.DCF_INST_MERGE_SEND_FILE_NAME def validate(self): @@ -40,9 +38,5 @@ class DCFInstMergeEnvironment(JskultBatchEnvironment): super()._assert_variable_not_empty( self.JSK_DATA_SEND_FOLDER, 'JSK_DATA_SEND_FOLDER') super()._assert_variable_not_empty(self.JSK_BACKUP_FOLDER, 'JSK_BACKUP_FOLDER') - super()._assert_variable_not_empty( - self.TRANSFER_RESULT_FOLDER, 'TRANSFER_RESULT_FOLDER') - super()._assert_variable_not_empty( - self.TRANSFER_RESULT_FILE_NAME, 'TRANSFER_RESULT_FILE_NAME') super()._assert_variable_not_empty(self.DCF_INST_MERGE_SEND_FILE_NAME, 'DCF_INST_MERGE_SEND_FILE_NAME') diff --git a/ecs/jskult-batch/src/batch/update_business_day.py b/ecs/jskult-batch/src/batch/update_business_day.py index deafcde2..396abbe4 100644 --- a/ecs/jskult-batch/src/batch/update_business_day.py +++ b/ecs/jskult-batch/src/batch/update_business_day.py @@ -4,14 +4,10 @@ from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint from src.aws.s3 import (JskTransferListBucket, JskIOBucket) from src.batch.environment.update_business_day_environment import \ UpdateBusinessDayEnvironment -from src.batch import archive_jsk_data from src.manager.jskult_batch_run_manager import JskultBatchRunManager from src.manager.jskult_batch_status_manager import JskultBatchStatusManager from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager -from src.manager.jskult_archive_manager import JskultArchiveManager -from src.error.exceptions import (NotReadyException, - EnvironmentVariableNotSetException, - ) +from src.error.exceptions import EnvironmentVariableNotSetException from src.logging.get_logger import get_logger from src.system_var import constants @@ -35,8 +31,6 @@ class UpdateBusinessDay(JskultBatchEntrypoint): logger.info('I-1 処理開始: 実消化＆アルトマーク_日付テーブル更新') jskult_hdke_tbl_manager = JskultHdkeTblManager() - jskult_archive_manager = JskultArchiveManager( - ) jskult_batch_status_manager = JskultBatchStatusManager( self.environment.PROCESS_NAME, @@ -58,7 +52,6 @@ class UpdateBusinessDay(JskultBatchEntrypoint): '日次バッチ処理中でない、もしくはdump取得が正常終了していないため、日次バッチ処理を異常終了します。') # バッチ実行管理テーブルをfailedで登録 jskult_batch_run_manager.batch_failed() - raise NotReadyException() if not jskult_batch_status_manager.can_run_business_day_update(): # 後続処理の起動条件を満たしていない場合 @@ -73,19 +66,18 @@ class UpdateBusinessDay(JskultBatchEntrypoint): return - _, _, archive_date, _, _, _ = jskult_archive_manager.get_archive_manage() - _, _, process_date = jskult_hdke_tbl_manager.get_batch_statuses() + + # archive取得日calendarの取得 + get_archive_calendar = None try: - if archive_date == process_date: + if get_archive_calendar == process_date: logger.info('[NOTICE]実消化データアーカイブ取得処理を実行します。') # 処理ステータスを「処理中」に設定 jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_DOING) - jskult_archive_manager.get_archive_data() - archive_jsk_data.exec() dt = datetime.strptime(process_date, "%Y/%m/%d") # 日付テーブルの処理年月日が月曜日の場合 diff --git a/ecs/jskult-batch/src/error/exceptions.py b/ecs/jskult-batch/src/error/exceptions.py index b4673b28..a6db7ed7 100644 --- a/ecs/jskult-batch/src/error/exceptions.py +++ b/ecs/jskult-batch/src/error/exceptions.py @@ -16,6 +16,3 @@ class MaxRunCountReachedException(MeDaCaException): class EnvironmentVariableNotSetException(MeDaCaException): pass - -class NotReadyException(MeDaCaException): - pass diff --git a/ecs/jskult-batch/src/manager/jskult_archive_manager.py b/ecs/jskult-batch/src/manager/jskult_archive_manager.py deleted file mode 100644 index 9bb25341..00000000 --- a/ecs/jskult-batch/src/manager/jskult_archive_manager.py +++ /dev/null @@ -1,112 +0,0 @@ -from src.db.database import Database -from src.logging.get_logger import get_logger - -logger = get_logger("アーカイブ管理テーブル操作") - - -class JskultArchiveManager: - _db: Database = None - - def __init__(self): - self._db = Database.get_instance() - - def get_archive_manage(self): - """対象テーブル、対象項目、対象年月、実行間隔（月）、前回対象年月、保存先を取得""" - try: - logger.info("処理開始：get_archive_manage") - sql = """ - select - target_table - , filter_column - , filter_date - , run_interval_months - , prev_filter_date - , archive_storage - from - internal07.jskult_archive_manage; - """ - self._db.connect() - jskult_archive_manage_data = self._db.execute_select(sql) - logger.info("処理終了：get_archive_manage") - return jskult_archive_manage_data - except Exception as e: - logger.info("異常終了：get_archive_manage") - raise - finally: - self._db.disconnect() - - def get_archive_data(self,target_table:str, filter_column:str, filter_date:str): - """アーカイブするデータを取得""" - try: - logger.info("処理開始：get_archive_data") - sql = f""" - select - * - from - src07.{target_table} - where - str_to_date({filter_column},'%Y%m%d') <= :filter_date; - """ - self._db.connect() - parameter_dict = {'filter_date' : filter_date} - target_table_data = self._db.execute_select(sql, parameter_dict) - logger.info("処理終了：get_archive_data") - return target_table_data - except Exception as e: - logger.info("異常終了：get_archive_data") - raise - finally: - self._db.disconnect() - - def delete_archive_data(self, target_table:str, filter_column:str, filter_date:str): - """アーカイブしたデータを削除""" - try: - logger.info("処理開始：delete_archive_data") - sql = f""" - delete from - src07.{target_table} - where - str_to_date({filter_column},'%Y%m%d') <= :filter_date; - """ - self._db.connect() - self._db.begin() - parameter_dict = {'filter_date' : filter_date} - self._db.execute(sql, parameter_dict) - self._db.commit() - logger.info("処理終了：delete_archive_data") - return - except Exception as e: - self._db.rollback() - logger.info("異常終了：delete_archive_data") - raise - finally: - self._db.disconnect() - - def update_archive_manage(self, target_table:str): - """アーカイブ管理テーブルの指定した対象テーブルのレコードを更新する""" - try: - logger.info("処理開始：update_archive_manage") - sql = f""" - update internal07.jskult_archive_manage - set - prev_filter_date = filter_date - , filter_date = LAST_DAY( - DATE_ADD(filter_date, INTERVAL run_interval_months MONTH) - ) - , upd_user = CURRENT_USER() - , upd_date = NOW() - where - target_table = '{target_table}'; - """ - self._db.connect() - self._db.begin() - self._db.execute(sql) - self._db.commit() - logger.info("処理終了：update_archive_manage") - return - except Exception as e: - self._db.rollback() - logger.info("異常終了：update_archive_manage") - raise - finally: - self._db.disconnect() diff --git a/ecs/jskult-batch/src/system_var/environment.py b/ecs/jskult-batch/src/system_var/environment.py index afe7c8d9..492c9661 100644 --- a/ecs/jskult-batch/src/system_var/environment.py +++ b/ecs/jskult-batch/src/system_var/environment.py @@ -22,12 +22,8 @@ JSKULT_BACKUP_BUCKET = os.environ.get('JSKULT_BACKUP_BUCKET', None) JSK_IO_BUCKET = os.environ.get('JSK_IO_BUCKET', None) JSK_BACKUP_FOLDER = os.environ.get('JSK_BACKUP_FOLDER', None) JSK_DATA_SEND_FOLDER = os.environ.get('JSK_DATA_SEND_FOLDER', None) -JSK_RECEIVE_FOLDER = os.environ.get('JSK_RECEIVE_FOLDER', None) BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get( 'BATCH_MANAGE_DYNAMODB_TABLE_NAME', None) -DATA_IMPORT_BUCKET = os.environ.get('DATA_IMPORT_BUCKET', None) -DATA_IMPORT_FOLDER = os.environ.get('DATA_IMPORT_FOLDER', None) -JSKULT_ARCHIVE_BUCKET = os.environ.get('JSKULT_ARCHIVE_BUCKET', None) # 初期値がある環境変数 From fae67ae4e775059e384791ee708faa83d9d2c086 Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Tue, 3 Jun 2025 19:33:00 +0900 Subject: [PATCH 04/11] =?UTF-8?q?=E5=AE=9F=E6=B6=88=E5=8C=96=E3=83=87?= =?UTF-8?q?=E3=83=BC=E3=82=BF=E3=82=A2=E3=83=BC=E3=82=AB=E3=82=A4=E3=83=96?= =?UTF-8?q?=E5=8F=96=E5=BE=97=E5=87=A6=E7=90=86=E3=82=92=E5=AE=9F=E8=A1=8C?= =?UTF-8?q?=E3=81=99=E3=82=8B=E5=87=A6=E7=90=86=E3=81=AE=E5=AE=9F=E8=A3=85?= =?UTF-8?q?=E3=81=A8TODO=E3=81=AE=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../update_business_day_environment.py | 7 ++- .../src/batch/update_business_day.py | 53 ++++++++----------- .../src/system_var/environment.py | 4 ++ 3 files changed, 29 insertions(+), 35 deletions(-) diff --git a/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py b/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py index 1056f83f..5a5f3770 100644 --- a/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py +++ b/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py @@ -15,8 +15,7 @@ class UpdateBusinessDayEnvironment(JskultBatchEnvironment): self.TRANSFER_RESULT_FOLDER = environment.TRANSFER_RESULT_FOLDER self.TRANSFER_RESULT_FILE_NAME = environment.TRANSFER_RESULT_FILE_NAME self.JSK_RECEIVE_FOLDER = environment.JSK_RECEIVE_FOLDER - self.DATA_IMPORT_BUCKET = environment.DATA_IMPORT_BUCKET - self.DATA_IMPORT_FOLDER = environment.DATA_IMPORT_FOLDER + self.ARCHIVE_STATEMACHINE_ARN = environment.ARCHIVE_STATEMACHINE_ARN def validate(self): """ @@ -39,5 +38,5 @@ class UpdateBusinessDayEnvironment(JskultBatchEnvironment): self.TRANSFER_RESULT_FOLDER, 'TRANSFER_RESULT_FOLDER') super()._assert_variable_not_empty( self.TRANSFER_RESULT_FILE_NAME, 'TRANSFER_RESULT_FILE_NAME') - super()._assert_variable_not_empty(self.DATA_IMPORT_BUCKET, 'DATA_IMPORT_BUCKET') - super()._assert_variable_not_empty(self.DATA_IMPORT_FOLDER, 'DATA_IMPORT_FOLDER') + super()._assert_variable_not_empty(self.JSK_RECEIVE_FOLDER, 'JSK_RECEIVE_FOLDER') + super()._assert_variable_not_empty(self.ARCHIVE_STATEMACHINE_ARN, 'ARCHIVE_STATEMACHINE_ARN') diff --git a/ecs/jskult-batch/src/batch/update_business_day.py b/ecs/jskult-batch/src/batch/update_business_day.py index 396abbe4..cdac4dbf 100644 --- a/ecs/jskult-batch/src/batch/update_business_day.py +++ b/ecs/jskult-batch/src/batch/update_business_day.py @@ -1,7 +1,8 @@ import json +import boto3 from datetime import datetime from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint -from src.aws.s3 import (JskTransferListBucket, JskIOBucket) +from src.aws.s3 import JskTransferListBucket from src.batch.environment.update_business_day_environment import \ UpdateBusinessDayEnvironment from src.manager.jskult_batch_run_manager import JskultBatchRunManager @@ -65,19 +66,24 @@ class UpdateBusinessDay(JskultBatchEntrypoint): logger.info('処理待：実消化＆アルトマーク_日付テーブル更新') return - - _, _, process_date = jskult_hdke_tbl_manager.get_batch_statuses() - - - # archive取得日calendarの取得 - get_archive_calendar = None try: - if get_archive_calendar == process_date: - logger.info('[NOTICE]実消化データアーカイブ取得処理を実行します。') - # 処理ステータスを「処理中」に設定 - jskult_batch_status_manager.set_process_status( - constants.PROCESS_STATUS_DOING) + # TODO アーカイブ取得日カレンダーを取得する + get_archive_calendar = None + # 日付テーブルの処理年月日を取得する + _, _, process_date = jskult_hdke_tbl_manager.get_batch_statuses() + + + if get_archive_calendar == process_date: + logger.info('I-7 [NOTICE]実消化データアーカイブ取得処理を実行します。') + + # 実消化データアーカイブ取得処理を起動する + sfn_client = boto3.client('stepfunctions') + + # StepFunctionsを起動する + response = sfn_client.start_execution( + stateMachineArn=self.environment.ARCHIVE_STATEMACHINE_ARN + ) dt = datetime.strptime(process_date, "%Y/%m/%d") # 日付テーブルの処理年月日が月曜日の場合 @@ -85,25 +91,10 @@ class UpdateBusinessDay(JskultBatchEntrypoint): if not JskultBatchStatusManager.is_done_ultmarc_import(): logger.info('[NOTICE]アルトマーク取込稼働日でしたが、アルトマーク取込が行われませんでした。') - # 転送ファイル一覧を取得 - try: - transfer_list_bucket = JskTransferListBucket() - transfer_list_file_path = transfer_list_bucket.download_transfer_result_file( - process_date) - except Exception as e: - logger.exception(f'転送ファイル一覧の取得に失敗しました。 {e}') - # バッチ実行管理テーブルをfailedで登録 - jskult_batch_run_manager.batch_failed() - - with open(transfer_list_file_path) as f: - transfer_list = json.load(f) - - try: - jsk_io_bucket = JskIOBucket() - jsk_receive_file_list: str = jsk_io_bucket.get_file_list() - except Exception as e: - logger.exception(f'実消化データリスト取得に失敗しました。{e}') - return constants.BATCH_EXIT_CODE_SUCCESS + # TODO 実消化転送データ一覧を取得する + jsk_receive_file_list = None + # TODO 実消化&アルトマーク_実消化受領予定データリストを取得する + transfer_list = None if transfer_list is not None: # TODO jsk_receive_file_listと比較して結果不足ファイルがあった場合(I-5)ログ出力 diff --git a/ecs/jskult-batch/src/system_var/environment.py b/ecs/jskult-batch/src/system_var/environment.py index 492c9661..9732ed5b 100644 --- a/ecs/jskult-batch/src/system_var/environment.py +++ b/ecs/jskult-batch/src/system_var/environment.py @@ -22,8 +22,12 @@ JSKULT_BACKUP_BUCKET = os.environ.get('JSKULT_BACKUP_BUCKET', None) JSK_IO_BUCKET = os.environ.get('JSK_IO_BUCKET', None) JSK_BACKUP_FOLDER = os.environ.get('JSK_BACKUP_FOLDER', None) JSK_DATA_SEND_FOLDER = os.environ.get('JSK_DATA_SEND_FOLDER', None) +JSK_RECEIVE_FOLDER = os.environ.get('JSK_RECEIVE_FOLDER', None) BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get( 'BATCH_MANAGE_DYNAMODB_TABLE_NAME', None) +ARCHIVE_STATEMACHINE_ARN = os.environ( + 'ARCHIVE_STATEMACHINE_ARN', None +) # 初期値がある環境変数 From ce1c7b6527051d484d99b344d5cb8d7a0cc23d5f Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Wed, 4 Jun 2025 10:27:59 +0900 Subject: [PATCH 05/11] =?UTF-8?q?=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E6=8C=87=E6=91=98=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../environment/dcf_inst_merge_environment.py | 6 +++ .../update_business_day_environment.py | 43 +++++++++++++------ .../src/batch/update_business_day.py | 40 ++++++++++------- .../src/system_var/environment.py | 25 ++++++----- 4 files changed, 74 insertions(+), 40 deletions(-) diff --git a/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py b/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py index 5688fa84..3da34501 100644 --- a/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py +++ b/ecs/jskult-batch/src/batch/environment/dcf_inst_merge_environment.py @@ -15,6 +15,8 @@ class DCFInstMergeEnvironment(JskultBatchEnvironment): self.PROCESS_NAME = environment.PROCESS_NAME self.JSK_DATA_SEND_FOLDER = environment.JSK_DATA_SEND_FOLDER self.JSK_BACKUP_FOLDER = environment.JSK_BACKUP_FOLDER + self.TRANSFER_RESULT_FOLDER = environment.TRANSFER_RESULT_FOLDER + self.TRANSFER_RESULT_FILE_NAME = environment.TRANSFER_RESULT_FILE_NAME self.DCF_INST_MERGE_SEND_FILE_NAME = environment.DCF_INST_MERGE_SEND_FILE_NAME def validate(self): @@ -38,5 +40,9 @@ class DCFInstMergeEnvironment(JskultBatchEnvironment): super()._assert_variable_not_empty( self.JSK_DATA_SEND_FOLDER, 'JSK_DATA_SEND_FOLDER') super()._assert_variable_not_empty(self.JSK_BACKUP_FOLDER, 'JSK_BACKUP_FOLDER') + super()._assert_variable_not_empty( + self.TRANSFER_RESULT_FOLDER, 'TRANSFER_RESULT_FOLDER') + super()._assert_variable_not_empty( + self.TRANSFER_RESULT_FILE_NAME, 'TRANSFER_RESULT_FILE_NAME') super()._assert_variable_not_empty(self.DCF_INST_MERGE_SEND_FILE_NAME, 'DCF_INST_MERGE_SEND_FILE_NAME') diff --git a/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py b/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py index 5a5f3770..cc826e93 100644 --- a/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py +++ b/ecs/jskult-batch/src/batch/environment/update_business_day_environment.py @@ -7,16 +7,17 @@ class UpdateBusinessDayEnvironment(JskultBatchEnvironment): """実消化&アルトマークのバッチ処理で使用する環境変数を管理するクラス""" def __init__(self): + self.PROCESS_NAME = environment.PROCESS_NAME + self.JSKULT_CONFIG_BUCKET = environment.JSKULT_CONFIG_BUCKET self.JSKULT_BACKUP_BUCKET = environment.JSKULT_BACKUP_BUCKET self.BATCH_MANAGE_DYNAMODB_TABLE_NAME = environment.BATCH_MANAGE_DYNAMODB_TABLE_NAME + self.ARCHIVE_STATEMACHINE_ARN = environment.ARCHIVE_STATEMACHINE_ARN self.BATCH_EXECUTION_ID = environment.BATCH_EXECUTION_ID self.MAX_RUN_COUNT = environment.MAX_RUN_COUNT - self.PROCESS_NAME = environment.PROCESS_NAME - self.TRANSFER_RESULT_FOLDER = environment.TRANSFER_RESULT_FOLDER - self.TRANSFER_RESULT_FILE_NAME = environment.TRANSFER_RESULT_FILE_NAME - self.JSK_RECEIVE_FOLDER = environment.JSK_RECEIVE_FOLDER - self.ARCHIVE_STATEMACHINE_ARN = environment.ARCHIVE_STATEMACHINE_ARN - + self.JSKULT_CONFIG_CALENDAR_FOLDER = environment.JSKULT_CONFIG_CALENDAR_FOLDER + self.JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME = environment.JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME + self.JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER = environment.JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER + self.JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME = environment.JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME def validate(self): """ 必須の環境変数が設定されているかどうか検査する。 @@ -25,18 +26,34 @@ class UpdateBusinessDayEnvironment(JskultBatchEnvironment): Raises: EnvironmentVariableNotSetException: 環境変数の設定ミス """ + + super()._assert_variable_not_empty( + self.PROCESS_NAME, 'PROCESS_NAME') super()._assert_variable_not_empty( self.JSKULT_BACKUP_BUCKET, 'JSKULT_BACKUP_BUCKET') + super()._assert_variable_not_empty( + self.JSKULT_CONFIG_BUCKET, 'JSKULT_CONFIG_BUCKET') super()._assert_variable_not_empty( self.BATCH_MANAGE_DYNAMODB_TABLE_NAME, 'BATCH_MANAGE_DYNAMODB_TABLE_NAME') - super()._assert_variable_not_empty(self.BATCH_EXECUTION_ID, 'BATCH_EXECUTION_ID') - super()._assert_variable_is_int(self.MAX_RUN_COUNT, 'MAX_RUN_COUNT') + super()._assert_variable_not_empty( + self.ARCHIVE_STATEMACHINE_ARN, 'ARCHIVE_STATEMACHINE_ARN') + super()._assert_variable_not_empty( + self.BATCH_EXECUTION_ID, 'BATCH_EXECUTION_ID') + super()._assert_variable_is_int( + self.MAX_RUN_COUNT, 'MAX_RUN_COUNT') # MAX_RUN_COUNTは数値として扱うため、検査後に変換 self.MAX_RUN_COUNT = int(self.MAX_RUN_COUNT) - super()._assert_variable_not_empty(self.PROCESS_NAME, 'PROCESS_NAME') + super()._assert_variable_not_empty( - self.TRANSFER_RESULT_FOLDER, 'TRANSFER_RESULT_FOLDER') + self.JSKULT_CONFIG_CALENDAR_FOLDER, + 'JSKULT_CONFIG_CALENDAR_FOLDER') super()._assert_variable_not_empty( - self.TRANSFER_RESULT_FILE_NAME, 'TRANSFER_RESULT_FILE_NAME') - super()._assert_variable_not_empty(self.JSK_RECEIVE_FOLDER, 'JSK_RECEIVE_FOLDER') - super()._assert_variable_not_empty(self.ARCHIVE_STATEMACHINE_ARN, 'ARCHIVE_STATEMACHINE_ARN') + self.JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME, + 'JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME') + super()._assert_variable_not_empty( + self.JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER, + 'JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER') + super()._assert_variable_not_empty( + self.JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME, + 'JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME') + diff --git a/ecs/jskult-batch/src/batch/update_business_day.py b/ecs/jskult-batch/src/batch/update_business_day.py index cdac4dbf..75bd20af 100644 --- a/ecs/jskult-batch/src/batch/update_business_day.py +++ b/ecs/jskult-batch/src/batch/update_business_day.py @@ -8,7 +8,8 @@ from src.batch.environment.update_business_day_environment import \ from src.manager.jskult_batch_run_manager import JskultBatchRunManager from src.manager.jskult_batch_status_manager import JskultBatchStatusManager from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager -from src.error.exceptions import EnvironmentVariableNotSetException +from src.error.exceptions import (EnvironmentVariableNotSetException, + MaxRunCountReachedException) from src.logging.get_logger import get_logger from src.system_var import constants @@ -35,9 +36,9 @@ class UpdateBusinessDay(JskultBatchEntrypoint): jskult_batch_status_manager = JskultBatchStatusManager( self.environment.PROCESS_NAME, - # TODO チケットNEWDWH2021-1847の実装で作成した定数に置き換え - 'update_business_day', + constants.PROCESS_NAME_UPDATE_BUSINESS_DAY, self.environment.MAX_RUN_COUNT, + # 日付テーブル更新では連携ファイル数を参照しないため、0を指定する。 0 ) @@ -53,19 +54,26 @@ class UpdateBusinessDay(JskultBatchEntrypoint): '日次バッチ処理中でない、もしくはdump取得が正常終了していないため、日次バッチ処理を異常終了します。') # バッチ実行管理テーブルをfailedで登録 jskult_batch_run_manager.batch_failed() - - if not jskult_batch_status_manager.can_run_business_day_update(): - # 後続処理の起動条件を満たしていない場合 - # 処理ステータスを「処理待」に設定 + + # バッチステータス管理テーブルをエラーに更新 jskult_batch_status_manager.set_process_status( - constants.PROCESS_STATUS_WAITING) - - # バッチ実行管理テーブルに「retry」で登録 - jskult_batch_run_manager.batch_retry() - - logger.info('処理待：実消化＆アルトマーク_日付テーブル更新') - + constants.PROCESS_STATUS_ERROR) return + + try: + if not jskult_batch_status_manager.can_run_post_process(): + # 後続処理の起動条件を満たしていない場合 + # 処理ステータスを「処理待」に設定 + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_WAITING) + + # バッチ実行管理テーブルに「retry」で登録 + jskult_batch_run_manager.batch_retry() + + return + + except MaxRunCountReachedException: + logger.info('後続処理の実行が完了していないため、日付テーブル更新処理を異常終了します。') try: # TODO アーカイブ取得日カレンダーを取得する get_archive_calendar = None @@ -102,12 +110,12 @@ class UpdateBusinessDay(JskultBatchEntrypoint): # TODO jsk_receive_file_listと比較して受領予定にないファイルがあった場合(I-6)ログ出力 pass - JskultHdkeTblManager.update_batch_process_complete() + jskult_hdke_tbl_manager.update_batch_process_complete() # 処理ステータスを「処理済」に設定 jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_DONE) - JskultBatchRunManager.batch_success() + jskult_batch_run_manager.batch_success() logger.info('[NOTICE]処理終了: 実消化＆アルトマーク_日付テーブル更新') except Exception as e: diff --git a/ecs/jskult-batch/src/system_var/environment.py b/ecs/jskult-batch/src/system_var/environment.py index 9732ed5b..63fcc236 100644 --- a/ecs/jskult-batch/src/system_var/environment.py +++ b/ecs/jskult-batch/src/system_var/environment.py @@ -12,21 +12,24 @@ PROCESS_NAME = os.environ.get('PROCESS_NAME', None) # AWS JSKULT_CONFIG_BUCKET = os.environ.get('JSKULT_CONFIG_BUCKET', None) +JSKULT_BACKUP_BUCKET = os.environ.get('JSKULT_BACKUP_BUCKET', None) +BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get('BATCH_MANAGE_DYNAMODB_TABLE_NAME', None) +ARCHIVE_STATEMACHINE_ARN = os.environ('ARCHIVE_STATEMACHINE_ARN', None) BATCH_EXECUTION_ID = os.environ.get('BATCH_EXECUTION_ID', None) MAX_RUN_COUNT = os.environ.get('MAX_RUN_COUNT', None) TRANSFER_RESULT_FOLDER = os.environ.get('TRANSFER_RESULT_FOLDER', None) TRANSFER_RESULT_FILE_NAME = os.environ.get('TRANSFER_RESULT_FILE_NAME', None) -DCF_INST_MERGE_SEND_FILE_NAME = os.environ.get( - 'DCF_INST_MERGE_SEND_FILE_NAME', None) -JSKULT_BACKUP_BUCKET = os.environ.get('JSKULT_BACKUP_BUCKET', None) -JSK_IO_BUCKET = os.environ.get('JSK_IO_BUCKET', None) -JSK_BACKUP_FOLDER = os.environ.get('JSK_BACKUP_FOLDER', None) -JSK_DATA_SEND_FOLDER = os.environ.get('JSK_DATA_SEND_FOLDER', None) -JSK_RECEIVE_FOLDER = os.environ.get('JSK_RECEIVE_FOLDER', None) -BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get( - 'BATCH_MANAGE_DYNAMODB_TABLE_NAME', None) -ARCHIVE_STATEMACHINE_ARN = os.environ( - 'ARCHIVE_STATEMACHINE_ARN', None +JSKULT_CONFIG_CALENDAR_FOLDER = os.environ.get( + 'JSKULT_CONFIG_CALENDAR_FOLDER', None +) +JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME = os.environ.get( + 'JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME', None +) +JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER = os.environ.get( + 'JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER', None +) +JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME = os.environ.get( + 'JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME', None ) From 41ae0841f2495981e6ff95cf9251106cddd4d9ad Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Wed, 4 Jun 2025 10:52:57 +0900 Subject: [PATCH 06/11] =?UTF-8?q?=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E6=8C=87=E6=91=98=E5=AF=BE=E5=BF=9C:=E5=BE=8C=E7=B6=9A?= =?UTF-8?q?=E5=87=A6=E7=90=86=E3=81=AE=E5=AE=9F=E8=A1=8C=E3=81=8C=E5=AE=8C?= =?UTF-8?q?=E4=BA=86=E3=81=97=E3=81=A6=E3=81=84=E3=81=AA=E3=81=84=E5=A0=B4?= =?UTF-8?q?=E5=90=88=E3=81=AE=E5=87=A6=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch/src/batch/update_business_day.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/ecs/jskult-batch/src/batch/update_business_day.py b/ecs/jskult-batch/src/batch/update_business_day.py index 75bd20af..65b8916d 100644 --- a/ecs/jskult-batch/src/batch/update_business_day.py +++ b/ecs/jskult-batch/src/batch/update_business_day.py @@ -73,7 +73,14 @@ class UpdateBusinessDay(JskultBatchEntrypoint): return except MaxRunCountReachedException: - logger.info('後続処理の実行が完了していないため、日付テーブル更新処理を異常終了します。') + logger.error('後続処理の実行が完了していないため、日付テーブル更新処理を異常終了します。') + + # 何らかのエラーが発生した際に、バッチ実行管理テーブルに「failed」で登録 + jskult_batch_run_manager.batch_failed() + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_ERROR) + logger.error(f'E-1 異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') + return try: # TODO アーカイブ取得日カレンダーを取得する get_archive_calendar = None @@ -123,4 +130,4 @@ class UpdateBusinessDay(JskultBatchEntrypoint): jskult_batch_run_manager.batch_failed() jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_ERROR) - logger.exception(f'異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') + logger.error(f'E-1 異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') From 0cb7ac281adc3dc118f22c4d46f311627e7b21ca Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Wed, 4 Jun 2025 16:43:37 +0900 Subject: [PATCH 07/11] =?UTF-8?q?=E5=AE=9F=E6=B6=88=E5=8C=96=E8=BB=A2?= =?UTF-8?q?=E9=80=81=E3=83=87=E3=83=BC=E3=82=BF=E4=B8=80=E8=A6=A7=E3=81=AE?= =?UTF-8?q?=E5=8F=96=E5=BE=97=E3=80=81=E4=B8=8D=E8=B6=B3=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=A4=E3=83=AB=E3=81=AE=E5=88=A4=E5=AE=9A=E3=81=A8=E5=8F=97?= =?UTF-8?q?=E9=A0=98=E4=BA=88=E5=AE=9A=E3=81=AB=E3=81=AA=E3=81=84=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=AE=E5=88=A4=E5=AE=9A=E3=81=AE?= =?UTF-8?q?=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch/src/aws/s3.py | 33 ++--- .../src/batch/common/calendar_file.py | 32 +++++ .../src/batch/update_business_day.py | 118 ++++++++++++++---- .../src/system_var/environment.py | 7 +- 4 files changed, 138 insertions(+), 52 deletions(-) create mode 100644 ecs/jskult-batch/src/batch/common/calendar_file.py diff --git a/ecs/jskult-batch/src/aws/s3.py b/ecs/jskult-batch/src/aws/s3.py index b37b38d2..27701f7b 100644 --- a/ecs/jskult-batch/src/aws/s3.py +++ b/ecs/jskult-batch/src/aws/s3.py @@ -53,46 +53,31 @@ class S3Bucket(): class ConfigBucket(S3Bucket): - # TODO 日付更新処理で内容の修正を行う _bucket_name = environment.JSKULT_CONFIG_BUCKET - def download_holiday_list(self): + def download_jsk_archive_run_day_list(self): # 一時ファイルとして保存する temporary_dir = tempfile.mkdtemp() temporary_file_path = path.join( - temporary_dir, environment.JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME) - holiday_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME}' + temporary_dir, environment.JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME) + jsk_archive_run_day_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_RUN_ARCHIVE_DAY_FILE_NAME}' with open(temporary_file_path, mode='wb') as f: self._s3_client.download_file( - self._bucket_name, holiday_list_key, f) + self._bucket_name, jsk_archive_run_day_list_key, f) f.seek(0) return temporary_file_path - - def download_wholesaler_stock_input_day_list(self): - # 一時ファイルとして保存する + + def download_jsk_expected_data_list(self): temporary_dir = tempfile.mkdtemp() temporary_file_path = path.join( - temporary_dir, environment.JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME) - wholesaler_stock_input_day_list_key = \ - f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME}' - + temporary_dir, environment.JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME) + jsk_expected_data_list_key = f'{environment.JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER}/{environment.JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME}' with open(temporary_file_path, mode='wb') as f: self._s3_client.download_file( - self._bucket_name, wholesaler_stock_input_day_list_key, f) + self._bucket_name, jsk_expected_data_list_key, f) f.seek(0) return temporary_file_path - def download_ultmarc_hex_convert_config(self): - # 一時ファイルとして保存する - temporary_dir = tempfile.mkdtemp() - temporary_file_path = path.join( - temporary_dir, environment.JSKULT_ULTMARC_HEX_CONVERT_CONFIG_FILE_NAME) - hex_convert_config_key = f'{environment.JSKULT_CONFIG_CONVERT_FOLDER}/{environment.JSKULT_ULTMARC_HEX_CONVERT_CONFIG_FILE_NAME}' - with open(temporary_file_path, mode='wb') as f: - self._s3_client.download_file( - self._bucket_name, hex_convert_config_key, f) - f.seek(0) - return temporary_file_path class JskUltBackupBucket(S3Bucket): diff --git a/ecs/jskult-batch/src/batch/common/calendar_file.py b/ecs/jskult-batch/src/batch/common/calendar_file.py new file mode 100644 index 00000000..b456f03c --- /dev/null +++ b/ecs/jskult-batch/src/batch/common/calendar_file.py @@ -0,0 +1,32 @@ +from src.system_var import constants + + +class CalendarFile: + """カレンダーファイル""" + + __calendar_file_lines: list[str] + + def __init__(self, calendar_file_path): + with open(calendar_file_path) as f: + self.__calendar_file_lines: list[str] = f.readlines() + + def compare_date(self, date_str: str) -> bool: + """与えられた日付がカレンダーファイル内に含まれているかどうか + カレンダーファイル内の日付はyyyy/mm/ddで書かれている前提 + コメント（#）が含まれている行は無視される + + Args: + date_str (str): yyyy/mm/dd文字列 + + Returns: + bool: 含まれていればTrue + """ + for calendar_date in self.__calendar_file_lines: + # コメント行が含まれている場合はスキップ + if constants.CALENDAR_COMMENT_SYMBOL in calendar_date: + continue + + if date_str in calendar_date: + return True + + return False diff --git a/ecs/jskult-batch/src/batch/update_business_day.py b/ecs/jskult-batch/src/batch/update_business_day.py index 65b8916d..298eb7c9 100644 --- a/ecs/jskult-batch/src/batch/update_business_day.py +++ b/ecs/jskult-batch/src/batch/update_business_day.py @@ -2,7 +2,9 @@ import json import boto3 from datetime import datetime from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint -from src.aws.s3 import JskTransferListBucket +from src.aws.s3 import (ConfigBucket, + JskTransferListBucket) +from src.batch.common.calendar_file import CalendarFile from src.batch.environment.update_business_day_environment import \ UpdateBusinessDayEnvironment from src.manager.jskult_batch_run_manager import JskultBatchRunManager @@ -54,10 +56,10 @@ class UpdateBusinessDay(JskultBatchEntrypoint): '日次バッチ処理中でない、もしくはdump取得が正常終了していないため、日次バッチ処理を異常終了します。') # バッチ実行管理テーブルをfailedで登録 jskult_batch_run_manager.batch_failed() - + # バッチステータス管理テーブルをエラーに更新 jskult_batch_status_manager.set_process_status( - constants.PROCESS_STATUS_ERROR) + constants.PROCESS_STATUS_ERROR) return try: @@ -71,7 +73,7 @@ class UpdateBusinessDay(JskultBatchEntrypoint): jskult_batch_run_manager.batch_retry() return - + except MaxRunCountReachedException: logger.error('後続処理の実行が完了していないため、日付テーブル更新処理を異常終了します。') @@ -82,40 +84,65 @@ class UpdateBusinessDay(JskultBatchEntrypoint): logger.error(f'E-1 異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') return try: - # TODO アーカイブ取得日カレンダーを取得する - get_archive_calendar = None - # 日付テーブルの処理年月日を取得する - _, _, process_date = jskult_hdke_tbl_manager.get_batch_statuses() - - - if get_archive_calendar == process_date: + _, _, syor_date = jskult_hdke_tbl_manager.get_batch_statuses() + + if self._is_archive_run_day(syor_date): logger.info('I-7 [NOTICE]実消化データアーカイブ取得処理を実行します。') - + # 実消化データアーカイブ取得処理を起動する sfn_client = boto3.client('stepfunctions') # StepFunctionsを起動する - response = sfn_client.start_execution( + sfn_client.start_execution( stateMachineArn=self.environment.ARCHIVE_STATEMACHINE_ARN ) - dt = datetime.strptime(process_date, "%Y/%m/%d") + dt = datetime.strptime(syor_date, "%Y/%m/%d") # 日付テーブルの処理年月日が月曜日の場合 - if dt.weekday() == 0: - if not JskultBatchStatusManager.is_done_ultmarc_import(): - logger.info('[NOTICE]アルトマーク取込稼働日でしたが、アルトマーク取込が行われませんでした。') + if dt.weekday() == constants.WEEKDAY_MONDAY: - # TODO 実消化転送データ一覧を取得する - jsk_receive_file_list = None - # TODO 実消化&アルトマーク_実消化受領予定データリストを取得する - transfer_list = None + if not jskult_batch_status_manager.is_done_ultmarc_import(): + logger.info( + 'I-3 [NOTICE]アルトマーク取込稼働日でしたが、アルトマーク取込が行われませんでした。') - if transfer_list is not None: - # TODO jsk_receive_file_listと比較して結果不足ファイルがあった場合(I-5)ログ出力 + # 実消化転送データ一覧を取得する + try: + transfer_list_bucket = JskTransferListBucket() - # TODO jsk_receive_file_listと比較して受領予定にないファイルがあった場合(I-6)ログ出力 - pass + transfer_list_file_path = transfer_list_bucket.download_transfer_result_file( + syor_date) + except Exception as e: + logger.exception(f'転送ファイル一覧の取得に失敗しました。 {e}') + # バッチ実行管理テーブルをfailedで登録 + jskult_batch_run_manager.batch_failed() + + with open(transfer_list_file_path) as f: + transfer_data_list = json.load(f) + + jsk_transfer_data_list = transfer_data_list['jsk_transfer_list'] + + # 実消化&アルトマーク_実消化受領予定データリストを取得する + try: + config_bucket = ConfigBucket() + jsk_expected_data_path = config_bucket.download_jsk_expected_data_list() + except Exception as e: + logger.exception(f'転送ファイル一覧の取得に失敗しました。 {e}') + # バッチ実行管理テーブルをfailedで登録 + jskult_batch_run_manager.batch_failed() + + with open(jsk_expected_data_path) as f: + jsk_expected_data_list = json.load(f) + + jsk_transfer_expected_data_list = jsk_expected_data_list['jsk_transfer_list'] + + if len(jsk_transfer_data_list) != 0: + # 不足ファイルがあった場合ログ出力 + self._check_missing_received_files( + jsk_transfer_expected_data_list, jsk_transfer_data_list) + # 受領予定にないファイルがあった場合ログ出力 + self._check_extra_received_files( + jsk_transfer_expected_data_list, jsk_transfer_data_list) jskult_hdke_tbl_manager.update_batch_process_complete() # 処理ステータスを「処理済」に設定 @@ -124,10 +151,49 @@ class UpdateBusinessDay(JskultBatchEntrypoint): jskult_batch_run_manager.batch_success() - logger.info('[NOTICE]処理終了: 実消化＆アルトマーク_日付テーブル更新') + logger.info('I-4 [NOTICE]処理終了: 実消化＆アルトマーク_日付テーブル更新') except Exception as e: # 何らかのエラーが発生した際に、バッチ実行管理テーブルに「failed」で登録 jskult_batch_run_manager.batch_failed() jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_ERROR) logger.error(f'E-1 異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') + + def _is_archive_run_day(self, syor_date: str): + # アーカイブ取得日カレンダーを取得する + archive_calendar_file_path = ConfigBucket().download_jsk_archive_run_day_list() + + # 設定ファイル「実消化データアーカイブ起動日カレンダーファイル」の定義内容を取得する + target_days = CalendarFile(archive_calendar_file_path) + + # 処理日付が、設定ファイル「実消化データアーカイブ起動日カレンダーファイル」の定義に該当するかを判定する + return target_days.compare_date(syor_date) + + def _check_missing_received_files(self, jsk_transfer_expected_data_list: list, jsk_transfer_data_list: list): + # jsk_transfer_data_listと比較し、結果不足ファイルがあった場合ログ出力 + missing_files = [ + jsk_transfer_expected_data + for jsk_transfer_expected_data in jsk_transfer_expected_data_list + if not any( + jsk_transfer_data.startswith( + jsk_transfer_expected_data) + for jsk_transfer_data in jsk_transfer_data_list + ) + ] + if missing_files != 0: + logger.info( + f'I-5 [NOTICE]日次連携受領ファイルに不足がありました。ファイル名：{missing_files}') + + def _check_extra_received_files(self, jsk_transfer_expected_data_list: list, jsk_transfer_data_list: list): + unexpected_data = [ + jsk_transfer_data + for jsk_transfer_data in jsk_transfer_data_list + if not any( + jsk_transfer_data.startswith( + jsk_transfer_expected_data) + for jsk_transfer_expected_data in jsk_transfer_expected_data_list + ) + ] + if len(unexpected_data) != 0: + logger.info( + f'I-6 [NOTICE]受領ファイルに日次連携ファイルでないファイルがありました。ファイル名：{unexpected_data}') diff --git a/ecs/jskult-batch/src/system_var/environment.py b/ecs/jskult-batch/src/system_var/environment.py index 63fcc236..fbb334f3 100644 --- a/ecs/jskult-batch/src/system_var/environment.py +++ b/ecs/jskult-batch/src/system_var/environment.py @@ -13,8 +13,9 @@ PROCESS_NAME = os.environ.get('PROCESS_NAME', None) # AWS JSKULT_CONFIG_BUCKET = os.environ.get('JSKULT_CONFIG_BUCKET', None) JSKULT_BACKUP_BUCKET = os.environ.get('JSKULT_BACKUP_BUCKET', None) -BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get('BATCH_MANAGE_DYNAMODB_TABLE_NAME', None) -ARCHIVE_STATEMACHINE_ARN = os.environ('ARCHIVE_STATEMACHINE_ARN', None) +BATCH_MANAGE_DYNAMODB_TABLE_NAME = os.environ.get( + 'BATCH_MANAGE_DYNAMODB_TABLE_NAME', None) +ARCHIVE_STATEMACHINE_ARN = os.environ.get('ARCHIVE_STATEMACHINE_ARN', None) BATCH_EXECUTION_ID = os.environ.get('BATCH_EXECUTION_ID', None) MAX_RUN_COUNT = os.environ.get('MAX_RUN_COUNT', None) TRANSFER_RESULT_FOLDER = os.environ.get('TRANSFER_RESULT_FOLDER', None) @@ -31,6 +32,8 @@ JSKULT_CONFIG_EXPECTED_DATA_LIST_FOLDER = os.environ.get( JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME = os.environ.get( 'JSKULT_CONFIG_EXPECTED_DATA_LIST_FILE_NAME', None ) +JSK_IO_BUCKET = os.environ.get('JSK_IO_BUCKET', None) +JSK_DATA_SEND_FOLDER = os.environ.get('JSK_DATA_SEND_FOLDER', None) # 初期値がある環境変数 From 5bde7acbd8b14d0b1c99e7dfb424d1d3cfa6e735 Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Thu, 5 Jun 2025 09:54:34 +0900 Subject: [PATCH 08/11] =?UTF-8?q?=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC?= =?UTF-8?q?=E6=8C=87=E6=91=98=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/update_business_day.py | 19 +++++++++- s3/config/jsk_archive_run_day.txt | 12 ++++++ s3/config/jsk_expected_data_list.json | 37 +++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 s3/config/jsk_archive_run_day.txt create mode 100644 s3/config/jsk_expected_data_list.json diff --git a/ecs/jskult-batch/src/batch/update_business_day.py b/ecs/jskult-batch/src/batch/update_business_day.py index 298eb7c9..20b99f5d 100644 --- a/ecs/jskult-batch/src/batch/update_business_day.py +++ b/ecs/jskult-batch/src/batch/update_business_day.py @@ -77,7 +77,6 @@ class UpdateBusinessDay(JskultBatchEntrypoint): except MaxRunCountReachedException: logger.error('後続処理の実行が完了していないため、日付テーブル更新処理を異常終了します。') - # 何らかのエラーが発生した際に、バッチ実行管理テーブルに「failed」で登録 jskult_batch_run_manager.batch_failed() jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_ERROR) @@ -116,6 +115,10 @@ class UpdateBusinessDay(JskultBatchEntrypoint): logger.exception(f'転送ファイル一覧の取得に失敗しました。 {e}') # バッチ実行管理テーブルをfailedで登録 jskult_batch_run_manager.batch_failed() + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_ERROR) + logger.error(f'E-1 異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') + return with open(transfer_list_file_path) as f: transfer_data_list = json.load(f) @@ -130,6 +133,11 @@ class UpdateBusinessDay(JskultBatchEntrypoint): logger.exception(f'転送ファイル一覧の取得に失敗しました。 {e}') # バッチ実行管理テーブルをfailedで登録 jskult_batch_run_manager.batch_failed() + jskult_batch_status_manager.set_process_status( + constants.PROCESS_STATUS_ERROR) + logger.error(f'E-1 異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') + return + with open(jsk_expected_data_path) as f: jsk_expected_data_list = json.load(f) @@ -149,6 +157,7 @@ class UpdateBusinessDay(JskultBatchEntrypoint): jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_DONE) + # 日付テーブルに対しバッチ処理中ステータスとダンプ取得状態区分を0に、処理日を+1日に更新 jskult_batch_run_manager.batch_success() logger.info('I-4 [NOTICE]処理終了: 実消化＆アルトマーク_日付テーブル更新') @@ -160,7 +169,12 @@ class UpdateBusinessDay(JskultBatchEntrypoint): logger.error(f'E-1 異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') def _is_archive_run_day(self, syor_date: str): - # アーカイブ取得日カレンダーを取得する + """アーカイブ取得日カレンダーを取得し、処理日がアーカイブ起動日かを確認 + + Returns: + bool + + """ archive_calendar_file_path = ConfigBucket().download_jsk_archive_run_day_list() # 設定ファイル「実消化データアーカイブ起動日カレンダーファイル」の定義内容を取得する @@ -185,6 +199,7 @@ class UpdateBusinessDay(JskultBatchEntrypoint): f'I-5 [NOTICE]日次連携受領ファイルに不足がありました。ファイル名：{missing_files}') def _check_extra_received_files(self, jsk_transfer_expected_data_list: list, jsk_transfer_data_list: list): + # jsk_transfer_data_listと比較し、受領予定にないファイルがあった場合ログ出力 unexpected_data = [ jsk_transfer_data for jsk_transfer_data in jsk_transfer_data_list diff --git a/s3/config/jsk_archive_run_day.txt b/s3/config/jsk_archive_run_day.txt new file mode 100644 index 00000000..0af50d99 --- /dev/null +++ b/s3/config/jsk_archive_run_day.txt @@ -0,0 +1,12 @@ +2025/01/01 +2025/02/01 +2025/03/01 +2025/04/01 +2025/05/01 +2025/06/01 +2025/07/01 +2025/08/01 +2025/09/01 +2025/10/01 +2025/11/01 +2025/12/01 \ No newline at end of file diff --git a/s3/config/jsk_expected_data_list.json b/s3/config/jsk_expected_data_list.json new file mode 100644 index 00000000..0b8183c0 --- /dev/null +++ b/s3/config/jsk_expected_data_list.json @@ -0,0 +1,37 @@ +{ + "jsk_transfer_list":[ + "TRN_RESULT_DATA", + "TRN_RESULT_DATA_BIO", + "TRN_Recive_Inventry", + "INST_IM_PLN", + "MR_YR_PLN_INST_KY", + "MST_WHLSLR", + "MST_SALES_PLACE", + "WHLSLR_LVL4", + "WHLSLR_LVL3", + "WHLSLR_LVL2", + "WHLSLR_LVL1", + "MST_COMPANY", + "MST_COMPANY_GRP", + "MST_DEAL_DIV", + "MST_PROD_PKG", + "PROD_PRICE", + "IM_PLN_PROD", + "MST_ASSN_PROD_GRP", + "PROD_ASSN_TEAM", + "MST_ATC_PHARM_PROD_GRP", + "CUSTOMER_LOTNO_ALL", + "MST_INST", + "ATC_PHARM", + "MST_WAREHOUSE", + "EMP", + "ORG_LVL4", + "ORG_LVL3", + "ORG_LVL2", + "ORG_LVL1", + "MST_JIS_PREF", + "MST_JIS_CITY", + "MST_INST_ASSN", + "MST_GENERAL" + ] +} \ No newline at end of file From c8b1bf19a8aa5f89a13702166a4c2079ef383f28 Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Thu, 5 Jun 2025 10:52:58 +0900 Subject: [PATCH 09/11] =?UTF-8?q?=E5=86=8D=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E7=B5=90=E6=9E=9C=E3=81=AE=E5=8F=8D=E6=98=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/update_business_day.py | 13 +++---- s3/config/jsk_expected_data_list.json | 37 ------------------- .../calendar}/jsk_archive_run_day.txt | 0 .../jsk_expected_data_list.json | 37 +++++++++++++++++++ 4 files changed, 42 insertions(+), 45 deletions(-) delete mode 100644 s3/config/jsk_expected_data_list.json rename s3/config/{ => jskult/calendar}/jsk_archive_run_day.txt (100%) create mode 100644 s3/config/jskult/expected_data_list/jsk_expected_data_list.json diff --git a/ecs/jskult-batch/src/batch/update_business_day.py b/ecs/jskult-batch/src/batch/update_business_day.py index 20b99f5d..fc4d9855 100644 --- a/ecs/jskult-batch/src/batch/update_business_day.py +++ b/ecs/jskult-batch/src/batch/update_business_day.py @@ -80,7 +80,6 @@ class UpdateBusinessDay(JskultBatchEntrypoint): jskult_batch_run_manager.batch_failed() jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_ERROR) - logger.error(f'E-1 異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') return try: # 日付テーブルの処理年月日を取得する @@ -117,7 +116,6 @@ class UpdateBusinessDay(JskultBatchEntrypoint): jskult_batch_run_manager.batch_failed() jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_ERROR) - logger.error(f'E-1 異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') return with open(transfer_list_file_path) as f: @@ -135,7 +133,6 @@ class UpdateBusinessDay(JskultBatchEntrypoint): jskult_batch_run_manager.batch_failed() jskult_batch_status_manager.set_process_status( constants.PROCESS_STATUS_ERROR) - logger.error(f'E-1 異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') return @@ -145,10 +142,8 @@ class UpdateBusinessDay(JskultBatchEntrypoint): jsk_transfer_expected_data_list = jsk_expected_data_list['jsk_transfer_list'] if len(jsk_transfer_data_list) != 0: - # 不足ファイルがあった場合ログ出力 self._check_missing_received_files( jsk_transfer_expected_data_list, jsk_transfer_data_list) - # 受領予定にないファイルがあった場合ログ出力 self._check_extra_received_files( jsk_transfer_expected_data_list, jsk_transfer_data_list) @@ -168,7 +163,7 @@ class UpdateBusinessDay(JskultBatchEntrypoint): constants.PROCESS_STATUS_ERROR) logger.error(f'E-1 異常終了：実消化＆アルトマーク_日付テーブル更新。{e}') - def _is_archive_run_day(self, syor_date: str): + def _is_archive_run_day(self, syor_date: str) -> bool: """アーカイブ取得日カレンダーを取得し、処理日がアーカイブ起動日かを確認 Returns: @@ -184,7 +179,8 @@ class UpdateBusinessDay(JskultBatchEntrypoint): return target_days.compare_date(syor_date) def _check_missing_received_files(self, jsk_transfer_expected_data_list: list, jsk_transfer_data_list: list): - # jsk_transfer_data_listと比較し、結果不足ファイルがあった場合ログ出力 + """jsk_transfer_data_listと比較し、結果不足ファイルがあった場合ログ出力 + """ missing_files = [ jsk_transfer_expected_data for jsk_transfer_expected_data in jsk_transfer_expected_data_list @@ -199,7 +195,8 @@ class UpdateBusinessDay(JskultBatchEntrypoint): f'I-5 [NOTICE]日次連携受領ファイルに不足がありました。ファイル名：{missing_files}') def _check_extra_received_files(self, jsk_transfer_expected_data_list: list, jsk_transfer_data_list: list): - # jsk_transfer_data_listと比較し、受領予定にないファイルがあった場合ログ出力 + """jsk_transfer_data_listと比較し、受領予定にないファイルがあった場合ログ出力 + """ unexpected_data = [ jsk_transfer_data for jsk_transfer_data in jsk_transfer_data_list diff --git a/s3/config/jsk_expected_data_list.json b/s3/config/jsk_expected_data_list.json deleted file mode 100644 index 0b8183c0..00000000 --- a/s3/config/jsk_expected_data_list.json +++ /dev/null @@ -1,37 +0,0 @@ -{ - "jsk_transfer_list":[ - "TRN_RESULT_DATA", - "TRN_RESULT_DATA_BIO", - "TRN_Recive_Inventry", - "INST_IM_PLN", - "MR_YR_PLN_INST_KY", - "MST_WHLSLR", - "MST_SALES_PLACE", - "WHLSLR_LVL4", - "WHLSLR_LVL3", - "WHLSLR_LVL2", - "WHLSLR_LVL1", - "MST_COMPANY", - "MST_COMPANY_GRP", - "MST_DEAL_DIV", - "MST_PROD_PKG", - "PROD_PRICE", - "IM_PLN_PROD", - "MST_ASSN_PROD_GRP", - "PROD_ASSN_TEAM", - "MST_ATC_PHARM_PROD_GRP", - "CUSTOMER_LOTNO_ALL", - "MST_INST", - "ATC_PHARM", - "MST_WAREHOUSE", - "EMP", - "ORG_LVL4", - "ORG_LVL3", - "ORG_LVL2", - "ORG_LVL1", - "MST_JIS_PREF", - "MST_JIS_CITY", - "MST_INST_ASSN", - "MST_GENERAL" - ] -} \ No newline at end of file diff --git a/s3/config/jsk_archive_run_day.txt b/s3/config/jskult/calendar/jsk_archive_run_day.txt similarity index 100% rename from s3/config/jsk_archive_run_day.txt rename to s3/config/jskult/calendar/jsk_archive_run_day.txt diff --git a/s3/config/jskult/expected_data_list/jsk_expected_data_list.json b/s3/config/jskult/expected_data_list/jsk_expected_data_list.json new file mode 100644 index 00000000..ff6a05ed --- /dev/null +++ b/s3/config/jskult/expected_data_list/jsk_expected_data_list.json @@ -0,0 +1,37 @@ +{ + "jsk_transfer_list": [ + "TRN_RESULT_DATA", + "TRN_RESULT_DATA_BIO", + "TRN_Recive_Inventry", + "INST_IM_PLN", + "MR_YR_PLN_INST_KY", + "MST_WHLSLR", + "MST_SALES_PLACE", + "WHLSLR_LVL4", + "WHLSLR_LVL3", + "WHLSLR_LVL2", + "WHLSLR_LVL1", + "MST_COMPANY", + "MST_COMPANY_GRP", + "MST_DEAL_DIV", + "MST_PROD_PKG", + "PROD_PRICE", + "IM_PLN_PROD", + "MST_ASSN_PROD_GRP", + "PROD_ASSN_TEAM", + "MST_ATC_PHARM_PROD_GRP", + "CUSTOMER_LOTNO_ALL", + "MST_INST", + "ATC_PHARM", + "MST_WAREHOUSE", + "EMP", + "ORG_LVL4", + "ORG_LVL3", + "ORG_LVL2", + "ORG_LVL1", + "MST_JIS_PREF", + "MST_JIS_CITY", + "MST_INST_ASSN", + "MST_GENERAL" + ] +} From 80b4562a3255f429bb3d4c9c779f7ea4fb44ea81 Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Thu, 5 Jun 2025 11:55:53 +0900 Subject: [PATCH 10/11] =?UTF-8?q?=E3=82=A2=E3=83=BC=E3=82=AB=E3=82=A4?= =?UTF-8?q?=E3=83=96=E5=8F=96=E5=BE=97=E6=97=A5=E3=82=92=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/config/jskult/calendar/jsk_archive_run_day.txt | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/s3/config/jskult/calendar/jsk_archive_run_day.txt b/s3/config/jskult/calendar/jsk_archive_run_day.txt index 0af50d99..cc8eee46 100644 --- a/s3/config/jskult/calendar/jsk_archive_run_day.txt +++ b/s3/config/jskult/calendar/jsk_archive_run_day.txt @@ -1,12 +1 @@ -2025/01/01 -2025/02/01 -2025/03/01 -2025/04/01 -2025/05/01 -2025/06/01 -2025/07/01 -2025/08/01 -2025/09/01 -2025/10/01 -2025/11/01 -2025/12/01 \ No newline at end of file +2026/01/09 \ No newline at end of file From bb64d5d732d187a3663f9e11e9e5bdefc8a1ed2a Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 5 Jun 2025 13:43:40 +0900 Subject: [PATCH 11/11] =?UTF-8?q?feat:=20=E3=83=95=E3=82=A9=E3=83=BC?= =?UTF-8?q?=E3=83=9E=E3=83=83=E3=83=88=E3=80=81=E3=82=B3=E3=83=A1=E3=83=B3?= =?UTF-8?q?=E3=83=88=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/update_business_day.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/ecs/jskult-batch/src/batch/update_business_day.py b/ecs/jskult-batch/src/batch/update_business_day.py index fc4d9855..963c17f4 100644 --- a/ecs/jskult-batch/src/batch/update_business_day.py +++ b/ecs/jskult-batch/src/batch/update_business_day.py @@ -1,18 +1,19 @@ import json -import boto3 from datetime import datetime -from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint -from src.aws.s3 import (ConfigBucket, - JskTransferListBucket) + +import boto3 + +from src.aws.s3 import ConfigBucket, JskTransferListBucket from src.batch.common.calendar_file import CalendarFile from src.batch.environment.update_business_day_environment import \ UpdateBusinessDayEnvironment -from src.manager.jskult_batch_run_manager import JskultBatchRunManager -from src.manager.jskult_batch_status_manager import JskultBatchStatusManager -from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager +from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint from src.error.exceptions import (EnvironmentVariableNotSetException, MaxRunCountReachedException) from src.logging.get_logger import get_logger +from src.manager.jskult_batch_run_manager import JskultBatchRunManager +from src.manager.jskult_batch_status_manager import JskultBatchStatusManager +from src.manager.jskult_hdke_tbl_manager import JskultHdkeTblManager from src.system_var import constants logger = get_logger('日付テーブル更新') @@ -135,15 +136,16 @@ class UpdateBusinessDay(JskultBatchEntrypoint): constants.PROCESS_STATUS_ERROR) return - with open(jsk_expected_data_path) as f: jsk_expected_data_list = json.load(f) jsk_transfer_expected_data_list = jsk_expected_data_list['jsk_transfer_list'] if len(jsk_transfer_data_list) != 0: + # 不足ファイルがあった場合ログ出力 self._check_missing_received_files( jsk_transfer_expected_data_list, jsk_transfer_data_list) + # 受領予定にないファイルがあった場合ログ出力 self._check_extra_received_files( jsk_transfer_expected_data_list, jsk_transfer_data_list)