diff --git a/ecs/jskult-batch-daily/.env.example b/ecs/jskult-batch-daily/.env.example index d95322fb..2c1cb5a7 100644 --- a/ecs/jskult-batch-daily/.env.example +++ b/ecs/jskult-batch-daily/.env.example @@ -8,6 +8,7 @@ ULTMARC_DATA_BUCKET=**************** ULTMARC_DATA_FOLDER=recv JSKULT_BACKUP_BUCKET=**************** ULTMARC_BACKUP_FOLDER=ultmarc +VJSK_BACKUP_FOLDER=vjsk JSKULT_CONFIG_BUCKET=********************** JSKULT_CONFIG_CALENDAR_FOLDER=jskult/calendar JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME=jskult_holiday_list.txt diff --git a/ecs/jskult-batch-daily/src/aws/s3.py b/ecs/jskult-batch-daily/src/aws/s3.py index 2e98eaac..0df364d4 100644 --- a/ecs/jskult-batch-daily/src/aws/s3.py +++ b/ecs/jskult-batch-daily/src/aws/s3.py @@ -112,6 +112,10 @@ class UltmarcBackupBucket(JskUltBackupBucket): _folder = environment.ULTMARC_BACKUP_FOLDER +class VjskBackupBucket(JskUltBackupBucket): + _folder = environment.VJSK_BACKUP_FOLDER + + class VjskReceiveBucket(S3Bucket): _bucket_name = environment.JSKULT_DATA_BUCKET _recv_folder = environment.JSKULT_DATA_FOLDER_RECV @@ -140,3 +144,13 @@ class VjskReceiveBucket(S3Bucket): file = os.path.join(temp_dir, extracted_file) ret.append(file) return ret + + def backup_dat_file(self, target_files: list, datetime_key: str): + jskult_backup_bucket = VjskBackupBucket() + for target_file in target_files: + backup_from_file_path = target_file.get("filename") + backup_to_filename = backup_from_file_path.replace(f"{self._recv_folder}/", "") + backup_key = f'{jskult_backup_bucket._folder}/{datetime_key}/{backup_to_filename}' + self._s3_client.copy(self._bucket_name, backup_from_file_path, + jskult_backup_bucket._bucket_name, backup_key) + self._s3_client.delete_file(self._bucket_name, backup_from_file_path) diff --git a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_importer.py b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_importer.py index 2737ffca..23fc8e5d 100644 --- a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_importer.py +++ b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_importer.py @@ -30,9 +30,10 @@ def exec(): # V実消化データファイル受領チェック logger.debug('V実消化データファイル受領チェック:開始') + received_s3_files = [] try: # S3バケットにある受領済のV実消化データファイルの存在チェックをする - _check_received_files() + received_s3_files = _check_received_files() except BatchOperationException as e: logger.debug('受領したV実消化データファイルに未受領もものがあります') @@ -43,7 +44,7 @@ def exec(): logger.debug('V実消化データ取込:開始') try: # S3バケットにある受領済のV実消化データファイルをデータベースに登録する - _import_file_to_db() + _import_file_to_db(received_s3_files) except Exception as e: logger.debug(f'データベース登録失敗 {e}') raise e @@ -52,7 +53,7 @@ def exec(): logger.debug('V実消化データ受領ファイルバックアップ退避:開始') try: # 取込が完了したS3バケットにある受領ファイルをバックアップ用S3バケットに移動する - _backup_received_files() + vjsk_recv_bucket.backup_dat_file(received_s3_files, batch_context.syor_date) except BatchOperationException as e: logger.debug('V実消化データ受領ファイルのバックアップ退避が失敗しました') @@ -80,7 +81,7 @@ def _check_if_file_exists(src_list: list, condkey: str) -> bool: return ret -def _check_received_files(): +def _check_received_files() -> list: """V実消化連携データファイル受領確認処理""" logger.debug('_check_received_files start') @@ -151,16 +152,13 @@ def _check_received_files(): logger.debug('_check_received_files done') - return + return received_s3_files -def _import_file_to_db(): +def _import_file_to_db(received_s3_files: list): """V実消化連携データ取込処理""" logger.debug('_import_file_to_db start') - # S3バケット「実消化&アルトマーク V実消化データ受領バケット」にある受領ファイル一覧を取得 - received_s3_files = vjsk_recv_bucket.get_s3_file_list() - # S3バケット「実消化&アルトマーク V実消化データ受領バケット」の受領ファイルをローカルストレージにdownloadして辞書化する target_dict = {} for s3_file_path in received_s3_files: @@ -253,11 +251,3 @@ def _determine_today_is_stockslipdata_target(): raise e logger.debug("_determine_today_is_stockslipdata_target done") return ret - - -def _backup_received_files(): - logger.debug("_backup_received_files start") - - # TODO 受領バケットのファイル → バックアップバケット - - logger.debug("_backup_received_files done") diff --git a/ecs/jskult-batch-daily/src/system_var/environment.py b/ecs/jskult-batch-daily/src/system_var/environment.py index 6a2fca0b..5973a181 100644 --- a/ecs/jskult-batch-daily/src/system_var/environment.py +++ b/ecs/jskult-batch-daily/src/system_var/environment.py @@ -12,6 +12,7 @@ ULTMARC_DATA_BUCKET = os.environ['ULTMARC_DATA_BUCKET'] ULTMARC_DATA_FOLDER = os.environ['ULTMARC_DATA_FOLDER'] JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] ULTMARC_BACKUP_FOLDER = os.environ['ULTMARC_BACKUP_FOLDER'] +VJSK_BACKUP_FOLDER = os.environ['VJSK_BACKUP_FOLDER'] JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] JSKULT_CONFIG_CALENDAR_FOLDER = os.environ['JSKULT_CONFIG_CALENDAR_FOLDER'] JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME = os.environ['JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME']