nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

名称変更

Browse Source
This commit is contained in:
野間 2023-07-05 18:05:44 +09:00
parent 34a4357988
commit 518c84695e
2 changed files with 24 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
ecs/jskult-batch-daily/src/aws/s3.py
View File

@ -1,4 +1,3 @@
import io
import os
import os.path as path
import tarfile
@ -80,28 +79,6 @@ class UltmarcBucket(S3Bucket):
self._s3_client.delete_file(self._bucket_name, dat_file_key)
class VjskBucket(S3Bucket):
_bucket_name = environment.VJSK_DATA_BUCKET
_folder = environment.VJSK_DATA_SEND_FOLDER
def list_dat_file(self):
return self._s3_client.list_objects(self._bucket_name, self._folder)
def upload_dat_file(self, vjsk_create_csv: str, csv_file_path: str):
# S3バケットにファイルを移動
csv_file_name = f'{self._folder}/{vjsk_create_csv}'
s3_client = S3Client()
s3_client.upload_file(csv_file_path, self._bucket_name, csv_file_name)
return
def backup_dat_file(self, dat_file_key: str, datetime_key: str):
# バックアップバケットにコピー
vjsk_backup_bucket = VjskBackupBucket()
dat_key = f'{self._folder}/{dat_file_key}'
backup_key = f'{vjsk_backup_bucket._folder}/{self._folder}/{datetime_key}/{dat_file_key.replace(f"{self._folder}/", "")}'
self._s3_client.copy(self._bucket_name, dat_key, vjsk_backup_bucket._bucket_name, backup_key)
class ConfigBucket(S3Bucket):
_bucket_name = environment.JSKULT_CONFIG_BUCKET
@ -139,7 +116,7 @@ class VjskBackupBucket(JskUltBackupBucket):
class VjskReceiveBucket(S3Bucket):
_bucket_name = environment.JSKULT_DATA_BUCKET
_bucket_name = environment.VJSK_DATA_BUCKET
_recv_folder = environment.JSKULT_DATA_FOLDER_RECV
_s3_file_list = None
@ -176,3 +153,22 @@ class VjskReceiveBucket(S3Bucket):
self._s3_client.copy(self._bucket_name, backup_from_file_path,
jskult_backup_bucket._bucket_name, backup_key)
self._s3_client.delete_file(self._bucket_name, backup_from_file_path)
class VjskSendBucket(S3Bucket):
_bucket_name = environment.VJSK_DATA_BUCKET
_send_folder = environment.VJSK_DATA_SEND_FOLDER
def upload_vjsk_csv_file(self, vjsk_create_csv: str, csv_file_path: str):
# S3バケットにファイルを移動
csv_file_name = f'{self._send_folder}/{vjsk_create_csv}'
s3_client = S3Client()
s3_client.upload_file(csv_file_path, self._bucket_name, csv_file_name)
return
def backup_vjsk_csv_file(self, dat_file_key: str, datetime_key: str):
# バックアップバケットにコピー
vjsk_backup_bucket = VjskBackupBucket()
dat_key = f'{self._send_folder}/{dat_file_key}'
backup_key = f'{vjsk_backup_bucket._folder}/{self._send_folder}/{datetime_key}/{dat_file_key.replace(f"{self._send_folder}/", "")}'
self._s3_client.copy(self._bucket_name, dat_key, vjsk_backup_bucket._bucket_name, backup_key)

8
ecs/jskult-batch-daily/src/batch/ultmarc/output_vjsk_inst_pharm_data.py
View File

@ -1,6 +1,6 @@
"""output_vjsk_inst_pharm_data"""
from src.aws.s3 import VjskBucket
from src.aws.s3 import VjskSendBucket
from src.batch.common.batch_context import BatchContext
from src.db.database import Database
@ -42,10 +42,10 @@ def exec():
# CSVファイル作成
csv_file_path = make_csv_data(record_inst, record_pharm, vjsk_csv_file_name)
vjsk_bucket = VjskBucket()
vjsk_bucket = VjskSendBucket()
try:
# s3へデータ移動
vjsk_bucket.upload_dat_file(vjsk_csv_file_name, csv_file_path)
vjsk_bucket.upload_vjsk_csv_file(vjsk_csv_file_name, csv_file_path)
except Exception as e:
logger.info('S3バケットにCSVデータを作成できませんでした。')
raise e
@ -53,7 +53,7 @@ def exec():
try:
# 処理後ファイルをバックアップ
batch_context = BatchContext.get_instance()
vjsk_bucket.backup_dat_file(vjsk_csv_file_name, batch_context.syor_date)
vjsk_bucket.backup_vjsk_csv_file(vjsk_csv_file_name, batch_context.syor_date)
except Exception as e:
logger.info('バックアップバケットへCSVデータをコピーできませんでした。')
raise e