refactor: ファイル情報をdictで取り回していたが、意味ないためstrで取り回すように修正。不要なメソッドを削除。
This commit is contained in:
parent
a3be8d39e3
commit
f05ec64f41
@ -1,11 +1,7 @@
|
|||||||
import gzip
|
|
||||||
import json
|
import json
|
||||||
import os
|
|
||||||
import os.path as path
|
|
||||||
import shutil
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
import boto3
|
import boto3
|
||||||
|
|
||||||
from src.system_var import environment
|
from src.system_var import environment
|
||||||
|
|
||||||
|
|
||||||
@ -20,7 +16,7 @@ class S3Client:
|
|||||||
return []
|
return []
|
||||||
contents = response['Contents']
|
contents = response['Contents']
|
||||||
# 末尾がスラッシュで終わるものはフォルダとみなしてスキップする
|
# 末尾がスラッシュで終わるものはフォルダとみなしてスキップする
|
||||||
objects = [{'filename': content['Key'], 'size': content['Size']}
|
objects = [content['Key']
|
||||||
for content in contents if not content['Key'].endswith('/')]
|
for content in contents if not content['Key'].endswith('/')]
|
||||||
return objects
|
return objects
|
||||||
|
|
||||||
@ -60,46 +56,26 @@ class JskIOBucket(S3Bucket):
|
|||||||
self._bucket_name, self._recv_folder)
|
self._bucket_name, self._recv_folder)
|
||||||
return self._s3_file_list
|
return self._s3_file_list
|
||||||
|
|
||||||
def download_data_file(self, data_filename: str):
|
def transfer_file_to_import(self, target_file: str):
|
||||||
temporary_dir = tempfile.mkdtemp()
|
|
||||||
temporary_file_path = path.join(
|
|
||||||
temporary_dir, f'{data_filename.replace(f"{self._recv_folder}/", "")}')
|
|
||||||
with open(temporary_file_path, mode='wb') as f:
|
|
||||||
self._s3_client.download_file(self._bucket_name, data_filename, f)
|
|
||||||
f.seek(0)
|
|
||||||
return temporary_file_path
|
|
||||||
|
|
||||||
def unzip_data_file(self, filename: str):
|
|
||||||
temp_dir = os.path.dirname(filename)
|
|
||||||
decompress_filename = os.path.basename(filename).replace('.gz', '')
|
|
||||||
decompress_file_path = os.path.join(temp_dir, decompress_filename)
|
|
||||||
with gzip.open(filename, 'rb') as gz:
|
|
||||||
with open(decompress_file_path, 'wb') as decompressed_file:
|
|
||||||
shutil.copyfileobj(gz, decompressed_file)
|
|
||||||
|
|
||||||
ret = [decompress_file_path]
|
|
||||||
return ret
|
|
||||||
|
|
||||||
def transfer_file_to_import(self, target_file: dict):
|
|
||||||
data_import_bucket = DataImportBucket()
|
data_import_bucket = DataImportBucket()
|
||||||
transfer_from_file_path = target_file.get("filename")
|
transfer_from_file_path = target_file
|
||||||
transfer_to_filename = transfer_from_file_path.replace(
|
transfer_to_filename = transfer_from_file_path.replace(
|
||||||
f"{self._recv_folder}/", "")
|
f"{self._recv_folder}/", "")
|
||||||
data_import_key = f'{data_import_bucket._folder}/{transfer_to_filename}'
|
data_import_key = f'{data_import_bucket._folder}/{transfer_to_filename}'
|
||||||
self._s3_client.copy(self._bucket_name, transfer_from_file_path,
|
self._s3_client.copy(self._bucket_name, transfer_from_file_path,
|
||||||
data_import_bucket._bucket_name, data_import_key)
|
data_import_bucket._bucket_name, data_import_key)
|
||||||
|
|
||||||
def backup_file(self, target_file: dict, datetime_key: str):
|
def backup_file(self, target_file: str, datetime_key: str):
|
||||||
jsk_backup_bucket = JskBackupBucket()
|
jsk_backup_bucket = JskBackupBucket()
|
||||||
backup_from_file_path = target_file.get("filename")
|
backup_from_file_path = target_file
|
||||||
backup_to_filename = backup_from_file_path.replace(
|
backup_to_filename = backup_from_file_path.replace(
|
||||||
f"{self._recv_folder}/", "")
|
f"{self._recv_folder}/", "")
|
||||||
backup_key = f'{jsk_backup_bucket._folder}/{datetime_key}/{backup_to_filename}'
|
backup_key = f'{jsk_backup_bucket._folder}/{datetime_key}/{backup_to_filename}'
|
||||||
self._s3_client.copy(self._bucket_name, backup_from_file_path,
|
self._s3_client.copy(self._bucket_name, backup_from_file_path,
|
||||||
jsk_backup_bucket._bucket_name, backup_key)
|
jsk_backup_bucket._bucket_name, backup_key)
|
||||||
|
|
||||||
def delete_file(self, target_file: dict):
|
def delete_file(self, target_file: str):
|
||||||
delete_path = target_file.get("filename")
|
delete_path = target_file
|
||||||
self._s3_client.delete_file(
|
self._s3_client.delete_file(
|
||||||
self._bucket_name, delete_path)
|
self._bucket_name, delete_path)
|
||||||
|
|
||||||
@ -116,16 +92,16 @@ class UltmarcBucket(S3Bucket):
|
|||||||
def get_file_list(self):
|
def get_file_list(self):
|
||||||
return self._s3_client.list_objects(self._bucket_name, self._folder)
|
return self._s3_client.list_objects(self._bucket_name, self._folder)
|
||||||
|
|
||||||
def backup_file(self, target_file: dict, datetime_key: str):
|
def backup_file(self, target_file: str, datetime_key: str):
|
||||||
# バックアップバケットにコピー
|
# バックアップバケットにコピー
|
||||||
ultmarc_backup_bucket = UltmarcBackupBucket()
|
ultmarc_backup_bucket = UltmarcBackupBucket()
|
||||||
target_file_path = target_file.get("filename")
|
target_file_path = target_file
|
||||||
backup_key = f'{ultmarc_backup_bucket._folder}/{datetime_key}/{target_file_path.replace(f"{self._folder}/", "")}'
|
backup_key = f'{ultmarc_backup_bucket._folder}/{datetime_key}/{target_file_path.replace(f"{self._folder}/", "")}'
|
||||||
self._s3_client.copy(self._bucket_name, target_file_path,
|
self._s3_client.copy(self._bucket_name, target_file_path,
|
||||||
ultmarc_backup_bucket._bucket_name, backup_key)
|
ultmarc_backup_bucket._bucket_name, backup_key)
|
||||||
|
|
||||||
def delete_file(self, target_file: dict):
|
def delete_file(self, target_file: str):
|
||||||
delete_path = target_file.get("filename")
|
delete_path = target_file
|
||||||
self._s3_client.delete_file(
|
self._s3_client.delete_file(
|
||||||
self._bucket_name, delete_path)
|
self._bucket_name, delete_path)
|
||||||
|
|
||||||
@ -134,8 +110,8 @@ class UltmarcImportBucket(S3Bucket):
|
|||||||
_bucket_name = environment.ULTMARC_DATA_BUCKET
|
_bucket_name = environment.ULTMARC_DATA_BUCKET
|
||||||
_folder = environment.ULTMARC_IMPORT_FOLDER
|
_folder = environment.ULTMARC_IMPORT_FOLDER
|
||||||
|
|
||||||
def transfer_file_to_import(self, target_file: dict):
|
def transfer_file_to_import(self, target_file: str):
|
||||||
from_file_path = target_file.get("filename")
|
from_file_path = target_file
|
||||||
to_filename = from_file_path.replace(
|
to_filename = from_file_path.replace(
|
||||||
f"{UltmarcBucket()._folder}/", "")
|
f"{UltmarcBucket()._folder}/", "")
|
||||||
data_import_key = f'{self._folder}/{to_filename}'
|
data_import_key = f'{self._folder}/{to_filename}'
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user