99 lines
3.7 KiB
Python
99 lines
3.7 KiB
Python
import boto3
|
|
from src.system_var.constants import (AWS_RESOURCE_S3, S3_CHAR_CODE,
|
|
S3_RESPONSE_BODY)
|
|
from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET,
|
|
CRM_IMPORT_DATA_BACKUP_FOLDER,
|
|
CRM_IMPORT_DATA_FOLDER,
|
|
IMPORT_DATA_BUCKET,
|
|
LAST_FETCH_DATE_FOLDER,
|
|
OBJECT_INFO_FILENAME,
|
|
OBJECT_INFO_FOLDER,
|
|
PROCESS_RESULT_FOLDER,
|
|
RESPONSE_JSON_BACKUP_FOLDER)
|
|
|
|
|
|
class S3Resource:
|
|
def __init__(self, bucket_name: str) -> None:
|
|
self.__s3_resource = boto3.resource(AWS_RESOURCE_S3)
|
|
self.__s3_bucket = self.__s3_resource.Bucket(bucket_name)
|
|
|
|
def get_object(self, object_key: str) -> str:
|
|
response = self.__s3_bucket.Object(object_key).get()
|
|
body = response[S3_RESPONSE_BODY].read()
|
|
return body.decode(S3_CHAR_CODE)
|
|
|
|
def put_object(self, object_key: str, local_file_path: str) -> None:
|
|
self.__s3_bucket.upload_file(Key=object_key, Filename=local_file_path)
|
|
return
|
|
|
|
def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None:
|
|
copy_source = {'Bucket': src_bucket, 'Key': src_key}
|
|
self.__s3_resource.meta.client.copy(copy_source, dest_bucket, dest_key)
|
|
return
|
|
|
|
|
|
class ConfigBucket:
|
|
__s3_resource: S3Resource = None
|
|
|
|
def __init__(self) -> None:
|
|
self.__s3_resource = S3Resource(CRM_CONFIG_BUCKET)
|
|
|
|
def __str__(self) -> str:
|
|
return CRM_CONFIG_BUCKET
|
|
|
|
def get_object_info_file(self) -> str:
|
|
return self.__s3_resource.get_object(f'{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}')
|
|
|
|
def get_last_fetch_datetime_file(self, file_key: str) -> str:
|
|
return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_key}')
|
|
|
|
def put_last_fetch_datetime_file(self, file_key: str, local_file_path: str) -> None:
|
|
self.__s3_resource.put_object(
|
|
f'{LAST_FETCH_DATE_FOLDER}/{file_key}', local_file_path)
|
|
return
|
|
|
|
|
|
class DataBucket:
|
|
__s3_resource: S3Resource = None
|
|
|
|
def __init__(self) -> None:
|
|
self.__s3_resource = S3Resource(IMPORT_DATA_BUCKET)
|
|
|
|
def __str__(self) -> str:
|
|
return IMPORT_DATA_BUCKET
|
|
|
|
def put_csv(self, file_key: str, local_file_path: str) -> None:
|
|
object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_key}'
|
|
self.__s3_resource.put_object(object_key, local_file_path)
|
|
return
|
|
|
|
def put_csv_from(self, src_bucket: str, src_key: str):
|
|
dest_filename = src_key.split('/')[-1]
|
|
self.__s3_resource.copy(src_bucket, src_key, str(self), f'{CRM_IMPORT_DATA_FOLDER}/{dest_filename}')
|
|
return
|
|
|
|
|
|
class BackupBucket:
|
|
__s3_resource: S3Resource = None
|
|
|
|
def __init__(self) -> None:
|
|
self.__s3_resource = S3Resource(CRM_BACKUP_BUCKET)
|
|
|
|
def __str__(self) -> str:
|
|
return CRM_BACKUP_BUCKET
|
|
|
|
def put_response_json(self, file_key: str, local_file_path: str) -> None:
|
|
object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_key}'
|
|
self.__s3_resource.put_object(object_key, local_file_path)
|
|
return
|
|
|
|
def put_csv(self, file_key: str, local_file_path: str) -> None:
|
|
object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_key}'
|
|
self.__s3_resource.put_object(object_key, local_file_path)
|
|
return
|
|
|
|
def put_result_json(self, file_key: str, local_file_path: str) -> None:
|
|
object_key = f'{PROCESS_RESULT_FOLDER}/{file_key}'
|
|
self.__s3_resource.put_object(object_key, local_file_path)
|
|
return
|