nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

getとread処理分けた・インターフェース命名変更

Browse Source
This commit is contained in:
Nik Afiq 2024-05-15 11:19:00 +09:00
parent e55e95a6e9
commit de1e82273e
5 changed files with 23 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
ecs/crm-datafetch/src/aws/s3.py
View File

@ -12,7 +12,7 @@ from src.system_var.environments import (CRM_BACKUP_BUCKET, CRM_CONFIG_BUCKET,
RESPONSE_JSON_BACKUP_FOLDER) RESPONSE_JSON_BACKUP_FOLDER)
class S3Resource: class S3Client:
def __init__(self, bucket_name: str) -> None: def __init__(self, bucket_name: str) -> None:
self.__s3_client = boto3.client(AWS_RESOURCE_S3) self.__s3_client = boto3.client(AWS_RESOURCE_S3)
self.__s3_bucket = bucket_name self.__s3_bucket = bucket_name
@ -33,66 +33,66 @@ class S3Resource:
class ConfigBucket: class ConfigBucket:
__s3_resource: S3Resource = None __s3_client: S3Client = None
def __init__(self) -> None: def __init__(self) -> None:
self.__s3_resource = S3Resource(CRM_CONFIG_BUCKET) self.__s3_client = S3Client(CRM_CONFIG_BUCKET)
def __str__(self) -> str: def __str__(self) -> str:
return CRM_CONFIG_BUCKET return CRM_CONFIG_BUCKET
def get_object_info_file(self) -> str: def get_object_info_file(self) -> str:
return self.__s3_resource.get_object(f'{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}') return self.__s3_client.get_object(f'{OBJECT_INFO_FOLDER}/{OBJECT_INFO_FILENAME}')
def get_last_fetch_datetime_file(self, file_key: str) -> str: def get_last_fetch_datetime_file(self, file_key: str) -> str:
return self.__s3_resource.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_key}') return self.__s3_client.get_object(f'{LAST_FETCH_DATE_FOLDER}/{file_key}')
def put_last_fetch_datetime_file(self, file_key: str, local_file_path: str) -> None: def put_last_fetch_datetime_file(self, file_key: str, local_file_path: str) -> None:
self.__s3_resource.put_object( self.__s3_client.put_object(
f'{LAST_FETCH_DATE_FOLDER}/{file_key}', local_file_path) f'{LAST_FETCH_DATE_FOLDER}/{file_key}', local_file_path)
return return
class DataBucket: class DataBucket:
__s3_resource: S3Resource = None __s3_client: S3Client = None
def __init__(self) -> None: def __init__(self) -> None:
self.__s3_resource = S3Resource(IMPORT_DATA_BUCKET) self.__s3_client = S3Client(IMPORT_DATA_BUCKET)
def __str__(self) -> str: def __str__(self) -> str:
return IMPORT_DATA_BUCKET return IMPORT_DATA_BUCKET
def put_csv(self, file_key: str, local_file_path: str) -> None: def put_csv(self, file_key: str, local_file_path: str) -> None:
object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_key}' object_key = f'{CRM_IMPORT_DATA_FOLDER}/{file_key}'
self.__s3_resource.put_object(object_key, local_file_path) self.__s3_client.put_object(object_key, local_file_path)
return return
def put_csv_from(self, src_bucket: str, src_key: str): def put_csv_from(self, src_bucket: str, src_key: str):
dest_filename = src_key.split('/')[-1] dest_filename = src_key.split('/')[-1]
self.__s3_resource.copy(src_bucket, src_key, str(self), f'{CRM_IMPORT_DATA_FOLDER}/{dest_filename}') self.__s3_client.copy(src_bucket, src_key, str(self), f'{CRM_IMPORT_DATA_FOLDER}/{dest_filename}')
return return
class BackupBucket: class BackupBucket:
__s3_resource: S3Resource = None __s3_client: S3Client = None
def __init__(self) -> None: def __init__(self) -> None:
self.__s3_resource = S3Resource(CRM_BACKUP_BUCKET) self.__s3_client = S3Client(CRM_BACKUP_BUCKET)
def __str__(self) -> str: def __str__(self) -> str:
return CRM_BACKUP_BUCKET return CRM_BACKUP_BUCKET
def put_response_json(self, file_key: str, local_file_path: str) -> None: def put_response_json(self, file_key: str, local_file_path: str) -> None:
object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_key}' object_key = f'{RESPONSE_JSON_BACKUP_FOLDER}/{file_key}'
self.__s3_resource.put_object(object_key, local_file_path) self.__s3_client.put_object(object_key, local_file_path)
return return
def put_csv(self, file_key: str, local_file_path: str) -> None: def put_csv(self, file_key: str, local_file_path: str) -> None:
object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_key}' object_key = f'{CRM_IMPORT_DATA_BACKUP_FOLDER}/{file_key}'
self.__s3_resource.put_object(object_key, local_file_path) self.__s3_client.put_object(object_key, local_file_path)
return return
def put_result_json(self, file_key: str, local_file_path: str) -> None: def put_result_json(self, file_key: str, local_file_path: str) -> None:
object_key = f'{PROCESS_RESULT_FOLDER}/{file_key}' object_key = f'{PROCESS_RESULT_FOLDER}/{file_key}'
self.__s3_resource.put_object(object_key, local_file_path) self.__s3_client.put_object(object_key, local_file_path)
return return

3
lambda/sap-fin-receive-check-daily/sap-fin-receive-check-daily.py
View File

@ -123,7 +123,8 @@ def lambda_handler(event, context):
logger.info('I-04-04 取得したオブジェクトリストと日次I/Fファイルネーム設定ファイルの突き合わせを開始します') logger.info('I-04-04 取得したオブジェクトリストと日次I/Fファイルネーム設定ファイルの突き合わせを開始します')
receive_daily_file_name_body = io.TextIOWrapper(io.BytesIO(receive_daily_file_name_response["Body"].read()), encoding='utf-8') receive_daily_file_name_body = io.TextIOWrapper(io.BytesIO(receive_daily_file_name_response["Body"].read()), encoding='utf-8')
match_count = 0 match_count = 0
row_count = sum(1 for line in io.BytesIO(s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_DAILY_FILE_NAME_LIST_PATH)["Body"].read())) receive_daily_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_DAILY_FILE_NAME_LIST_PATH)
row_count = sum(1 for line in io.BytesIO(receive_daily_file_name_response["Body"].read()))
for row in csv.reader(receive_daily_file_name_body, delimiter='\t'): for row in csv.reader(receive_daily_file_name_body, delimiter='\t'):
file_exists = False file_exists = False
for file_name in file_list: for file_name in file_list:

3
lambda/sap-fin-receive-check-monthly/sap-fin-receive-check-monthly.py
View File

@ -127,7 +127,8 @@ def lambda_handler(event, context):
logger.info('I-04-04 取得したオブジェクトリストと月次I/Fファイルネーム設定ファイルの突き合わせを開始します') logger.info('I-04-04 取得したオブジェクトリストと月次I/Fファイルネーム設定ファイルの突き合わせを開始します')
receive_monthly_file_name_body = io.TextIOWrapper(io.BytesIO(receive_monthly_file_name_response["Body"].read()), encoding='utf-8') receive_monthly_file_name_body = io.TextIOWrapper(io.BytesIO(receive_monthly_file_name_response["Body"].read()), encoding='utf-8')
match_count = 0 match_count = 0
row_count = sum(1 for line in io.BytesIO(s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH)["Body"].read())) receive_monthly_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH)
row_count = sum(1 for line in io.BytesIO(receive_monthly_file_name_response["Body"].read()))
for row in csv.reader(receive_monthly_file_name_body, delimiter='\t'): for row in csv.reader(receive_monthly_file_name_body, delimiter='\t'):
file_exists = False file_exists = False
for file_name in file_list: for file_name in file_list:

3
lambda/sap-sup-receive-check-daily/sap-sup-receive-check-daily.py
View File

@ -123,7 +123,8 @@ def lambda_handler(event, context):
logger.info('I-04-04 取得したオブジェクトリストと日次I/Fファイルネーム設定ファイルの突き合わせを開始します') logger.info('I-04-04 取得したオブジェクトリストと日次I/Fファイルネーム設定ファイルの突き合わせを開始します')
receive_daily_file_name_body = io.TextIOWrapper(io.BytesIO(receive_daily_file_name_response["Body"].read()), encoding='utf-8') receive_daily_file_name_body = io.TextIOWrapper(io.BytesIO(receive_daily_file_name_response["Body"].read()), encoding='utf-8')
match_count = 0 match_count = 0
row_count = sum(1 for line in io.BytesIO(s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_DAILY_FILE_NAME_LIST_PATH)["Body"].read())) receive_daily_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_DAILY_FILE_NAME_LIST_PATH)
row_count = sum(1 for line in io.BytesIO(receive_daily_file_name_response["Body"].read()))
for row in csv.reader(receive_daily_file_name_body, delimiter='\t'): for row in csv.reader(receive_daily_file_name_body, delimiter='\t'):
file_exists = False file_exists = False
for file_name in file_list: for file_name in file_list:

3
lambda/sap-sup-receive-check-monthly/sap-sup-receive-check-monthly.py
View File

@ -126,7 +126,8 @@ def lambda_handler(event, context):
logger.info('I-04-04 取得したオブジェクトリストと月次I/Fファイルネーム設定ファイルの突き合わせを開始します') logger.info('I-04-04 取得したオブジェクトリストと月次I/Fファイルネーム設定ファイルの突き合わせを開始します')
receive_monthly_file_name_body = io.TextIOWrapper(io.BytesIO(receive_monthly_file_name_response["Body"].read()), encoding='utf-8') receive_monthly_file_name_body = io.TextIOWrapper(io.BytesIO(receive_monthly_file_name_response["Body"].read()), encoding='utf-8')
match_count = 0 match_count = 0
row_count = sum(1 for line in io.BytesIO(s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH)["Body"].read())) receive_monthly_file_name_response = s3_client.get_object(Bucket=CONFIG_BUCKET_NAME, Key=RECEIVE_MONTHLY_FILE_NAME_LIST_PATH)
row_count = sum(1 for line in io.BytesIO(receive_monthly_file_name_response["Body"].read()))
for row in csv.reader(receive_monthly_file_name_body, delimiter='\t'): for row in csv.reader(receive_monthly_file_name_body, delimiter='\t'):
file_exists = False file_exists = False
for file_name in file_list: for file_name in file_list: