nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: データ登録処理 チェック処理内のresourceインタフェースをクライアントインタフェースに変更

Browse Source
This commit is contained in:
shimoda.m@nds-tyo.co.jp 2023-01-27 16:19:44 +09:00
parent 1a1499a189
commit d5c6485a72
1 changed files with 5 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
ecs/dataimport/dataimport/chk.py
View File

@ -4,7 +4,6 @@ import sys
from datetime import datetime
import boto3
from common import convert_quotechar, debug_log
from end import end
from error import error
@ -41,7 +40,7 @@ LINE_FEED_CODE = {
}
# クラス変数
s3_resource = boto3.resource('s3')
s3_client = boto3.client('s3')
# チェック例外クラス
@ -74,16 +73,14 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-01 - チェック処理を開始します')
# データ読込
settings_obj = s3_resource.Object(bucket_name, settings_key)
settings_response = settings_obj.get()
settings_obj_response = s3_client.get_object(Bucket=bucket_name, Key=settings_key)
settings_list = []
for line in io.TextIOWrapper(io.BytesIO(settings_response["Body"].read()), encoding='utf-8'):
for line in io.TextIOWrapper(io.BytesIO(settings_obj_response["Body"].read()), encoding='utf-8'):
settings_list.append(line.rstrip('\n'))
work_key = target_data_source + DIRECTORY_WORK + target_file_name
work_obj = s3_resource.Object(bucket_name, work_key)
work_response = work_obj.get()
work_data = io.TextIOWrapper(io.BytesIO(work_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]])
work_obj_response = s3_client.get_object(Bucket=bucket_name, Key=work_key)
work_data = io.TextIOWrapper(io.BytesIO(work_obj_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]])
work_csv_row = []
for i, line in enumerate(csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]])):
# ヘッダあり、かつ、1行目の場合