From b8975ad45334d82baba222b9e2ae95fa50f325c8 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Mon, 30 Jan 2023 11:17:41 +0900 Subject: [PATCH] =?UTF-8?q?feat:=20=E3=83=87=E3=83=BC=E3=82=BF=E7=99=BB?= =?UTF-8?q?=E9=8C=B2=E5=87=A6=E7=90=86=20=E3=83=A1=E3=82=A4=E3=83=B3?= =?UTF-8?q?=E5=87=A6=E7=90=86=E5=86=85=E3=81=AEresource=E3=82=A4=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9=E3=82=92=E3=82=AF?= =?UTF-8?q?=E3=83=A9=E3=82=A4=E3=82=A2=E3=83=B3=E3=83=88=E3=82=A4=E3=83=B3?= =?UTF-8?q?=E3=82=BF=E3=83=95=E3=82=A7=E3=83=BC=E3=82=B9=E3=81=AB=E5=A4=89?= =?UTF-8?q?=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index ef90f598..3233472c 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -5,10 +5,9 @@ from datetime import datetime import boto3 import pymysql -from pymysql.constants import CLIENT - from common import convert_quotechar, debug_log from error import error +from pymysql.constants import CLIENT # 定数 DIRECTORY_WORK = '/work/' @@ -47,7 +46,6 @@ INVALID_CONFIG_EXCEPTION_MESSAGE = f'個別設定ファイルのインポート # クラス変数 s3_client = boto3.client('s3') -s3_resource = boto3.resource('s3') def main(bucket_name, target_data_source, target_file_name, settings_key, db_info, log_info, mode): @@ -91,8 +89,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # ④ 個別設定ファイルのロードスキーマのテーブル名に記載されているテーブルをTRUNCATEする # 個別設定ファイルの読み込み - settings_obj = s3_resource.Object(bucket_name, settings_key) - settings_response = settings_obj.get() + settings_response = s3_client.get_object(Bucket=bucket_name, Key=settings_key) settings_list = [] for line in io.TextIOWrapper(io.BytesIO(settings_response["Body"].read()), encoding='utf-8'): settings_list.append(line.rstrip('\n')) @@ -110,8 +107,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # ⑤ 投入データファイルを１行ごとにループする print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-05 - 投入データ {target_file_name} の読み込みを開始します') work_key = target_data_source + DIRECTORY_WORK + target_file_name - work_obj = s3_resource.Object(bucket_name, work_key) - work_response = work_obj.get() + work_response = s3_client.get_object(Bucket=bucket_name, Key=work_key) work_data = io.TextIOWrapper(io.BytesIO(work_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]]) process_count = 0 # 処理件数カウンタ @@ -261,10 +257,9 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf try: if ex_sql_file_exists: # 拡張SQLファイルからSQL文生成 - ex_sqls_obj = s3_resource.Object(bucket_name, ex_sql_key) - ex_sql_response = ex_sqls_obj.get() + ex_sql_obj_response = s3_client.get_object(Bucket=bucket_name, Key=ex_sql_key) ex_sql = '' - for line in io.TextIOWrapper(io.BytesIO(ex_sql_response["Body"].read()), encoding='utf-8'): + for line in io.TextIOWrapper(io.BytesIO(ex_sql_obj_response["Body"].read()), encoding='utf-8'): ex_sql = f'{ex_sql} {line.rstrip()}' # トランザクション開始 @@ -358,3 +353,18 @@ def truncate_judge(settings_list): class InvalidConfigException(Exception): pass + + +# ローカル実行用コード +# 値はよしなに変えてください +# if __name__ == '__main__': +# DB_INFO = {"host": '127.0.0.1', "name": 'org02', "pass": 'user', "user": 'user'} +# main( +# bucket_name='バケット名', +# target_data_source='投入データのディレクトリ名よりデータソースに該当する部分', +# target_file_name='投入データのファイル名', +# settings_key='投入データに該当する個別設定ファイルのフルパス', +# db_info=DB_INFO, +# log_info='info', +# mode='i' +# )