From 4226ad9db1dc9d07cb3798b4f6da5e28e1958e59 Mon Sep 17 00:00:00 2001 From: "x.azuma.m@nds-tyo.co.jp" Date: Mon, 8 May 2023 18:13:23 +0900 Subject: [PATCH] =?UTF-8?q?org=E3=81=AE=E3=83=86=E3=83=BC=E3=83=96?= =?UTF-8?q?=E3=83=AB=E3=81=ABLOAD=20DATA=20LOCAL=20INFILE=E3=81=8C?= =?UTF-8?q?=E9=80=9A=E3=81=A3=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/vjsk/vjsk_data_load_manager.py | 9 +++------ ecs/jskult-batch-daily/src/db/database.py | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py index 9aa80070..af35a2aa 100644 --- a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py +++ b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py @@ -23,13 +23,10 @@ class JjskDataLoadManager: # orgをtruncate db.execute(f"TRUNCATE TABLE {table_name_org};") - # load DATA local infileステートメント実行許可設定 - db.execute("SET GLOBAL local_infile=on;") - # orgにload　※warningは1148エラーになるらしい - sql = f"LOAD DATA LOCAL INFILE '{src_file_name}' INTO TABLE {table_name_org} FIELDS TERMINATED BY '\\t' ENCLOSED BY \"'\" IGNORE 1 LINES;" - cnt = db.execute(sql) - logger.info(f'tsvデータをorgテーブルにLOAD : 件数({cnt})') + sql = f"LOAD DATA LOCAL INFILE :src_file_name INTO TABLE {table_name_org} FIELDS TERMINATED BY '\\t' ENCLOSED BY '\"' IGNORE 1 LINES;" + result = db.execute(sql, {"src_file_name": src_file_name}) + logger.info(f'tsvデータをorgテーブルにLOAD : 件数({result.rowcount})') # org→srcにinsert select # TODO: INTO句とSELECT句はmapperに持たせてcondkeyで引っ張ってくるようにしたい diff --git a/ecs/jskult-batch-daily/src/db/database.py b/ecs/jskult-batch-daily/src/db/database.py index f67a21b9..b67c3a3d 100644 --- a/ecs/jskult-batch-daily/src/db/database.py +++ b/ecs/jskult-batch-daily/src/db/database.py @@ -44,7 +44,7 @@ class Database: host=self.__host, port=self.__port, database=self.__schema, - query={"charset": "utf8mb4"} + query={"charset": "utf8mb4", "local_infile": "1"}, ) self.__engine = create_engine(