From 4431ca3e4224ba3673b926d96ade41e9ba95d054 Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 15 Sep 2022 14:47:12 +0900 Subject: [PATCH 1/8] =?UTF-8?q?feat:setting=5Fdb=5Fcolumn=5Flist=E3=81=A8l?= =?UTF-8?q?ine=E3=81=A7=E4=BD=BF=E3=82=8F=E3=82=8C=E3=81=A6=E3=81=84?= =?UTF-8?q?=E3=82=8B=E5=A4=89=E6=95=B0=E3=80=8Ci=E3=80=8D=E3=81=AE?= =?UTF-8?q?=E9=87=8D=E8=A4=87=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 94379370..221a7132 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -129,8 +129,8 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # SQL文生成 sql = f'INSERT INTO {settings_list[SETTINGS_ITEM["loadSchemaName"]]} (' - for i in range(len(settings_db_columu_list)): - sql = f'{sql} {settings_db_columu_list[i]},' + for db_column in settings_db_columu_list: + sql = f'{sql} {db_column},' sql = f'{sql} file_name,' # システム項目：取込ファイル名 sql = f'{sql} file_row_cnt,' # システム項目：取込ファイル行番号 sql = f'{sql} delete_flg,' # システム項目：論理削除フラグ From e0a1f7365bf803f5283cd3b2afd6722717e9e0ef Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 15 Sep 2022 14:48:15 +0900 Subject: [PATCH 2/8] =?UTF-8?q?fix:=E3=82=BC=E3=83=AD=E3=83=90=E3=82=A4?= =?UTF-8?q?=E3=83=88=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF=E3=81=AE=E5=88=A4?= =?UTF-8?q?=E6=96=AD=E6=96=B9=E6=B3=95=E3=81=AE=E4=B8=8D=E5=82=99=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/chk.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ecs/dataimport/dataimport/chk.py b/ecs/dataimport/dataimport/chk.py index 263008cd..52e1cb4a 100644 --- a/ecs/dataimport/dataimport/chk.py +++ b/ecs/dataimport/dataimport/chk.py @@ -84,6 +84,9 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i work_data = io.TextIOWrapper(io.BytesIO(work_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]]) work_header_list = [] for line in csv.reader(work_data, quotechar=settings_list[SETTINGS_ITEM["quotechar"]], delimiter=settings_list[SETTINGS_ITEM["delimiter"]]): + if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == True: + continue + work_header_list = line break From 6c2524fbe0d6a91922924a0b665270074c44e73a Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 15 Sep 2022 17:46:33 +0900 Subject: [PATCH 3/8] =?UTF-8?q?fix:=E5=9B=B2=E3=81=84=E6=96=87=E5=AD=97?= =?UTF-8?q?=E3=81=AE=E8=A8=AD=E5=AE=9A=E5=80=A4=E3=81=8C=E7=A9=BA=E6=96=87?= =?UTF-8?q?=E5=AD=97=E3=81=BE=E3=81=9F=E3=81=AF=E5=8D=8A=E8=A7=92=E3=82=B9?= =?UTF-8?q?=E3=83=9A=E3=83=BC=E3=82=B9=E3=81=AE=E5=A0=B4=E5=90=88=E3=80=8C?= =?UTF-8?q?None=E3=80=8D=E3=81=AB=E5=A4=89=E6=8F=9B=E3=81=99=E3=82=8B?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/chk.py | 9 +++++---- ecs/dataimport/dataimport/common.py | 16 ++++++++++++++++ ecs/dataimport/dataimport/main.py | 3 ++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/ecs/dataimport/dataimport/chk.py b/ecs/dataimport/dataimport/chk.py index 52e1cb4a..d3d8801c 100644 --- a/ecs/dataimport/dataimport/chk.py +++ b/ecs/dataimport/dataimport/chk.py @@ -6,6 +6,7 @@ import sys from end import end from error import error from common import debug_log +from common import convert_quotechar # 定数 DIRECTORY_WORK = '/work/' @@ -82,12 +83,12 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i work_obj = s3_resource.Object(bucket_name, work_key) work_response = work_obj.get() work_data = io.TextIOWrapper(io.BytesIO(work_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]]) - work_header_list = [] - for line in csv.reader(work_data, quotechar=settings_list[SETTINGS_ITEM["quotechar"]], delimiter=settings_list[SETTINGS_ITEM["delimiter"]]): - if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == True: + work_header_list = [] + for i, line in enumerate(csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]])): + if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == True and i == 0: + work_header_list = line continue - work_header_list = line break # ② C-0のデータ件数チェックを開始する diff --git a/ecs/dataimport/dataimport/common.py b/ecs/dataimport/dataimport/common.py index 8b8eeed6..946defe4 100644 --- a/ecs/dataimport/dataimport/common.py +++ b/ecs/dataimport/dataimport/common.py @@ -1,5 +1,6 @@ from datetime import datetime + # 定数 LOG_LEVEL = {"d": 'Debug'} MODE_TYPE = { @@ -11,3 +12,18 @@ MODE_TYPE = { def debug_log(log, log_info, mode): if MODE_TYPE['d'] == mode: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["d"]} {log}') + +def convert_quotechar(quotechar): + """csvモジュールの囲い文字を変換する + + Args: + quotechar : 項目囲い文字の設定値 + + Returns: + 空文字、空白文字の場合→None + それ以外→設定値をそのまま帰す + """ + if (quotechar.strip(' ') == ''): + return None + + return quotechar diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 221a7132..c0894209 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -7,6 +7,7 @@ import io import csv from error import error from common import debug_log +from common import convert_quotechar # 定数 DIRECTORY_WORK = '/work/' @@ -117,7 +118,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf settings_db_columu_list = settings_list[SETTINGS_ITEM["dbColumuName"]].rstrip().split(',') settings_replace_comma_list = settings_list[SETTINGS_ITEM["commaReplaceColumns"]].rstrip().split(',') - for line in csv.reader(work_data, quotechar=settings_list[SETTINGS_ITEM["quotechar"]], delimiter=settings_list[SETTINGS_ITEM["delimiter"]]): + for line in csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]]): try: if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == True and index == 0: index += 1 From 23727bdf45a4713e8030a5eb5fad15c41549efbf Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 15 Sep 2022 17:52:41 +0900 Subject: [PATCH 4/8] =?UTF-8?q?fix:=E3=83=87=E3=83=BC=E3=82=BF=E5=8F=96?= =?UTF-8?q?=E8=BE=BC=E3=81=AE=E5=9B=B2=E3=81=84=E6=96=87=E5=AD=97=E3=81=AE?= =?UTF-8?q?=E8=A8=AD=E5=AE=9A=E5=80=A4=E4=BF=AE=E6=AD=A3=E5=AF=BE=E5=BF=9C?= =?UTF-8?q?=E3=81=AB=E3=82=88=E3=82=8B=E3=80=81=E8=A8=AD=E5=AE=9A=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/SAP_fin/settings/CostReport.txt | 2 +- s3/data/SAP_fin/settings/IOReport.txt | 2 +- s3/data/SAP_fin/settings/Invoice.txt | 2 +- s3/data/SAP_fin/settings/WBSList.txt | 2 +- s3/data/SAP_fin/settings/WBSReport.txt | 2 +- s3/data/SAP_sup/settings/ConfReport.txt | 2 +- s3/data/SAP_sup/settings/GMReport.txt | 2 +- s3/data/SAP_sup/settings/GRReport.txt | 2 +- s3/data/SAP_sup/settings/MLCReport.txt | 2 +- s3/data/SAP_sup/settings/POReport.txt | 2 +- s3/data/SAP_sup/settings/QAReport.txt | 2 +- s3/data/SAP_sup/settings/StockList.txt | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/s3/data/SAP_fin/settings/CostReport.txt b/s3/data/SAP_fin/settings/CostReport.txt index 82e3a04b..be3dcc5e 100644 --- a/s3/data/SAP_fin/settings/CostReport.txt +++ b/s3/data/SAP_fin/settings/CostReport.txt @@ -1,7 +1,7 @@ SAP_fin utf-8 - + LF 1 18 diff --git a/s3/data/SAP_fin/settings/IOReport.txt b/s3/data/SAP_fin/settings/IOReport.txt index fb3dbda9..f7023c20 100644 --- a/s3/data/SAP_fin/settings/IOReport.txt +++ b/s3/data/SAP_fin/settings/IOReport.txt @@ -1,7 +1,7 @@ SAP_fin utf-8 - + LF 1 18 diff --git a/s3/data/SAP_fin/settings/Invoice.txt b/s3/data/SAP_fin/settings/Invoice.txt index 582e8df1..da1bb360 100644 --- a/s3/data/SAP_fin/settings/Invoice.txt +++ b/s3/data/SAP_fin/settings/Invoice.txt @@ -1,7 +1,7 @@ SAP_fin utf-8 - + LF 1 25 diff --git a/s3/data/SAP_fin/settings/WBSList.txt b/s3/data/SAP_fin/settings/WBSList.txt index ad66730e..1729a641 100644 --- a/s3/data/SAP_fin/settings/WBSList.txt +++ b/s3/data/SAP_fin/settings/WBSList.txt @@ -1,7 +1,7 @@ SAP_fin utf-8 - + LF 1 10 diff --git a/s3/data/SAP_fin/settings/WBSReport.txt b/s3/data/SAP_fin/settings/WBSReport.txt index 1d622c2e..1a899d2d 100644 --- a/s3/data/SAP_fin/settings/WBSReport.txt +++ b/s3/data/SAP_fin/settings/WBSReport.txt @@ -1,7 +1,7 @@ SAP_fin utf-8 - + LF 1 18 diff --git a/s3/data/SAP_sup/settings/ConfReport.txt b/s3/data/SAP_sup/settings/ConfReport.txt index 706e64d7..cd11e609 100644 --- a/s3/data/SAP_sup/settings/ConfReport.txt +++ b/s3/data/SAP_sup/settings/ConfReport.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 11 diff --git a/s3/data/SAP_sup/settings/GMReport.txt b/s3/data/SAP_sup/settings/GMReport.txt index 920ee02c..a9b15d1e 100644 --- a/s3/data/SAP_sup/settings/GMReport.txt +++ b/s3/data/SAP_sup/settings/GMReport.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 13 diff --git a/s3/data/SAP_sup/settings/GRReport.txt b/s3/data/SAP_sup/settings/GRReport.txt index 3ddaec9d..d43f139e 100644 --- a/s3/data/SAP_sup/settings/GRReport.txt +++ b/s3/data/SAP_sup/settings/GRReport.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 24 diff --git a/s3/data/SAP_sup/settings/MLCReport.txt b/s3/data/SAP_sup/settings/MLCReport.txt index 26953eac..bd2cbfd7 100644 --- a/s3/data/SAP_sup/settings/MLCReport.txt +++ b/s3/data/SAP_sup/settings/MLCReport.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 24 diff --git a/s3/data/SAP_sup/settings/POReport.txt b/s3/data/SAP_sup/settings/POReport.txt index 27dfbc37..ba46619b 100644 --- a/s3/data/SAP_sup/settings/POReport.txt +++ b/s3/data/SAP_sup/settings/POReport.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 12 diff --git a/s3/data/SAP_sup/settings/QAReport.txt b/s3/data/SAP_sup/settings/QAReport.txt index e9beb8ff..1cb09f0f 100644 --- a/s3/data/SAP_sup/settings/QAReport.txt +++ b/s3/data/SAP_sup/settings/QAReport.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 24 diff --git a/s3/data/SAP_sup/settings/StockList.txt b/s3/data/SAP_sup/settings/StockList.txt index f571083c..eb6cef3b 100644 --- a/s3/data/SAP_sup/settings/StockList.txt +++ b/s3/data/SAP_sup/settings/StockList.txt @@ -1,7 +1,7 @@ SAP_sup utf-8 - + LF 1 13 From 3a177ae18c5585bd54984f793cf8156811fc73f8 Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Fri, 16 Sep 2022 09:26:33 +0900 Subject: [PATCH 5/8] =?UTF-8?q?feat:=E4=B8=8B=E7=94=B0=E3=81=95=E3=82=93?= =?UTF-8?q?=E3=81=AE=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC=E6=8C=87=E6=91=98?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/chk.py | 8 ++++---- ecs/dataimport/dataimport/common.py | 1 - ecs/dataimport/dataimport/main.py | 3 +-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ecs/dataimport/dataimport/chk.py b/ecs/dataimport/dataimport/chk.py index d3d8801c..125dbb2a 100644 --- a/ecs/dataimport/dataimport/chk.py +++ b/ecs/dataimport/dataimport/chk.py @@ -5,8 +5,7 @@ import csv import sys from end import end from error import error -from common import debug_log -from common import convert_quotechar +from common import debug_log, convert_quotechar # 定数 DIRECTORY_WORK = '/work/' @@ -83,12 +82,13 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i work_obj = s3_resource.Object(bucket_name, work_key) work_response = work_obj.get() work_data = io.TextIOWrapper(io.BytesIO(work_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]]) - work_header_list = [] + work_header_list = [] + # ヘッダ行のみのファイルは、0バイトファイル同等とみなす for i, line in enumerate(csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]])): if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == True and i == 0: work_header_list = line continue - + break # ② C-0のデータ件数チェックを開始する diff --git a/ecs/dataimport/dataimport/common.py b/ecs/dataimport/dataimport/common.py index 946defe4..1361ff53 100644 --- a/ecs/dataimport/dataimport/common.py +++ b/ecs/dataimport/dataimport/common.py @@ -1,6 +1,5 @@ from datetime import datetime - # 定数 LOG_LEVEL = {"d": 'Debug'} MODE_TYPE = { diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index c0894209..23932c6f 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -6,8 +6,7 @@ from pymysql.constants import CLIENT import io import csv from error import error -from common import debug_log -from common import convert_quotechar +from common import debug_log, convert_quotechar # 定数 DIRECTORY_WORK = '/work/' From cda1a79e7ab0be67b8a776752a810a908ebdcc5e Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 16 Sep 2022 12:47:14 +0900 Subject: [PATCH 6/8] =?UTF-8?q?fix:=200=E3=83=90=E3=82=A4=E3=83=88?= =?UTF-8?q?=E5=88=A4=E5=AE=9A=E3=82=92=E4=BF=AE=E6=AD=A3=E3=80=82=E3=83=98?= =?UTF-8?q?=E3=83=83=E3=83=80=E8=A1=8C=E3=81=AF=E5=88=A5=E3=81=AE=E3=81=A8?= =?UTF-8?q?=E3=81=93=E3=82=8D=E3=81=A7=E3=82=82=E4=BD=BF=E3=81=A3=E3=81=A6?= =?UTF-8?q?=E3=81=84=E3=81=9F=E3=81=9F=E3=82=81=E3=80=81=E5=88=A4=E5=AE=9A?= =?UTF-8?q?=E5=87=A6=E7=90=86=E3=81=AE=E3=81=BF=E9=96=A2=E6=95=B0=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit （本当は他のも関数化したいけど我慢） --- ecs/dataimport/dataimport/chk.py | 37 +++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/ecs/dataimport/dataimport/chk.py b/ecs/dataimport/dataimport/chk.py index 125dbb2a..9358db30 100644 --- a/ecs/dataimport/dataimport/chk.py +++ b/ecs/dataimport/dataimport/chk.py @@ -1,11 +1,13 @@ -from datetime import datetime -import boto3 -import io import csv +import io import sys +from datetime import datetime + +import boto3 + +from common import convert_quotechar, debug_log from end import end from error import error -from common import debug_log, convert_quotechar # 定数 DIRECTORY_WORK = '/work/' @@ -83,17 +85,13 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i work_response = work_obj.get() work_data = io.TextIOWrapper(io.BytesIO(work_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]]) work_header_list = [] - # ヘッダ行のみのファイルは、0バイトファイル同等とみなす for i, line in enumerate(csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]])): - if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == True and i == 0: - work_header_list = line - continue - + work_header_list = line break # ② C-0のデータ件数チェックを開始する print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-02 - C-0のチェックを開始します') - if not len(work_header_list): + if is_empty_file(work_header_list, settings_list): print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-03 - 投入ファイルが０バイトのため処理を終了します') end(bucket_name, target_data_source, target_file_name, '', log_info, mode) print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-04 - 終了処理完了') @@ -126,3 +124,22 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-CHK-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) + + +def is_empty_file(work_header_list: list, settings_list: list): + """② C-0のデータ件数チェック + ヘッダ行がある場合は、1行目を読み飛ばして判定する + + Args: + work_header_list (list): CSVファイルの1行目 + settings_list (list): 個別設定ファイルのリスト + + Returns: + bool: CSVファイルの1行目が0件だった場合はTrue + """ + has_header = settings_list[SETTINGS_ITEM["headerFlag"]] + # ヘッダのみのファイルも0バイトファイルをみなす + if has_header: + return len(work_header_list[1:]) == 0 + + return len(work_header_list) == 0 From 89b5028919eab6fc6b43b8ce9fe47f185f6dd285 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 16 Sep 2022 12:48:35 +0900 Subject: [PATCH 7/8] =?UTF-8?q?refactor:=20=E6=B7=BB=E5=AD=97=E4=BD=BF?= =?UTF-8?q?=E3=82=8F=E3=81=AA=E3=81=8F=E3=81=AA=E3=81=A3=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/chk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/dataimport/dataimport/chk.py b/ecs/dataimport/dataimport/chk.py index 9358db30..b8df1eb0 100644 --- a/ecs/dataimport/dataimport/chk.py +++ b/ecs/dataimport/dataimport/chk.py @@ -85,7 +85,7 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i work_response = work_obj.get() work_data = io.TextIOWrapper(io.BytesIO(work_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]]) work_header_list = [] - for i, line in enumerate(csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]])): + for line in csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]]): work_header_list = line break From 22c68fa7a52ba77c1e6559a804e62b002bd39427 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Fri, 16 Sep 2022 13:37:59 +0900 Subject: [PATCH 8/8] =?UTF-8?q?fix:=20=E3=83=98=E3=83=83=E3=83=80=E3=82=92?= =?UTF-8?q?=E5=90=AB=E3=82=93=E3=81=A00Byte=E5=88=A4=E5=AE=9A=E3=81=AE?= =?UTF-8?q?=E8=AA=A4=E3=82=8A=E3=82=92=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/chk.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/ecs/dataimport/dataimport/chk.py b/ecs/dataimport/dataimport/chk.py index b8df1eb0..d90e2fb5 100644 --- a/ecs/dataimport/dataimport/chk.py +++ b/ecs/dataimport/dataimport/chk.py @@ -84,14 +84,18 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i work_obj = s3_resource.Object(bucket_name, work_key) work_response = work_obj.get() work_data = io.TextIOWrapper(io.BytesIO(work_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]]) - work_header_list = [] - for line in csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]]): - work_header_list = line + work_csv_row = [] + for i, line in enumerate(csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]])): + # ヘッダあり、かつ、1行目の場合 + if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == 1 and i == 0: + work_csv_row.append(line) + continue + work_csv_row.append(line) break # ② C-0のデータ件数チェックを開始する print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-02 - C-0のチェックを開始します') - if is_empty_file(work_header_list, settings_list): + if is_empty_file(work_csv_row, settings_list): print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-03 - 投入ファイルが０バイトのため処理を終了します') end(bucket_name, target_data_source, target_file_name, '', log_info, mode) print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-04 - 終了処理完了') @@ -100,16 +104,17 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i # ③ C-1の項目数チェックを開始する print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-06 - C-1のチェックを開始します') - work_header_list_len = len(work_header_list) - if work_header_list_len == int(settings_list[SETTINGS_ITEM["csvNumItems"]]): + work_csv_row_item_len = len(work_csv_row[0]) + if work_csv_row_item_len == int(settings_list[SETTINGS_ITEM["csvNumItems"]]): print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-07 - C-1：正常終了') else: - raise CheckError(f'E-CHK-01 - 項目数が一致しません 個別設定ファイル項目数：{settings_list[SETTINGS_ITEM["csvNumItems"]]} 投入データ項目数：{work_header_list_len}') + raise CheckError(f'E-CHK-01 - 項目数が一致しません 個別設定ファイル項目数：{settings_list[SETTINGS_ITEM["csvNumItems"]]} 投入データ項目数：{work_csv_row_item_len}') # ④ C-2の項目並び順チェック開始する if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == True: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-08 - C-2のチェックを開始します') settings_header_list = settings_list[SETTINGS_ITEM["csvNameItems"]].rstrip().split(',') + work_header_list = work_csv_row[0] for i in range(len(settings_header_list)): if not settings_header_list[i] == work_header_list[i]: raise CheckError(f'E-CHK-02 - 項目順序が一致しません {i + 1}番目の項目 個別設定ファイル項目：{settings_header_list[i]} 投入データ項目：{work_header_list[i]}') @@ -126,20 +131,20 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i error(bucket_name, target_data_source, target_file_name, log_info) -def is_empty_file(work_header_list: list, settings_list: list): +def is_empty_file(work_csv_row: list, settings_list: list): """② C-0のデータ件数チェック ヘッダ行がある場合は、1行目を読み飛ばして判定する Args: - work_header_list (list): CSVファイルの1行目 + work_csv_row (list): CSVファイルの1行目(ヘッダを含む場合は2行目まで) settings_list (list): 個別設定ファイルのリスト Returns: bool: CSVファイルの1行目が0件だった場合はTrue """ - has_header = settings_list[SETTINGS_ITEM["headerFlag"]] + has_header = int(settings_list[SETTINGS_ITEM["headerFlag"]]) == 1 # ヘッダのみのファイルも0バイトファイルをみなす if has_header: - return len(work_header_list[1:]) == 0 + return len(work_csv_row[1:]) == 0 - return len(work_header_list) == 0 + return len(work_csv_row) == 0