nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #102 feature-NEWDWH2021-724 into develop-8sap

Browse Source
This commit is contained in:
下田雅人 2022-09-16 14:48:00 +09:00
commit ba80d5b900
15 changed files with 68 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
ecs/dataimport/dataimport/chk.py
View File

@ -1,11 +1,13 @@
from datetime import datetime
import boto3
import io
import csv
import io
import sys
from datetime import datetime
import boto3
from common import convert_quotechar, debug_log
from end import end
from error import error
from common import debug_log
# 定数
DIRECTORY_WORK = '/work/'
@ -82,14 +84,18 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i
work_obj = s3_resource.Object(bucket_name, work_key)
work_response = work_obj.get()
work_data = io.TextIOWrapper(io.BytesIO(work_response["Body"].read()), encoding=settings_list[SETTINGS_ITEM["charCode"]], newline=LINE_FEED_CODE[settings_list[SETTINGS_ITEM["lineFeedCode"]]])
work_header_list = []
for line in csv.reader(work_data, quotechar=settings_list[SETTINGS_ITEM["quotechar"]], delimiter=settings_list[SETTINGS_ITEM["delimiter"]]):
work_header_list = line
work_csv_row = []
for i, line in enumerate(csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]])):
# ヘッダあり、かつ、1行目の場合
if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == 1 and i == 0:
work_csv_row.append(line)
continue
work_csv_row.append(line)
break
# ② C-0のデータ件数チェックを開始する
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-02 - C-0のチェックを開始します')
if not len(work_header_list):
if is_empty_file(work_csv_row, settings_list):
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-03 - 投入ファイルが０バイトのため処理を終了します')
end(bucket_name, target_data_source, target_file_name, '', log_info, mode)
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-04 - 終了処理完了')
@ -98,16 +104,17 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i
# ③ C-1の項目数チェックを開始する
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-06 - C-1のチェックを開始します')
work_header_list_len = len(work_header_list)
if work_header_list_len == int(settings_list[SETTINGS_ITEM["csvNumItems"]]):
work_csv_row_item_len = len(work_csv_row[0])
if work_csv_row_item_len == int(settings_list[SETTINGS_ITEM["csvNumItems"]]):
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-07 - C-1：正常終了')
else:
raise CheckError(f'E-CHK-01 - 項目数が一致しません 個別設定ファイル項目数：{settings_list[SETTINGS_ITEM["csvNumItems"]]} 投入データ項目数：{work_header_list_len}')
raise CheckError(f'E-CHK-01 - 項目数が一致しません 個別設定ファイル項目数：{settings_list[SETTINGS_ITEM["csvNumItems"]]} 投入データ項目数：{work_csv_row_item_len}')
# ④ C-2の項目並び順チェック開始する
if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == True:
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-CHK-08 - C-2のチェックを開始します')
settings_header_list = settings_list[SETTINGS_ITEM["csvNameItems"]].rstrip().split(',')
work_header_list = work_csv_row[0]
for i in range(len(settings_header_list)):
if not settings_header_list[i] == work_header_list[i]:
raise CheckError(f'E-CHK-02 - 項目順序が一致しません {i + 1}番目の項目 個別設定ファイル項目：{settings_header_list[i]} 投入データ項目：{work_header_list[i]}')
@ -122,3 +129,22 @@ def check(bucket_name, target_data_source, target_file_name, settings_key, log_i
except Exception as e:
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-CHK-99 - エラー内容：{e}')
error(bucket_name, target_data_source, target_file_name, log_info)
def is_empty_file(work_csv_row: list, settings_list: list):
"""② C-0のデータ件数チェック
ヘッダ行がある場合は、1行目を読み飛ばして判定する
Args:
work_csv_row (list): CSVファイルの1行目(ヘッダを含む場合は2行目まで)
settings_list (list): 個別設定ファイルのリスト
Returns:
bool: CSVファイルの1行目が0件だった場合はTrue
"""
has_header = int(settings_list[SETTINGS_ITEM["headerFlag"]]) == 1
# ヘッダのみのファイルも0バイトファイルをみなす
if has_header:
return len(work_csv_row[1:]) == 0
return len(work_csv_row) == 0

15
ecs/dataimport/dataimport/common.py
View File

@ -11,3 +11,18 @@ MODE_TYPE = {
def debug_log(log, log_info, mode):
if MODE_TYPE['d'] == mode:
print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["d"]} {log}')
def convert_quotechar(quotechar):
"""csvモジュールの囲い文字を変換する
Args:
quotechar : 項目囲い文字の設定値
Returns:
空文字、空白文字の場合→None
それ以外→設定値をそのまま帰す
"""
if (quotechar.strip(' ') == ''):
return None
return quotechar

8
ecs/dataimport/dataimport/main.py
View File

@ -6,7 +6,7 @@ from pymysql.constants import CLIENT
import io
import csv
from error import error
from common import debug_log
from common import debug_log, convert_quotechar
# 定数
DIRECTORY_WORK = '/work/'
@ -117,7 +117,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf
settings_db_columu_list = settings_list[SETTINGS_ITEM["dbColumuName"]].rstrip().split(',')
settings_replace_comma_list = settings_list[SETTINGS_ITEM["commaReplaceColumns"]].rstrip().split(',')
for line in csv.reader(work_data, quotechar=settings_list[SETTINGS_ITEM["quotechar"]], delimiter=settings_list[SETTINGS_ITEM["delimiter"]]):
for line in csv.reader(work_data, quotechar=convert_quotechar(settings_list[SETTINGS_ITEM["quotechar"]]), delimiter=settings_list[SETTINGS_ITEM["delimiter"]]):
try:
if int(settings_list[SETTINGS_ITEM["headerFlag"]]) == True and index == 0:
index += 1
@ -129,8 +129,8 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf
# SQL文生成
sql = f'INSERT INTO {settings_list[SETTINGS_ITEM["loadSchemaName"]]} ('
for i in range(len(settings_db_columu_list)):
sql = f'{sql} {settings_db_columu_list[i]},'
for db_column in settings_db_columu_list:
sql = f'{sql} {db_column},'
sql = f'{sql} file_name,' # システム項目：取込ファイル名
sql = f'{sql} file_row_cnt,' # システム項目：取込ファイル行番号
sql = f'{sql} delete_flg,' # システム項目：論理削除フラグ

2
s3/data/SAP_fin/settings/CostReport.txt
View File

@ -1,7 +1,7 @@
SAP_fin
utf-8
LF
1
18

2
s3/data/SAP_fin/settings/IOReport.txt
View File

@ -1,7 +1,7 @@
SAP_fin
utf-8
LF
1
18

2
s3/data/SAP_fin/settings/Invoice.txt
View File

@ -1,7 +1,7 @@
SAP_fin
utf-8
LF
1
25

2
s3/data/SAP_fin/settings/WBSList.txt
View File

@ -1,7 +1,7 @@
SAP_fin
utf-8
LF
1
10

2
s3/data/SAP_fin/settings/WBSReport.txt
View File

@ -1,7 +1,7 @@
SAP_fin
utf-8
LF
1
18

2
s3/data/SAP_sup/settings/ConfReport.txt
View File

@ -1,7 +1,7 @@
SAP_sup
utf-8
LF
1
11

2
s3/data/SAP_sup/settings/GMReport.txt
View File

@ -1,7 +1,7 @@
SAP_sup
utf-8
LF
1
13

2
s3/data/SAP_sup/settings/GRReport.txt
View File

@ -1,7 +1,7 @@
SAP_sup
utf-8
LF
1
24

2
s3/data/SAP_sup/settings/MLCReport.txt
View File

@ -1,7 +1,7 @@
SAP_sup
utf-8
LF
1
24

2
s3/data/SAP_sup/settings/POReport.txt
View File

@ -1,7 +1,7 @@
SAP_sup
utf-8
LF
1
12

2
s3/data/SAP_sup/settings/QAReport.txt
View File

@ -1,7 +1,7 @@
SAP_sup
utf-8
LF
1
24

2
s3/data/SAP_sup/settings/StockList.txt
View File

@ -1,7 +1,7 @@
SAP_sup
utf-8
LF
1
13