From fcdc50569be741d73f40de2e01a036987a41fe47 Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 14 Jul 2022 16:38:25 +0900 Subject: [PATCH 1/4] =?UTF-8?q?feat:=E8=A8=AD=E5=AE=9A=E3=83=95=E3=82=A1?= =?UTF-8?q?=E3=82=A4=E3=83=AB=E3=82=AB=E3=83=B3=E3=83=9E=E9=99=A4=E5=8E=BB?= =?UTF-8?q?=E5=AF=BE=E5=BF=9C=E3=81=AE=E3=81=9F=E3=82=81=E6=95=B0=E5=80=A4?= =?UTF-8?q?=E5=9E=8B=E3=81=AEDB=E3=82=AB=E3=83=A9=E3=83=A0=E7=89=A9?= =?UTF-8?q?=E7=90=86=E5=90=8D=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- s3/data/SAP_fin/settings/CostReport.txt | 1 + s3/data/SAP_fin/settings/IOReport.txt | 1 + s3/data/SAP_fin/settings/Invoice.txt | 1 + s3/data/SAP_fin/settings/WBSList.txt | 3 +-- s3/data/SAP_fin/settings/WBSReport.txt | 1 + s3/data/SAP_sup/settings/ConfReport.txt | 1 + s3/data/SAP_sup/settings/GMReport.txt | 1 + s3/data/SAP_sup/settings/GRReport.txt | 1 + s3/data/SAP_sup/settings/MLCReport.txt | 1 + s3/data/SAP_sup/settings/POReport.txt | 1 + s3/data/SAP_sup/settings/QAReport.txt | 1 + s3/data/SAP_sup/settings/StockList.txt | 1 + 12 files changed, 12 insertions(+), 2 deletions(-) diff --git a/s3/data/SAP_fin/settings/CostReport.txt b/s3/data/SAP_fin/settings/CostReport.txt index 98eb615b..a3b33f26 100644 --- a/s3/data/SAP_fin/settings/CostReport.txt +++ b/s3/data/SAP_fin/settings/CostReport.txt @@ -10,3 +10,4 @@ vt,fiscal_year,fiscal_period,cost_center,cost_element,cost_element_name,vbl_valu src03a.sapf_costreport org03a.sapf_costreport +vbl_value \ No newline at end of file diff --git a/s3/data/SAP_fin/settings/IOReport.txt b/s3/data/SAP_fin/settings/IOReport.txt index 0157c6e9..a2ab0774 100644 --- a/s3/data/SAP_fin/settings/IOReport.txt +++ b/s3/data/SAP_fin/settings/IOReport.txt @@ -10,3 +10,4 @@ fiscal_year,fiscal_period,internal_order,cost_element,cost_element_name,vbl_valu src03a.sapf_ioreport org03a.sapf_ioreport +vbl_value \ No newline at end of file diff --git a/s3/data/SAP_fin/settings/Invoice.txt b/s3/data/SAP_fin/settings/Invoice.txt index 61488a51..52ed7636 100644 --- a/s3/data/SAP_fin/settings/Invoice.txt +++ b/s3/data/SAP_fin/settings/Invoice.txt @@ -10,3 +10,4 @@ billing_type,condition_type,distribution_channel,invoiced_number,invoice_item,in src03b.sapf_invoice org03b.sapf_invoice +vbl_value \ No newline at end of file diff --git a/s3/data/SAP_fin/settings/WBSList.txt b/s3/data/SAP_fin/settings/WBSList.txt index a64e1390..9247ca7f 100644 --- a/s3/data/SAP_fin/settings/WBSList.txt +++ b/s3/data/SAP_fin/settings/WBSList.txt @@ -8,5 +8,4 @@ LF WBS element,Level,Profit center,Project Definition,Description,Control area,Responsibility,Basic finish date,Basic start date wbs_element,level,profit_center,project_definition,description,control_area,responsibility,basic_finish_date,basic_start_date src03a.sapf_wbslist -org03a.sapf_wbslist - +org03a.sapf_wbslist \ No newline at end of file diff --git a/s3/data/SAP_fin/settings/WBSReport.txt b/s3/data/SAP_fin/settings/WBSReport.txt index 91c75389..2d29d606 100644 --- a/s3/data/SAP_fin/settings/WBSReport.txt +++ b/s3/data/SAP_fin/settings/WBSReport.txt @@ -10,3 +10,4 @@ fiscal_year,fiscal_period,wbs_element,cost_element,cost_element_name,vbl_value,o src03a.sapf_wbsreport org03a.sapf_wbsreport +vbl_value \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/ConfReport.txt b/s3/data/SAP_sup/settings/ConfReport.txt index 299632cf..7c7ecefe 100644 --- a/s3/data/SAP_sup/settings/ConfReport.txt +++ b/s3/data/SAP_sup/settings/ConfReport.txt @@ -10,3 +10,4 @@ process_order,operation_activity,yield,uom,posting_date,finish_execution_date,ma src04.saps_confreport org04.saps_confreport +yield \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/GMReport.txt b/s3/data/SAP_sup/settings/GMReport.txt index c20b6aa9..c254e937 100644 --- a/s3/data/SAP_sup/settings/GMReport.txt +++ b/s3/data/SAP_sup/settings/GMReport.txt @@ -10,3 +10,4 @@ process_order,material_document,material_document_item,material,material_descrip src04.saps_gmreport org04.saps_gmreport +qty \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/GRReport.txt b/s3/data/SAP_sup/settings/GRReport.txt index 3ed56da7..aecae348 100644 --- a/s3/data/SAP_sup/settings/GRReport.txt +++ b/s3/data/SAP_sup/settings/GRReport.txt @@ -10,3 +10,4 @@ plant,material,material_description,batch,posting_date,qty_in_un_of_entry,moveme src04.saps_grreport org04.saps_grreport +qty_in_un_of_entry,total_valuated_stock_before_the_posting,quantity,amount \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/MLCReport.txt b/s3/data/SAP_sup/settings/MLCReport.txt index ed71a72f..db3af777 100644 --- a/s3/data/SAP_sup/settings/MLCReport.txt +++ b/s3/data/SAP_sup/settings/MLCReport.txt @@ -10,3 +10,4 @@ plant,material,material_description,batch,posting_date,qty_in_un_of_entry,moveme src04.saps_mlcreport org04.saps_mlcreport +qty_in_un_of_entry,total_valuated_stock_before_the_posting,quantity,amount \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/POReport.txt b/s3/data/SAP_sup/settings/POReport.txt index eb6de20d..5fe24ce6 100644 --- a/s3/data/SAP_sup/settings/POReport.txt +++ b/s3/data/SAP_sup/settings/POReport.txt @@ -10,3 +10,4 @@ purchasing_document,vendor,item,material,short_text,document_date,order_quantity src04.saps_poreport org04.saps_poreport +order_quantity,net_order_value \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/QAReport.txt b/s3/data/SAP_sup/settings/QAReport.txt index 96b6f6aa..fca0ca72 100644 --- a/s3/data/SAP_sup/settings/QAReport.txt +++ b/s3/data/SAP_sup/settings/QAReport.txt @@ -10,3 +10,4 @@ plant,material,material_description,batch,posting_date,qty_in_un_of_entry,moveme src04.saps_qareport org04.saps_qareport +qty_in_un_of_entry,total_valuated_stock_before_the_posting,quantity,amount \ No newline at end of file diff --git a/s3/data/SAP_sup/settings/StockList.txt b/s3/data/SAP_sup/settings/StockList.txt index 8bc8985d..f571083c 100644 --- a/s3/data/SAP_sup/settings/StockList.txt +++ b/s3/data/SAP_sup/settings/StockList.txt @@ -10,3 +10,4 @@ spl_stock_indic,material_num,material_desc,storage_location,batch_num,expired_da src04.saps_stocklist org04.saps_stocklist StockList_ex.sql +unrestricted_stock,in_quality_stock,blocked_stock,consign_stock,total_stock_quantity \ No newline at end of file From 7a8700ac3e2aff3e7d5d5735e85b3995b30c6114 Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Thu, 14 Jul 2022 18:49:32 +0900 Subject: [PATCH 2/4] =?UTF-8?q?feat:=E3=82=AB=E3=83=B3=E3=83=9E=E5=89=8A?= =?UTF-8?q?=E9=99=A4=E5=87=A6=E7=90=86=E3=81=AE=E8=A8=AD=E5=AE=9A=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E4=BF=AE=E6=AD=A3=E3=81=AB=E4=BC=B4?= =?UTF-8?q?=E3=81=86=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/chk.py | 8 ++++++ ecs/dataimport/dataimport/main.py | 41 ++++++++++++++++++++++++++----- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/ecs/dataimport/dataimport/chk.py b/ecs/dataimport/dataimport/chk.py index a0bf2ced..ad1b9af7 100644 --- a/ecs/dataimport/dataimport/chk.py +++ b/ecs/dataimport/dataimport/chk.py @@ -23,6 +23,14 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, + 'removeCommas': 12, + 'reserved0': 13, + 'reserved1': 14, + 'reserved2': 15, + 'reserved3': 16, + 'reserved4': 17, + 'reserved5': 18, + 'reserved6': 19 } LINE_FEED_CODE = { 'CR': '\r', diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 730a7be8..0ffe224f 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -1,4 +1,5 @@ from datetime import datetime +import re import boto3 import pymysql from pymysql.constants import CLIENT @@ -23,6 +24,14 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, + 'replaceCommas': 12, + 'reserved0': 13, + 'reserved1': 14, + 'reserved2': 15, + 'reserved3': 16, + 'reserved4': 17, + 'reserved5': 18, + 'reserved6': 19 } LINE_FEED_CODE = { 'CR': '\r', @@ -76,12 +85,18 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["i"]} I-MAIN-03 - タイムゾーンを変更しました') # ④ 個別設定ファイルのロードスキーマのテーブル名に記載されているテーブルをTRUNCATEする + # 個別設定ファイルの読み込み settings_obj = s3_resource.Object(bucket_name, settings_key) settings_response = settings_obj.get() settings_list = [] for line in io.TextIOWrapper(io.BytesIO(settings_response["Body"].read()), encoding='utf-8'): settings_list.append(line.rstrip('\n')) + # 予約行挿入のためsetting_listとSETTINGS_ITEMの要素数を揃える + for _ in range(len(SETTINGS_ITEM) - len(settings_list)): + settings_list.append('') + + # ロードスキーマのTRUNCATE with conn.cursor() as cur: sql_truncate = f'TRUNCATE table {settings_list[SETTINGS_ITEM["loadSchemaName"]]}' cur.execute(sql_truncate) @@ -100,6 +115,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf warning_info = '' # ワーニング情報 index = 0 # ループインデックス settings_db_columu_list = settings_list[SETTINGS_ITEM["dbColumuName"]].rstrip().split(',') + settings_replace_comma_list = settings_list[SETTINGS_ITEM["replaceCommas"]].rstrip().split(',') for line in csv.reader(work_data, quotechar=settings_list[SETTINGS_ITEM["quotechar"]], delimiter=settings_list[SETTINGS_ITEM["delimiter"]]): try: @@ -125,13 +141,15 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf sql = f'{sql} VALUES (' for i in range(len(line)): # データ項目値が0桁より大きいかチェックする - if len(line[i]) > 0: - # 0桁より大きい場合 - replace_line = line[i].replace('\\', '\\\\') - sql = f'{sql} "{replace_line}",' - else: - # 上記以外の場合 + if len(line[i]) == 0: + # 0桁の場合 sql = f'{sql} NULL,' + continue + + # データ項目値の変換処理 + column_value = replace_column_value(line,settings_db_columu_list,settings_replace_comma_list,i) + sql = f'{sql} "{column_value}",' + sql = f'{sql} "{target_file_name}",' # システム項目：取込ファイル名 sql = f'{sql} "{index + 1}",' # システム項目：取込ファイル行番号 sql = f'{sql} "0",' # システム項目：論理削除フラグ @@ -272,3 +290,14 @@ def connection_close(conn, bucket_name, target_data_source, target_file_name, lo except Exception as e: print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) + +def replace_column_value(line,settings_db_columu_list,settings_replace_comma_list,i): + org_column_value = line[i] + + # 投入データのDB物理カラム名が設定ファイルの数値型のDBカラム物理名に含まれている場合 + if settings_db_columu_list[i] in settings_replace_comma_list: + org_column_value = org_column_value.replace(',', '') + + org_column_value = org_column_value.replace('\\', '\\\\') + return org_column_value + \ No newline at end of file From 9709ba1180a017bdfa59651a926555675bf17ded Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Fri, 15 Jul 2022 14:31:18 +0900 Subject: [PATCH 3/4] =?UTF-8?q?feat:=E4=B8=8B=E7=94=B0=E3=81=95=E3=82=93?= =?UTF-8?q?=E3=81=AE=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC=E6=8C=87=E6=91=98?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 25 ++++++++++++++----------- s3/data/SAP_fin/settings/Invoice.txt | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 0ffe224f..8d82789e 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -24,7 +24,7 @@ SETTINGS_ITEM = { 'storageSchemaName': 9, 'loadSchemaName': 10, 'exSqlFileName': 11, - 'replaceCommas': 12, + 'commaReplaceColumns': 12, 'reserved0': 13, 'reserved1': 14, 'reserved2': 15, @@ -92,7 +92,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf for line in io.TextIOWrapper(io.BytesIO(settings_response["Body"].read()), encoding='utf-8'): settings_list.append(line.rstrip('\n')) - # 予約行挿入のためsetting_listとSETTINGS_ITEMの要素数を揃える + # 設定ファイルに記載のない行を空文字として扱い、予約行とする for _ in range(len(SETTINGS_ITEM) - len(settings_list)): settings_list.append('') @@ -115,7 +115,7 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf warning_info = '' # ワーニング情報 index = 0 # ループインデックス settings_db_columu_list = settings_list[SETTINGS_ITEM["dbColumuName"]].rstrip().split(',') - settings_replace_comma_list = settings_list[SETTINGS_ITEM["replaceCommas"]].rstrip().split(',') + settings_replace_comma_list = settings_list[SETTINGS_ITEM["commaReplaceColumns"]].rstrip().split(',') for line in csv.reader(work_data, quotechar=settings_list[SETTINGS_ITEM["quotechar"]], delimiter=settings_list[SETTINGS_ITEM["delimiter"]]): try: @@ -147,7 +147,9 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf continue # データ項目値の変換処理 - column_value = replace_column_value(line,settings_db_columu_list,settings_replace_comma_list,i) + org_column_value = line[i] + current_settings_db_columu_name = settings_db_columu_list[i] + column_value = convert_column_value(org_column_value,current_settings_db_columu_name,settings_replace_comma_list) sql = f'{sql} "{column_value}",' sql = f'{sql} "{target_file_name}",' # システム項目：取込ファイル名 @@ -291,13 +293,14 @@ def connection_close(conn, bucket_name, target_data_source, target_file_name, lo print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) -def replace_column_value(line,settings_db_columu_list,settings_replace_comma_list,i): - org_column_value = line[i] +def convert_column_value(org_column_value,current_settings_db_columu_name,settings_replace_comma_list): - # 投入データのDB物理カラム名が設定ファイルの数値型のDBカラム物理名に含まれている場合 - if settings_db_columu_list[i] in settings_replace_comma_list: - org_column_value = org_column_value.replace(',', '') + # データ内の「\\」という文字がpythonで読んだ場合に「\\\\」となるため、「\\」に戻す + converted_column_value = org_column_value.replace('\\', '\\\\') + + # 投入データのDB物理カラム名が設定ファイルの数値型のDBカラム物理名に含まれている場合、データ項目値の「,」を取り除く + if current_settings_db_columu_name in settings_replace_comma_list: + converted_column_value = converted_column_value.replace(',', '') - org_column_value = org_column_value.replace('\\', '\\\\') - return org_column_value + return converted_column_value \ No newline at end of file diff --git a/s3/data/SAP_fin/settings/Invoice.txt b/s3/data/SAP_fin/settings/Invoice.txt index 52ed7636..6fcac702 100644 --- a/s3/data/SAP_fin/settings/Invoice.txt +++ b/s3/data/SAP_fin/settings/Invoice.txt @@ -10,4 +10,4 @@ billing_type,condition_type,distribution_channel,invoiced_number,invoice_item,in src03b.sapf_invoice org03b.sapf_invoice -vbl_value \ No newline at end of file +quantity_invoiced,extended_amount_invoiced_amount,accrual_value_rebate_1,accrual_percent_rebate_1,accrual_value_rebate_2,accrual_percent_rebate_2,unit_selling_price \ No newline at end of file From 13d55634dbc98f07b30276f45e7b25a8c59af38d Mon Sep 17 00:00:00 2001 From: yuusuke_kanamura Date: Tue, 19 Jul 2022 09:33:28 +0900 Subject: [PATCH 4/4] =?UTF-8?q?feat:=E4=B8=8B=E7=94=B0=E3=81=95=E3=82=93?= =?UTF-8?q?=E3=81=AE=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC=E6=8C=87=E6=91=98?= =?UTF-8?q?=E5=86=8D=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/dataimport/dataimport/main.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ecs/dataimport/dataimport/main.py b/ecs/dataimport/dataimport/main.py index 8d82789e..94379370 100644 --- a/ecs/dataimport/dataimport/main.py +++ b/ecs/dataimport/dataimport/main.py @@ -148,8 +148,8 @@ def main(bucket_name, target_data_source, target_file_name, settings_key, db_inf # データ項目値の変換処理 org_column_value = line[i] - current_settings_db_columu_name = settings_db_columu_list[i] - column_value = convert_column_value(org_column_value,current_settings_db_columu_name,settings_replace_comma_list) + current_settings_db_column_name = settings_db_columu_list[i] + column_value = convert_column_value(org_column_value, current_settings_db_column_name, settings_replace_comma_list) sql = f'{sql} "{column_value}",' sql = f'{sql} "{target_file_name}",' # システム項目：取込ファイル名 @@ -293,13 +293,22 @@ def connection_close(conn, bucket_name, target_data_source, target_file_name, lo print(f'{datetime.now():%Y-%m-%d %H:%M:%S} {log_info} {LOG_LEVEL["e"]} E-MAIN-99 - エラー内容：{e}') error(bucket_name, target_data_source, target_file_name, log_info) -def convert_column_value(org_column_value,current_settings_db_columu_name,settings_replace_comma_list): - +def convert_column_value(org_column_value, current_settings_db_column_name, settings_replace_comma_list): + """データ項目値変換処理 + + Args: + org_column_value : 投入データの値 + current_settings_db_column_name : 投入データのDBカラム物理名 + settings_replace_comma_list : 投入データの数値型のDBカラム物理名のリスト + + Returns: + converted_column_value:変換処理を行った投入データの値 + """ # データ内の「\\」という文字がpythonで読んだ場合に「\\\\」となるため、「\\」に戻す converted_column_value = org_column_value.replace('\\', '\\\\') # 投入データのDB物理カラム名が設定ファイルの数値型のDBカラム物理名に含まれている場合、データ項目値の「,」を取り除く - if current_settings_db_columu_name in settings_replace_comma_list: + if current_settings_db_column_name in settings_replace_comma_list: converted_column_value = converted_column_value.replace(',', '') return converted_column_value