Merge branch 'develop-8sap' into feature-NEWDWH2021-541

.gitignore

@@ -2,5 +2,9 @@ lambda/mbj-newdwh2021-staging-NoticeToSlack/package-lock.json
 lambda/mbj-newdwh2021-staging-NoticeToSlack/node_modules/*
 lambda/mbj-newdwh2021-staging-PublishFromLog/package-lock.json
 lambda/mbj-newdwh2021-staging-PublishFromLog/node_modules/*
-__pycache__/
+
+# ローカルの環境変数
 .env
+# Python関連
+.venv
+__pycache__

lambda/sap-data-decrypt/datadecrypt/main.py

@@ -6,7 +6,6 @@ import datetime
 import logging
 from abc import *
 from zoneinfo import ZoneInfo
-import traceback
 
 # 環境変数
 SECRET_KEY_FILE_BUCKET_NAME = os.environ["SECRET_KEY_FILE_BUCKET_NAME"]
@@ -23,6 +22,7 @@ LOG_LEVEL = os.environ["LOG_LEVEL"]
 # 定数
 DIRECTORY_RECV = '/recv/'
 DIRECTORY_RECV_ERROR = '/recv_error/'
+DIRECTORY_TARGET = '/target/'
 EXTENSION_ERROR = '.error'
 EXTENSION_DECRYPT_ERROR = '.decrypt_error'
 INDEX_EXTENSION_DELETE_NUM = -4
@@ -78,7 +78,7 @@ def handler(event, context):
             s3_client.download_file(s3_event.bucket_name, s3_event.file_path, PATH_TEMP_ENCRYPT_FILE)
             logger.info('I-03-02 PGP暗号化ファイルを読み込みました')
         except Exception as e:
-            logger.error(f'E-03-01 PGP暗号化ファイルの読み込みに失敗しました　エラー内容：{e}')
+            logger.exception(f'E-03-01 PGP暗号化ファイルの読み込みに失敗しました　エラー内容：{e}')
             raise EncryptFileReadException('E-03-01', EXTENSION_ERROR, e)
 
         # ④ S3から秘密鍵ファイルを読み込む
@@ -87,7 +87,7 @@ def handler(event, context):
             s3_client.download_file(SECRET_KEY_FILE_BUCKET_NAME, SECRET_KEY_FILE_PATH, PATH_TEMP_PRIVATE_KEY)
             logger.info('I-04-02 秘密鍵ファイルを読み込みました')
         except Exception as e:
-            logger.error(f'E-04-01 秘密鍵ファイルの読み込みに失敗しました　エラー内容：{e}')
+            logger.exception(f'E-04-01 秘密鍵ファイルの読み込みに失敗しました　エラー内容：{e}')
             raise FileReadException('E-04-01', EXTENSION_ERROR, e)
 
         # ⑤ 「③」で読み込んだ秘密鍵ファイルをPGPライブラリにインポートを行う
@@ -98,7 +98,7 @@ def handler(event, context):
                 gpg.import_keys(key_file.read())
             logger.info('I-05-02 秘密鍵ファイルをインポートしました')
         except Exception as e:
-            logger.error(f'E-05-01 秘密鍵ファイルのインポートに失敗しました　エラー内容：{e}')
+            logger.exception(f'E-05-01 秘密鍵ファイルのインポートに失敗しました　エラー内容：{e}')
             raise PrivateKeyImportException('E-05-01', EXTENSION_ERROR, e)
 
         # ⑥ 「④」で読み込んだPGP暗号化ファイルを「⑤」でインポートした秘密鍵ファイルで復号する
@@ -109,7 +109,7 @@ def handler(event, context):
             decrypt_file = open(PATH_TEMP_DECRYPT_FILE, 'rb')
             logger.info('I-06-02 PGP暗号化ファイルを復号しました')
         except Exception as e:
-            logger.error(f'E-06-01 PGP暗号化ファイルの復号化に失敗しました　エラー内容：{e}')
+            logger.exception(f'E-06-01 PGP暗号化ファイルの復号化に失敗しました　エラー内容：{e}')
             raise DecryptException('E-06-01', EXTENSION_DECRYPT_ERROR, e)
 
         # ⑦ 各ファイルをS3に出力する
@@ -126,7 +126,7 @@ def handler(event, context):
             decrypt_file.close
             logger.info('I-07-03 復号化ファイルをS3に出力しました')
         except Exception as e:
-            logger.error(f'E-07-01 復号化ファイルのS3出力に失敗しました　エラー内容：{e}')
+            logger.exception(f'E-07-01 復号化ファイルのS3出力に失敗しました　エラー内容：{e}')
             raise FileOutputException('E-07-01', EXTENSION_ERROR, e)
 
         # 「④」で読み込んだPGP暗号化ファイルを以下に移動（コピー削除）する
@@ -142,23 +142,34 @@ def handler(event, context):
             s3_client.delete_object(Bucket=s3_event.bucket_name, Key=s3_event.file_path)
             logger.info('I-07-05 PGP暗号化ファイルをバックアップ用バケットに移動しました')
         except Exception as e:
-            logger.error(f'E-07-02 PGP暗号化ファイルのS3出力に失敗しました　エラー内容：{e}')
+            logger.exception(f'E-07-02 PGP暗号化ファイルのS3出力に失敗しました　エラー内容：{e}')
+            raise FileOutputException('E-07-02', EXTENSION_ERROR, e)
+        # 「⑥」で復号化したファイルをデータ取込用バケットに出力する
+        try:
+            copy_source = {
+                'Bucket': decrypt_bucket_name,
+                'Key': decrypt_file_key
+            }
+            import_file_folder = f'{s3_event.data_source_name}{DIRECTORY_TARGET}'
+            import_file_key = f'{import_file_folder}{decrypt_file_name}'
+            logger.info(f'I-07-06 復号化ファイル出力　ファイル名：{decrypt_file_name}　出力先：{import_file_folder}')
+            import_file_obj = s3_resource.Object(s3_event.bucket_name, import_file_key)
+            import_file_obj.copy(copy_source)
+            logger.info(f'I-07-07 復号化ファイルをS3に出力しました')
+        except Exception as e:
+            logger.exception(f'E-07-03 復号化ファイルのS3出力に失敗しました　エラー内容：{e}')
             raise FileOutputException('E-07-02', EXTENSION_ERROR, e)
-
         # ⑧ 処理終了ログを出力する
         logger.info('I-08-01 処理終了　SAPデータ復号処理')
     except EncryptFileReadException as e:
-        traceback.print_exc()
         create_status_file(s3_event, e.extension)
         error_notice(e.id, e.arg)
     except CustomException as e:
-        traceback.print_exc()
         create_status_file(s3_event, e.extension)
         move_encrypt_file(s3_event)
         error_notice(e.id, e.arg)
     except Exception as e:
-        logger.error(f'E-99 想定外のエラーが発生しました　エラー内容：{e}')
-        traceback.print_exc()
+        logger.exception(f'E-99 想定外のエラーが発生しました　エラー内容：{e}')
         create_status_file(s3_event, EXTENSION_ERROR)
         move_encrypt_file(s3_event)
         error_notice('E-99', e)
@@ -174,8 +185,7 @@ def create_status_file(s3_event, extension) -> None:
         result_error_obj.put(Body='')
         logger.error(f'E-ERR-01 recvディレクトリにエラーファイルを作成しました　ファイル名：{result_error_file_name}　出力先：{s3_event.bucket_name}/{result_error_key}')
     except Exception as e:
-        logger.error(f'E-96 エラーステータスファイルの作成に失敗しました　エラー内容：{e}')
-        traceback.print_exc()
+        logger.exception(f'E-96 エラーステータスファイルの作成に失敗しました　エラー内容：{e}')
     return
 
 
@@ -193,8 +203,7 @@ def move_encrypt_file(s3_event) -> None:
         s3_client.delete_object(Bucket=s3_event.bucket_name, Key=s3_event.file_path)
         logger.error(f'E-ERR-02 recv_errorディレクトリにファイルを移動しました　移動元：{s3_event.bucket_name}/{s3_event.file_path}　移動先：{s3_event.bucket_name}/{error_key}')
     except Exception as e:
-        logger.error(f'E-97 PGP暗号化ファイルの移動に失敗しました　エラー内容：{e}')
-        traceback.print_exc()
+        logger.exception(f'E-97 PGP暗号化ファイルの移動に失敗しました　エラー内容：{e}')
     return
 
 
@@ -210,8 +219,7 @@ def error_notice(error_log_id, exception) -> None:
         sns_client.publish(**params)
         logger.error(f'E-ERR-03 処理異常通知の送信指示をしました　通知先トピック：{NDS_NOTICE_TOPIC}')
     except Exception as e:
-        logger.error(f'E-98 処理異常通知の送信指示に失敗しました　エラー内容：{e}')
-        traceback.print_exc()
+        logger.exception(f'E-98 処理異常通知の送信指示に失敗しました　エラー内容：{e}')
     return
 
 

s3/data/SAP_fin/settings/CostReport.txt

@@ -0,0 +1,11 @@
+SAP_fin
+	
+utf-8
+ 
+LF
+1
+18
+VT,Fiscal Year,Fiscal Period,Cost Center,Cost element,Cost element name,Vbl.value,Offsetting account,Offsetting account name,Name,Document header text,Posting date,Document date,Purchasing document,Document number,Posting row,Ref doc number,Rev
+vt,fiscal_year,fiscal_period,cost_center,cost_element,cost_element_name,vbl_value,offsetting_account,offsetting_account_name,name,document_header_text,posting_date,document_date,purchasing_document,document_number,posting_row,ref_doc_number,rev
+src03a.sapf_costreport
+org03a.sapf_costreport

s3/data/SAP_fin/settings/IOReport.txt

@@ -0,0 +1,11 @@
+SAP_fin
+	
+utf-8
+ 
+LF
+1
+17
+Fiscal Year,Fiscal Period,Internal Order,Cost element,Cost element name,Vbl.value,Offsetting account,Offsetting account name,AuxAccAs1,Name,Document header text,Posting date,Document date,Purchasing document,Document number,Posting row,Ref doc number
+fiscal_year,fiscal_period,internal_order,cost_element,cost_element_name,vbl_value,offsetting_account,offsetting_account_name,auxaccas1,name,document_header_text,posting_date,document_date,purchasing_document,document_number,posting_row,ref_doc_number
+src03a.sapf_ioreport
+org03a.sapf_ioreport

s3/data/SAP_fin/settings/Invoice.txt

@@ -0,0 +1,11 @@
+SAP_fin
+	
+utf-8
+ 
+LF
+1
+25
+Billing Type,Condition Type,Distribution channel,Invoiced number,Invoice item,Invoice date,Account number (Sold-to),Customer name,Ship-to,Ship-to(Name),Quantity invoiced,Extended amount (invoiced amount),Accrual value (rebate 1),Accrual % (rebate 1),Accrual value (rebate 2),Accrual % (rebate 2),Unit selling price,Item code,Item name,Tax,Order reason,Reference doc,Sales order No.,Rejected RSN,Batch number
+billing_type,condition_type,distribution_channel,invoiced_number,invoice_item,invoice_date,account_number_sold_to,customer_name,ship_to,ship_to_name,quantity_invoiced,extended_amount_invoiced_amount,accrual_value_rebate_1,accrual_percent_rebate_1,accrual_value_rebate_2,accrual_percent_rebate_2,unit_selling_price,item_code,item_name,tax,order_reason,reference_doc,sales_order_no,rejected_rsn,batch_number
+src03b.sapf_invoice
+org03b.sapf_invoice

s3/data/SAP_fin/settings/WBSList.txt

@@ -0,0 +1,11 @@
+SAP_fin
+	
+utf-8
+ 
+LF
+1
+9
+WBS element,Level,Profit center,Project Definition,Description,Control area,Responsibility,Basic finish date,Basic start date
+wbs_element,level,profit_center,project_definition,description,control_area,responsibility,basic_finish_date,basic_start_date
+src03a.sapf_wbslist
+org03a.sapf_wbslist

s3/data/SAP_fin/settings/WBSReport.txt

@@ -0,0 +1,11 @@
+SAP_fin
+	
+utf-8
+ 
+LF
+1
+17
+Fiscal Year,Fiscal Period,WBS element,Cost element,Cost element name,Vbl.value,Offsetting account,Offsetting account name,AuxAccAs1,Name,Document header text,Posting date,Document date,Purchasing document,Document number,Posting row,Ref doc number
+fiscal_year,fiscal_period,wbs_element,cost_element,cost_element_name,vbl_value,offsetting_account,offsetting_account_name,auxaccas1,name,document_header_text,posting_date,document_date,purchasing_document,document_number,posting_row,ref_doc_number
+src03a.sapf_wbsreport
+org03a.sapf_wbsreport

s3/data/SAP_fin/settings/configmap.config

@@ -0,0 +1,7 @@
+/* SAPデータ取込:Finance */
+CostReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	CostReport.txt
+IOReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	IOReport.txt
+WBSReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	WBSReport.txt
+WBSList_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	WBSList.txt
+Invoice_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	Invoice.txt
+

s3/data/SAP_sup/settings/ConfReport.txt

@@ -0,0 +1,11 @@
+SAP_sup
+	
+utf-8
+ 
+LF
+1
+11
+Process Order,Operation / activity,Yield,UoM,Posting date,Finish execution date,Material,Material description,Confirmation,cofirm.counter,cancelled. conf
+process_order,operation_activity,yield,uom,posting_date,finish_execution_date,material,material_description,confirmation,cofirm_counter,cancelled_conf
+src04.saps_gmreport
+org04.saps_gmreport

s3/data/SAP_sup/settings/GMReport.txt

@@ -0,0 +1,11 @@
+SAP_sup
+	
+utf-8
+ 
+LF
+1
+12
+Process Order,Material document,Material document item,Material,Material description,Goods movement,Posting date,Movement type,D/C indicator,Storage location,Batch,Qty
+process_order,material_document,material_document_item,material,material_description,goods_movement,posting_date,movement_type,d_c_indicator,storage_location,batch,qty
+src04.saps_gmreport
+org04.saps_gmreport

s3/data/SAP_sup/settings/GRReport.txt

@@ -0,0 +1,11 @@
+SAP_sup
+	
+utf-8
+ 
+LF
+1
+24
+Plant,Material,Material Description,Batch,Posting Date,Qty in Un. of Entry,Movement Type,Unit of Entry,Entry Date,Material Doc. Year,Document Date,Total valuated stock before the posting,Base Unit of Measure,Quantity,Reference,Purchase Order,Customer,Amount,Amount in LC,Vendor,Item,Material Document,Storage Location,Movement Type Text
+plant,material,material_description4,batch,posting_date,qty_in_un_of_entry,movement_type,unit_of_entry,entry_date,material_doc_year,document_date,total_valuated_stock_before_the_posting,base_unit_of_measure,quantity,reference,purchase_order,customer,amount,amount_in_lc,vendor,item,material_document,storage_location,movement_type_text
+src04.saps_grreport
+org04.saps_grreport

s3/data/SAP_sup/settings/MLCReport.txt

@@ -0,0 +1,11 @@
+SAP_sup
+	
+utf-8
+ 
+LF
+1
+24
+Plant,Material,Material Description,Batch,Posting Date,Qty in Un. of Entry,Movement Type,Unit of Entry,Entry Date,Material Doc. Year,Document Date,Total valuated stock before the posting,Base Unit of Measure,Quantity,Reference,Purchase Order,Customer,Amount,Amount in LC,Vendor,Item,Material Document,Storage Location,Movement Type Text
+plant,material,material_description,batch,posting_date,qty_in_un_of_entry,movement_type,unit_of_entry,entry_date,material_doc_year,document_date,total_valuated_stock_before_the_posting,base_unit_of_measure,quantity,reference,purchase_order,customer,amount,amount_in_lc,vendor,item,material_document,storage_location,movement_type_text
+src04.saps_mlcreport
+org04.saps_mlcreport

s3/data/SAP_sup/settings/POReport.txt

@@ -0,0 +1,11 @@
+SAP_sup
+	
+utf-8
+ 
+LF
+1
+12
+Purchasing Document,Vendor,Item,Material,Short Text,Document Date,Order Quantity,Order Unit,Net Order Value,Currency,Order Price Unit,Deletion Flag
+purchasing_document,vendor,item,material,short_text,document_date,order_quantity,order_unit,net_order_value,currency,order_price_unit,deletion_flag
+src04.saps_poreport
+org04.saps_poreport

s3/data/SAP_sup/settings/QAReport.txt

@@ -0,0 +1,11 @@
+SAP_sup
+	  
+utf-8
+ 
+LF
+1
+24
+Plant,Material,Material Description,Batch,Posting Date,Qty in Un. of Entry,Movement Type,Unit of Entry,Entry Date,Material Doc. Year,Document Date,Total valuated stock before the posting,Base Unit of Measure,Quantity,Reference,Purchase Order,Customer,Amount,Amount in LC,Vendor,Item,Material Document,Storage Location,Movement Type Text
+plant,material,material_description,batch,posting_date,qty_in_un_of_entry,movement_type,unit_of_entry,entry_date,material_doc_year,document_date,total_valuated_stock_before_the_posting,base_unit_of_measure,quantity,reference,purchase_order,customer,amount,amount_in_lc,vendor,item,material_document,storage_location,movement_type_text
+src04.saps_qareport
+org04.saps_qareport

s3/data/SAP_sup/settings/StockList.txt

@@ -0,0 +1,12 @@
+SAP_sup
+	
+utf-8
+ 
+LF
+1
+13
+SPL.stock Indic,Material Num.,Material Desc.,Storage Location,Batch Num.,Expired Date,Unrestricted Stock,In Quality Stock,Blocked Stock,Consign Stock,Total Stock Quantity,Sold to,Name
+spl_stock_indic,material_num,material_desc,storage_location,batch_num,expired_date,unrestricted_stock,in_quality_stock,blocked_stock,consign_stock,total_stock_quantity,sold_to,name
+src04.saps_stocklist
+org04.saps_stocklist
+StockList_ex.sql

s3/data/SAP_sup/settings/StockList_ex.sql

@@ -0,0 +1,7 @@
+/* 蓄積スキーマ */
+/* execute_dateがnullのレコードを抽出し、取込ファイル名のyyyymmdd部分を切り出しセットする */
+update src04.saps_stocklist 
+set
+    execute_date = STR_TO_DATE(SUBSTRING(file_name,11,8),'%Y%m%d') 
+where
+    execute_date is null

s3/data/SAP_sup/settings/configmap.config

@@ -0,0 +1,9 @@
+/* SAPデータ取込:SupplyChain */
+GRReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	GRReport.txt
+QAReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	QAReport.txt
+MLCReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	MLCReport.txt
+MLCReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	MLCReport.txt
+POReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	POReport.txt
+StockList_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	StockList.txt
+ConfReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	ConfReport.txt
+GMReport_[0-9]{8}_[0-9]{6}\.(TSV|tsv)	GMReport.txt