Merge pull request #514 feature-NEWDWH2021-1927 into develop

ecs/jskult-batch-archive-jsk-data/src/batch/archive_jsk_data.py

@@ -23,7 +23,11 @@ def exec():
         for jskult_archive_manage_data in jskult_archive_manage_data_list:
             # 対象テーブルで条件項目が条件年月以前のデータを取得
             archive_data = jskult_archive_manager.get_archive_data(
-                jskult_archive_manage_data["target_table"], jskult_archive_manage_data["filter_column"], jskult_archive_manage_data["filter_date"])
+                jskult_archive_manage_data["target_table"], 
+                jskult_archive_manage_data["filter_column_name"],
+                jskult_archive_manage_data["filter_column_type"],
+                jskult_archive_manage_data["filter_column_digit"],
+                jskult_archive_manage_data["filter_date"])
             # 取得データが0件の場合、スキップする
             if not archive_data:
                 logger.info(
@@ -48,7 +52,8 @@ def exec():
                 # 作成したCSVをzip形式に圧縮
                 zip_file_path = path.join(temporary_dir, f"{file_name}.zip")
                 with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
-                    zipf.write(csv_file_path)
+                    arcname = path.basename(csv_file_path)
+                    zipf.write(csv_file_path, arcname=arcname)
                 logger.info(f"zip形式への圧縮に成功しました。{file_name}.zip")
 
                 # 圧縮したCSVを保存先へアップロード
@@ -60,7 +65,9 @@ def exec():
                 # アーカイブしたデータをDBから削除
                 jskult_archive_manager.delete_archive_data(
                     jskult_archive_manage_data["target_table"],
-                    jskult_archive_manage_data["filter_column"],
+                    jskult_archive_manage_data["filter_column_name"],
+                    jskult_archive_manage_data["filter_column_type"],
+                    jskult_archive_manage_data["filter_column_digit"],
                     jskult_archive_manage_data["filter_date"])
                 logger.info(
                     f"アーカイブしたデータのDBから削除に成功しました。対象テーブル：{jskult_archive_manage_data['target_table']} 条件年月：{jskult_archive_manage_data['filter_date']}")

ecs/jskult-batch-archive-jsk-data/src/batch/jskult_archive_manager.py

@@ -17,7 +17,9 @@ class JskultArchiveManager:
             sql = """
                 select
                     target_table
-                    , filter_column
+                    , filter_column_name
+                    , filter_column_type
+                    , filter_column_digit
                     , filter_date
                     , run_interval_months
                     , prev_filter_date
@@ -34,18 +36,44 @@ class JskultArchiveManager:
             raise
         finally:
             self._db.disconnect()
+    
+    def _filter_column_to_date(self,filter_column_name:str , filter_column_type:str, filter_column_digit:int):
+        """ filter_columnを日付型へ変換"""
+        try: 
+            logger.info("処理開始：_filter_column_to_date")
+            if filter_column_type == 'date':
+                filter_column = filter_column_name
 
-    def get_archive_data(self,target_table:str, filter_column:str, filter_date:str):
+            elif filter_column_type == 'varchar' and filter_column_digit == 8:
+                filter_column = f"str_to_date({filter_column_name},'%Y%m%d')"
+
+            elif filter_column_type == 'varchar' and filter_column_digit == 6:
+                filter_column = f"str_to_date(concat({filter_column_name} , '01'),'%Y%m%d')"
+
+            else:
+                raise Exception("条件項目データ型もしくは条件項目桁数が不正です。")
+            
+            logger.info("処理終了：_filter_column_to_date")
+            return filter_column
+            
+        except Exception as e:
+            logger.info("異常終了：_filter_column_to_date")
+            raise
+
+    def get_archive_data(self,target_table:str, filter_column_name:str, filter_column_type:str, filter_column_digit:int, filter_date:str):
         """アーカイブするデータを取得"""
         try:
             logger.info("処理開始：get_archive_data")
+
+            filter_column = self._filter_column_to_date(filter_column_name, filter_column_type, filter_column_digit)
+
             sql = f"""
                 select
                     *
                 from
                     src07.{target_table}
                 where
-                    str_to_date({filter_column},'%Y%m%d') <= :filter_date;
+                    {filter_column} <= :filter_date;
             """
             self._db.connect()
             parameter_dict = {'filter_date' : filter_date}
@@ -58,15 +86,18 @@ class JskultArchiveManager:
         finally:
             self._db.disconnect()
 
-    def delete_archive_data(self, target_table:str, filter_column:str, filter_date:str):
+    def delete_archive_data(self, target_table:str, filter_column_name:str, filter_column_type:str, filter_column_digit:int, filter_date:str):
         """アーカイブしたデータを削除"""
         try:
             logger.info("処理開始：delete_archive_data")
+
+            filter_column = self._filter_column_to_date(filter_column_name, filter_column_type, filter_column_digit)
+            
             sql = f"""
                 delete from
                     src07.{target_table}
                 where
-                    str_to_date({filter_column},'%Y%m%d') <= :filter_date;
+                    {filter_column} <= :filter_date;
             """
             self._db.connect()
             self._db.begin()