Merge commit 'ac99c40ad0635cf49c42739e421afb5ec40c1a62' into feature-NEWDWH2021-1166

ecs/jskult-batch-daily/src/aws/s3.py

@@ -1,6 +1,7 @@
+import gzip
 import os
 import os.path as path
-import tarfile
+import shutil
 import tempfile
 
 import boto3
@@ -134,14 +135,14 @@ class VjskReceiveBucket(S3Bucket):
         return temporary_file_path
 
     def unzip_data_file(self, filename: str):
-        ret = []
-        with tarfile.open(filename) as tar:
-            temp_dir = os.path.dirname(filename)
-            tar.extractall(path=temp_dir)
-            extracted_files = tar.getnames()
-            for extracted_file in extracted_files:
-                file = os.path.join(temp_dir, extracted_file)
-                ret.append(file)
+        temp_dir = os.path.dirname(filename)
+        decompress_filename = os.path.basename(filename).replace('.gz', '')
+        decompress_file_path = os.path.join(temp_dir, decompress_filename)
+        with gzip.open(filename, 'rb') as gz:
+            with open(decompress_file_path, 'wb') as decompressed_file:
+                shutil.copyfileobj(gz, decompressed_file)
+
+        ret = [decompress_file_path]
         return ret
 
     def backup_dat_file(self, target_files: list, datetime_key: str):

ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py

@@ -1159,7 +1159,7 @@ class TestImportFileToDb:
             _import_file_to_db(received_s3_files)
 
         # 検証
-        assert str(e.value) == "file could not be opened successfully"
+        assert str(e.value) == "Not a gzipped file (b'7z')"
 
         # teardown
         for test_file in test_files:

ecs/jskult-webapp/Dockerfile

@@ -16,4 +16,4 @@ RUN \
 
 COPY src ./src
 
-CMD ["gunicorn", "src.main:app", "-w", "4", "-k" ,"uvicorn.workers.UvicornWorker", "-b", "0.0.0.0:80"]
+CMD ["gunicorn", "src.main:app", "-w", "4", "-k" ,"uvicorn.workers.UvicornWorker", "-b", "0.0.0.0:80", "--timeout", "300"]

ecs/jskult-webapp/src/controller/master_mainte.py

@@ -4,9 +4,12 @@ from typing import Optional
 from fastapi import APIRouter, Depends, HTTPException, Request
 from fastapi.responses import HTMLResponse
 from starlette import status
+
 from src.depends.services import get_service
 from src.logging.get_logger import get_logger
 from src.model.internal.session import UserSession
+from src.model.request.master_mainte_csvdl import MasterMainteCsvDlModel
+from src.model.request.master_mainte_csvup import MasterMainteCsvUpModel
 from src.model.view.inst_emp_csv_download_view_model import \
     InstEmpCsvDownloadViewModel
 from src.model.view.inst_emp_csv_upload_view_model import \
@@ -20,9 +23,6 @@ from src.services.master_mainte_service import MasterMainteService
 from src.services.session_service import set_session
 from src.system_var import constants
 from src.templates import templates
-from src.model.request.master_mainte_csvup import MasterMainteCsvUpModel
-from src.model.request.master_mainte_csvdl import MasterMainteCsvDlModel
-
 
 logger = get_logger('マスターメンテ')
 
@@ -117,7 +117,7 @@ def inst_emp_csv_upload_view(
 @router.post('/instEmpCsvUL', response_class=HTMLResponse)
 async def inst_emp_csv_upload(
     request: Request,
-    csv_upload_form: Optional[MasterMainteCsvUpModel] = Depends(MasterMainteCsvUpModel.as_form),
+    csv_upload_form: MasterMainteCsvUpModel = Depends(MasterMainteCsvUpModel.as_form),
     master_mainte_service: MasterMainteService = Depends(get_service(MasterMainteService)),
     batch_status_service: BatchStatusService = Depends(get_service(BatchStatusService))
 ):
@@ -137,13 +137,13 @@ async def inst_emp_csv_upload(
 
     # 画面表示用のモデル
     error_message_list = []
-    content_type: str = csv_upload_form.csv_file.content_type.strip()
+    csv_filename: str = csv_upload_form.csv_file.filename
     if csv_upload_form.csv_file.size == 0:
         error_message_list.append('選択されたファイルが見つかりませんでした。')
-    elif content_type != 'text/csv' and content_type != 'application/vnd.ms-excel' and content_type != 'application/octet-stream':
+    elif not csv_filename.endswith('.csv'):
         error_message_list.append('選択されたファイル形式が"csv"ではありません。')
     elif csv_upload_form.csv_file.size >= constants.MENTE_CSV_UPLOAD_MAX_FILE_SIZE_BYTE:
-        error_message_list.append('選択されたCSVファイルサイズが大きいです。100MB未満にしてください。')
+        error_message_list.append('選択されたCSVファイルサイズが大きいです。20MB未満にしてください。')
     else:
         mainte_csv_up = master_mainte_service.prepare_mainte_csv_up_view(
             TextIOWrapper(BytesIO(await csv_upload_form.csv_file.read()), encoding='utf-8'),

ecs/jskult-webapp/src/system_var/constants.py

@@ -50,31 +50,6 @@ BIO_CSV_HEADER = [
     '2017年11月以前データ'
 ]
 
-SLIP_ORG_KBN_FULL_NAME = {
-    'J': 'JD-NET',
-    'N': 'NHI',
-    'H': '手入力'
-}
-DATA_KBN_JP_NAME = {
-    '0': '正常',
-    '1': 'ロットエラー',
-    '2': '除外',
-    '3': 'ロット不明',
-    '9': 'エラー（解消済）',
-}
-LOT_NO_ERR_FLG_JP_NAME = {
-    '0': '正常',
-    '1': 'ロットエラー',
-    '2': '日付エラー',
-    '3': 'ロットエラー（解消済）',
-    '4': 'ロットエラー（調査不能）',
-    '5': '日付エラー（解消済）',
-    '6': '日付エラー（調査不能）',
-    '7': '除外（卸都合）',
-    '8': '除外（再送信）',
-    'Z': '過去データ'
-}
-
 LOGOUT_REASON_DO_LOGOUT = 'do_logout'
 LOGOUT_REASON_LOGIN_ERROR = 'login_error'
 LOGOUT_REASON_BATCH_PROCESSING = 'batch_processing'
@@ -212,5 +187,5 @@ MENTE_CSV_DOWNLOAD_HEADER = [
 
 MENTE_CSV_DOWNLOAD_FILE_NAME = 'instEmpData.csv'
 
-# CSVアップロードの制限サイズ＝100MB
-MENTE_CSV_UPLOAD_MAX_FILE_SIZE_BYTE = 104857600
+# CSVアップロードの制限サイズ＝20MB
+MENTE_CSV_UPLOAD_MAX_FILE_SIZE_BYTE = 20971520