diff --git a/ecs/jskult-batch-daily/src/aws/s3.py b/ecs/jskult-batch-daily/src/aws/s3.py index a4ebc30a..3a5fe240 100644 --- a/ecs/jskult-batch-daily/src/aws/s3.py +++ b/ecs/jskult-batch-daily/src/aws/s3.py @@ -1,6 +1,7 @@ +import gzip import os import os.path as path -import tarfile +import shutil import tempfile import boto3 @@ -134,14 +135,14 @@ class VjskReceiveBucket(S3Bucket): return temporary_file_path def unzip_data_file(self, filename: str): - ret = [] - with tarfile.open(filename) as tar: - temp_dir = os.path.dirname(filename) - tar.extractall(path=temp_dir) - extracted_files = tar.getnames() - for extracted_file in extracted_files: - file = os.path.join(temp_dir, extracted_file) - ret.append(file) + temp_dir = os.path.dirname(filename) + decompress_filename = os.path.basename(filename).replace('.gz', '') + decompress_file_path = os.path.join(temp_dir, decompress_filename) + with gzip.open(filename, 'rb') as gz: + with open(decompress_file_path, 'wb') as decompressed_file: + shutil.copyfileobj(gz, decompressed_file) + + ret = [decompress_file_path] return ret def backup_dat_file(self, target_files: list, datetime_key: str): diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py index 6b45f96e..c15f39b7 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py @@ -1159,7 +1159,7 @@ class TestImportFileToDb: _import_file_to_db(received_s3_files) # 検証 - assert str(e.value) == "file could not be opened successfully" + assert str(e.value) == "Not a gzipped file (b'7z')" # teardown for test_file in test_files: diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/bio_slip_data_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/bio_slip_data_202304270000.gz old mode 100644 new mode 100755 index 36d1af45..6ea16a9c Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/bio_slip_data_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/bio_slip_data_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/fcl_mst_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/fcl_mst_202304270000.gz old mode 100644 new mode 100755 index 24234bfe..fffb9215 Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/fcl_mst_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/fcl_mst_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/hld_mst_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/hld_mst_202304270000.gz old mode 100644 new mode 100755 index efedd921..20c194a2 Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/hld_mst_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/hld_mst_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/lot_num_mst_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/lot_num_mst_202304270000.gz old mode 100644 new mode 100755 index 615d1451..6d6f7dd9 Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/lot_num_mst_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/lot_num_mst_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/mdb_conv_mst_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/mdb_conv_mst_202304270000.gz old mode 100644 new mode 100755 index 994a80b8..9b177904 Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/mdb_conv_mst_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/mdb_conv_mst_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/mkr_org_horizon_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/mkr_org_horizon_202304270000.gz old mode 100644 new mode 100755 index 835d2360..9eb2cd34 Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/mkr_org_horizon_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/mkr_org_horizon_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/org_cnv_mst_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/org_cnv_mst_202304270000.gz old mode 100644 new mode 100755 index 89eb4a65..72b636f5 Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/org_cnv_mst_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/org_cnv_mst_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/phm_prd_mst_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/phm_prd_mst_202304270000.gz old mode 100644 new mode 100755 index 169f6761..7d864d6f Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/phm_prd_mst_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/phm_prd_mst_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/phm_price_mst_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/phm_price_mst_202304270000.gz old mode 100644 new mode 100755 index f7658c88..7cf9f449 Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/phm_price_mst_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/phm_price_mst_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/slip_data_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/slip_data_202304270000.gz old mode 100644 new mode 100755 index 5a90e676..82465aea Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/slip_data_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/slip_data_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/stock_slip_data_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/stock_slip_data_202304270000.gz old mode 100644 new mode 100755 index e2cfdfdf..f36e25ca Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/stock_slip_data_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/stock_slip_data_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/tran_kbn_mst_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/tran_kbn_mst_202304270000.gz old mode 100644 new mode 100755 index b68d5099..77e7eac6 Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/tran_kbn_mst_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/tran_kbn_mst_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/vop_hco_merge_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/vop_hco_merge_202304270000.gz old mode 100644 new mode 100755 index 58fbc63d..3671b2a6 Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/vop_hco_merge_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/vop_hco_merge_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/whs_customer_mst_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/whs_customer_mst_202304270000.gz old mode 100644 new mode 100755 index fd0ed8cd..7c3bd6d4 Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/whs_customer_mst_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/whs_customer_mst_202304270000.gz differ diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/whs_mst_202304270000.gz b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/whs_mst_202304270000.gz old mode 100644 new mode 100755 index 013eae2c..d41f560a Binary files a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/whs_mst_202304270000.gz and b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestImportFileToDb/whs_mst_202304270000.gz differ diff --git a/ecs/jskult-webapp/Dockerfile b/ecs/jskult-webapp/Dockerfile index ef14f718..d7590655 100644 --- a/ecs/jskult-webapp/Dockerfile +++ b/ecs/jskult-webapp/Dockerfile @@ -16,4 +16,4 @@ RUN \ COPY src ./src -CMD ["gunicorn", "src.main:app", "-w", "4", "-k" ,"uvicorn.workers.UvicornWorker", "-b", "0.0.0.0:80"] +CMD ["gunicorn", "src.main:app", "-w", "4", "-k" ,"uvicorn.workers.UvicornWorker", "-b", "0.0.0.0:80", "--timeout", "300"] diff --git a/ecs/jskult-webapp/src/controller/master_mainte.py b/ecs/jskult-webapp/src/controller/master_mainte.py index 046fc4ab..e2a54928 100644 --- a/ecs/jskult-webapp/src/controller/master_mainte.py +++ b/ecs/jskult-webapp/src/controller/master_mainte.py @@ -4,9 +4,12 @@ from typing import Optional from fastapi import APIRouter, Depends, HTTPException, Request from fastapi.responses import HTMLResponse from starlette import status + from src.depends.services import get_service from src.logging.get_logger import get_logger from src.model.internal.session import UserSession +from src.model.request.master_mainte_csvdl import MasterMainteCsvDlModel +from src.model.request.master_mainte_csvup import MasterMainteCsvUpModel from src.model.view.inst_emp_csv_download_view_model import \ InstEmpCsvDownloadViewModel from src.model.view.inst_emp_csv_upload_view_model import \ @@ -20,9 +23,6 @@ from src.services.master_mainte_service import MasterMainteService from src.services.session_service import set_session from src.system_var import constants from src.templates import templates -from src.model.request.master_mainte_csvup import MasterMainteCsvUpModel -from src.model.request.master_mainte_csvdl import MasterMainteCsvDlModel - logger = get_logger('マスターメンテ') @@ -117,7 +117,7 @@ def inst_emp_csv_upload_view( @router.post('/instEmpCsvUL', response_class=HTMLResponse) async def inst_emp_csv_upload( request: Request, - csv_upload_form: Optional[MasterMainteCsvUpModel] = Depends(MasterMainteCsvUpModel.as_form), + csv_upload_form: MasterMainteCsvUpModel = Depends(MasterMainteCsvUpModel.as_form), master_mainte_service: MasterMainteService = Depends(get_service(MasterMainteService)), batch_status_service: BatchStatusService = Depends(get_service(BatchStatusService)) ): @@ -137,13 +137,13 @@ async def inst_emp_csv_upload( # 画面表示用のモデル error_message_list = [] - content_type: str = csv_upload_form.csv_file.content_type.strip() + csv_filename: str = csv_upload_form.csv_file.filename if csv_upload_form.csv_file.size == 0: error_message_list.append('選択されたファイルが見つかりませんでした。') - elif content_type != 'text/csv' and content_type != 'application/vnd.ms-excel' and content_type != 'application/octet-stream': + elif not csv_filename.endswith('.csv'): error_message_list.append('選択されたファイル形式が"csv"ではありません。') elif csv_upload_form.csv_file.size >= constants.MENTE_CSV_UPLOAD_MAX_FILE_SIZE_BYTE: - error_message_list.append('選択されたCSVファイルサイズが大きいです。100MB未満にしてください。') + error_message_list.append('選択されたCSVファイルサイズが大きいです。20MB未満にしてください。') else: mainte_csv_up = master_mainte_service.prepare_mainte_csv_up_view( TextIOWrapper(BytesIO(await csv_upload_form.csv_file.read()), encoding='utf-8'), diff --git a/ecs/jskult-webapp/src/system_var/constants.py b/ecs/jskult-webapp/src/system_var/constants.py index 7be637b9..276556b0 100644 --- a/ecs/jskult-webapp/src/system_var/constants.py +++ b/ecs/jskult-webapp/src/system_var/constants.py @@ -50,31 +50,6 @@ BIO_CSV_HEADER = [ '2017年11月以前データ' ] -SLIP_ORG_KBN_FULL_NAME = { - 'J': 'JD-NET', - 'N': 'NHI', - 'H': '手入力' -} -DATA_KBN_JP_NAME = { - '0': '正常', - '1': 'ロットエラー', - '2': '除外', - '3': 'ロット不明', - '9': 'エラー(解消済)', -} -LOT_NO_ERR_FLG_JP_NAME = { - '0': '正常', - '1': 'ロットエラー', - '2': '日付エラー', - '3': 'ロットエラー(解消済)', - '4': 'ロットエラー(調査不能)', - '5': '日付エラー(解消済)', - '6': '日付エラー(調査不能)', - '7': '除外(卸都合)', - '8': '除外(再送信)', - 'Z': '過去データ' -} - LOGOUT_REASON_DO_LOGOUT = 'do_logout' LOGOUT_REASON_LOGIN_ERROR = 'login_error' LOGOUT_REASON_BATCH_PROCESSING = 'batch_processing' @@ -212,5 +187,5 @@ MENTE_CSV_DOWNLOAD_HEADER = [ MENTE_CSV_DOWNLOAD_FILE_NAME = 'instEmpData.csv' -# CSVアップロードの制限サイズ=100MB -MENTE_CSV_UPLOAD_MAX_FILE_SIZE_BYTE = 104857600 +# CSVアップロードの制限サイズ=20MB +MENTE_CSV_UPLOAD_MAX_FILE_SIZE_BYTE = 20971520