From 143fda6e26efd59818bcad134971159f7c046d3f Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 24 Aug 2023 15:47:43 +0900 Subject: [PATCH] =?UTF-8?q?fix:=20=E7=94=9F=E7=89=A9=E7=94=B1=E6=9D=A5Exce?= =?UTF-8?q?l=E3=81=AE=E5=87=BA=E5=8A=9B=E4=BB=B6=E6=95=B0=E5=88=B6?= =?UTF-8?q?=E9=99=90=E3=82=92=E5=AE=9F=E8=A3=85=EF=BC=88=E3=83=A1=E3=83=A2?= =?UTF-8?q?=E3=83=AA=E3=82=AA=E3=83=BC=E3=83=90=E3=83=BC=E5=AF=BE=E7=AD=96?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-webapp/.env.example | 1 + ecs/jskult-webapp/src/controller/bio_download.py | 15 ++++++++++++--- .../src/repositories/bio_sales_lot_repository.py | 4 ++-- .../src/services/bio_view_service.py | 10 ++++++++-- ecs/jskult-webapp/src/system_var/environment.py | 3 ++- .../jskult/task_settings/web_task_settings.env | 1 + 6 files changed, 26 insertions(+), 8 deletions(-) diff --git a/ecs/jskult-webapp/.env.example b/ecs/jskult-webapp/.env.example index 10844a21..d1770156 100644 --- a/ecs/jskult-webapp/.env.example +++ b/ecs/jskult-webapp/.env.example @@ -23,6 +23,7 @@ AWS_REGION=ap-northeast-1 AUTHORIZE_ENDPOINT=oauth2/authorize TOKEN_ENDPOINT=oauth2/token BIO_SEARCH_RESULT_MAX_COUNT=35000 +BIO_EXCEL_RESULT_MAX_COUNT=6500 ULTMARC_SEARCH_RESULT_MAX_COUNT=500 SESSION_EXPIRE_MINUTE=20 LOG_LEVEL=DEBUG \ No newline at end of file diff --git a/ecs/jskult-webapp/src/controller/bio_download.py b/ecs/jskult-webapp/src/controller/bio_download.py index 454ff857..9b4fb607 100644 --- a/ecs/jskult-webapp/src/controller/bio_download.py +++ b/ecs/jskult-webapp/src/controller/bio_download.py @@ -51,7 +51,7 @@ async def download_bio_data( return {'status': 'batch_processing'} # 生物由来データを検索 # 検索に使用したクエリも取得 - search_result_df, query = _search_bio_data(bio_service, search_param, download_param.user_id) + search_result_df, query = _search_bio_data(bio_service, search_param, download_param) # アクセスログを記録 bio_service.write_access_log(query, search_param, download_param.user_id, current_timestamp, download_file_name) @@ -99,10 +99,19 @@ async def download_bio_data( return json_response -def _search_bio_data(bio_service: BioViewService, search_param: BioModel, user_id: str) -> pd.DataFrame: +def _search_bio_data( + bio_service: BioViewService, + search_param: BioModel, + download_param: BioDownloadModel +) -> pd.DataFrame: try: # 生物由来データを検索 - search_result_df, query = bio_service.search_download_bio_data(search_param) + # Excelの場合、出力件数を絞る + if download_param.ext == 'xlsx': + search_result_df, query = bio_service.search_download_bio_data( + search_param, limitation=environment.BIO_EXCEL_RESULT_MAX_COUNT) + elif download_param.ext == 'csv': + search_result_df, query = bio_service.search_download_bio_data(search_param) except DBException as e: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/ecs/jskult-webapp/src/repositories/bio_sales_lot_repository.py b/ecs/jskult-webapp/src/repositories/bio_sales_lot_repository.py index f3cbab0b..8816636b 100644 --- a/ecs/jskult-webapp/src/repositories/bio_sales_lot_repository.py +++ b/ecs/jskult-webapp/src/repositories/bio_sales_lot_repository.py @@ -81,12 +81,12 @@ class BioSalesLotRepository(BaseRepository): finally: self._database.disconnect() - def fetch_as_data_frame(self, parameter: BioModel): + def fetch_as_data_frame(self, parameter: BioModel, limitation: int): try: self._database.connect() logger.debug('DB参照実行') where_clause = self.__build_condition(parameter) - query = self.FETCH_SQL.format(where_clause=where_clause, limit=environment.BIO_SEARCH_RESULT_MAX_COUNT + 1) + query = self.FETCH_SQL.format(where_clause=where_clause, limit=limitation) logger.debug(f'SQL: {query}') df = self._to_data_frame(query, parameter) logger.debug(f'count= {len(df.index)}') diff --git a/ecs/jskult-webapp/src/services/bio_view_service.py b/ecs/jskult-webapp/src/services/bio_view_service.py index dd348d06..79446a44 100644 --- a/ecs/jskult-webapp/src/services/bio_view_service.py +++ b/ecs/jskult-webapp/src/services/bio_view_service.py @@ -69,9 +69,15 @@ class BioViewService(BaseService): return display_bio_data - def search_download_bio_data(self, search_params: BioModel): + def search_download_bio_data( + self, + search_params: BioModel, + limitation=(environment.BIO_SEARCH_RESULT_MAX_COUNT + 1) + ): # 生物由来データをダウンロードするために、DBから検索した結果をデータフレームに変換 - bio_sales_data_frame, query = self.bio_sales_repository.fetch_as_data_frame(parameter=search_params) + bio_sales_data_frame, query = self.bio_sales_repository.fetch_as_data_frame( + parameter=search_params, limitation=limitation + ) return bio_sales_data_frame, query def write_access_log( diff --git a/ecs/jskult-webapp/src/system_var/environment.py b/ecs/jskult-webapp/src/system_var/environment.py index 6e7cf79d..d5fe4630 100644 --- a/ecs/jskult-webapp/src/system_var/environment.py +++ b/ecs/jskult-webapp/src/system_var/environment.py @@ -20,7 +20,8 @@ DB_PASSWORD = os.environ['DB_PASSWORD'] DB_SCHEMA = os.environ['DB_SCHEMA'] BIO_SEARCH_RESULT_MAX_COUNT = int(os.environ.get('BIO_SEARCH_RESULT_MAX_COUNT', 35000)) -ULTMARC_SEARCH_RESULT_MAX_COUNT = int(os.environ['ULTMARC_SEARCH_RESULT_MAX_COUNT']) +BIO_EXCEL_RESULT_MAX_COUNT = int(os.environ.get('BIO_EXCEL_RESULT_MAX_COUNT', 6500)) +ULTMARC_SEARCH_RESULT_MAX_COUNT = int(os.environ.get('ULTMARC_SEARCH_RESULT_MAX_COUNT', 500)) SESSION_EXPIRE_MINUTE = int(os.environ['SESSION_EXPIRE_MINUTE']) LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') diff --git a/s3/config/jskult/task_settings/web_task_settings.env b/s3/config/jskult/task_settings/web_task_settings.env index 9f6b542a..a17c6bdd 100644 --- a/s3/config/jskult/task_settings/web_task_settings.env +++ b/s3/config/jskult/task_settings/web_task_settings.env @@ -3,5 +3,6 @@ AWS_REGION=ap-northeast-1 AUTHORIZE_ENDPOINT=oauth2/authorize TOKEN_ENDPOINT=oauth2/token BIO_SEARCH_RESULT_MAX_COUNT=35000 +BIO_EXCEL_RESULT_MAX_COUNT=6500 SESSION_EXPIRE_MINUTE=20 ULTMARC_SEARCH_RESULT_MAX_COUNT=500