nik/newdwh2021
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: ダウンロード処理のパフォーマンス向上、async-awaitでメモリ効率を向上

Browse Source
This commit is contained in:
shimoda.m@nds-tyo.co.jp 2023-08-28 19:27:27 +09:00
parent 57fcd3cbc7
commit 5a0a0770d3
2 changed files with 19 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
ecs/jskult-webapp/src/controller/bio_download.py
View File

@ -29,7 +29,7 @@ router = APIRouter()
@router.post('/download')
def download_bio_data(
async def download_bio_data(
search_param: BioModel = Depends(BioModel.as_body),
download_param: BioDownloadModel = Depends(BioDownloadModel.as_body),
bio_service: BioViewService = Depends(get_service(BioViewService)),
@ -61,7 +61,7 @@ def download_bio_data(
return {'status': 'ok', 'download_url': None}
# ファイルを書き出し(Excel or CSV)
local_file_path = _write_bio_data_to_file(bio_service, download_param, search_result_df, download_file_name)
local_file_path = await _write_bio_data_to_file(bio_service, download_param, search_result_df, download_file_name)
logger.info('ファイル書き出し完了')
@ -123,7 +123,7 @@ def _search_bio_data(
return search_result_df, query
def _write_bio_data_to_file(
async def _write_bio_data_to_file(
bio_service: BioViewService,
download_param: BioDownloadModel,
df: pd.DataFrame,
@ -133,11 +133,11 @@ def _write_bio_data_to_file(
local_file_path = ''
if download_param.ext == 'xlsx':
logger.info('今回はExcelファイルに出力する')
local_file_path = bio_service.write_excel_file(
local_file_path = await bio_service.write_excel_file(
df, download_param.user_id, download_file_name=download_file_name)
elif download_param.ext == 'csv':
logger.info('今回はCSVファイルに出力する')
local_file_path = bio_service.write_csv_file(
local_file_path = await bio_service.write_csv_file(
df, download_param.user_id, header=constants.BIO_CSV_HEADER, download_file_name=download_file_name)
return local_file_path

18
ecs/jskult-webapp/src/services/bio_view_service.py
View File

@ -1,3 +1,5 @@
import asyncio
import functools
import os
import shutil
from datetime import datetime
@ -131,7 +133,9 @@ class BioViewService(BaseService):
# S3にアップロード
self.upload_bio_access_log_file(access_log_file_path)
def write_excel_file(self, data_frame: pd.DataFrame, user_id: str, download_file_name: str):
async def write_excel_file(self, data_frame: pd.DataFrame, user_id: str, download_file_name: str):
# 非同期処理用のイベントループを取得
loop = asyncio.get_event_loop()
# Excelに書き込み
output_file_path = os.path.join(constants.BIO_TEMPORARY_FILE_DIR_PATH, download_file_name)
@ -159,11 +163,15 @@ class BioViewService(BaseService):
if is_first:
is_first = False
start_row = 1
chunk_df.to_excel(writer, header=False, index=False, startrow=start_row, startcol=0)
await loop.run_in_executor(None, functools.partial(
chunk_df.to_excel, writer,
header=False, index=False, startrow=start_row, startcol=0))
return output_file_path
def write_csv_file(self, data_frame: pd.DataFrame, user_id: str, header: list[str], download_file_name: str):
async def write_csv_file(self, data_frame: pd.DataFrame, user_id: str, header: list[str], download_file_name: str):
# 非同期処理用のイベントループを取得
loop = asyncio.get_event_loop()
# csvに書き込み
output_file_path = os.path.join(constants.BIO_TEMPORARY_FILE_DIR_PATH, download_file_name)
# 横長のDataFrameとするため、ヘッダーの加工処理
@ -174,7 +182,9 @@ class BioViewService(BaseService):
header_df = pd.DataFrame([header_data], index=None)
output_df = pd.concat([header_df, data_frame])
# ヘッダー行としてではなく、1レコードとして出力する
output_df.to_csv(output_file_path, index=False, header=False, encoding="utf-8_sig")
await loop.run_in_executor(None, functools.partial(
output_df.to_csv, output_file_path,
index=False, header=False, encoding="utf-8_sig"))
return output_file_path