From 21e42d8826d812d4b62c8f1a361b6a664a9969ed Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Tue, 20 May 2025 14:42:14 +0900 Subject: [PATCH 01/13] =?UTF-8?q?=E3=82=A8=E3=83=B3=E3=83=88=E3=83=AA?= =?UTF-8?q?=E3=83=BC=E3=83=9D=E3=82=A4=E3=83=B3=E3=83=88=E3=81=AE=E4=BD=9C?= =?UTF-8?q?=E6=88=90=E3=81=A8=E6=8C=AF=E3=82=8A=E5=88=86=E3=81=91=E5=87=A6?= =?UTF-8?q?=E7=90=86=E4=BD=9C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch/.dockerignore | 12 + ecs/jskult-batch/.env.example | 26 + ecs/jskult-batch/.gitignore | 10 + ecs/jskult-batch/.vscode/launch.json | 16 + .../.vscode/recommended_settings.json | 31 ++ ecs/jskult-batch/Dockerfile | 20 + ecs/jskult-batch/Pipfile | 29 ++ ecs/jskult-batch/Pipfile.lock | 443 ++++++++++++++++++ ecs/jskult-batch/entrypoint.py | 16 + ecs/jskult-batch/pytest.ini | 3 + ecs/jskult-batch/src/__init__.py | 0 ecs/jskult-batch/src/aws/__init__.py | 0 ecs/jskult-batch/src/aws/s3.py | 185 ++++++++ ecs/jskult-batch/src/batch/common/__init__.py | 0 .../common/jskult_batch_entrypoint_factory.py | 30 ++ .../src/batch/dcf_inst_merge_io.py | 10 + .../src/batch/jskult_batch_entrypoint.py | 8 + ecs/jskult-batch/src/batch/mst_inst.py | 10 + .../src/batch/trn_result_data_bio_lot.py | 10 + .../src/batch/update_business_day.py | 10 + ecs/jskult-batch/src/db/__init__.py | 0 ecs/jskult-batch/src/db/database.py | 195 ++++++++ ecs/jskult-batch/src/error/__init__.py | 0 ecs/jskult-batch/src/error/exceptions.py | 10 + ecs/jskult-batch/src/logging/get_logger.py | 37 ++ ecs/jskult-batch/src/system_var/__init__.py | 0 ecs/jskult-batch/src/system_var/constants.py | 17 + .../src/system_var/environment.py | 18 + 28 files changed, 1146 insertions(+) create mode 100644 ecs/jskult-batch/.dockerignore create mode 100644 ecs/jskult-batch/.env.example create mode 100644 ecs/jskult-batch/.gitignore create mode 100644 ecs/jskult-batch/.vscode/launch.json create mode 100644 ecs/jskult-batch/.vscode/recommended_settings.json create mode 100644 ecs/jskult-batch/Dockerfile create mode 100644 ecs/jskult-batch/Pipfile create mode 100644 ecs/jskult-batch/Pipfile.lock create mode 100644 ecs/jskult-batch/entrypoint.py create mode 100644 ecs/jskult-batch/pytest.ini create mode 100644 ecs/jskult-batch/src/__init__.py create mode 100644 ecs/jskult-batch/src/aws/__init__.py create mode 100644 ecs/jskult-batch/src/aws/s3.py create mode 100644 ecs/jskult-batch/src/batch/common/__init__.py create mode 100644 ecs/jskult-batch/src/batch/common/jskult_batch_entrypoint_factory.py create mode 100644 ecs/jskult-batch/src/batch/dcf_inst_merge_io.py create mode 100644 ecs/jskult-batch/src/batch/jskult_batch_entrypoint.py create mode 100644 ecs/jskult-batch/src/batch/mst_inst.py create mode 100644 ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py create mode 100644 ecs/jskult-batch/src/batch/update_business_day.py create mode 100644 ecs/jskult-batch/src/db/__init__.py create mode 100644 ecs/jskult-batch/src/db/database.py create mode 100644 ecs/jskult-batch/src/error/__init__.py create mode 100644 ecs/jskult-batch/src/error/exceptions.py create mode 100644 ecs/jskult-batch/src/logging/get_logger.py create mode 100644 ecs/jskult-batch/src/system_var/__init__.py create mode 100644 ecs/jskult-batch/src/system_var/constants.py create mode 100644 ecs/jskult-batch/src/system_var/environment.py diff --git a/ecs/jskult-batch/.dockerignore b/ecs/jskult-batch/.dockerignore new file mode 100644 index 00000000..8b9da402 --- /dev/null +++ b/ecs/jskult-batch/.dockerignore @@ -0,0 +1,12 @@ +tests/* +.coverage +.env +.env.example +.report/* +.vscode/* +.pytest_cache/* +*/__pychache__/* +Dockerfile +pytest.ini +README.md +*.sql diff --git a/ecs/jskult-batch/.env.example b/ecs/jskult-batch/.env.example new file mode 100644 index 00000000..500f843d --- /dev/null +++ b/ecs/jskult-batch/.env.example @@ -0,0 +1,26 @@ +DB_HOST=************ +DB_PORT=************ +DB_USERNAME=************ +DB_PASSWORD=************ +DB_SCHEMA=src05 +LOG_LEVEL=INFO +ULTMARC_DATA_BUCKET=**************** +ULTMARC_DATA_FOLDER=recv +JSKULT_BACKUP_BUCKET=**************** +ULTMARC_BACKUP_FOLDER=ultmarc +VJSK_BACKUP_FOLDER=vjsk +JSKULT_CONFIG_BUCKET=********************** +JSKULT_CONFIG_CALENDAR_FOLDER=jskult/calendar +JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME=jskult_holiday_list.txt +VJSK_DATA_SEND_FOLDER=send +VJSK_DATA_RECEIVE_FOLDER=recv +VJSK_DATA_BUCKET=************* +JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME=jskult_wholesaler_stock_input_day_list.txt +JSKULT_CONFIG_CONVERT_FOLDER=jskult/convert +JSKULT_ULTMARC_HEX_CONVERT_CONFIG_FILE_NAME=ultmarc_hex_convert_config.json +# 連携データ抽出期間 +SALES_LAUNDERING_EXTRACT_DATE_PERIOD=0 +# 洗替対象テーブル名 +SALES_LAUNDERING_TARGET_TABLE_NAME=src05.sales_lau +# 卸実績洗替で作成するデータの期間(年単位) +SALES_LAUNDERING_TARGET_YEAR_OFFSET=5 diff --git a/ecs/jskult-batch/.gitignore b/ecs/jskult-batch/.gitignore new file mode 100644 index 00000000..bd0b37f8 --- /dev/null +++ b/ecs/jskult-batch/.gitignore @@ -0,0 +1,10 @@ +.vscode/settings.json +.env + +# python +__pycache__ + +# python test +.pytest_cache +.coverage +.report/ \ No newline at end of file diff --git a/ecs/jskult-batch/.vscode/launch.json b/ecs/jskult-batch/.vscode/launch.json new file mode 100644 index 00000000..bcd1c6dd --- /dev/null +++ b/ecs/jskult-batch/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // IntelliSense を使用して利用可能な属性を学べます。 + // 既存の属性の説明をホバーして表示します。 + // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "(DEBUG)jskult batch", + "type": "python", + "request": "launch", + "program": "entrypoint.py", + "console": "integratedTerminal", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/ecs/jskult-batch/.vscode/recommended_settings.json b/ecs/jskult-batch/.vscode/recommended_settings.json new file mode 100644 index 00000000..2fde8732 --- /dev/null +++ b/ecs/jskult-batch/.vscode/recommended_settings.json @@ -0,0 +1,31 @@ +{ + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + // 自身の環境に合わせて変えてください + "python.defaultInterpreterPath": "", + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": [ + "--max-line-length=200", + "--ignore=F541" + ], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Path": "autopep8", + "python.formatting.autopep8Args": [ + "--max-line-length", "200", + "--ignore=F541" + ], + "python.testing.pytestArgs": [ + "tests/batch/" + ], + + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} diff --git a/ecs/jskult-batch/Dockerfile b/ecs/jskult-batch/Dockerfile new file mode 100644 index 00000000..fc0fde90 --- /dev/null +++ b/ecs/jskult-batch/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.12-slim-bookworm + +ENV TZ="Asia/Tokyo" +# pythonの標準出力をバッファリングしないフラグ +ENV PYTHONUNBUFFERED=1 +# pythonのバイトコードを生成しないフラグ +ENV PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /usr/src/app +COPY Pipfile Pipfile.lock ./ +RUN \ + apt update -y && \ + pip install pipenv --no-cache-dir && \ + pipenv install --system --deploy && \ + pip uninstall -y pipenv virtualenv-clone virtualenv + +COPY src ./src +COPY entrypoint.py entrypoint.py + +CMD ["python", "entrypoint.py"] diff --git a/ecs/jskult-batch/Pipfile b/ecs/jskult-batch/Pipfile new file mode 100644 index 00000000..2b56d8c0 --- /dev/null +++ b/ecs/jskult-batch/Pipfile @@ -0,0 +1,29 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[scripts] +"test:ultmarc" = "pytest tests/batch/ultmarc/" +"test:ultmarc:cov" = "pytest --cov=src/batch/ultmarc/ --cov-branch --cov-report=term-missing tests/batch/ultmarc/" +"test:vjsk" = "pytest tests/batch/vjsk/" +"test:vjsk:cov" = "pytest --cov=src/batch/vjsk/ --cov-branch --cov-report=term-missing tests/batch/vjsk/" + +[packages] +boto3 = "*" +PyMySQL = "*" +sqlalchemy = "*" +tenacity = "*" + +[dev-packages] +autopep8 = "*" +flake8 = "*" +pytest = "*" +pytest-cov = "*" +boto3 = "*" + +[requires] +python_version = "3.12" + +[pipenv] +allow_prereleases = true diff --git a/ecs/jskult-batch/Pipfile.lock b/ecs/jskult-batch/Pipfile.lock new file mode 100644 index 00000000..eaffb317 --- /dev/null +++ b/ecs/jskult-batch/Pipfile.lock @@ -0,0 +1,443 @@ +{ + "_meta": { + "hash": { + "sha256": "aa2d1d97600fea225b7d249dae0d065190d00fdadbf85b20773e0c1d9862f5c1" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.12" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "boto3": { + "hashes": [ + "sha256:d57d8040d04b8fefb7439062529653701cc90d5b2734cc4f1144fedb75ba94a7", + "sha256:fdd69f23e6216a508bbc1fbda9486791c161f3ecd5933ac7090d7290f6f2d0f5" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==1.38.19" + }, + "botocore": { + "hashes": [ + "sha256:796b948c05017eb33385b798990cd91ed4af0e881eb9eb1ee6e17666be02abc9", + "sha256:f937a20e75889215a99280ea0fdd4e1716ffede23e4f9af7bc9c64af9bc63e61" + ], + "markers": "python_version >= '3.9'", + "version": "==1.38.19" + }, + "greenlet": { + "hashes": [ + "sha256:00cd814b8959b95a546e47e8d589610534cfb71f19802ea8a2ad99d95d702057", + "sha256:02a98600899ca1ca5d3a2590974c9e3ec259503b2d6ba6527605fcd74e08e207", + "sha256:02f5972ff02c9cf615357c17ab713737cccfd0eaf69b951084a9fd43f39833d3", + "sha256:055916fafad3e3388d27dd68517478933a97edc2fc54ae79d3bec827de2c64c4", + "sha256:0a16fb934fcabfdfacf21d79e6fed81809d8cd97bc1be9d9c89f0e4567143d7b", + "sha256:1592a615b598643dbfd566bac8467f06c8c8ab6e56f069e573832ed1d5d528cc", + "sha256:1919cbdc1c53ef739c94cf2985056bcc0838c1f217b57647cbf4578576c63825", + "sha256:1e4747712c4365ef6765708f948acc9c10350719ca0545e362c24ab973017370", + "sha256:1e76106b6fc55fa3d6fe1c527f95ee65e324a13b62e243f77b48317346559708", + "sha256:1f72667cc341c95184f1c68f957cb2d4fc31eef81646e8e59358a10ce6689457", + "sha256:2593283bf81ca37d27d110956b79e8723f9aa50c4bcdc29d3c0543d4743d2763", + "sha256:2dc5c43bb65ec3669452af0ab10729e8fdc17f87a1f2ad7ec65d4aaaefabf6bf", + "sha256:3091bc45e6b0c73f225374fefa1536cd91b1e987377b12ef5b19129b07d93ebe", + "sha256:354f67445f5bed6604e493a06a9a49ad65675d3d03477d38a4db4a427e9aad0e", + "sha256:3885f85b61798f4192d544aac7b25a04ece5fe2704670b4ab73c2d2c14ab740d", + "sha256:3ab7194ee290302ca15449f601036007873028712e92ca15fc76597a0aeb4c59", + "sha256:3aeca9848d08ce5eb653cf16e15bb25beeab36e53eb71cc32569f5f3afb2a3aa", + "sha256:44671c29da26539a5f142257eaba5110f71887c24d40df3ac87f1117df589e0e", + "sha256:45f9f4853fb4cc46783085261c9ec4706628f3b57de3e68bae03e8f8b3c0de51", + "sha256:4bd139e4943547ce3a56ef4b8b1b9479f9e40bb47e72cc906f0f66b9d0d5cab3", + "sha256:4fefc7aa68b34b9224490dfda2e70ccf2131368493add64b4ef2d372955c207e", + "sha256:6629311595e3fe7304039c67f00d145cd1d38cf723bb5b99cc987b23c1433d61", + "sha256:6fadd183186db360b61cb34e81117a096bff91c072929cd1b529eb20dd46e6c5", + "sha256:71566302219b17ca354eb274dfd29b8da3c268e41b646f330e324e3967546a74", + "sha256:7409796591d879425997a518138889d8d17e63ada7c99edc0d7a1c22007d4907", + "sha256:752f0e79785e11180ebd2e726c8a88109ded3e2301d40abced2543aa5d164275", + "sha256:7791dcb496ec53d60c7f1c78eaa156c21f402dda38542a00afc3e20cae0f480f", + "sha256:782743700ab75716650b5238a4759f840bb2dcf7bff56917e9ffdf9f1f23ec59", + "sha256:7c9896249fbef2c615853b890ee854f22c671560226c9221cfd27c995db97e5c", + "sha256:85f3e248507125bf4af607a26fd6cb8578776197bd4b66e35229cdf5acf1dfbf", + "sha256:89c69e9a10670eb7a66b8cef6354c24671ba241f46152dd3eed447f79c29fb5b", + "sha256:8cb8553ee954536500d88a1a2f58fcb867e45125e600e80f586ade399b3f8819", + "sha256:9ae572c996ae4b5e122331e12bbb971ea49c08cc7c232d1bd43150800a2d6c65", + "sha256:9c7b15fb9b88d9ee07e076f5a683027bc3befd5bb5d25954bb633c385d8b737e", + "sha256:9ea5231428af34226c05f927e16fc7f6fa5e39e3ad3cd24ffa48ba53a47f4240", + "sha256:a31ead8411a027c2c4759113cf2bd473690517494f3d6e4bf67064589afcd3c5", + "sha256:a8fa80665b1a29faf76800173ff5325095f3e66a78e62999929809907aca5659", + "sha256:ad053d34421a2debba45aa3cc39acf454acbcd025b3fc1a9f8a0dee237abd485", + "sha256:b24c7844c0a0afc3ccbeb0b807adeefb7eff2b5599229ecedddcfeb0ef333bec", + "sha256:b50a8c5c162469c3209e5ec92ee4f95c8231b11db6a04db09bbe338176723bb8", + "sha256:ba30e88607fb6990544d84caf3c706c4b48f629e18853fc6a646f82db9629418", + "sha256:bf3fc9145141250907730886b031681dfcc0de1c158f3cc51c092223c0f381ce", + "sha256:c23ea227847c9dbe0b3910f5c0dd95658b607137614eb821e6cbaecd60d81cc6", + "sha256:c3cc1a3ed00ecfea8932477f729a9f616ad7347a5e55d50929efa50a86cb7be7", + "sha256:c49e9f7c6f625507ed83a7485366b46cbe325717c60837f7244fc99ba16ba9d6", + "sha256:d0cb7d47199001de7658c213419358aa8937df767936506db0db7ce1a71f4a2f", + "sha256:d8009ae46259e31bc73dc183e402f548e980c96f33a6ef58cc2e7865db012e13", + "sha256:da956d534a6d1b9841f95ad0f18ace637668f680b1339ca4dcfb2c1837880a0b", + "sha256:dcb9cebbf3f62cb1e5afacae90761ccce0effb3adaa32339a0670fe7805d8068", + "sha256:decb0658ec19e5c1f519faa9a160c0fc85a41a7e6654b3ce1b44b939f8bf1325", + "sha256:df4d1509efd4977e6a844ac96d8be0b9e5aa5d5c77aa27ca9f4d3f92d3fcf330", + "sha256:eeb27bece45c0c2a5842ac4c5a1b5c2ceaefe5711078eed4e8043159fa05c834", + "sha256:efcdfb9df109e8a3b475c016f60438fcd4be68cd13a365d42b35914cdab4bb2b", + "sha256:fd9fb7c941280e2c837b603850efc93c999ae58aae2b40765ed682a6907ebbc5", + "sha256:fe46d4f8e94e637634d54477b0cfabcf93c53f29eedcbdeecaf2af32029b4421" + ], + "markers": "python_version >= '3.9'", + "version": "==3.2.2" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "pymysql": { + "hashes": [ + "sha256:4de15da4c61dc132f4fb9ab763063e693d521a80fd0e87943b9a453dd4c19d6c", + "sha256:e127611aaf2b417403c60bf4dc570124aeb4a57f5f37b8e95ae399a42f904cd0" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==1.1.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.9.0.post0" + }, + "s3transfer": { + "hashes": [ + "sha256:35b314d7d82865756edab59f7baebc6b477189e6ab4c53050e28c1de4d9cce18", + "sha256:8ac58bc1989a3fdb7c7f3ee0918a66b160d038a147c7b5db1500930a607e9a1c" + ], + "markers": "python_version >= '3.9'", + "version": "==0.12.0" + }, + "six": { + "hashes": [ + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.17.0" + }, + "sqlalchemy": { + "hashes": [ + "sha256:023b3ee6169969beea3bb72312e44d8b7c27c75b347942d943cf49397b7edeb5", + "sha256:03968a349db483936c249f4d9cd14ff2c296adfa1290b660ba6516f973139582", + "sha256:05132c906066142103b83d9c250b60508af556982a385d96c4eaa9fb9720ac2b", + "sha256:087b6b52de812741c27231b5a3586384d60c353fbd0e2f81405a814b5591dc8b", + "sha256:0b3dbf1e7e9bc95f4bac5e2fb6d3fb2f083254c3fdd20a1789af965caf2d2348", + "sha256:118c16cd3f1b00c76d69343e38602006c9cfb9998fa4f798606d28d63f23beda", + "sha256:1936af879e3db023601196a1684d28e12f19ccf93af01bf3280a3262c4b6b4e5", + "sha256:1e3f196a0c59b0cae9a0cd332eb1a4bda4696e863f4f1cf84ab0347992c548c2", + "sha256:23a8825495d8b195c4aa9ff1c430c28f2c821e8c5e2d98089228af887e5d7e29", + "sha256:293cd444d82b18da48c9f71cd7005844dbbd06ca19be1ccf6779154439eec0b8", + "sha256:32f9dc8c44acdee06c8fc6440db9eae8b4af8b01e4b1aee7bdd7241c22edff4f", + "sha256:34ea30ab3ec98355235972dadc497bb659cc75f8292b760394824fab9cf39826", + "sha256:3d3549fc3e40667ec7199033a4e40a2f669898a00a7b18a931d3efb4c7900504", + "sha256:41836fe661cc98abfae476e14ba1906220f92c4e528771a8a3ae6a151242d2ae", + "sha256:4d44522480e0bf34c3d63167b8cfa7289c1c54264c2950cc5fc26e7850967e45", + "sha256:4eeb195cdedaf17aab6b247894ff2734dcead6c08f748e617bfe05bd5a218443", + "sha256:4f67766965996e63bb46cfbf2ce5355fc32d9dd3b8ad7e536a920ff9ee422e23", + "sha256:57df5dc6fdb5ed1a88a1ed2195fd31927e705cad62dedd86b46972752a80f576", + "sha256:598d9ebc1e796431bbd068e41e4de4dc34312b7aa3292571bb3674a0cb415dd1", + "sha256:5b14e97886199c1f52c14629c11d90c11fbb09e9334fa7bb5f6d068d9ced0ce0", + "sha256:5e22575d169529ac3e0a120cf050ec9daa94b6a9597993d1702884f6954a7d71", + "sha256:60c578c45c949f909a4026b7807044e7e564adf793537fc762b2489d522f3d11", + "sha256:6145afea51ff0af7f2564a05fa95eb46f542919e6523729663a5d285ecb3cf5e", + "sha256:6375cd674fe82d7aa9816d1cb96ec592bac1726c11e0cafbf40eeee9a4516b5f", + "sha256:6854175807af57bdb6425e47adbce7d20a4d79bbfd6f6d6519cd10bb7109a7f8", + "sha256:6ab60a5089a8f02009f127806f777fca82581c49e127f08413a66056bd9166dd", + "sha256:725875a63abf7c399d4548e686debb65cdc2549e1825437096a0af1f7e374814", + "sha256:7492967c3386df69f80cf67efd665c0f667cee67032090fe01d7d74b0e19bb08", + "sha256:81965cc20848ab06583506ef54e37cf15c83c7e619df2ad16807c03100745dea", + "sha256:81c24e0c0fde47a9723c81d5806569cddef103aebbf79dbc9fcbb617153dea30", + "sha256:81eedafa609917040d39aa9332e25881a8e7a0862495fcdf2023a9667209deda", + "sha256:81f413674d85cfd0dfcd6512e10e0f33c19c21860342a4890c3a2b59479929f9", + "sha256:8280856dd7c6a68ab3a164b4a4b1c51f7691f6d04af4d4ca23d6ecf2261b7923", + "sha256:82ca366a844eb551daff9d2e6e7a9e5e76d2612c8564f58db6c19a726869c1df", + "sha256:8b4af17bda11e907c51d10686eda89049f9ce5669b08fbe71a29747f1e876036", + "sha256:90144d3b0c8b139408da50196c5cad2a6909b51b23df1f0538411cd23ffa45d3", + "sha256:906e6b0d7d452e9a98e5ab8507c0da791856b2380fdee61b765632bb8698026f", + "sha256:90c11ceb9a1f482c752a71f203a81858625d8df5746d787a4786bca4ffdf71c6", + "sha256:911cc493ebd60de5f285bcae0491a60b4f2a9f0f5c270edd1c4dbaef7a38fc04", + "sha256:9a420a91913092d1e20c86a2f5f1fc85c1a8924dbcaf5e0586df8aceb09c9cc2", + "sha256:9f8c9fdd15a55d9465e590a402f42082705d66b05afc3ffd2d2eb3c6ba919560", + "sha256:a104c5694dfd2d864a6f91b0956eb5d5883234119cb40010115fd45a16da5e70", + "sha256:a373a400f3e9bac95ba2a06372c4fd1412a7cee53c37fc6c05f829bf672b8769", + "sha256:a62448526dd9ed3e3beedc93df9bb6b55a436ed1474db31a2af13b313a70a7e1", + "sha256:a8808d5cf866c781150d36a3c8eb3adccfa41a8105d031bf27e92c251e3969d6", + "sha256:b1f09b6821406ea1f94053f346f28f8215e293344209129a9c0fcc3578598d7b", + "sha256:b2ac41acfc8d965fb0c464eb8f44995770239668956dc4cdf502d1b1ffe0d747", + "sha256:b46fa6eae1cd1c20e6e6f44e19984d438b6b2d8616d21d783d150df714f44078", + "sha256:b50eab9994d64f4a823ff99a0ed28a6903224ddbe7fef56a6dd865eec9243440", + "sha256:bfc9064f6658a3d1cadeaa0ba07570b83ce6801a1314985bf98ec9b95d74e15f", + "sha256:c0b0e5e1b5d9f3586601048dd68f392dc0cc99a59bb5faf18aab057ce00d00b2", + "sha256:c153265408d18de4cc5ded1941dcd8315894572cddd3c58df5d5b5705b3fa28d", + "sha256:d4ae769b9c1c7757e4ccce94b0641bc203bbdf43ba7a2413ab2523d8d047d8dc", + "sha256:dc56c9788617b8964ad02e8fcfeed4001c1f8ba91a9e1f31483c0dffb207002a", + "sha256:dd5ec3aa6ae6e4d5b5de9357d2133c07be1aff6405b136dad753a16afb6717dd", + "sha256:edba70118c4be3c2b1f90754d308d0b79c6fe2c0fdc52d8ddf603916f83f4db9", + "sha256:ff8e80c4c4932c10493ff97028decfdb622de69cae87e0f127a7ebe32b4069c6" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==2.0.41" + }, + "tenacity": { + "hashes": [ + "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", + "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==9.1.2" + }, + "typing-extensions": { + "hashes": [ + "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", + "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef" + ], + "markers": "python_version >= '3.8'", + "version": "==4.13.2" + }, + "urllib3": { + "hashes": [ + "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466", + "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813" + ], + "markers": "python_version >= '3.9'", + "version": "==2.4.0" + } + }, + "develop": { + "autopep8": { + "hashes": [ + "sha256:89440a4f969197b69a995e4ce0661b031f455a9f776d2c5ba3dbd83466931758", + "sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==2.3.2" + }, + "boto3": { + "hashes": [ + "sha256:d57d8040d04b8fefb7439062529653701cc90d5b2734cc4f1144fedb75ba94a7", + "sha256:fdd69f23e6216a508bbc1fbda9486791c161f3ecd5933ac7090d7290f6f2d0f5" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==1.38.19" + }, + "botocore": { + "hashes": [ + "sha256:796b948c05017eb33385b798990cd91ed4af0e881eb9eb1ee6e17666be02abc9", + "sha256:f937a20e75889215a99280ea0fdd4e1716ffede23e4f9af7bc9c64af9bc63e61" + ], + "markers": "python_version >= '3.9'", + "version": "==1.38.19" + }, + "coverage": { + "extras": [ + "toml" + ], + "hashes": [ + "sha256:042e7841a26498fff7a37d6fda770d17519982f5b7d8bf5278d140b67b61095f", + "sha256:04bfec25a8ef1c5f41f5e7e5c842f6b615599ca8ba8391ec33a9290d9d2db3a3", + "sha256:0915742f4c82208ebf47a2b154a5334155ed9ef9fe6190674b8a46c2fb89cb05", + "sha256:18c5ae6d061ad5b3e7eef4363fb27a0576012a7447af48be6c75b88494c6cf25", + "sha256:2931f66991175369859b5fd58529cd4b73582461877ecfd859b6549869287ffe", + "sha256:2e4b6b87bb0c846a9315e3ab4be2d52fac905100565f4b92f02c445c8799e257", + "sha256:3043ba1c88b2139126fc72cb48574b90e2e0546d4c78b5299317f61b7f718b78", + "sha256:379fe315e206b14e21db5240f89dc0774bdd3e25c3c58c2c733c99eca96f1ada", + "sha256:42421e04069fb2cbcbca5a696c4050b84a43b05392679d4068acbe65449b5c64", + "sha256:4dfd9a93db9e78666d178d4f08a5408aa3f2474ad4d0e0378ed5f2ef71640cb6", + "sha256:52a523153c568d2c0ef8826f6cc23031dc86cffb8c6aeab92c4ff776e7951b28", + "sha256:554fec1199d93ab30adaa751db68acec2b41c5602ac944bb19187cb9a41a8067", + "sha256:581a40c7b94921fffd6457ffe532259813fc68eb2bdda60fa8cc343414ce3733", + "sha256:5a26c0c795c3e0b63ec7da6efded5f0bc856d7c0b24b2ac84b4d1d7bc578d676", + "sha256:5a570cd9bd20b85d1a0d7b009aaf6c110b52b5755c17be6962f8ccd65d1dbd23", + "sha256:5aaeb00761f985007b38cf463b1d160a14a22c34eb3f6a39d9ad6fc27cb73008", + "sha256:5ac46d0c2dd5820ce93943a501ac5f6548ea81594777ca585bf002aa8854cacd", + "sha256:5c8a5c139aae4c35cbd7cadca1df02ea8cf28a911534fc1b0456acb0b14234f3", + "sha256:6b8af63b9afa1031c0ef05b217faa598f3069148eeee6bb24b79da9012423b82", + "sha256:769773614e676f9d8e8a0980dd7740f09a6ea386d0f383db6821df07d0f08545", + "sha256:771eb7587a0563ca5bb6f622b9ed7f9d07bd08900f7589b4febff05f469bea00", + "sha256:77af0f6447a582fdc7de5e06fa3757a3ef87769fbb0fdbdeba78c23049140a47", + "sha256:7a3d62b3b03b4b6fd41a085f3574874cf946cb4604d2b4d3e8dca8cd570ca501", + "sha256:821f7bcbaa84318287115d54becb1915eece6918136c6f91045bb84e2f88739d", + "sha256:89b1f4af0d4afe495cd4787a68e00f30f1d15939f550e869de90a86efa7e0814", + "sha256:8a1d96e780bdb2d0cbb297325711701f7c0b6f89199a57f2049e90064c29f6bd", + "sha256:8a40fcf208e021eb14b0fac6bdb045c0e0cab53105f93ba0d03fd934c956143a", + "sha256:8f99eb72bf27cbb167b636eb1726f590c00e1ad375002230607a844d9e9a2318", + "sha256:90e7fbc6216ecaffa5a880cdc9c77b7418c1dcb166166b78dbc630d07f278cc3", + "sha256:94ec0be97723ae72d63d3aa41961a0b9a6f5a53ff599813c324548d18e3b9e8c", + "sha256:95aa6ae391a22bbbce1b77ddac846c98c5473de0372ba5c463480043a07bff42", + "sha256:96121edfa4c2dfdda409877ea8608dd01de816a4dc4a0523356067b305e4e17a", + "sha256:a1f406a8e0995d654b2ad87c62caf6befa767885301f3b8f6f73e6f3c31ec3a6", + "sha256:a321c61477ff8ee705b8a5fed370b5710c56b3a52d17b983d9215861e37b642a", + "sha256:a5761c70c017c1b0d21b0815a920ffb94a670c8d5d409d9b38857874c21f70d7", + "sha256:a9abbccd778d98e9c7e85038e35e91e67f5b520776781d9a1e2ee9d400869487", + "sha256:ad80e6b4a0c3cb6f10f29ae4c60e991f424e6b14219d46f1e7d442b938ee68a4", + "sha256:b44674870709017e4b4036e3d0d6c17f06a0e6d4436422e0ad29b882c40697d2", + "sha256:b571bf5341ba8c6bc02e0baeaf3b061ab993bf372d982ae509807e7f112554e9", + "sha256:b8194fb8e50d556d5849753de991d390c5a1edeeba50f68e3a9253fbd8bf8ccd", + "sha256:b87eb6fc9e1bb8f98892a2458781348fa37e6925f35bb6ceb9d4afd54ba36c73", + "sha256:bbb5cc845a0292e0c520656d19d7ce40e18d0e19b22cb3e0409135a575bf79fc", + "sha256:be945402e03de47ba1872cd5236395e0f4ad635526185a930735f66710e1bd3f", + "sha256:bf13d564d310c156d1c8e53877baf2993fb3073b2fc9f69790ca6a732eb4bfea", + "sha256:cf60dd2696b457b710dd40bf17ad269d5f5457b96442f7f85722bdb16fa6c899", + "sha256:d1ba00ae33be84066cfbe7361d4e04dec78445b2b88bdb734d0d1cbab916025a", + "sha256:d39fc4817fd67b3915256af5dda75fd4ee10621a3d484524487e33416c6f3543", + "sha256:d766a4f0e5aa1ba056ec3496243150698dc0481902e2b8559314368717be82b1", + "sha256:dbf364b4c5e7bae9250528167dfe40219b62e2d573c854d74be213e1e52069f7", + "sha256:dd19608788b50eed889e13a5d71d832edc34fc9dfce606f66e8f9f917eef910d", + "sha256:e013b07ba1c748dacc2a80e69a46286ff145935f260eb8c72df7185bf048f502", + "sha256:e5d2b9be5b0693cf21eb4ce0ec8d211efb43966f6657807f6859aab3814f946b", + "sha256:e5ff52d790c7e1628241ffbcaeb33e07d14b007b6eb00a19320c7b8a7024c040", + "sha256:e75a2ad7b647fd8046d58c3132d7eaf31b12d8a53c0e4b21fa9c4d23d6ee6d3c", + "sha256:e7ac22a0bb2c7c49f441f7a6d46c9c80d96e56f5a8bc6972529ed43c8b694e27", + "sha256:ed2144b8a78f9d94d9515963ed273d620e07846acd5d4b0a642d4849e8d91a0c", + "sha256:f017a61399f13aa6d1039f75cd467be388d157cd81f1a119b9d9a68ba6f2830d", + "sha256:f1d8a2a57b47142b10374902777e798784abf400a004b14f1b0b9eaf1e528ba4", + "sha256:f2d32f95922927186c6dbc8bc60df0d186b6edb828d299ab10898ef3f40052fe", + "sha256:f319bae0321bc838e205bf9e5bc28f0a3165f30c203b610f17ab5552cff90323", + "sha256:f3c38e4e5ccbdc9198aecc766cedbb134b2d89bf64533973678dfcf07effd883", + "sha256:f9983d01d7705b2d1f7a95e10bbe4091fabc03a46881a256c2787637b087003f", + "sha256:fa260de59dfb143af06dcf30c2be0b200bed2a73737a8a59248fcb9fa601ef0f" + ], + "markers": "python_version >= '3.9'", + "version": "==7.8.0" + }, + "flake8": { + "hashes": [ + "sha256:93b92ba5bdb60754a6da14fa3b93a9361fd00a59632ada61fd7b130436c40343", + "sha256:fa558ae3f6f7dbf2b4f22663e5343b6b6023620461f8d4ff2019ef4b5ee70426" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==7.2.0" + }, + "iniconfig": { + "hashes": [ + "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", + "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760" + ], + "markers": "python_version >= '3.8'", + "version": "==2.1.0" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "mccabe": { + "hashes": [ + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "packaging": { + "hashes": [ + "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", + "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" + ], + "markers": "python_version >= '3.8'", + "version": "==25.0" + }, + "pluggy": { + "hashes": [ + "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", + "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746" + ], + "markers": "python_version >= '3.9'", + "version": "==1.6.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:35863c5974a271c7a726ed228a14a4f6daf49df369d8c50cd9a6f58a5e143ba9", + "sha256:c8415bf09abe81d9c7f872502a6eee881fbe85d8763dd5b9924bb0a01d67efae" + ], + "markers": "python_version >= '3.9'", + "version": "==2.13.0" + }, + "pyflakes": { + "hashes": [ + "sha256:5039c8339cbb1944045f4ee5466908906180f13cc99cc9949348d10f82a5c32a", + "sha256:6dfd61d87b97fba5dcfaaf781171ac16be16453be6d816147989e7f6e6a9576b" + ], + "markers": "python_version >= '3.9'", + "version": "==3.3.2" + }, + "pytest": { + "hashes": [ + "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", + "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==8.3.5" + }, + "pytest-cov": { + "hashes": [ + "sha256:46935f7aaefba760e716c2ebfbe1c216240b9592966e7da99ea8292d4d3e2a0a", + "sha256:bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==6.1.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.9.0.post0" + }, + "s3transfer": { + "hashes": [ + "sha256:35b314d7d82865756edab59f7baebc6b477189e6ab4c53050e28c1de4d9cce18", + "sha256:8ac58bc1989a3fdb7c7f3ee0918a66b160d038a147c7b5db1500930a607e9a1c" + ], + "markers": "python_version >= '3.9'", + "version": "==0.12.0" + }, + "six": { + "hashes": [ + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.17.0" + }, + "urllib3": { + "hashes": [ + "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466", + "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813" + ], + "markers": "python_version >= '3.9'", + "version": "==2.4.0" + } + } +} diff --git a/ecs/jskult-batch/entrypoint.py b/ecs/jskult-batch/entrypoint.py new file mode 100644 index 00000000..421053fa --- /dev/null +++ b/ecs/jskult-batch/entrypoint.py @@ -0,0 +1,16 @@ +"""実消化&アルトマーク データ取込後続処理バッチエントリーポイント""" +from src.batch.common.jskult_batch_entrypoint_factory import JskultBatchEntrypointFactory +from src.system_var.environment import PROCESS_NAME + +if __name__ == '__main__': + try: + jkultBatchEntrypointFactory = JskultBatchEntrypointFactory( + PROCESS_NAME) + module = jkultBatchEntrypointFactory.create() + module.execute() + exit(0) + + except Exception: + # エラーが起きても、正常系のコードで返す。 + # エラーが起きた事実はbatch_process内でログを出す。 + exit(0) diff --git a/ecs/jskult-batch/pytest.ini b/ecs/jskult-batch/pytest.ini new file mode 100644 index 00000000..5dbe2661 --- /dev/null +++ b/ecs/jskult-batch/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +log_format = %(levelname)s %(asctime)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S diff --git a/ecs/jskult-batch/src/__init__.py b/ecs/jskult-batch/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch/src/aws/__init__.py b/ecs/jskult-batch/src/aws/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch/src/aws/s3.py b/ecs/jskult-batch/src/aws/s3.py new file mode 100644 index 00000000..6203868d --- /dev/null +++ b/ecs/jskult-batch/src/aws/s3.py @@ -0,0 +1,185 @@ +import gzip +import os +import os.path as path +import shutil +import tempfile + +import boto3 + +from src.system_var import environment + + +class S3Client: + __s3_client = boto3.client('s3') + _bucket_name: str + + def list_objects(self, bucket_name: str, folder_name: str): + response = self.__s3_client.list_objects_v2(Bucket=bucket_name, Prefix=folder_name) + if response['KeyCount'] == 0: + return [] + contents = response['Contents'] + # 末尾がスラッシュで終わるものはフォルダとみなしてスキップする + objects = [{'filename': content['Key'], 'size': content['Size']} + for content in contents if not content['Key'].endswith('/')] + return objects + + def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: + copy_source = {'Bucket': src_bucket, 'Key': src_key} + self.__s3_client.copy(copy_source, dest_bucket, dest_key) + return + + def download_file(self, bucket_name: str, file_key: str, file): + self.__s3_client.download_fileobj( + Bucket=bucket_name, + Key=file_key, + Fileobj=file + ) + return + + def upload_file(self, local_file_path: str, bucket_name: str, file_key: str): + self.__s3_client.upload_file( + local_file_path, + Bucket=bucket_name, + Key=file_key + ) + + def delete_file(self, bucket_name: str, file_key: str): + self.__s3_client.delete_object( + Bucket=bucket_name, + Key=file_key + ) + + +class S3Bucket(): + _s3_client = S3Client() + _bucket_name: str = None + + +class UltmarcBucket(S3Bucket): + _bucket_name = environment.ULTMARC_DATA_BUCKET + _folder = environment.ULTMARC_DATA_FOLDER + + def list_dat_file(self): + return self._s3_client.list_objects(self._bucket_name, self._folder) + + def download_dat_file(self, dat_filename: str): + # 一時ファイルとして保存する + temporary_dir = tempfile.mkdtemp() + temporary_file_path = path.join(temporary_dir, f'{dat_filename.replace(f"{self._folder}/", "")}') + with open(temporary_file_path, mode='wb') as f: + self._s3_client.download_file(self._bucket_name, dat_filename, f) + f.seek(0) + return temporary_file_path + + def backup_dat_file(self, dat_file_key: str, datetime_key: str): + # バックアップバケットにコピー + ultmarc_backup_bucket = UltmarcBackupBucket() + backup_key = f'{ultmarc_backup_bucket._folder}/{datetime_key}/{dat_file_key.replace(f"{self._folder}/", "")}' + self._s3_client.copy(self._bucket_name, dat_file_key, ultmarc_backup_bucket._bucket_name, backup_key) + # コピー元のファイルを削除 + self._s3_client.delete_file(self._bucket_name, dat_file_key) + + +class ConfigBucket(S3Bucket): + _bucket_name = environment.JSKULT_CONFIG_BUCKET + + def download_holiday_list(self): + # 一時ファイルとして保存する + temporary_dir = tempfile.mkdtemp() + temporary_file_path = path.join(temporary_dir, environment.JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME) + holiday_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME}' + with open(temporary_file_path, mode='wb') as f: + self._s3_client.download_file(self._bucket_name, holiday_list_key, f) + f.seek(0) + return temporary_file_path + + def download_wholesaler_stock_input_day_list(self): + # 一時ファイルとして保存する + temporary_dir = tempfile.mkdtemp() + temporary_file_path = path.join(temporary_dir, environment.JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME) + wholesaler_stock_input_day_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME}' + with open(temporary_file_path, mode='wb') as f: + self._s3_client.download_file(self._bucket_name, wholesaler_stock_input_day_list_key, f) + f.seek(0) + return temporary_file_path + + def download_ultmarc_hex_convert_config(self): + # 一時ファイルとして保存する + temporary_dir = tempfile.mkdtemp() + temporary_file_path = path.join(temporary_dir, environment.JSKULT_ULTMARC_HEX_CONVERT_CONFIG_FILE_NAME) + hex_convert_config_key = f'{environment.JSKULT_CONFIG_CONVERT_FOLDER}/{environment.JSKULT_ULTMARC_HEX_CONVERT_CONFIG_FILE_NAME}' + with open(temporary_file_path, mode='wb') as f: + self._s3_client.download_file(self._bucket_name, hex_convert_config_key, f) + f.seek(0) + return temporary_file_path + + +class JskUltBackupBucket(S3Bucket): + _bucket_name = environment.JSKULT_BACKUP_BUCKET + + +class UltmarcBackupBucket(JskUltBackupBucket): + _folder = environment.ULTMARC_BACKUP_FOLDER + + +class VjskBackupBucket(JskUltBackupBucket): + _folder = environment.VJSK_BACKUP_FOLDER + + +class VjskReceiveBucket(S3Bucket): + _bucket_name = environment.VJSK_DATA_BUCKET + _recv_folder = environment.VJSK_DATA_RECEIVE_FOLDER + + _s3_file_list = None + + def get_s3_file_list(self): + self._s3_file_list = self._s3_client.list_objects(self._bucket_name, self._recv_folder) + return self._s3_file_list + + def download_data_file(self, data_filename: str): + temporary_dir = tempfile.mkdtemp() + temporary_file_path = path.join(temporary_dir, f'{data_filename.replace(f"{self._recv_folder}/", "")}') + with open(temporary_file_path, mode='wb') as f: + self._s3_client.download_file(self._bucket_name, data_filename, f) + f.seek(0) + return temporary_file_path + + def unzip_data_file(self, filename: str): + temp_dir = os.path.dirname(filename) + decompress_filename = os.path.basename(filename).replace('.gz', '') + decompress_file_path = os.path.join(temp_dir, decompress_filename) + with gzip.open(filename, 'rb') as gz: + with open(decompress_file_path, 'wb') as decompressed_file: + shutil.copyfileobj(gz, decompressed_file) + + ret = [decompress_file_path] + return ret + + def backup_dat_file(self, target_files: list, datetime_key: str): + jskult_backup_bucket = VjskBackupBucket() + for target_file in target_files: + backup_from_file_path = target_file.get("filename") + backup_to_filename = backup_from_file_path.replace(f"{self._recv_folder}/", "") + backup_key = f'{jskult_backup_bucket._folder}/{datetime_key}/{backup_to_filename}' + self._s3_client.copy(self._bucket_name, backup_from_file_path, + jskult_backup_bucket._bucket_name, backup_key) + self._s3_client.delete_file(self._bucket_name, backup_from_file_path) + + +class VjskSendBucket(S3Bucket): + _bucket_name = environment.VJSK_DATA_BUCKET + _send_folder = environment.VJSK_DATA_SEND_FOLDER + + def upload_inst_pharm_csv_file(self, vjsk_create_csv: str, csv_file_path: str): + # S3バケットにファイルを移動 + csv_file_name = f'{self._send_folder}/{vjsk_create_csv}' + s3_client = S3Client() + s3_client.upload_file(csv_file_path, self._bucket_name, csv_file_name) + return + + def backup_inst_pharm_csv_file(self, dat_file_key: str, datetime_key: str): + # バックアップバケットにコピー + vjsk_backup_bucket = VjskBackupBucket() + dat_key = f'{self._send_folder}/{dat_file_key}' + backup_key = f'{vjsk_backup_bucket._folder}/{self._send_folder}/{datetime_key}/{dat_file_key.replace(f"{self._send_folder}/", "")}' + self._s3_client.copy(self._bucket_name, dat_key, vjsk_backup_bucket._bucket_name, backup_key) diff --git a/ecs/jskult-batch/src/batch/common/__init__.py b/ecs/jskult-batch/src/batch/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch/src/batch/common/jskult_batch_entrypoint_factory.py b/ecs/jskult-batch/src/batch/common/jskult_batch_entrypoint_factory.py new file mode 100644 index 00000000..6c297fe9 --- /dev/null +++ b/ecs/jskult-batch/src/batch/common/jskult_batch_entrypoint_factory.py @@ -0,0 +1,30 @@ +from src.batch.update_business_day import UpdateBusinessDay +from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint +from src.batch.trn_result_data_bio_lot import TrnResultDataBioLot +from src.batch.mst_inst import MstInst +from src.batch.dcf_inst_merge_io import DcfInstMergeIO + +from src.logging.get_logger import get_logger + +logger = get_logger("後続処理/日付更新処理振り分け") + + +class JskultBatchEntrypointFactory: + + def __init__(self, entrypoint_module_name: str): + self._entrypoint_module_name: str = entrypoint_module_name + + def create(self) -> JskultBatchEntrypoint: + + if self._entrypoint_module_name == "jskult-batch-trn-result-data-bio-lot": + return TrnResultDataBioLot() + if self._entrypoint_module_name == "jskult-batch-mst-inst": + return MstInst() + if self._entrypoint_module_name == "jskult-batch-dcf-inst-merge-io": + return DcfInstMergeIO() + if self._entrypoint_module_name == "jskult-batch-update-business-day": + return UpdateBusinessDay() + + logger.error( + f"一致するエントリーポイント識別子ではありませんでした。エントリーポイント識別子:{self._entrypoint_module_name}") + raise ValueError() diff --git a/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py new file mode 100644 index 00000000..710380c9 --- /dev/null +++ b/ecs/jskult-batch/src/batch/dcf_inst_merge_io.py @@ -0,0 +1,10 @@ +from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint + + +class DcfInstMergeIO(JskultBatchEntrypoint): + def __init__(self): + super().__init__() + + def execute(self): + # TODO: ここでDCF削除新規マスタ作成/データ出力処理を実行する + pass diff --git a/ecs/jskult-batch/src/batch/jskult_batch_entrypoint.py b/ecs/jskult-batch/src/batch/jskult_batch_entrypoint.py new file mode 100644 index 00000000..291a8d1f --- /dev/null +++ b/ecs/jskult-batch/src/batch/jskult_batch_entrypoint.py @@ -0,0 +1,8 @@ +import abc +# 実消化&アルトマークの後続処理/日付更新実行クラスの基底クラス + + +class JskultBatchEntrypoint(metaclass=abc.ABCMeta): + @abc.abstractmethod() + def execute(self): + pass diff --git a/ecs/jskult-batch/src/batch/mst_inst.py b/ecs/jskult-batch/src/batch/mst_inst.py new file mode 100644 index 00000000..c14fd1e7 --- /dev/null +++ b/ecs/jskult-batch/src/batch/mst_inst.py @@ -0,0 +1,10 @@ +from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint + + +class MstInst(JskultBatchEntrypoint): + def __init__(self): + super().__init__() + + def execute(self): + # TODO: ここでメルク施設マスタ作成処理を実行する + pass diff --git a/ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py b/ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py new file mode 100644 index 00000000..d0142cbc --- /dev/null +++ b/ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py @@ -0,0 +1,10 @@ +from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint + + +class TrnResultDataBioLot(JskultBatchEntrypoint): + def __init__(self): + super().__init__() + + def execute(self): + # TODO: ここで生物由来ロット分解処理を実行する + pass diff --git a/ecs/jskult-batch/src/batch/update_business_day.py b/ecs/jskult-batch/src/batch/update_business_day.py new file mode 100644 index 00000000..57e9372f --- /dev/null +++ b/ecs/jskult-batch/src/batch/update_business_day.py @@ -0,0 +1,10 @@ +from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint + + +class UpdateBusinessDay(JskultBatchEntrypoint): + def __init__(self): + super().__init__() + + def execute(self): + # TODO: ここで日付更新処理を実行する + pass diff --git a/ecs/jskult-batch/src/db/__init__.py b/ecs/jskult-batch/src/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch/src/db/database.py b/ecs/jskult-batch/src/db/database.py new file mode 100644 index 00000000..5ddaba4e --- /dev/null +++ b/ecs/jskult-batch/src/db/database.py @@ -0,0 +1,195 @@ +from sqlalchemy import (Connection, CursorResult, Engine, QueuePool, + create_engine, text) +from sqlalchemy.engine.url import URL +from tenacity import retry, stop_after_attempt, wait_exponential + +from src.error.exceptions import DBException +from src.logging.get_logger import get_logger +from src.system_var import environment + +logger = get_logger(__name__) + + +class Database: + """データベース操作クラス""" + __connection: Connection = None + __transactional_engine: Engine = None + __autocommit_engine: Engine = None + __host: str = None + __port: str = None + __username: str = None + __password: str = None + __schema: str = None + __autocommit: bool = None + __connection_string: str = None + + def __init__(self, username: str, password: str, host: str, port: int, schema: str, autocommit: bool = False) -> None: + """このクラスの新たなインスタンスを初期化します + + Args: + username (str): DBユーザー名 + password (str): DBパスワード + host (str): DBホスト名 + port (int): DBポート + schema (str): DBスキーマ名 + autocommit(bool): 自動コミットモードで接続するかどうか(Trueの場合、トランザクションの有無に限らず即座にコミットされる). Defaults to False. + """ + self.__username = username + self.__password = password + self.__host = host + self.__port = int(port) + self.__schema = schema + self.__autocommit = autocommit + + self.__connection_string = URL.create( + drivername='mysql+pymysql', + username=self.__username, + password=self.__password, + host=self.__host, + port=self.__port, + database=self.__schema, + query={"charset": "utf8mb4", "local_infile": "1"}, + ) + + self.__transactional_engine = create_engine( + self.__connection_string, + pool_timeout=5, + poolclass=QueuePool + ) + + self.__autocommit_engine = self.__transactional_engine.execution_options(isolation_level='AUTOCOMMIT') + + @classmethod + def get_instance(cls, autocommit=False): + """インスタンスを取得します + + Args: + autocommit (bool, optional): 自動コミットモードで接続するかどうか(Trueの場合、トランザクションの有無に限らず即座にコミットされる). Defaults to False. + Returns: + Database: DB操作クラスインスタンス + """ + return cls( + username=environment.DB_USERNAME, + password=environment.DB_PASSWORD, + host=environment.DB_HOST, + port=environment.DB_PORT, + schema=environment.DB_SCHEMA, + autocommit=autocommit + ) + + @retry( + wait=wait_exponential( + multiplier=environment.DB_CONNECTION_RETRY_INTERVAL_INIT, + min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, + max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS + ), + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), + retry_error_cls=DBException + ) + def connect(self): + """ + DBに接続します。接続に失敗した場合、リトライします。\n + インスタンスのautocommitがTrueの場合、自動コミットモードで接続する。（明示的なトランザクションも無視される） + Raises: + DBException: 接続失敗 + """ + try: + self.__connection = ( + self.__autocommit_engine.connect() if self.__autocommit is True + else self.__transactional_engine.connect()) + except Exception as e: + raise DBException(e) + + def execute_select(self, select_query: str, parameters=None) -> list[dict]: + """SELECTクエリを実行します。 + + Args: + select_query (str): SELECT文 + parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None. + + Raises: + DBException: DBエラー + + Returns: + list[dict]: カラム名: 値の辞書リスト + """ + if self.__connection is None: + raise DBException('DBに接続していません') + + result = None + try: + # トランザクションが開始している場合は、トランザクションを引き継ぐ + if self.__connection.in_transaction(): + result = self.__connection.execute(text(select_query), parameters) + else: + # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。 + result = self.__execute_with_transaction(select_query, parameters) + except Exception as e: + raise DBException(f'SQL Error: {e}') + + result_rows = result.mappings().all() + return result_rows + + def execute(self, query: str, parameters=None) -> CursorResult: + """SQLクエリを実行します。 + + Args: + query (str): SQL文 + parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None. + + Raises: + DBException: DBエラー + + Returns: + CursorResult: 取得結果 + """ + if self.__connection is None: + raise DBException('DBに接続していません') + + result = None + try: + # トランザクションが開始している場合は、トランザクションを引き継ぐ + if self.__connection.in_transaction(): + result = self.__connection.execute(text(query), parameters) + else: + # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。 + result = self.__execute_with_transaction(query, parameters) + except Exception as e: + raise DBException(f'SQL Error: {e}') + + return result + + def begin(self): + """トランザクションを開始します。""" + if not self.__connection.in_transaction(): + self.__connection.begin() + + def commit(self): + """トランザクションをコミットします""" + if self.__connection.in_transaction(): + self.__connection.commit() + + def rollback(self): + """トランザクションをロールバックします""" + if self.__connection.in_transaction(): + self.__connection.rollback() + + def disconnect(self): + """DB接続を切断します。""" + if self.__connection is not None: + self.__connection.close() + self.__connection = None + + def to_jst(self): + self.execute('SET time_zone = "+9:00"') + + def __execute_with_transaction(self, query: str, parameters: dict): + # トランザクションを開始してクエリを実行する + with self.__connection.begin(): + try: + result = self.__connection.execute(text(query), parameters=parameters) + except Exception as e: + self.__connection.rollback() + raise e + # ここでコミットされる + return result diff --git a/ecs/jskult-batch/src/error/__init__.py b/ecs/jskult-batch/src/error/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch/src/error/exceptions.py b/ecs/jskult-batch/src/error/exceptions.py new file mode 100644 index 00000000..055c24f6 --- /dev/null +++ b/ecs/jskult-batch/src/error/exceptions.py @@ -0,0 +1,10 @@ +class MeDaCaException(Exception): + pass + + +class DBException(MeDaCaException): + pass + + +class BatchOperationException(MeDaCaException): + pass diff --git a/ecs/jskult-batch/src/logging/get_logger.py b/ecs/jskult-batch/src/logging/get_logger.py new file mode 100644 index 00000000..f36f1199 --- /dev/null +++ b/ecs/jskult-batch/src/logging/get_logger.py @@ -0,0 +1,37 @@ +import logging + +from src.system_var.environment import LOG_LEVEL + +# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する +for name in ["boto3", "botocore", "s3transfer", "urllib3"]: + logging.getLogger(name).setLevel(logging.WARNING) + + +def get_logger(log_name: str) -> logging.Logger: + """一意のログ出力モジュールを取得します。 + + Args: + log_name (str): ロガー名 + + Returns: + _type_: _description_ + """ + logger = logging.getLogger(log_name) + level = logging.getLevelName(LOG_LEVEL) + if not isinstance(level, int): + level = logging.INFO + logger.setLevel(level) + + if not logger.hasHandlers(): + handler = logging.StreamHandler() + logger.addHandler(handler) + + formatter = logging.Formatter( + '%(name)s\t[%(levelname)s]\t%(asctime)s\t%(message)s', + '%Y-%m-%d %H:%M:%S' + ) + + for handler in logger.handlers: + handler.setFormatter(formatter) + + return logger diff --git a/ecs/jskult-batch/src/system_var/__init__.py b/ecs/jskult-batch/src/system_var/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch/src/system_var/constants.py b/ecs/jskult-batch/src/system_var/constants.py new file mode 100644 index 00000000..8a0ccbb3 --- /dev/null +++ b/ecs/jskult-batch/src/system_var/constants.py @@ -0,0 +1,17 @@ +# バッチ正常終了コード +BATCH_EXIT_CODE_SUCCESS = 0 + +# バッチ処理中フラグ：未処理 +BATCH_ACTF_BATCH_UNPROCESSED = '0' +# バッチ処理中フラグ：処理中 +BATCH_ACTF_BATCH_IN_PROCESSING = '1' +# dump取得状態区分：未処理 +DUMP_STATUS_KBN_UNPROCESSED = '0' +# dump取得状態区分：dump取得正常終了 +DUMP_STATUS_KBN_COMPLETE = '2' + +# カレンダーファイルのコメントシンボル +CALENDAR_COMMENT_SYMBOL = '#' + +# 月曜日(datetime.weekday()で月曜日を表す数字) +WEEKDAY_MONDAY = 0 diff --git a/ecs/jskult-batch/src/system_var/environment.py b/ecs/jskult-batch/src/system_var/environment.py new file mode 100644 index 00000000..e70d8bb4 --- /dev/null +++ b/ecs/jskult-batch/src/system_var/environment.py @@ -0,0 +1,18 @@ +import os + +# Database +DB_HOST = os.environ['DB_HOST'] +DB_PORT = int(os.environ['DB_PORT']) +DB_USERNAME = os.environ['DB_USERNAME'] +DB_PASSWORD = os.environ['DB_PASSWORD'] +DB_SCHEMA = os.environ['DB_SCHEMA'] + +# 処理名 +PROCESS_NAME = os.environ['PROCESS_NAME'] + +# 初期値がある環境変数 +LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') +DB_CONNECTION_MAX_RETRY_ATTEMPT = int(os.environ.get('DB_CONNECTION_MAX_RETRY_ATTEMPT', 4)) +DB_CONNECTION_RETRY_INTERVAL_INIT = int(os.environ.get('DB_CONNECTION_RETRY_INTERVAL', 5)) +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MIN_SECONDS', 5)) +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MAX_SECONDS', 50)) From 52d9b80649ad99af2cd659d731b07c692c7718fb Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Tue, 20 May 2025 18:48:03 +0900 Subject: [PATCH 02/13] =?UTF-8?q?=E3=83=90=E3=83=83=E3=83=81=E3=82=B9?= =?UTF-8?q?=E3=83=86=E3=83=BC=E3=82=BF=E3=82=B9=E7=AE=A1=E7=90=86=E3=83=86?= =?UTF-8?q?=E3=83=BC=E3=83=96=E3=83=ABupsert=E3=82=B9=E3=83=88=E3=82=A2?= =?UTF-8?q?=E3=83=89=20=E9=80=94=E4=B8=AD=E3=82=B3=E3=83=9F=E3=83=83?= =?UTF-8?q?=E3=83=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../upsert_jskult_batch_status_manage.sql | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql diff --git a/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql b/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql new file mode 100644 index 00000000..b7c04294 --- /dev/null +++ b/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql @@ -0,0 +1,93 @@ +-- A5M2で実行時に[SQL] - [スラッシュ(/)のみの行でSQLを区切る]に変えてから実行する +-- $$から始まる文字は後からREPLACEする文字を示す独自ルール +-- 当プロシージャは、同一セッション内での並列処理を実行することができない +-- 実行者の権限でストアドプロシージャを実行するために、「SQL SECURITY INVOKER」を付与している +CREATE PROCEDURE `internal07`.`upsert_jskult_batch_status_manage`(IN process_name varchar(100), IN process_type varchar(50), IN process_status varchar(50), IN total_run_count INT, max_run_count_flg TINYINT(1)) + +-- 振り分けスキーマ +DECLARE distribution_schema VARCHAR(20); +-- 振り分けカラム +DECLARE target_column_value VARCHAR(100); +-- 振り分け先テーブルID +DECLARE temp_table_id VARCHAR(18); +-- カーソルフェッチステータス +DECLARE fetch_done BOOLEAN DEFAULT FALSE; + +-- カーソル設定 +DECLARE table_cursor CURSOR FOR SELECT Id FROM internal07.; +DECLARE CONTINUE HANDLER FOR NOT FOUND SET fetch_done = TRUE; + +-- エラー処理 +DECLARE EXIT HANDLER FOR SQLEXCEPTION + BEGIN + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + ROLLBACK; + SET @error_msg = ( + CASE + WHEN LENGTH(@error_msg) > 128 THEN CONCAT(SUBSTRING(@error_msg, 1, 125), '...') + ELSE @error_msg + END + ); + SIGNAL SQLSTATE '45000' + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; + END; + +SQL SECURITY INVOKER +BEGIN + +START TRANSACTION; + +-- UPSERT STATEMENT設定 +SET @upsert_statement_base = + 'INSERT INTO $$distribution_schema$$.jskult_archive_manager + (process_name, process_date, process_type, process_status, + total_run_count, max_run_count_flg, ins_user, ins_date, upd_user, upd_date) + SELECT + ?, ?, ?, ?, total_run_count = IF(? IS NULL, total_run_count, ?), + max_run_count_flg = IF(? IS NULL, total_run_count, ?), + ins_user, ins_date, upd_user, upd_date) + FROM + internal07.jskult_archive_manager AS internaltb + WHERE + internaltb.process_name = ? AND + internaltb.process_date = ? + + ON DUPLICATE KEY UPDATE + process_type = IF(? IS NULL, process_type, ?), + process_status = IF(? IS NULL, process_status, ?), + total_run_count = IF(? IS NULL, total_run_count, ?), + max_run_count_flg = IF(? IS NULL, max_run_count_flg, ?), + upd_user = CURRENT_USER(), + upd_date = CURRENT_TIMESTAMP();'; + +OPEN table_cursor; + +-- ループ +TableCursorLoop: LOOP + -- スキーマ取得 + FETCH table_cursor INTO temp_table_id, target_column_value; + + -- 終了条件ループ抜き + IF fetch_done THEN + LEAVE TableCursorLoop; + END IF; + + SET @distribution_schema = internal07.get_distribution_to_schema('jskult_archive_manager', 'process_name', target_column_value); + SET @temp_table_id = temp_table_id; + SET @upsert_statement = REPLACE(@upsert_statement_base, "$$distribution_schema$$", @distribution_schema); + + -- INSERT時に引数がNULLである時用の変換 + SET @total_run_count = IFNULL(total_run_count, 0); + SET @max_run_count_flg = IFNULL(max_run_count_flg, 0); + + -- UPSERT実行 + PREPARE stmt FROM @upsert_statement; + EXECUTE stmt USING @temp_table_id, @process_name, @process_type, @process_status, @total_run_count, @max_run_count_flg; + DEALLOCATE PREPARE stmt; + +END LOOP; +CLOSE table_cursor; +COMMIT; + +END \ No newline at end of file From fce17870ce238cac3de1c919a05de25d6be022ab Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Wed, 21 May 2025 14:01:54 +0900 Subject: [PATCH 03/13] =?UTF-8?q?=E5=86=85=E5=AE=B9=E3=81=AE=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=E3=81=A8=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E3=82=92?= =?UTF-8?q?=E8=A8=98=E8=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../upsert_jskult_batch_status_manage.sql | 120 ++++++++++-------- 1 file changed, 70 insertions(+), 50 deletions(-) diff --git a/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql b/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql index b7c04294..5384fa86 100644 --- a/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql +++ b/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql @@ -2,20 +2,7 @@ -- $$から始まる文字は後からREPLACEする文字を示す独自ルール -- 当プロシージャは、同一セッション内での並列処理を実行することができない -- 実行者の権限でストアドプロシージャを実行するために、「SQL SECURITY INVOKER」を付与している -CREATE PROCEDURE `internal07`.`upsert_jskult_batch_status_manage`(IN process_name varchar(100), IN process_type varchar(50), IN process_status varchar(50), IN total_run_count INT, max_run_count_flg TINYINT(1)) - --- 振り分けスキーマ -DECLARE distribution_schema VARCHAR(20); --- 振り分けカラム -DECLARE target_column_value VARCHAR(100); --- 振り分け先テーブルID -DECLARE temp_table_id VARCHAR(18); --- カーソルフェッチステータス -DECLARE fetch_done BOOLEAN DEFAULT FALSE; - --- カーソル設定 -DECLARE table_cursor CURSOR FOR SELECT Id FROM internal07.; -DECLARE CONTINUE HANDLER FOR NOT FOUND SET fetch_done = TRUE; +CREATE PROCEDURE `internal07`.`upsert_jskult_batch_status_manage`(process_name varchar(100), process_type varchar(50), process_status varchar(50), total_run_count INT, max_run_count_flg TINYINT(1)) -- エラー処理 DECLARE EXIT HANDLER FOR SQLEXCEPTION @@ -40,54 +27,87 @@ START TRANSACTION; -- UPSERT STATEMENT設定 SET @upsert_statement_base = - 'INSERT INTO $$distribution_schema$$.jskult_archive_manager - (process_name, process_date, process_type, process_status, - total_run_count, max_run_count_flg, ins_user, ins_date, upd_user, upd_date) - SELECT - ?, ?, ?, ?, total_run_count = IF(? IS NULL, total_run_count, ?), - max_run_count_flg = IF(? IS NULL, total_run_count, ?), + 'INSERT INTO src07.jskult_batch_status_manage + (process_name, process_date, process_type, process_status, total_run_count,max_run_count_flg, + ins_user, ins_date, upd_user, upd_date) + VALUES( + (?, ?, ?, ?, ?, ?, ins_user, ins_date, upd_user, upd_date) - FROM - internal07.jskult_archive_manager AS internaltb - WHERE - internaltb.process_name = ? AND - internaltb.process_date = ? ON DUPLICATE KEY UPDATE - process_type = IF(? IS NULL, process_type, ?), - process_status = IF(? IS NULL, process_status, ?), - total_run_count = IF(? IS NULL, total_run_count, ?), - max_run_count_flg = IF(? IS NULL, max_run_count_flg, ?), + process_type = ?, + process_status = ?, + total_run_count = ?, + max_run_count_flg = ?, upd_user = CURRENT_USER(), upd_date = CURRENT_TIMESTAMP();'; -OPEN table_cursor; --- ループ -TableCursorLoop: LOOP - -- スキーマ取得 - FETCH table_cursor INTO temp_table_id, target_column_value; +-- 日付テーブルから処理年月日を取得 +SET @select_process_date_statement = 'SELECT syor_date INTO get_value FROM src07.hdke_tbl'; - -- 終了条件ループ抜き - IF fetch_done THEN - LEAVE TableCursorLoop; - END IF; +PREPARE stmt FROM @select_process_date_statement; +EXECUTE stmt: +DEALLOCATE PREPARE stmt; +SET @insert_process_date = get_value; - SET @distribution_schema = internal07.get_distribution_to_schema('jskult_archive_manager', 'process_name', target_column_value); - SET @temp_table_id = temp_table_id; - SET @upsert_statement = REPLACE(@upsert_statement_base, "$$distribution_schema$$", @distribution_schema); - - -- INSERT時に引数がNULLである時用の変換 - SET @total_run_count = IFNULL(total_run_count, 0); - SET @max_run_count_flg = IFNULL(max_run_count_flg, 0); +-- INSERT時に引数がNULLであるとき0を設定 +SET @insert_total_run_count = IFNULL(total_run_count, 0); +SET @insert_max_run_count_flg = IFNULL(max_run_count_flg, 0); - -- UPSERT実行 - PREPARE stmt FROM @upsert_statement; - EXECUTE stmt USING @temp_table_id, @process_name, @process_type, @process_status, @total_run_count, @max_run_count_flg; + +-- 元々の値を取得するためのステートメント +SET @select_statement_base = 'SELECT $$select_column$$ INTO get_value FROM src07.jskult_batch_status_manage WHERE process_name = ?' + +-- UPDATE時にprocess_typeがNULLであるとき更新前の値を設定 +IF process_type = NULL + SET @select_statement = REPLACE(@select_statement_base, "$$select_column$$", "process_type"); + PREPARE stmt FROM @select_statement; + EXECUTE stmt USING @process_name; DEALLOCATE PREPARE stmt; + SET @update_process_type = get_value; +ELSE + SET @update_process_type = process_type; +END IF; + +-- UPDATE時にprocess_statusがNULLであるとき更新前の値を設定 +IF process_status = NULL + SET @select_statement = REPLACE(@select_statement_base, "$$select_column$$", "process_status"); + PREPARE stmt FROM @select_statement; + EXECUTE stmt USING @process_name; + DEALLOCATE PREPARE stmt; + SET @update_process_status = get_value; +ELSE + SET @update_process_status = process_status; +END IF; + +-- UPDATE時にtotal_run_countがNULLであるとき更新前の値を設定 +IF total_run_count = NULL + SET @select_statement = REPLACE(@select_statement_base, "$$select_column$$","total_run_count"); + PREPARE stmt FROM @select_statement; + EXECUTE stmt USING @process_name; + DEALLOCATE PREPARE stmt; + SET @update_total_run_count = get_value +ELSE + SET @update_total_run_count = total_run_count; +END IF; + +-- UPDATE時にmax_run_count_flgがNULLであるとき更新前の値を設定 +IF max_run_count_flg = NULL + SET @select_statement = REPLACE(@select_statement_base, "$$select_column$$","max_run_count_flg"); + PREPARE stmt FROM @select_statement; + EXECUTE stmt USING @process_name; + DEALLOCATE PREPARE stmt; + SET @update_max_run_count_flg = get_value; +ELSE + SET @update_max_run_count_flg = max_run_count_flg; +END IF; + +-- UPSERT実行 +PREPARE stmt FROM @upsert_statement; +EXECUTE stmt USING @process_name, @insert_process_date , @process_type, @process_status, @insert_total_run_count, @insert_max_run_count_flg, @update_process_type, @update_process_status, @update_total_run_count, @update_max_run_count_flg; +DEALLOCATE PREPARE stmt; -END LOOP; -CLOSE table_cursor; COMMIT; END \ No newline at end of file From 44764a7a2661907e4ce414df7c7613a0b8d9974b Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Wed, 21 May 2025 14:13:25 +0900 Subject: [PATCH 04/13] =?UTF-8?q?update=E6=99=82=E3=81=AE=E5=80=A4?= =?UTF-8?q?=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal07/upsert_jskult_batch_status_manage.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql b/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql index 5384fa86..d67be48e 100644 --- a/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql +++ b/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql @@ -26,13 +26,13 @@ BEGIN START TRANSACTION; -- UPSERT STATEMENT設定 -SET @upsert_statement_base = +SET @upsert_statement = 'INSERT INTO src07.jskult_batch_status_manage (process_name, process_date, process_type, process_status, total_run_count,max_run_count_flg, ins_user, ins_date, upd_user, upd_date) VALUES( (?, ?, ?, ?, ?, ?, - ins_user, ins_date, upd_user, upd_date) + ins_user = CURRENT_USER(), ins_date = CURRENT_TIMESTAMP(), NULL, NULL) ON DUPLICATE KEY UPDATE process_type = ?, @@ -56,7 +56,7 @@ SET @insert_total_run_count = IFNULL(total_run_count, 0); SET @insert_max_run_count_flg = IFNULL(max_run_count_flg, 0); --- 元々の値を取得するためのステートメント +-- 引数がNULLであるときに更新前の値を取得するためのステートメント設定 SET @select_statement_base = 'SELECT $$select_column$$ INTO get_value FROM src07.jskult_batch_status_manage WHERE process_name = ?' -- UPDATE時にprocess_typeがNULLであるとき更新前の値を設定 From b6e3252dac0b827c2a27a6cd7e05f6c1cf92ab71 Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Wed, 21 May 2025 17:49:26 +0900 Subject: [PATCH 05/13] =?UTF-8?q?=E6=8C=87=E6=91=98=E4=BA=8B=E9=A0=85?= =?UTF-8?q?=E3=81=AE=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../upsert_jskult_batch_status_manage.sql | 119 ++++++------------ 1 file changed, 38 insertions(+), 81 deletions(-) diff --git a/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql b/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql index d67be48e..545cc64f 100644 --- a/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql +++ b/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql @@ -2,112 +2,69 @@ -- $$から始まる文字は後からREPLACEする文字を示す独自ルール -- 当プロシージャは、同一セッション内での並列処理を実行することができない -- 実行者の権限でストアドプロシージャを実行するために、「SQL SECURITY INVOKER」を付与している -CREATE PROCEDURE `internal07`.`upsert_jskult_batch_status_manage`(process_name varchar(100), process_type varchar(50), process_status varchar(50), total_run_count INT, max_run_count_flg TINYINT(1)) +CREATE PROCEDURE `internal07`.`upsert_jskult_batch_status_manage`( + process_name varchar(100), + process_type varchar(50), + process_status varchar(50), + total_run_count INT, + max_run_count_flg TINYINT(1) +) +SQL SECURITY INVOKER +BEGIN -- エラー処理 DECLARE EXIT HANDLER FOR SQLEXCEPTION BEGIN - GET DIAGNOSTICS CONDITION 1 - @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; - ROLLBACK; + GET DIAGNOSTICS CONDITION 1 + @error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT; + ROLLBACK; SET @error_msg = ( CASE WHEN LENGTH(@error_msg) > 128 THEN CONCAT(SUBSTRING(@error_msg, 1, 125), '...') ELSE @error_msg END ); - SIGNAL SQLSTATE '45000' - SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; - END; - -SQL SECURITY INVOKER -BEGIN + SIGNAL SQLSTATE '45000' + SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; + END; START TRANSACTION; -- UPSERT STATEMENT設定 SET @upsert_statement = - 'INSERT INTO src07.jskult_batch_status_manage - (process_name, process_date, process_type, process_status, total_run_count,max_run_count_flg, - ins_user, ins_date, upd_user, upd_date) + 'INSERT INTO internal07.jskult_batch_status_manage( + process_name, process_date, process_type, process_status, + total_run_count,max_run_count_flg, + ins_user, ins_date, upd_user, upd_date + ) VALUES( - (?, ?, ?, ?, ?, ?, - ins_user = CURRENT_USER(), ins_date = CURRENT_TIMESTAMP(), NULL, NULL) - + ?, src07.get_syor_date(),?, ?, + IFNULL(?, 0), + IFNULL(?, 0), + CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL + ) ON DUPLICATE KEY UPDATE - process_type = ?, - process_status = ?, - total_run_count = ?, - max_run_count_flg = ?, + process_type = CASE WHEN ? is NULL THEN process_type ELSE ? END, + process_status = CASE WHEN ? is NULL THEN process_status ELSE ? END, + total_run_count = CASE WHEN ? is NULL THEN total_run_count ELSE ? END, + max_run_count_flg = CASE WHEN ? is NULL THEN max_run_count_flg ELSE ? END, upd_user = CURRENT_USER(), upd_date = CURRENT_TIMESTAMP();'; --- 日付テーブルから処理年月日を取得 -SET @select_process_date_statement = 'SELECT syor_date INTO get_value FROM src07.hdke_tbl'; - -PREPARE stmt FROM @select_process_date_statement; -EXECUTE stmt: -DEALLOCATE PREPARE stmt; -SET @insert_process_date = get_value; - --- INSERT時に引数がNULLであるとき0を設定 -SET @insert_total_run_count = IFNULL(total_run_count, 0); -SET @insert_max_run_count_flg = IFNULL(max_run_count_flg, 0); - - --- 引数がNULLであるときに更新前の値を取得するためのステートメント設定 -SET @select_statement_base = 'SELECT $$select_column$$ INTO get_value FROM src07.jskult_batch_status_manage WHERE process_name = ?' - --- UPDATE時にprocess_typeがNULLであるとき更新前の値を設定 -IF process_type = NULL - SET @select_statement = REPLACE(@select_statement_base, "$$select_column$$", "process_type"); - PREPARE stmt FROM @select_statement; - EXECUTE stmt USING @process_name; - DEALLOCATE PREPARE stmt; - SET @update_process_type = get_value; -ELSE - SET @update_process_type = process_type; -END IF; - --- UPDATE時にprocess_statusがNULLであるとき更新前の値を設定 -IF process_status = NULL - SET @select_statement = REPLACE(@select_statement_base, "$$select_column$$", "process_status"); - PREPARE stmt FROM @select_statement; - EXECUTE stmt USING @process_name; - DEALLOCATE PREPARE stmt; - SET @update_process_status = get_value; -ELSE - SET @update_process_status = process_status; -END IF; - --- UPDATE時にtotal_run_countがNULLであるとき更新前の値を設定 -IF total_run_count = NULL - SET @select_statement = REPLACE(@select_statement_base, "$$select_column$$","total_run_count"); - PREPARE stmt FROM @select_statement; - EXECUTE stmt USING @process_name; - DEALLOCATE PREPARE stmt; - SET @update_total_run_count = get_value -ELSE - SET @update_total_run_count = total_run_count; -END IF; - --- UPDATE時にmax_run_count_flgがNULLであるとき更新前の値を設定 -IF max_run_count_flg = NULL - SET @select_statement = REPLACE(@select_statement_base, "$$select_column$$","max_run_count_flg"); - PREPARE stmt FROM @select_statement; - EXECUTE stmt USING @process_name; - DEALLOCATE PREPARE stmt; - SET @update_max_run_count_flg = get_value; -ELSE - SET @update_max_run_count_flg = max_run_count_flg; -END IF; +SET @process_name = process_name; +SET @process_type = process_type; +SET @process_status = process_status; +SET @total_run_count = total_run_count; +SET @max_run_count_flg = max_run_count_flg; -- UPSERT実行 PREPARE stmt FROM @upsert_statement; -EXECUTE stmt USING @process_name, @insert_process_date , @process_type, @process_status, @insert_total_run_count, @insert_max_run_count_flg, @update_process_type, @update_process_status, @update_total_run_count, @update_max_run_count_flg; +EXECUTE stmt USING +@process_name, @process_type, @process_status, @total_run_count, @max_run_count_flg, +@process_type, @process_type, @process_status, @process_status, @total_run_count, @total_run_count, @max_run_count_flg, @max_run_count_flg; DEALLOCATE PREPARE stmt; COMMIT; -END \ No newline at end of file +END; \ No newline at end of file From 85647c25630be929af6eea58fa646fb7ebb47196 Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Wed, 21 May 2025 18:14:57 +0900 Subject: [PATCH 06/13] =?UTF-8?q?=E3=83=90=E3=83=83=E3=83=81=E5=AE=9F?= =?UTF-8?q?=E8=A1=8C=E7=AE=A1=E7=90=86=E3=83=86=E3=83=BC=E3=83=96=E3=83=AB?= =?UTF-8?q?(DynamoDB)=E6=93=8D=E4=BD=9C=E3=82=AF=E3=83=A9=E3=82=B9?= =?UTF-8?q?=E3=81=AE=E5=AE=9F=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/manager/jskult_batch_run_manager.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 ecs/jskult-batch/src/manager/jskult_batch_run_manager.py diff --git a/ecs/jskult-batch/src/manager/jskult_batch_run_manager.py b/ecs/jskult-batch/src/manager/jskult_batch_run_manager.py new file mode 100644 index 00000000..3a55f7b9 --- /dev/null +++ b/ecs/jskult-batch/src/manager/jskult_batch_run_manager.py @@ -0,0 +1,28 @@ +class JskultBatchRunManager: + def __init__(self, execution_id: str): + self._execution_id: str = execution_id + + def batch_success(self): + try: + self._put_dynamodb_record('success') + except Exception as e: + raise e + + def batch_failed(self): + try: + self._put_dynamodb_record('failed') + except Exception as e: + raise e + + def batch_retry(self): + try: + self._put_dynamodb_record('retry') + except Exception as e: + raise e + + def _put_dynamodb_record(self, execution_id: str): + try: + self._execution_id = execution_id + # TODO バッチ実行管理テーブルの登録、更新（upsert) + except Exception as e: + raise e \ No newline at end of file From db54af7252513afb144102d9576c903cd2f8fb10 Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Wed, 21 May 2025 18:40:40 +0900 Subject: [PATCH 07/13] =?UTF-8?q?=E5=A4=89=E6=95=B0=E3=81=AE=E3=82=B3?= =?UTF-8?q?=E3=83=A1=E3=83=B3=E3=83=88=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../upsert_jskult_batch_status_manage.sql | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql b/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql index 545cc64f..2aa2947f 100644 --- a/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql +++ b/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql @@ -51,7 +51,6 @@ SET @upsert_statement = upd_user = CURRENT_USER(), upd_date = CURRENT_TIMESTAMP();'; - SET @process_name = process_name; SET @process_type = process_type; SET @process_status = process_status; @@ -61,8 +60,25 @@ SET @max_run_count_flg = max_run_count_flg; -- UPSERT実行 PREPARE stmt FROM @upsert_statement; EXECUTE stmt USING -@process_name, @process_type, @process_status, @total_run_count, @max_run_count_flg, -@process_type, @process_type, @process_status, @process_status, @total_run_count, @total_run_count, @max_run_count_flg, @max_run_count_flg; +-- INSERT用処理名 +@process_name, +-- INSERT用管理区分 +@process_type, +-- INSERT用処理ステータス +@process_status, +-- INSERT用起動回数 +@total_run_count, +-- INSERT用最大起動回数フラグ +@max_run_count_flg, + +-- UPDATE用管理区分 +@process_type, @process_type, +-- UPDATE用処理ステータス +@process_status, @process_status, +-- UPDATE用起動回数 +@total_run_count, @total_run_count, +-- UPDATE用最大起動回数フラグ +@max_run_count_flg, @max_run_count_flg; DEALLOCATE PREPARE stmt; COMMIT; From e60124bec4dc171b19bc1225983b59eb3425931a Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Thu, 22 May 2025 17:16:53 +0900 Subject: [PATCH 08/13] =?UTF-8?q?START=20TRANSACTION=E3=81=A8commit?= =?UTF-8?q?=E3=81=AE=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../internal07/upsert_jskult_batch_status_manage.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql b/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql index 2aa2947f..ef9841b3 100644 --- a/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql +++ b/rds_mysql/stored_procedure/internal07/upsert_jskult_batch_status_manage.sql @@ -28,8 +28,6 @@ DECLARE EXIT HANDLER FOR SQLEXCEPTION SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg; END; -START TRANSACTION; - -- UPSERT STATEMENT設定 SET @upsert_statement = 'INSERT INTO internal07.jskult_batch_status_manage( @@ -81,6 +79,5 @@ EXECUTE stmt USING @max_run_count_flg, @max_run_count_flg; DEALLOCATE PREPARE stmt; -COMMIT; END; \ No newline at end of file From 8214de77e441df87affa04b43fc0877f948166dd Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Thu, 22 May 2025 19:48:39 +0900 Subject: [PATCH 09/13] =?UTF-8?q?=E3=83=90=E3=83=83=E3=83=81=E3=82=B9?= =?UTF-8?q?=E3=83=86=E3=83=BC=E3=82=BF=E3=82=B9=E7=AE=A1=E7=90=86=E3=83=86?= =?UTF-8?q?=E3=83=BC=E3=83=96=E3=83=AB=E7=AE=A1=E7=90=86=E3=82=AF=E3=83=A9?= =?UTF-8?q?=E3=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manager/jskult_batch_status_manager.py | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 ecs/jskult-batch/src/manager/jskult_batch_status_manager.py diff --git a/ecs/jskult-batch/src/manager/jskult_batch_status_manager.py b/ecs/jskult-batch/src/manager/jskult_batch_status_manager.py new file mode 100644 index 00000000..5ffbe0c4 --- /dev/null +++ b/ecs/jskult-batch/src/manager/jskult_batch_status_manager.py @@ -0,0 +1,199 @@ +from src.error.exceptions import MaxRunCountReachedException +from src.db.database import Database + +# 実消化&アルトマーク_バッチステータス管理テーブルを管理するクラス +class JskultBatchStatusManager: + def __init__(self, process_name : str, process_type : str, max_run_count_flg : int, receive_file_count : int): + + # 処理名 + self._process_name: str = process_name + + # 管理区分 + self._process_type: str = process_type + + # 最大起動回数 + self._max_run_count_flg: str = max_run_count_flg + + # 受信ファイル数 + self._receive_file_count: str = receive_file_count + + # DB接続モジュール(バッチ)のget_instanceを呼び出し設定 + self._db = Database().get_instance() + + # 処理ステータスの登録および更新を行う + def set_process_status (self, process_status : str): + try: + self._db.begin() + self._db.execute(f"CALL upsert_jskult_batch_status_manage({self._process_name} {self._process_type} {process_status} NULL NULL);") + self._db.commit() + + except Exception as e: + + # Exceptionをcatchした場合はrollback + self._db.rollback() + raise e + + # 後続処理を実行してよいか判定する + def can_run_post_process(self): + + # SELECTの結果からレコード数を取得 + record_count = self._db.execute_select( + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" + ).count() + + if record_count == 0: + raise ValueError("レコードの取得が出来ませんでした。") + + # 起動回数のインクリメント + self._increment_run_count() + + # データ取込が完了していた場合 + if self._is_done_data_import(): + return True + + #最大起動回数に到達していない場合 + if not self._is_max_run_count_reached: + return False + + # 最大起動回数フラグを立てる + self._activate_max_run_count_flg() + + return True + + # 日付テーブル更新を実行してよいか判定する + def can_run_business_day_update(self): + + # SELECTの結果からレコード数を取得 + record_count = self._db.execute_select( + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" + ).count() + + if record_count == 0: + raise ValueError("レコードの取得が出来ませんでした。") + + # 起動回数のインクリメント + self._increment_run_count() + + # 後続処理が完了していた場合 + if self._is_done_post_process(): + return True + + #最大起動回数に到達していない場合 + if not self._is_max_run_count_reached(): + return False + + # 最大起動回数フラグを立てる + self._activate_max_run_count_flg() + + # 最大起動回数に到達した場合にメッセージをスロー + raise MaxRunCountReachedException("最大起動回数に到達しました") + + + # アルトマークデータ連携があったかを確認する + def is_done_ultmarc_import(self): + + # SELECTの結果からレコード数を取得 + record_count = self._db.execute_select( + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = 'jskult-batch-ultmarc-io' AND process_date = src07.get_syor_date();" + ).count() + # アルトマークデータ連携が無かった場合 + if record_count == 0: + return False + + return True + + # 起動回数をインクリメントする + def _increment_run_count(self): + try: + self._db.begin() + # SELECTの結果からレコードを取得 + record = self._db.execute_select( + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" + ) + + run_count += record[0]['run_count'] + + self._db.execute(f"CALL upsert_jskult_batch_status_manage({self._process_name} {self._process_type} NULL {run_count} NULL);") + + self._db.commit() + + except Exception as e: + + # Exceptionをcatchした場合はrollbakc + self._db.rollback() + raise e + + # データ取込処理が完了しているかを判定する + def _is_done_data_import(self): + + # SELECTの結果からレコード数を取得 + record_count = self._db.execute_select( + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_type = {self._process_type} AND process_status = 'done' AND process_date = src07.get_syor_date();" + ).count() + + # データ取込数が一致している場合 + if(self._receive_file_count == record_count): + return True + + return False + + # 後続処理のすべての処理が完了しているかを判定する + def _is_done_post_process(self): + + if not self._is_done_process("jskult-batch-trn-result-data-bio-lot"): + return False + + if not self._is_done_process("jskult-batch-mst-inst"): + return False + + if not self._is_done_process("jskult-batch-dcf-inst-merge-io"): + return False + + # 全ての後続処理が完了している場合Trueを返す + return True + + + # データ取込処理が完了しているかを判定する + def _is_done_process(self, process_name : str): + + # SELECTの結果からレコード数を取得 + record_count = self._db.execute_select( + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_status = 'done' AND process_date = src07.get_syor_date();" + ).count() + + if(record_count == 0): + return False + + return True + + + # 起動回数が最大回数に到達しているか判定する + def _is_max_run_count_reached(self): + + # SELECTの結果からレコード数を取得 + record = self._db.execute_select( + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" + ) + + run_count = record[0]['run_count'] + + # 取得した起動回数とフィールド変数の最大起動回数が一致するか確認 + if run_count == self._max_run_count_flg: + return True + + return False + + + def _activate_max_run_count_flg(self): + try: + self._db.begin() + + # 最大起動回数フラグにフラグを立てる + self._db.execute(f"CALL upsert_jskult_batch_status_manage({self._process_name} {self._process_type} NULL NULL 1);") + + self._db.commit() + + except Exception as e: + + self._db.rollback() + raise e \ No newline at end of file From 800f79b8257b5b11edcea959dbba6d1e1bd87009 Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Thu, 22 May 2025 19:50:47 +0900 Subject: [PATCH 10/13] =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=89=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/manager/jskult_batch_run_manager.py | 51 +++++++++++-------- 1 file changed, 31 insertions(+), 20 deletions(-) diff --git a/ecs/jskult-batch/src/manager/jskult_batch_run_manager.py b/ecs/jskult-batch/src/manager/jskult_batch_run_manager.py index 3a55f7b9..2a727e10 100644 --- a/ecs/jskult-batch/src/manager/jskult_batch_run_manager.py +++ b/ecs/jskult-batch/src/manager/jskult_batch_run_manager.py @@ -1,28 +1,39 @@ +import os +from datetime import datetime, timezone + +import boto3 + +TABLE_NAME = 'mbj-newdwh2021-staging-jskult-batch-run-manage' + +dynamodb = boto3.client('dynamodb') + + class JskultBatchRunManager: def __init__(self, execution_id: str): self._execution_id: str = execution_id - + + # バッチ処理ステータスをsuccessで登録および更新を行う def batch_success(self): - try: - self._put_dynamodb_record('success') - except Exception as e: - raise e - + self._put_dynamodb_record('success') + + # バッチ処理ステータスをfailedで登録および更新を行う def batch_failed(self): - try: - self._put_dynamodb_record('failed') - except Exception as e: - raise e + self._put_dynamodb_record('failed') + # バッチ処理ステータスをretryで登録および更新を行う def batch_retry(self): - try: - self._put_dynamodb_record('retry') - except Exception as e: - raise e + self._put_dynamodb_record('retry') - def _put_dynamodb_record(self, execution_id: str): - try: - self._execution_id = execution_id - # TODO バッチ実行管理テーブルの登録、更新（upsert) - except Exception as e: - raise e \ No newline at end of file + def _put_dynamodb_record(self, record: str): + # バッチ実行管理テーブルの登録、更新（upsert) + now = int(datetime.now(timezone.utc).timestamp() * 1000) + + options = { + 'TableName': TABLE_NAME, + 'Item': { + 'execution_id': {'S': self._execution_id}, + 'status': {'S': record}, + 'createdAt': {'N': str(now)}, + }, + } + dynamodb.put_item(**options) From fa408d2a832a54181ce5528d969a851a75b5bd4e Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Thu, 22 May 2025 19:51:33 +0900 Subject: [PATCH 11/13] =?UTF-8?q?=E3=82=B3=E3=83=BC=E3=83=89=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manager/jskult_batch_status_manager.py | 117 ++++++++++-------- 1 file changed, 65 insertions(+), 52 deletions(-) diff --git a/ecs/jskult-batch/src/manager/jskult_batch_status_manager.py b/ecs/jskult-batch/src/manager/jskult_batch_status_manager.py index 5ffbe0c4..5792570d 100644 --- a/ecs/jskult-batch/src/manager/jskult_batch_status_manager.py +++ b/ecs/jskult-batch/src/manager/jskult_batch_status_manager.py @@ -1,10 +1,19 @@ +import os from src.error.exceptions import MaxRunCountReachedException from src.db.database import Database +DB_USERNAME = os.environ.get("DB_USERNAME") +DB_PASSWORD = os.environ.get("DB_PASSWORD") +DB_HOST = os.environ.get("DB_HOST") +DB_PORT = os.environ.get("DB_PORT") +DB_SCHEMA = os.environ.get("DB_SCHEMA") + # 実消化&アルトマーク_バッチステータス管理テーブルを管理するクラス + + class JskultBatchStatusManager: - def __init__(self, process_name : str, process_type : str, max_run_count_flg : int, receive_file_count : int): - + def __init__(self, process_name: str, process_type: str, max_run_count_flg: int, receive_file_count: int): + # 処理名 self._process_name: str = process_name @@ -18,43 +27,46 @@ class JskultBatchStatusManager: self._receive_file_count: str = receive_file_count # DB接続モジュール(バッチ)のget_instanceを呼び出し設定 - self._db = Database().get_instance() + self._db = Database(DB_USERNAME, DB_PASSWORD, DB_HOST, + DB_PORT, DB_SCHEMA).get_instance() # 処理ステータスの登録および更新を行う - def set_process_status (self, process_status : str): + def set_process_status(self, process_status: str): try: self._db.begin() - self._db.execute(f"CALL upsert_jskult_batch_status_manage({self._process_name} {self._process_type} {process_status} NULL NULL);") + self._db.execute( + f"CALL upsert_jskult_batch_status_manage({self._process_name}, {self._process_type}, {process_status}, NULL, NULL);" + ) self._db.commit() - + except Exception as e: - + # Exceptionをcatchした場合はrollback self._db.rollback() - raise e + raise e # 後続処理を実行してよいか判定する def can_run_post_process(self): # SELECTの結果からレコード数を取得 record_count = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" ).count() if record_count == 0: raise ValueError("レコードの取得が出来ませんでした。") - + # 起動回数のインクリメント self._increment_run_count() # データ取込が完了していた場合 if self._is_done_data_import(): return True - - #最大起動回数に到達していない場合 + + # 最大起動回数に到達していない場合 if not self._is_max_run_count_reached: return False - + # 最大起動回数フラグを立てる self._activate_max_run_count_flg() @@ -65,41 +77,41 @@ class JskultBatchStatusManager: # SELECTの結果からレコード数を取得 record_count = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" ).count() if record_count == 0: raise ValueError("レコードの取得が出来ませんでした。") - + # 起動回数のインクリメント self._increment_run_count() # 後続処理が完了していた場合 if self._is_done_post_process(): return True - - #最大起動回数に到達していない場合 + + # 最大起動回数に到達していない場合 if not self._is_max_run_count_reached(): return False - + # 最大起動回数フラグを立てる self._activate_max_run_count_flg() # 最大起動回数に到達した場合にメッセージをスロー raise MaxRunCountReachedException("最大起動回数に到達しました") - # アルトマークデータ連携があったかを確認する + def is_done_ultmarc_import(self): # SELECTの結果からレコード数を取得 record_count = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = 'jskult-batch-ultmarc-io' AND process_date = src07.get_syor_date();" + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = 'jskult-batch-ultmarc-io' AND process_date = src07.get_syor_date();" ).count() # アルトマークデータ連携が無かった場合 if record_count == 0: return False - + return True # 起動回数をインクリメントする @@ -108,12 +120,13 @@ class JskultBatchStatusManager: self._db.begin() # SELECTの結果からレコードを取得 record = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" ) run_count += record[0]['run_count'] - self._db.execute(f"CALL upsert_jskult_batch_status_manage({self._process_name} {self._process_type} NULL {run_count} NULL);") + self._db.execute( + f"CALL upsert_jskult_batch_status_manage({self._process_name}, {self._process_type}, NULL, {run_count}, NULL);") self._db.commit() @@ -122,78 +135,78 @@ class JskultBatchStatusManager: # Exceptionをcatchした場合はrollbakc self._db.rollback() raise e - + # データ取込処理が完了しているかを判定する def _is_done_data_import(self): - # SELECTの結果からレコード数を取得 + # SELECTの結果からレコード数を取得 record_count = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_type = {self._process_type} AND process_status = 'done' AND process_date = src07.get_syor_date();" + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_type = {self._process_type} AND process_status = 'done' AND process_date = src07.get_syor_date();" ).count() - + # データ取込数が一致している場合 - if(self._receive_file_count == record_count): + if (self._receive_file_count == record_count): return True - + return False # 後続処理のすべての処理が完了しているかを判定する def _is_done_post_process(self): - if not self._is_done_process("jskult-batch-trn-result-data-bio-lot"): - return False - - if not self._is_done_process("jskult-batch-mst-inst"): - return False - - if not self._is_done_process("jskult-batch-dcf-inst-merge-io"): - return False - - # 全ての後続処理が完了している場合Trueを返す - return True + if not self._is_done_process("jskult-batch-trn-result-data-bio-lot"): + return False + + if not self._is_done_process("jskult-batch-mst-inst"): + return False + + if not self._is_done_process("jskult-batch-dcf-inst-merge-io"): + return False + + # 全ての後続処理が完了している場合Trueを返す + return True - # データ取込処理が完了しているかを判定する - def _is_done_process(self, process_name : str): + + def _is_done_process(self): # SELECTの結果からレコード数を取得 record_count = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_status = 'done' AND process_date = src07.get_syor_date();" + f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_status = 'done' AND process_date = src07.get_syor_date();" ).count() - if(record_count == 0): + if (record_count == 0): return False - + return True - # 起動回数が最大回数に到達しているか判定する + def _is_max_run_count_reached(self): # SELECTの結果からレコード数を取得 record = self._db.execute_select( f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" - ) + ) run_count = record[0]['run_count'] - + # 取得した起動回数とフィールド変数の最大起動回数が一致するか確認 if run_count == self._max_run_count_flg: return True - - return False + return False def _activate_max_run_count_flg(self): try: self._db.begin() # 最大起動回数フラグにフラグを立てる - self._db.execute(f"CALL upsert_jskult_batch_status_manage({self._process_name} {self._process_type} NULL NULL 1);") + self._db.execute( + f"CALL upsert_jskult_batch_status_manage({self._process_name}, {self._process_type}, NULL, NULL, 1);") self._db.commit() except Exception as e: self._db.rollback() - raise e \ No newline at end of file + raise e From fdabeb9d3dac21c405aaf201cf2032159e45118f Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Fri, 23 May 2025 14:47:49 +0900 Subject: [PATCH 12/13] =?UTF-8?q?DB=E3=81=AEconnect=E3=81=AE=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3=E3=81=A8=E6=96=87=E4=BD=93=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../manager/jskult_batch_status_manager.py | 166 +++++++++++++++--- 1 file changed, 137 insertions(+), 29 deletions(-) diff --git a/ecs/jskult-batch/src/manager/jskult_batch_status_manager.py b/ecs/jskult-batch/src/manager/jskult_batch_status_manager.py index 5792570d..766701ff 100644 --- a/ecs/jskult-batch/src/manager/jskult_batch_status_manager.py +++ b/ecs/jskult-batch/src/manager/jskult_batch_status_manager.py @@ -2,12 +2,6 @@ import os from src.error.exceptions import MaxRunCountReachedException from src.db.database import Database -DB_USERNAME = os.environ.get("DB_USERNAME") -DB_PASSWORD = os.environ.get("DB_PASSWORD") -DB_HOST = os.environ.get("DB_HOST") -DB_PORT = os.environ.get("DB_PORT") -DB_SCHEMA = os.environ.get("DB_SCHEMA") - # 実消化&アルトマーク_バッチステータス管理テーブルを管理するクラス @@ -27,21 +21,33 @@ class JskultBatchStatusManager: self._receive_file_count: str = receive_file_count # DB接続モジュール(バッチ)のget_instanceを呼び出し設定 - self._db = Database(DB_USERNAME, DB_PASSWORD, DB_HOST, - DB_PORT, DB_SCHEMA).get_instance() + self._db = Database.get_instance() # 処理ステータスの登録および更新を行う + def set_process_status(self, process_status: str): try: + # DB接続開始 + self._db.connect() self._db.begin() + self._db.to_jst() self._db.execute( - f"CALL upsert_jskult_batch_status_manage({self._process_name}, {self._process_type}, {process_status}, NULL, NULL);" + f""" + CALL + internal07.upsert_jskult_batch_status_manage( + {self._process_name}, + {self._process_type}, + {process_status}, + NULL, + NULL + );""" ) self._db.commit() except Exception as e: # Exceptionをcatchした場合はrollback + print(self._db) self._db.rollback() raise e @@ -49,9 +55,20 @@ class JskultBatchStatusManager: def can_run_post_process(self): # SELECTの結果からレコード数を取得 - record_count = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" - ).count() + record = self._db.execute_select( + f""" + SELECT + COUNT(*) + FROM + internal07.jskult_batch_status_manage + WHERE + process_name = {self._process_name} + AND + process_date = src07.get_syor_date(); + """ + ) + + record_count = record[0]['count'] if record_count == 0: raise ValueError("レコードの取得が出来ませんでした。") @@ -76,9 +93,20 @@ class JskultBatchStatusManager: def can_run_business_day_update(self): # SELECTの結果からレコード数を取得 - record_count = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" - ).count() + record = self._db.execute_select( + f""" + SELECT + COUNT(*) + FROM + internal07.jskult_batch_status_manage + WHERE + process_name = {self._process_name} + AND + process_date = src07.get_syor_date(); + """ + ) + + record_count = record[0]['count'] if record_count == 0: raise ValueError("レコードの取得が出来ませんでした。") @@ -106,10 +134,19 @@ class JskultBatchStatusManager: # SELECTの結果からレコード数を取得 record_count = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = 'jskult-batch-ultmarc-io' AND process_date = src07.get_syor_date();" - ).count() + f""" + SELECT + COUNT(*) + FROM + internal07.jskult_batch_status_manage + WHERE + process_name = 'jskult-batch-ultmarc-io' + AND + process_date = src07.get_syor_date(); + """ + ) # アルトマークデータ連携が無かった場合 - if record_count == 0: + if record_count.pop() == 0: return False return True @@ -117,16 +154,37 @@ class JskultBatchStatusManager: # 起動回数をインクリメントする def _increment_run_count(self): try: + # DB接続開始 + self._db.connect() self._db.begin() + self._db.to_jst() # SELECTの結果からレコードを取得 record = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" + f""" + SELECT + * + FROM + internal07.jskult_batch_status_manage + WHERE + process_name = {self._process_name} + AND + process_date = src07.get_syor_date(); + """ ) run_count += record[0]['run_count'] self._db.execute( - f"CALL upsert_jskult_batch_status_manage({self._process_name}, {self._process_type}, NULL, {run_count}, NULL);") + f""" + CALL + upsert_jskult_batch_status_manage( + {self._process_name}, + {self._process_type}, + NULL, + {run_count}, + NULL); + """ + ) self._db.commit() @@ -140,9 +198,22 @@ class JskultBatchStatusManager: def _is_done_data_import(self): # SELECTの結果からレコード数を取得 - record_count = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_type = {self._process_type} AND process_status = 'done' AND process_date = src07.get_syor_date();" - ).count() + record = self._db.execute_select( + f""" + SELECT + COUNT(*) + FROM + internal07.jskult_batch_status_manage + WHERE + process_type = {self._process_type} + AND + process_status = 'done' + AND + process_date = src07.get_syor_date(); + """ + ) + + record_count = record[0]['count'] # データ取込数が一致している場合 if (self._receive_file_count == record_count): @@ -153,12 +224,15 @@ class JskultBatchStatusManager: # 後続処理のすべての処理が完了しているかを判定する def _is_done_post_process(self): + # 生物由来データロット分解のチェック if not self._is_done_process("jskult-batch-trn-result-data-bio-lot"): return False + # メルク施設マスタ作成のチェック if not self._is_done_process("jskult-batch-mst-inst"): return False + # DCF削除新規マスタ作成のチェック if not self._is_done_process("jskult-batch-dcf-inst-merge-io"): return False @@ -170,9 +244,22 @@ class JskultBatchStatusManager: def _is_done_process(self): # SELECTの結果からレコード数を取得 - record_count = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_status = 'done' AND process_date = src07.get_syor_date();" - ).count() + record = self._db.execute_select( + f""" + SELECT + * + FROM + internal07.jskult_batch_status_manage + WHERE + process_name = {self._process_name} + AND + process_status = 'done' + AND + process_date = src07.get_syor_date(); + """ + ) + + record_count = record[0]['count'] if (record_count == 0): return False @@ -183,9 +270,18 @@ class JskultBatchStatusManager: def _is_max_run_count_reached(self): - # SELECTの結果からレコード数を取得 + # SELECTの結果からレコードを取得 record = self._db.execute_select( - f"SELECT * FROM internal07.jskult_batch_status_manage WHERE process_name = {self._process_name} AND process_date = src07.get_syor_date();" + f""" + SELECT + * + FROM + internal07.jskult_batch_status_manage + WHERE + process_name = {self._process_name} + AND + process_date = src07.get_syor_date(); + """ ) run_count = record[0]['run_count'] @@ -198,11 +294,23 @@ class JskultBatchStatusManager: def _activate_max_run_count_flg(self): try: + # DB接続開始 + self._db.connect() self._db.begin() + self._db.to_jst() # 最大起動回数フラグにフラグを立てる self._db.execute( - f"CALL upsert_jskult_batch_status_manage({self._process_name}, {self._process_type}, NULL, NULL, 1);") + f""" + CALL + upsert_jskult_batch_status_manage( + {self._process_name}, + {self._process_type}, + NULL, + NULL, + 1); + """ + ) self._db.commit() From 991176167aa037951e7cf8349500ca7a935d4e53 Mon Sep 17 00:00:00 2001 From: "mori.k" Date: Fri, 23 May 2025 15:32:41 +0900 Subject: [PATCH 13/13] =?UTF-8?q?=E6=97=A5=E4=BB=98=E3=83=86=E3=83=BC?= =?UTF-8?q?=E3=83=96=E3=83=AB=E6=93=8D=E4=BD=9C=E3=82=AF=E3=83=A9=E3=82=B9?= =?UTF-8?q?=E3=81=AE=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/manager/jskult_hdke_tbl_manager.py | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 ecs/jskult-batch/src/manager/jskult_hdke_tbl_manager.py diff --git a/ecs/jskult-batch/src/manager/jskult_hdke_tbl_manager.py b/ecs/jskult-batch/src/manager/jskult_hdke_tbl_manager.py new file mode 100644 index 00000000..4a804ef3 --- /dev/null +++ b/ecs/jskult-batch/src/manager/jskult_hdke_tbl_manager.py @@ -0,0 +1,122 @@ +from datetime import datetime + +from src.db.database import Database +from src.error.exceptions import BatchOperationException, DBException +from src.system_var import constants + + +class JskultHdkeTblManager: + _db: Database + + def __init__(self): + self._db = Database.get_instance() + + def get_batch_statuses(self) -> tuple[str, str, str]: + """日次バッチ処理中フラグ、dump取得状況区分、処理日を取得する + + Raises: + BatchOperationException: DB操作の何らかのエラー + + Returns: + tuple[str, str, str]: [0]日次バッチ処理中フラグ、[1]dump取得状況区分、[2]処理日 + """ + sql = 'SELECT bch_actf, dump_sts_kbn, src07.get_syor_date() AS syor_date FROM src07.hdke_tbl' + try: + self._db.connect() + hdke_tbl_result = self._db.execute_select(sql) + except DBException as e: + raise BatchOperationException(e) + finally: + self._db.disconnect() + + if len(hdke_tbl_result) == 0: + raise BatchOperationException('日付テーブルが取得できませんでした') + + # 必ず1件取れる + hdke_tbl_record = hdke_tbl_result[0] + batch_processing_flag = hdke_tbl_record['bch_actf'] + dump_status_kbn = hdke_tbl_record['dump_sts_kbn'] + syor_date = hdke_tbl_record['syor_date'] + # 処理日を文字列に変換する + syor_date_str = datetime.strftime(syor_date, '%Y/%m/%d') + + return batch_processing_flag, dump_status_kbn, syor_date_str + + def update_batch_process_start(self): + """バッチ処理中フラグを処理中に更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + + sql = """\ + UPDATE src07.hdke_tbl + SET + bch_actf = :start, + updater = CURRENT_USER(), + update_date = NOW() + """ + try: + self._db.connect() + self._db.to_jst() + self._db.execute( + sql, {'start': constants.BATCH_ACTF_BATCH_START}) + except DBException as e: + raise BatchOperationException(e) + finally: + self._db.disconnect() + + return + + def update_batch_process_complete(self) -> None: + """バッチ正常終了処理時の更新処理 + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + + sql = """\ + UPDATE src07.hdke_tbl + SET + bch_actf = :batch_complete, + dump_sts_kbn = :dump_unprocessed, + syor_date = DATE_FORMAT((src07.get_syor_date() + interval 1 day), '%Y%m%d'), -- +1日 + updater = CURRENT_USER(), + update_date = NOW() + """ + try: + self._db.connect() + self._db.to_jst() + self._db.execute(sql, { + 'batch_complete': constants.BATCH_ACTF_BATCH_UNPROCESSED, + 'dump_unprocessed': constants.DUMP_STATUS_KBN_UNPROCESSED + }) + except DBException as e: + raise BatchOperationException(e) + finally: + self._db.disconnect() + + def can_run_process(self) -> bool: + """バッチ処理を起動してよいかを判定する + + Raises: + BatchOperationException: DB操作の何らかのエラー + + Returns: + bool: バッチ処理を実行して良い場合はTrue + """ + try: + # 日次バッチ処置中フラグ、dump処理状態区分を取得 + batch_processing_flag, dump_status_kbn, _ = self.get_batch_statuses() + except DBException as e: + raise BatchOperationException(e) + finally: + self._db.disconnect() + # 日次バッチ処理中ではない場合、後続の処理は行わない + if batch_processing_flag != constants.BATCH_ACTF_BATCH_START: + return False + # dump取得が正常終了していない場合、後続の処理は行わない + if dump_status_kbn != constants.DUMP_STATUS_KBN_COMPLETE: + return False + + return True