Merge pull request #476 feature-NEWDWH2021-1845 into develop
This commit is contained in:
下田雅人
commit
71b587e03c
12
ecs/jskult-batch/.dockerignore
Normal file
12
ecs/jskult-batch/.dockerignore
Normal file
@ -0,0 +1,12 @@
|
||||
tests/*
|
||||
.coverage
|
||||
.env
|
||||
.env.example
|
||||
.report/*
|
||||
.vscode/*
|
||||
.pytest_cache/*
|
||||
*/__pychache__/*
|
||||
Dockerfile
|
||||
pytest.ini
|
||||
README.md
|
||||
*.sql
|
||||
26
ecs/jskult-batch/.env.example
Normal file
26
ecs/jskult-batch/.env.example
Normal file
@ -0,0 +1,26 @@
|
||||
DB_HOST=************
|
||||
DB_PORT=************
|
||||
DB_USERNAME=************
|
||||
DB_PASSWORD=************
|
||||
DB_SCHEMA=src05
|
||||
LOG_LEVEL=INFO
|
||||
ULTMARC_DATA_BUCKET=****************
|
||||
ULTMARC_DATA_FOLDER=recv
|
||||
JSKULT_BACKUP_BUCKET=****************
|
||||
ULTMARC_BACKUP_FOLDER=ultmarc
|
||||
VJSK_BACKUP_FOLDER=vjsk
|
||||
JSKULT_CONFIG_BUCKET=**********************
|
||||
JSKULT_CONFIG_CALENDAR_FOLDER=jskult/calendar
|
||||
JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME=jskult_holiday_list.txt
|
||||
VJSK_DATA_SEND_FOLDER=send
|
||||
VJSK_DATA_RECEIVE_FOLDER=recv
|
||||
VJSK_DATA_BUCKET=*************
|
||||
JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME=jskult_wholesaler_stock_input_day_list.txt
|
||||
JSKULT_CONFIG_CONVERT_FOLDER=jskult/convert
|
||||
JSKULT_ULTMARC_HEX_CONVERT_CONFIG_FILE_NAME=ultmarc_hex_convert_config.json
|
||||
# 連携データ抽出期間
|
||||
SALES_LAUNDERING_EXTRACT_DATE_PERIOD=0
|
||||
# 洗替対象テーブル名
|
||||
SALES_LAUNDERING_TARGET_TABLE_NAME=src05.sales_lau
|
||||
# 卸実績洗替で作成するデータの期間(年単位)
|
||||
SALES_LAUNDERING_TARGET_YEAR_OFFSET=5
|
||||
10
ecs/jskult-batch/.gitignore
vendored
Normal file
10
ecs/jskult-batch/.gitignore
vendored
Normal file
@ -0,0 +1,10 @@
|
||||
.vscode/settings.json
|
||||
.env
|
||||
|
||||
# python
|
||||
__pycache__
|
||||
|
||||
# python test
|
||||
.pytest_cache
|
||||
.coverage
|
||||
.report/
|
||||
16
ecs/jskult-batch/.vscode/launch.json
vendored
Normal file
16
ecs/jskult-batch/.vscode/launch.json
vendored
Normal file
@ -0,0 +1,16 @@
|
||||
{
|
||||
// IntelliSense を使用して利用可能な属性を学べます。
|
||||
// 既存の属性の説明をホバーして表示します。
|
||||
// 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "(DEBUG)jskult batch",
|
||||
"type": "python",
|
||||
"request": "launch",
|
||||
"program": "entrypoint.py",
|
||||
"console": "integratedTerminal",
|
||||
"justMyCode": true
|
||||
}
|
||||
]
|
||||
}
|
||||
31
ecs/jskult-batch/.vscode/recommended_settings.json
vendored
Normal file
31
ecs/jskult-batch/.vscode/recommended_settings.json
vendored
Normal file
@ -0,0 +1,31 @@
|
||||
{
|
||||
"[python]": {
|
||||
"editor.defaultFormatter": null,
|
||||
"editor.formatOnSave": true,
|
||||
"editor.codeActionsOnSave": {
|
||||
"source.organizeImports": true
|
||||
}
|
||||
},
|
||||
// 自身の環境に合わせて変えてください
|
||||
"python.defaultInterpreterPath": "<pythonインタプリターのパス>",
|
||||
"python.linting.lintOnSave": true,
|
||||
"python.linting.enabled": true,
|
||||
"python.linting.pylintEnabled": false,
|
||||
"python.linting.flake8Enabled": true,
|
||||
"python.linting.flake8Args": [
|
||||
"--max-line-length=200",
|
||||
"--ignore=F541"
|
||||
],
|
||||
"python.formatting.provider": "autopep8",
|
||||
"python.formatting.autopep8Path": "autopep8",
|
||||
"python.formatting.autopep8Args": [
|
||||
"--max-line-length", "200",
|
||||
"--ignore=F541"
|
||||
],
|
||||
"python.testing.pytestArgs": [
|
||||
"tests/batch/"
|
||||
],
|
||||
|
||||
"python.testing.unittestEnabled": false,
|
||||
"python.testing.pytestEnabled": true
|
||||
}
|
||||
20
ecs/jskult-batch/Dockerfile
Normal file
20
ecs/jskult-batch/Dockerfile
Normal file
@ -0,0 +1,20 @@
|
||||
FROM python:3.12-slim-bookworm
|
||||
|
||||
ENV TZ="Asia/Tokyo"
|
||||
# pythonの標準出力をバッファリングしないフラグ
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
# pythonのバイトコードを生成しないフラグ
|
||||
ENV PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
WORKDIR /usr/src/app
|
||||
COPY Pipfile Pipfile.lock ./
|
||||
RUN \
|
||||
apt update -y && \
|
||||
pip install pipenv --no-cache-dir && \
|
||||
pipenv install --system --deploy && \
|
||||
pip uninstall -y pipenv virtualenv-clone virtualenv
|
||||
|
||||
COPY src ./src
|
||||
COPY entrypoint.py entrypoint.py
|
||||
|
||||
CMD ["python", "entrypoint.py"]
|
||||
29
ecs/jskult-batch/Pipfile
Normal file
29
ecs/jskult-batch/Pipfile
Normal file
@ -0,0 +1,29 @@
|
||||
[[source]]
|
||||
url = "https://pypi.org/simple"
|
||||
verify_ssl = true
|
||||
name = "pypi"
|
||||
|
||||
[scripts]
|
||||
"test:ultmarc" = "pytest tests/batch/ultmarc/"
|
||||
"test:ultmarc:cov" = "pytest --cov=src/batch/ultmarc/ --cov-branch --cov-report=term-missing tests/batch/ultmarc/"
|
||||
"test:vjsk" = "pytest tests/batch/vjsk/"
|
||||
"test:vjsk:cov" = "pytest --cov=src/batch/vjsk/ --cov-branch --cov-report=term-missing tests/batch/vjsk/"
|
||||
|
||||
[packages]
|
||||
boto3 = "*"
|
||||
PyMySQL = "*"
|
||||
sqlalchemy = "*"
|
||||
tenacity = "*"
|
||||
|
||||
[dev-packages]
|
||||
autopep8 = "*"
|
||||
flake8 = "*"
|
||||
pytest = "*"
|
||||
pytest-cov = "*"
|
||||
boto3 = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.12"
|
||||
|
||||
[pipenv]
|
||||
allow_prereleases = true
|
||||
443
ecs/jskult-batch/Pipfile.lock
generated
Normal file
443
ecs/jskult-batch/Pipfile.lock
generated
Normal file
@ -0,0 +1,443 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "aa2d1d97600fea225b7d249dae0d065190d00fdadbf85b20773e0c1d9862f5c1"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3.12"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
"name": "pypi",
|
||||
"url": "https://pypi.org/simple",
|
||||
"verify_ssl": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"default": {
|
||||
"boto3": {
|
||||
"hashes": [
|
||||
"sha256:d57d8040d04b8fefb7439062529653701cc90d5b2734cc4f1144fedb75ba94a7",
|
||||
"sha256:fdd69f23e6216a508bbc1fbda9486791c161f3ecd5933ac7090d7290f6f2d0f5"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==1.38.19"
|
||||
},
|
||||
"botocore": {
|
||||
"hashes": [
|
||||
"sha256:796b948c05017eb33385b798990cd91ed4af0e881eb9eb1ee6e17666be02abc9",
|
||||
"sha256:f937a20e75889215a99280ea0fdd4e1716ffede23e4f9af7bc9c64af9bc63e61"
|
||||
],
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==1.38.19"
|
||||
},
|
||||
"greenlet": {
|
||||
"hashes": [
|
||||
"sha256:00cd814b8959b95a546e47e8d589610534cfb71f19802ea8a2ad99d95d702057",
|
||||
"sha256:02a98600899ca1ca5d3a2590974c9e3ec259503b2d6ba6527605fcd74e08e207",
|
||||
"sha256:02f5972ff02c9cf615357c17ab713737cccfd0eaf69b951084a9fd43f39833d3",
|
||||
"sha256:055916fafad3e3388d27dd68517478933a97edc2fc54ae79d3bec827de2c64c4",
|
||||
"sha256:0a16fb934fcabfdfacf21d79e6fed81809d8cd97bc1be9d9c89f0e4567143d7b",
|
||||
"sha256:1592a615b598643dbfd566bac8467f06c8c8ab6e56f069e573832ed1d5d528cc",
|
||||
"sha256:1919cbdc1c53ef739c94cf2985056bcc0838c1f217b57647cbf4578576c63825",
|
||||
"sha256:1e4747712c4365ef6765708f948acc9c10350719ca0545e362c24ab973017370",
|
||||
"sha256:1e76106b6fc55fa3d6fe1c527f95ee65e324a13b62e243f77b48317346559708",
|
||||
"sha256:1f72667cc341c95184f1c68f957cb2d4fc31eef81646e8e59358a10ce6689457",
|
||||
"sha256:2593283bf81ca37d27d110956b79e8723f9aa50c4bcdc29d3c0543d4743d2763",
|
||||
"sha256:2dc5c43bb65ec3669452af0ab10729e8fdc17f87a1f2ad7ec65d4aaaefabf6bf",
|
||||
"sha256:3091bc45e6b0c73f225374fefa1536cd91b1e987377b12ef5b19129b07d93ebe",
|
||||
"sha256:354f67445f5bed6604e493a06a9a49ad65675d3d03477d38a4db4a427e9aad0e",
|
||||
"sha256:3885f85b61798f4192d544aac7b25a04ece5fe2704670b4ab73c2d2c14ab740d",
|
||||
"sha256:3ab7194ee290302ca15449f601036007873028712e92ca15fc76597a0aeb4c59",
|
||||
"sha256:3aeca9848d08ce5eb653cf16e15bb25beeab36e53eb71cc32569f5f3afb2a3aa",
|
||||
"sha256:44671c29da26539a5f142257eaba5110f71887c24d40df3ac87f1117df589e0e",
|
||||
"sha256:45f9f4853fb4cc46783085261c9ec4706628f3b57de3e68bae03e8f8b3c0de51",
|
||||
"sha256:4bd139e4943547ce3a56ef4b8b1b9479f9e40bb47e72cc906f0f66b9d0d5cab3",
|
||||
"sha256:4fefc7aa68b34b9224490dfda2e70ccf2131368493add64b4ef2d372955c207e",
|
||||
"sha256:6629311595e3fe7304039c67f00d145cd1d38cf723bb5b99cc987b23c1433d61",
|
||||
"sha256:6fadd183186db360b61cb34e81117a096bff91c072929cd1b529eb20dd46e6c5",
|
||||
"sha256:71566302219b17ca354eb274dfd29b8da3c268e41b646f330e324e3967546a74",
|
||||
"sha256:7409796591d879425997a518138889d8d17e63ada7c99edc0d7a1c22007d4907",
|
||||
"sha256:752f0e79785e11180ebd2e726c8a88109ded3e2301d40abced2543aa5d164275",
|
||||
"sha256:7791dcb496ec53d60c7f1c78eaa156c21f402dda38542a00afc3e20cae0f480f",
|
||||
"sha256:782743700ab75716650b5238a4759f840bb2dcf7bff56917e9ffdf9f1f23ec59",
|
||||
"sha256:7c9896249fbef2c615853b890ee854f22c671560226c9221cfd27c995db97e5c",
|
||||
"sha256:85f3e248507125bf4af607a26fd6cb8578776197bd4b66e35229cdf5acf1dfbf",
|
||||
"sha256:89c69e9a10670eb7a66b8cef6354c24671ba241f46152dd3eed447f79c29fb5b",
|
||||
"sha256:8cb8553ee954536500d88a1a2f58fcb867e45125e600e80f586ade399b3f8819",
|
||||
"sha256:9ae572c996ae4b5e122331e12bbb971ea49c08cc7c232d1bd43150800a2d6c65",
|
||||
"sha256:9c7b15fb9b88d9ee07e076f5a683027bc3befd5bb5d25954bb633c385d8b737e",
|
||||
"sha256:9ea5231428af34226c05f927e16fc7f6fa5e39e3ad3cd24ffa48ba53a47f4240",
|
||||
"sha256:a31ead8411a027c2c4759113cf2bd473690517494f3d6e4bf67064589afcd3c5",
|
||||
"sha256:a8fa80665b1a29faf76800173ff5325095f3e66a78e62999929809907aca5659",
|
||||
"sha256:ad053d34421a2debba45aa3cc39acf454acbcd025b3fc1a9f8a0dee237abd485",
|
||||
"sha256:b24c7844c0a0afc3ccbeb0b807adeefb7eff2b5599229ecedddcfeb0ef333bec",
|
||||
"sha256:b50a8c5c162469c3209e5ec92ee4f95c8231b11db6a04db09bbe338176723bb8",
|
||||
"sha256:ba30e88607fb6990544d84caf3c706c4b48f629e18853fc6a646f82db9629418",
|
||||
"sha256:bf3fc9145141250907730886b031681dfcc0de1c158f3cc51c092223c0f381ce",
|
||||
"sha256:c23ea227847c9dbe0b3910f5c0dd95658b607137614eb821e6cbaecd60d81cc6",
|
||||
"sha256:c3cc1a3ed00ecfea8932477f729a9f616ad7347a5e55d50929efa50a86cb7be7",
|
||||
"sha256:c49e9f7c6f625507ed83a7485366b46cbe325717c60837f7244fc99ba16ba9d6",
|
||||
"sha256:d0cb7d47199001de7658c213419358aa8937df767936506db0db7ce1a71f4a2f",
|
||||
"sha256:d8009ae46259e31bc73dc183e402f548e980c96f33a6ef58cc2e7865db012e13",
|
||||
"sha256:da956d534a6d1b9841f95ad0f18ace637668f680b1339ca4dcfb2c1837880a0b",
|
||||
"sha256:dcb9cebbf3f62cb1e5afacae90761ccce0effb3adaa32339a0670fe7805d8068",
|
||||
"sha256:decb0658ec19e5c1f519faa9a160c0fc85a41a7e6654b3ce1b44b939f8bf1325",
|
||||
"sha256:df4d1509efd4977e6a844ac96d8be0b9e5aa5d5c77aa27ca9f4d3f92d3fcf330",
|
||||
"sha256:eeb27bece45c0c2a5842ac4c5a1b5c2ceaefe5711078eed4e8043159fa05c834",
|
||||
"sha256:efcdfb9df109e8a3b475c016f60438fcd4be68cd13a365d42b35914cdab4bb2b",
|
||||
"sha256:fd9fb7c941280e2c837b603850efc93c999ae58aae2b40765ed682a6907ebbc5",
|
||||
"sha256:fe46d4f8e94e637634d54477b0cfabcf93c53f29eedcbdeecaf2af32029b4421"
|
||||
],
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==3.2.2"
|
||||
},
|
||||
"jmespath": {
|
||||
"hashes": [
|
||||
"sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980",
|
||||
"sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==1.0.1"
|
||||
},
|
||||
"pymysql": {
|
||||
"hashes": [
|
||||
"sha256:4de15da4c61dc132f4fb9ab763063e693d521a80fd0e87943b9a453dd4c19d6c",
|
||||
"sha256:e127611aaf2b417403c60bf4dc570124aeb4a57f5f37b8e95ae399a42f904cd0"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==1.1.1"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
"sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",
|
||||
"sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
|
||||
"version": "==2.9.0.post0"
|
||||
},
|
||||
"s3transfer": {
|
||||
"hashes": [
|
||||
"sha256:35b314d7d82865756edab59f7baebc6b477189e6ab4c53050e28c1de4d9cce18",
|
||||
"sha256:8ac58bc1989a3fdb7c7f3ee0918a66b160d038a147c7b5db1500930a607e9a1c"
|
||||
],
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==0.12.0"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274",
|
||||
"sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
|
||||
"version": "==1.17.0"
|
||||
},
|
||||
"sqlalchemy": {
|
||||
"hashes": [
|
||||
"sha256:023b3ee6169969beea3bb72312e44d8b7c27c75b347942d943cf49397b7edeb5",
|
||||
"sha256:03968a349db483936c249f4d9cd14ff2c296adfa1290b660ba6516f973139582",
|
||||
"sha256:05132c906066142103b83d9c250b60508af556982a385d96c4eaa9fb9720ac2b",
|
||||
"sha256:087b6b52de812741c27231b5a3586384d60c353fbd0e2f81405a814b5591dc8b",
|
||||
"sha256:0b3dbf1e7e9bc95f4bac5e2fb6d3fb2f083254c3fdd20a1789af965caf2d2348",
|
||||
"sha256:118c16cd3f1b00c76d69343e38602006c9cfb9998fa4f798606d28d63f23beda",
|
||||
"sha256:1936af879e3db023601196a1684d28e12f19ccf93af01bf3280a3262c4b6b4e5",
|
||||
"sha256:1e3f196a0c59b0cae9a0cd332eb1a4bda4696e863f4f1cf84ab0347992c548c2",
|
||||
"sha256:23a8825495d8b195c4aa9ff1c430c28f2c821e8c5e2d98089228af887e5d7e29",
|
||||
"sha256:293cd444d82b18da48c9f71cd7005844dbbd06ca19be1ccf6779154439eec0b8",
|
||||
"sha256:32f9dc8c44acdee06c8fc6440db9eae8b4af8b01e4b1aee7bdd7241c22edff4f",
|
||||
"sha256:34ea30ab3ec98355235972dadc497bb659cc75f8292b760394824fab9cf39826",
|
||||
"sha256:3d3549fc3e40667ec7199033a4e40a2f669898a00a7b18a931d3efb4c7900504",
|
||||
"sha256:41836fe661cc98abfae476e14ba1906220f92c4e528771a8a3ae6a151242d2ae",
|
||||
"sha256:4d44522480e0bf34c3d63167b8cfa7289c1c54264c2950cc5fc26e7850967e45",
|
||||
"sha256:4eeb195cdedaf17aab6b247894ff2734dcead6c08f748e617bfe05bd5a218443",
|
||||
"sha256:4f67766965996e63bb46cfbf2ce5355fc32d9dd3b8ad7e536a920ff9ee422e23",
|
||||
"sha256:57df5dc6fdb5ed1a88a1ed2195fd31927e705cad62dedd86b46972752a80f576",
|
||||
"sha256:598d9ebc1e796431bbd068e41e4de4dc34312b7aa3292571bb3674a0cb415dd1",
|
||||
"sha256:5b14e97886199c1f52c14629c11d90c11fbb09e9334fa7bb5f6d068d9ced0ce0",
|
||||
"sha256:5e22575d169529ac3e0a120cf050ec9daa94b6a9597993d1702884f6954a7d71",
|
||||
"sha256:60c578c45c949f909a4026b7807044e7e564adf793537fc762b2489d522f3d11",
|
||||
"sha256:6145afea51ff0af7f2564a05fa95eb46f542919e6523729663a5d285ecb3cf5e",
|
||||
"sha256:6375cd674fe82d7aa9816d1cb96ec592bac1726c11e0cafbf40eeee9a4516b5f",
|
||||
"sha256:6854175807af57bdb6425e47adbce7d20a4d79bbfd6f6d6519cd10bb7109a7f8",
|
||||
"sha256:6ab60a5089a8f02009f127806f777fca82581c49e127f08413a66056bd9166dd",
|
||||
"sha256:725875a63abf7c399d4548e686debb65cdc2549e1825437096a0af1f7e374814",
|
||||
"sha256:7492967c3386df69f80cf67efd665c0f667cee67032090fe01d7d74b0e19bb08",
|
||||
"sha256:81965cc20848ab06583506ef54e37cf15c83c7e619df2ad16807c03100745dea",
|
||||
"sha256:81c24e0c0fde47a9723c81d5806569cddef103aebbf79dbc9fcbb617153dea30",
|
||||
"sha256:81eedafa609917040d39aa9332e25881a8e7a0862495fcdf2023a9667209deda",
|
||||
"sha256:81f413674d85cfd0dfcd6512e10e0f33c19c21860342a4890c3a2b59479929f9",
|
||||
"sha256:8280856dd7c6a68ab3a164b4a4b1c51f7691f6d04af4d4ca23d6ecf2261b7923",
|
||||
"sha256:82ca366a844eb551daff9d2e6e7a9e5e76d2612c8564f58db6c19a726869c1df",
|
||||
"sha256:8b4af17bda11e907c51d10686eda89049f9ce5669b08fbe71a29747f1e876036",
|
||||
"sha256:90144d3b0c8b139408da50196c5cad2a6909b51b23df1f0538411cd23ffa45d3",
|
||||
"sha256:906e6b0d7d452e9a98e5ab8507c0da791856b2380fdee61b765632bb8698026f",
|
||||
"sha256:90c11ceb9a1f482c752a71f203a81858625d8df5746d787a4786bca4ffdf71c6",
|
||||
"sha256:911cc493ebd60de5f285bcae0491a60b4f2a9f0f5c270edd1c4dbaef7a38fc04",
|
||||
"sha256:9a420a91913092d1e20c86a2f5f1fc85c1a8924dbcaf5e0586df8aceb09c9cc2",
|
||||
"sha256:9f8c9fdd15a55d9465e590a402f42082705d66b05afc3ffd2d2eb3c6ba919560",
|
||||
"sha256:a104c5694dfd2d864a6f91b0956eb5d5883234119cb40010115fd45a16da5e70",
|
||||
"sha256:a373a400f3e9bac95ba2a06372c4fd1412a7cee53c37fc6c05f829bf672b8769",
|
||||
"sha256:a62448526dd9ed3e3beedc93df9bb6b55a436ed1474db31a2af13b313a70a7e1",
|
||||
"sha256:a8808d5cf866c781150d36a3c8eb3adccfa41a8105d031bf27e92c251e3969d6",
|
||||
"sha256:b1f09b6821406ea1f94053f346f28f8215e293344209129a9c0fcc3578598d7b",
|
||||
"sha256:b2ac41acfc8d965fb0c464eb8f44995770239668956dc4cdf502d1b1ffe0d747",
|
||||
"sha256:b46fa6eae1cd1c20e6e6f44e19984d438b6b2d8616d21d783d150df714f44078",
|
||||
"sha256:b50eab9994d64f4a823ff99a0ed28a6903224ddbe7fef56a6dd865eec9243440",
|
||||
"sha256:bfc9064f6658a3d1cadeaa0ba07570b83ce6801a1314985bf98ec9b95d74e15f",
|
||||
"sha256:c0b0e5e1b5d9f3586601048dd68f392dc0cc99a59bb5faf18aab057ce00d00b2",
|
||||
"sha256:c153265408d18de4cc5ded1941dcd8315894572cddd3c58df5d5b5705b3fa28d",
|
||||
"sha256:d4ae769b9c1c7757e4ccce94b0641bc203bbdf43ba7a2413ab2523d8d047d8dc",
|
||||
"sha256:dc56c9788617b8964ad02e8fcfeed4001c1f8ba91a9e1f31483c0dffb207002a",
|
||||
"sha256:dd5ec3aa6ae6e4d5b5de9357d2133c07be1aff6405b136dad753a16afb6717dd",
|
||||
"sha256:edba70118c4be3c2b1f90754d308d0b79c6fe2c0fdc52d8ddf603916f83f4db9",
|
||||
"sha256:ff8e80c4c4932c10493ff97028decfdb622de69cae87e0f127a7ebe32b4069c6"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==2.0.41"
|
||||
},
|
||||
"tenacity": {
|
||||
"hashes": [
|
||||
"sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb",
|
||||
"sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==9.1.2"
|
||||
},
|
||||
"typing-extensions": {
|
||||
"hashes": [
|
||||
"sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c",
|
||||
"sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==4.13.2"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466",
|
||||
"sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813"
|
||||
],
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==2.4.0"
|
||||
}
|
||||
},
|
||||
"develop": {
|
||||
"autopep8": {
|
||||
"hashes": [
|
||||
"sha256:89440a4f969197b69a995e4ce0661b031f455a9f776d2c5ba3dbd83466931758",
|
||||
"sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==2.3.2"
|
||||
},
|
||||
"boto3": {
|
||||
"hashes": [
|
||||
"sha256:d57d8040d04b8fefb7439062529653701cc90d5b2734cc4f1144fedb75ba94a7",
|
||||
"sha256:fdd69f23e6216a508bbc1fbda9486791c161f3ecd5933ac7090d7290f6f2d0f5"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==1.38.19"
|
||||
},
|
||||
"botocore": {
|
||||
"hashes": [
|
||||
"sha256:796b948c05017eb33385b798990cd91ed4af0e881eb9eb1ee6e17666be02abc9",
|
||||
"sha256:f937a20e75889215a99280ea0fdd4e1716ffede23e4f9af7bc9c64af9bc63e61"
|
||||
],
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==1.38.19"
|
||||
},
|
||||
"coverage": {
|
||||
"extras": [
|
||||
"toml"
|
||||
],
|
||||
"hashes": [
|
||||
"sha256:042e7841a26498fff7a37d6fda770d17519982f5b7d8bf5278d140b67b61095f",
|
||||
"sha256:04bfec25a8ef1c5f41f5e7e5c842f6b615599ca8ba8391ec33a9290d9d2db3a3",
|
||||
"sha256:0915742f4c82208ebf47a2b154a5334155ed9ef9fe6190674b8a46c2fb89cb05",
|
||||
"sha256:18c5ae6d061ad5b3e7eef4363fb27a0576012a7447af48be6c75b88494c6cf25",
|
||||
"sha256:2931f66991175369859b5fd58529cd4b73582461877ecfd859b6549869287ffe",
|
||||
"sha256:2e4b6b87bb0c846a9315e3ab4be2d52fac905100565f4b92f02c445c8799e257",
|
||||
"sha256:3043ba1c88b2139126fc72cb48574b90e2e0546d4c78b5299317f61b7f718b78",
|
||||
"sha256:379fe315e206b14e21db5240f89dc0774bdd3e25c3c58c2c733c99eca96f1ada",
|
||||
"sha256:42421e04069fb2cbcbca5a696c4050b84a43b05392679d4068acbe65449b5c64",
|
||||
"sha256:4dfd9a93db9e78666d178d4f08a5408aa3f2474ad4d0e0378ed5f2ef71640cb6",
|
||||
"sha256:52a523153c568d2c0ef8826f6cc23031dc86cffb8c6aeab92c4ff776e7951b28",
|
||||
"sha256:554fec1199d93ab30adaa751db68acec2b41c5602ac944bb19187cb9a41a8067",
|
||||
"sha256:581a40c7b94921fffd6457ffe532259813fc68eb2bdda60fa8cc343414ce3733",
|
||||
"sha256:5a26c0c795c3e0b63ec7da6efded5f0bc856d7c0b24b2ac84b4d1d7bc578d676",
|
||||
"sha256:5a570cd9bd20b85d1a0d7b009aaf6c110b52b5755c17be6962f8ccd65d1dbd23",
|
||||
"sha256:5aaeb00761f985007b38cf463b1d160a14a22c34eb3f6a39d9ad6fc27cb73008",
|
||||
"sha256:5ac46d0c2dd5820ce93943a501ac5f6548ea81594777ca585bf002aa8854cacd",
|
||||
"sha256:5c8a5c139aae4c35cbd7cadca1df02ea8cf28a911534fc1b0456acb0b14234f3",
|
||||
"sha256:6b8af63b9afa1031c0ef05b217faa598f3069148eeee6bb24b79da9012423b82",
|
||||
"sha256:769773614e676f9d8e8a0980dd7740f09a6ea386d0f383db6821df07d0f08545",
|
||||
"sha256:771eb7587a0563ca5bb6f622b9ed7f9d07bd08900f7589b4febff05f469bea00",
|
||||
"sha256:77af0f6447a582fdc7de5e06fa3757a3ef87769fbb0fdbdeba78c23049140a47",
|
||||
"sha256:7a3d62b3b03b4b6fd41a085f3574874cf946cb4604d2b4d3e8dca8cd570ca501",
|
||||
"sha256:821f7bcbaa84318287115d54becb1915eece6918136c6f91045bb84e2f88739d",
|
||||
"sha256:89b1f4af0d4afe495cd4787a68e00f30f1d15939f550e869de90a86efa7e0814",
|
||||
"sha256:8a1d96e780bdb2d0cbb297325711701f7c0b6f89199a57f2049e90064c29f6bd",
|
||||
"sha256:8a40fcf208e021eb14b0fac6bdb045c0e0cab53105f93ba0d03fd934c956143a",
|
||||
"sha256:8f99eb72bf27cbb167b636eb1726f590c00e1ad375002230607a844d9e9a2318",
|
||||
"sha256:90e7fbc6216ecaffa5a880cdc9c77b7418c1dcb166166b78dbc630d07f278cc3",
|
||||
"sha256:94ec0be97723ae72d63d3aa41961a0b9a6f5a53ff599813c324548d18e3b9e8c",
|
||||
"sha256:95aa6ae391a22bbbce1b77ddac846c98c5473de0372ba5c463480043a07bff42",
|
||||
"sha256:96121edfa4c2dfdda409877ea8608dd01de816a4dc4a0523356067b305e4e17a",
|
||||
"sha256:a1f406a8e0995d654b2ad87c62caf6befa767885301f3b8f6f73e6f3c31ec3a6",
|
||||
"sha256:a321c61477ff8ee705b8a5fed370b5710c56b3a52d17b983d9215861e37b642a",
|
||||
"sha256:a5761c70c017c1b0d21b0815a920ffb94a670c8d5d409d9b38857874c21f70d7",
|
||||
"sha256:a9abbccd778d98e9c7e85038e35e91e67f5b520776781d9a1e2ee9d400869487",
|
||||
"sha256:ad80e6b4a0c3cb6f10f29ae4c60e991f424e6b14219d46f1e7d442b938ee68a4",
|
||||
"sha256:b44674870709017e4b4036e3d0d6c17f06a0e6d4436422e0ad29b882c40697d2",
|
||||
"sha256:b571bf5341ba8c6bc02e0baeaf3b061ab993bf372d982ae509807e7f112554e9",
|
||||
"sha256:b8194fb8e50d556d5849753de991d390c5a1edeeba50f68e3a9253fbd8bf8ccd",
|
||||
"sha256:b87eb6fc9e1bb8f98892a2458781348fa37e6925f35bb6ceb9d4afd54ba36c73",
|
||||
"sha256:bbb5cc845a0292e0c520656d19d7ce40e18d0e19b22cb3e0409135a575bf79fc",
|
||||
"sha256:be945402e03de47ba1872cd5236395e0f4ad635526185a930735f66710e1bd3f",
|
||||
"sha256:bf13d564d310c156d1c8e53877baf2993fb3073b2fc9f69790ca6a732eb4bfea",
|
||||
"sha256:cf60dd2696b457b710dd40bf17ad269d5f5457b96442f7f85722bdb16fa6c899",
|
||||
"sha256:d1ba00ae33be84066cfbe7361d4e04dec78445b2b88bdb734d0d1cbab916025a",
|
||||
"sha256:d39fc4817fd67b3915256af5dda75fd4ee10621a3d484524487e33416c6f3543",
|
||||
"sha256:d766a4f0e5aa1ba056ec3496243150698dc0481902e2b8559314368717be82b1",
|
||||
"sha256:dbf364b4c5e7bae9250528167dfe40219b62e2d573c854d74be213e1e52069f7",
|
||||
"sha256:dd19608788b50eed889e13a5d71d832edc34fc9dfce606f66e8f9f917eef910d",
|
||||
"sha256:e013b07ba1c748dacc2a80e69a46286ff145935f260eb8c72df7185bf048f502",
|
||||
"sha256:e5d2b9be5b0693cf21eb4ce0ec8d211efb43966f6657807f6859aab3814f946b",
|
||||
"sha256:e5ff52d790c7e1628241ffbcaeb33e07d14b007b6eb00a19320c7b8a7024c040",
|
||||
"sha256:e75a2ad7b647fd8046d58c3132d7eaf31b12d8a53c0e4b21fa9c4d23d6ee6d3c",
|
||||
"sha256:e7ac22a0bb2c7c49f441f7a6d46c9c80d96e56f5a8bc6972529ed43c8b694e27",
|
||||
"sha256:ed2144b8a78f9d94d9515963ed273d620e07846acd5d4b0a642d4849e8d91a0c",
|
||||
"sha256:f017a61399f13aa6d1039f75cd467be388d157cd81f1a119b9d9a68ba6f2830d",
|
||||
"sha256:f1d8a2a57b47142b10374902777e798784abf400a004b14f1b0b9eaf1e528ba4",
|
||||
"sha256:f2d32f95922927186c6dbc8bc60df0d186b6edb828d299ab10898ef3f40052fe",
|
||||
"sha256:f319bae0321bc838e205bf9e5bc28f0a3165f30c203b610f17ab5552cff90323",
|
||||
"sha256:f3c38e4e5ccbdc9198aecc766cedbb134b2d89bf64533973678dfcf07effd883",
|
||||
"sha256:f9983d01d7705b2d1f7a95e10bbe4091fabc03a46881a256c2787637b087003f",
|
||||
"sha256:fa260de59dfb143af06dcf30c2be0b200bed2a73737a8a59248fcb9fa601ef0f"
|
||||
],
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==7.8.0"
|
||||
},
|
||||
"flake8": {
|
||||
"hashes": [
|
||||
"sha256:93b92ba5bdb60754a6da14fa3b93a9361fd00a59632ada61fd7b130436c40343",
|
||||
"sha256:fa558ae3f6f7dbf2b4f22663e5343b6b6023620461f8d4ff2019ef4b5ee70426"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==7.2.0"
|
||||
},
|
||||
"iniconfig": {
|
||||
"hashes": [
|
||||
"sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7",
|
||||
"sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==2.1.0"
|
||||
},
|
||||
"jmespath": {
|
||||
"hashes": [
|
||||
"sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980",
|
||||
"sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"
|
||||
],
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==1.0.1"
|
||||
},
|
||||
"mccabe": {
|
||||
"hashes": [
|
||||
"sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325",
|
||||
"sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"
|
||||
],
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==0.7.0"
|
||||
},
|
||||
"packaging": {
|
||||
"hashes": [
|
||||
"sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484",
|
||||
"sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==25.0"
|
||||
},
|
||||
"pluggy": {
|
||||
"hashes": [
|
||||
"sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3",
|
||||
"sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"
|
||||
],
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==1.6.0"
|
||||
},
|
||||
"pycodestyle": {
|
||||
"hashes": [
|
||||
"sha256:35863c5974a271c7a726ed228a14a4f6daf49df369d8c50cd9a6f58a5e143ba9",
|
||||
"sha256:c8415bf09abe81d9c7f872502a6eee881fbe85d8763dd5b9924bb0a01d67efae"
|
||||
],
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==2.13.0"
|
||||
},
|
||||
"pyflakes": {
|
||||
"hashes": [
|
||||
"sha256:5039c8339cbb1944045f4ee5466908906180f13cc99cc9949348d10f82a5c32a",
|
||||
"sha256:6dfd61d87b97fba5dcfaaf781171ac16be16453be6d816147989e7f6e6a9576b"
|
||||
],
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==3.3.2"
|
||||
},
|
||||
"pytest": {
|
||||
"hashes": [
|
||||
"sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820",
|
||||
"sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==8.3.5"
|
||||
},
|
||||
"pytest-cov": {
|
||||
"hashes": [
|
||||
"sha256:46935f7aaefba760e716c2ebfbe1c216240b9592966e7da99ea8292d4d3e2a0a",
|
||||
"sha256:bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde"
|
||||
],
|
||||
"index": "pypi",
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==6.1.1"
|
||||
},
|
||||
"python-dateutil": {
|
||||
"hashes": [
|
||||
"sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",
|
||||
"sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
|
||||
"version": "==2.9.0.post0"
|
||||
},
|
||||
"s3transfer": {
|
||||
"hashes": [
|
||||
"sha256:35b314d7d82865756edab59f7baebc6b477189e6ab4c53050e28c1de4d9cce18",
|
||||
"sha256:8ac58bc1989a3fdb7c7f3ee0918a66b160d038a147c7b5db1500930a607e9a1c"
|
||||
],
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==0.12.0"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274",
|
||||
"sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
|
||||
"version": "==1.17.0"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466",
|
||||
"sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813"
|
||||
],
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==2.4.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
16
ecs/jskult-batch/entrypoint.py
Normal file
16
ecs/jskult-batch/entrypoint.py
Normal file
@ -0,0 +1,16 @@
|
||||
"""実消化&アルトマーク データ取込後続処理バッチエントリーポイント"""
|
||||
from src.batch.common.jskult_batch_entrypoint_factory import JskultBatchEntrypointFactory
|
||||
from src.system_var.environment import PROCESS_NAME
|
||||
|
||||
if __name__ == '__main__':
|
||||
try:
|
||||
jkultBatchEntrypointFactory = JskultBatchEntrypointFactory(
|
||||
PROCESS_NAME)
|
||||
module = jkultBatchEntrypointFactory.create()
|
||||
module.execute()
|
||||
exit(0)
|
||||
|
||||
except Exception:
|
||||
# エラーが起きても、正常系のコードで返す。
|
||||
# エラーが起きた事実はbatch_process内でログを出す。
|
||||
exit(0)
|
||||
3
ecs/jskult-batch/pytest.ini
Normal file
3
ecs/jskult-batch/pytest.ini
Normal file
@ -0,0 +1,3 @@
|
||||
[pytest]
|
||||
log_format = %(levelname)s %(asctime)s %(message)s
|
||||
log_date_format = %Y-%m-%d %H:%M:%S
|
||||
0
ecs/jskult-batch/src/__init__.py
Normal file
0
ecs/jskult-batch/src/__init__.py
Normal file
0
ecs/jskult-batch/src/aws/__init__.py
Normal file
0
ecs/jskult-batch/src/aws/__init__.py
Normal file
185
ecs/jskult-batch/src/aws/s3.py
Normal file
185
ecs/jskult-batch/src/aws/s3.py
Normal file
@ -0,0 +1,185 @@
|
||||
import gzip
|
||||
import os
|
||||
import os.path as path
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
import boto3
|
||||
|
||||
from src.system_var import environment
|
||||
|
||||
|
||||
class S3Client:
|
||||
__s3_client = boto3.client('s3')
|
||||
_bucket_name: str
|
||||
|
||||
def list_objects(self, bucket_name: str, folder_name: str):
|
||||
response = self.__s3_client.list_objects_v2(Bucket=bucket_name, Prefix=folder_name)
|
||||
if response['KeyCount'] == 0:
|
||||
return []
|
||||
contents = response['Contents']
|
||||
# 末尾がスラッシュで終わるものはフォルダとみなしてスキップする
|
||||
objects = [{'filename': content['Key'], 'size': content['Size']}
|
||||
for content in contents if not content['Key'].endswith('/')]
|
||||
return objects
|
||||
|
||||
def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None:
|
||||
copy_source = {'Bucket': src_bucket, 'Key': src_key}
|
||||
self.__s3_client.copy(copy_source, dest_bucket, dest_key)
|
||||
return
|
||||
|
||||
def download_file(self, bucket_name: str, file_key: str, file):
|
||||
self.__s3_client.download_fileobj(
|
||||
Bucket=bucket_name,
|
||||
Key=file_key,
|
||||
Fileobj=file
|
||||
)
|
||||
return
|
||||
|
||||
def upload_file(self, local_file_path: str, bucket_name: str, file_key: str):
|
||||
self.__s3_client.upload_file(
|
||||
local_file_path,
|
||||
Bucket=bucket_name,
|
||||
Key=file_key
|
||||
)
|
||||
|
||||
def delete_file(self, bucket_name: str, file_key: str):
|
||||
self.__s3_client.delete_object(
|
||||
Bucket=bucket_name,
|
||||
Key=file_key
|
||||
)
|
||||
|
||||
|
||||
class S3Bucket():
|
||||
_s3_client = S3Client()
|
||||
_bucket_name: str = None
|
||||
|
||||
|
||||
class UltmarcBucket(S3Bucket):
|
||||
_bucket_name = environment.ULTMARC_DATA_BUCKET
|
||||
_folder = environment.ULTMARC_DATA_FOLDER
|
||||
|
||||
def list_dat_file(self):
|
||||
return self._s3_client.list_objects(self._bucket_name, self._folder)
|
||||
|
||||
def download_dat_file(self, dat_filename: str):
|
||||
# 一時ファイルとして保存する
|
||||
temporary_dir = tempfile.mkdtemp()
|
||||
temporary_file_path = path.join(temporary_dir, f'{dat_filename.replace(f"{self._folder}/", "")}')
|
||||
with open(temporary_file_path, mode='wb') as f:
|
||||
self._s3_client.download_file(self._bucket_name, dat_filename, f)
|
||||
f.seek(0)
|
||||
return temporary_file_path
|
||||
|
||||
def backup_dat_file(self, dat_file_key: str, datetime_key: str):
|
||||
# バックアップバケットにコピー
|
||||
ultmarc_backup_bucket = UltmarcBackupBucket()
|
||||
backup_key = f'{ultmarc_backup_bucket._folder}/{datetime_key}/{dat_file_key.replace(f"{self._folder}/", "")}'
|
||||
self._s3_client.copy(self._bucket_name, dat_file_key, ultmarc_backup_bucket._bucket_name, backup_key)
|
||||
# コピー元のファイルを削除
|
||||
self._s3_client.delete_file(self._bucket_name, dat_file_key)
|
||||
|
||||
|
||||
class ConfigBucket(S3Bucket):
|
||||
_bucket_name = environment.JSKULT_CONFIG_BUCKET
|
||||
|
||||
def download_holiday_list(self):
|
||||
# 一時ファイルとして保存する
|
||||
temporary_dir = tempfile.mkdtemp()
|
||||
temporary_file_path = path.join(temporary_dir, environment.JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME)
|
||||
holiday_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME}'
|
||||
with open(temporary_file_path, mode='wb') as f:
|
||||
self._s3_client.download_file(self._bucket_name, holiday_list_key, f)
|
||||
f.seek(0)
|
||||
return temporary_file_path
|
||||
|
||||
def download_wholesaler_stock_input_day_list(self):
|
||||
# 一時ファイルとして保存する
|
||||
temporary_dir = tempfile.mkdtemp()
|
||||
temporary_file_path = path.join(temporary_dir, environment.JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME)
|
||||
wholesaler_stock_input_day_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME}'
|
||||
with open(temporary_file_path, mode='wb') as f:
|
||||
self._s3_client.download_file(self._bucket_name, wholesaler_stock_input_day_list_key, f)
|
||||
f.seek(0)
|
||||
return temporary_file_path
|
||||
|
||||
def download_ultmarc_hex_convert_config(self):
|
||||
# 一時ファイルとして保存する
|
||||
temporary_dir = tempfile.mkdtemp()
|
||||
temporary_file_path = path.join(temporary_dir, environment.JSKULT_ULTMARC_HEX_CONVERT_CONFIG_FILE_NAME)
|
||||
hex_convert_config_key = f'{environment.JSKULT_CONFIG_CONVERT_FOLDER}/{environment.JSKULT_ULTMARC_HEX_CONVERT_CONFIG_FILE_NAME}'
|
||||
with open(temporary_file_path, mode='wb') as f:
|
||||
self._s3_client.download_file(self._bucket_name, hex_convert_config_key, f)
|
||||
f.seek(0)
|
||||
return temporary_file_path
|
||||
|
||||
|
||||
class JskUltBackupBucket(S3Bucket):
|
||||
_bucket_name = environment.JSKULT_BACKUP_BUCKET
|
||||
|
||||
|
||||
class UltmarcBackupBucket(JskUltBackupBucket):
|
||||
_folder = environment.ULTMARC_BACKUP_FOLDER
|
||||
|
||||
|
||||
class VjskBackupBucket(JskUltBackupBucket):
|
||||
_folder = environment.VJSK_BACKUP_FOLDER
|
||||
|
||||
|
||||
class VjskReceiveBucket(S3Bucket):
|
||||
_bucket_name = environment.VJSK_DATA_BUCKET
|
||||
_recv_folder = environment.VJSK_DATA_RECEIVE_FOLDER
|
||||
|
||||
_s3_file_list = None
|
||||
|
||||
def get_s3_file_list(self):
|
||||
self._s3_file_list = self._s3_client.list_objects(self._bucket_name, self._recv_folder)
|
||||
return self._s3_file_list
|
||||
|
||||
def download_data_file(self, data_filename: str):
|
||||
temporary_dir = tempfile.mkdtemp()
|
||||
temporary_file_path = path.join(temporary_dir, f'{data_filename.replace(f"{self._recv_folder}/", "")}')
|
||||
with open(temporary_file_path, mode='wb') as f:
|
||||
self._s3_client.download_file(self._bucket_name, data_filename, f)
|
||||
f.seek(0)
|
||||
return temporary_file_path
|
||||
|
||||
def unzip_data_file(self, filename: str):
|
||||
temp_dir = os.path.dirname(filename)
|
||||
decompress_filename = os.path.basename(filename).replace('.gz', '')
|
||||
decompress_file_path = os.path.join(temp_dir, decompress_filename)
|
||||
with gzip.open(filename, 'rb') as gz:
|
||||
with open(decompress_file_path, 'wb') as decompressed_file:
|
||||
shutil.copyfileobj(gz, decompressed_file)
|
||||
|
||||
ret = [decompress_file_path]
|
||||
return ret
|
||||
|
||||
def backup_dat_file(self, target_files: list, datetime_key: str):
|
||||
jskult_backup_bucket = VjskBackupBucket()
|
||||
for target_file in target_files:
|
||||
backup_from_file_path = target_file.get("filename")
|
||||
backup_to_filename = backup_from_file_path.replace(f"{self._recv_folder}/", "")
|
||||
backup_key = f'{jskult_backup_bucket._folder}/{datetime_key}/{backup_to_filename}'
|
||||
self._s3_client.copy(self._bucket_name, backup_from_file_path,
|
||||
jskult_backup_bucket._bucket_name, backup_key)
|
||||
self._s3_client.delete_file(self._bucket_name, backup_from_file_path)
|
||||
|
||||
|
||||
class VjskSendBucket(S3Bucket):
|
||||
_bucket_name = environment.VJSK_DATA_BUCKET
|
||||
_send_folder = environment.VJSK_DATA_SEND_FOLDER
|
||||
|
||||
def upload_inst_pharm_csv_file(self, vjsk_create_csv: str, csv_file_path: str):
|
||||
# S3バケットにファイルを移動
|
||||
csv_file_name = f'{self._send_folder}/{vjsk_create_csv}'
|
||||
s3_client = S3Client()
|
||||
s3_client.upload_file(csv_file_path, self._bucket_name, csv_file_name)
|
||||
return
|
||||
|
||||
def backup_inst_pharm_csv_file(self, dat_file_key: str, datetime_key: str):
|
||||
# バックアップバケットにコピー
|
||||
vjsk_backup_bucket = VjskBackupBucket()
|
||||
dat_key = f'{self._send_folder}/{dat_file_key}'
|
||||
backup_key = f'{vjsk_backup_bucket._folder}/{self._send_folder}/{datetime_key}/{dat_file_key.replace(f"{self._send_folder}/", "")}'
|
||||
self._s3_client.copy(self._bucket_name, dat_key, vjsk_backup_bucket._bucket_name, backup_key)
|
||||
0
ecs/jskult-batch/src/batch/common/__init__.py
Normal file
0
ecs/jskult-batch/src/batch/common/__init__.py
Normal file
@ -0,0 +1,30 @@
|
||||
from src.batch.update_business_day import UpdateBusinessDay
|
||||
from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint
|
||||
from src.batch.trn_result_data_bio_lot import TrnResultDataBioLot
|
||||
from src.batch.mst_inst import MstInst
|
||||
from src.batch.dcf_inst_merge_io import DcfInstMergeIO
|
||||
|
||||
from src.logging.get_logger import get_logger
|
||||
|
||||
logger = get_logger("後続処理/日付更新処理振り分け")
|
||||
|
||||
|
||||
class JskultBatchEntrypointFactory:
|
||||
|
||||
def __init__(self, entrypoint_module_name: str):
|
||||
self._entrypoint_module_name: str = entrypoint_module_name
|
||||
|
||||
def create(self) -> JskultBatchEntrypoint:
|
||||
|
||||
if self._entrypoint_module_name == "jskult-batch-trn-result-data-bio-lot":
|
||||
return TrnResultDataBioLot()
|
||||
if self._entrypoint_module_name == "jskult-batch-mst-inst":
|
||||
return MstInst()
|
||||
if self._entrypoint_module_name == "jskult-batch-dcf-inst-merge-io":
|
||||
return DcfInstMergeIO()
|
||||
if self._entrypoint_module_name == "jskult-batch-update-business-day":
|
||||
return UpdateBusinessDay()
|
||||
|
||||
logger.error(
|
||||
f"一致するエントリーポイント識別子ではありませんでした。エントリーポイント識別子:{self._entrypoint_module_name}")
|
||||
raise ValueError()
|
||||
10
ecs/jskult-batch/src/batch/dcf_inst_merge_io.py
Normal file
10
ecs/jskult-batch/src/batch/dcf_inst_merge_io.py
Normal file
@ -0,0 +1,10 @@
|
||||
from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint
|
||||
|
||||
|
||||
class DcfInstMergeIO(JskultBatchEntrypoint):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def execute(self):
|
||||
# TODO: ここでDCF削除新規マスタ作成/データ出力処理を実行する
|
||||
pass
|
||||
8
ecs/jskult-batch/src/batch/jskult_batch_entrypoint.py
Normal file
8
ecs/jskult-batch/src/batch/jskult_batch_entrypoint.py
Normal file
@ -0,0 +1,8 @@
|
||||
import abc
|
||||
# 実消化&アルトマークの後続処理/日付更新実行クラスの基底クラス
|
||||
|
||||
|
||||
class JskultBatchEntrypoint(metaclass=abc.ABCMeta):
|
||||
@abc.abstractmethod()
|
||||
def execute(self):
|
||||
pass
|
||||
10
ecs/jskult-batch/src/batch/mst_inst.py
Normal file
10
ecs/jskult-batch/src/batch/mst_inst.py
Normal file
@ -0,0 +1,10 @@
|
||||
from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint
|
||||
|
||||
|
||||
class MstInst(JskultBatchEntrypoint):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def execute(self):
|
||||
# TODO: ここでメルク施設マスタ作成処理を実行する
|
||||
pass
|
||||
10
ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py
Normal file
10
ecs/jskult-batch/src/batch/trn_result_data_bio_lot.py
Normal file
@ -0,0 +1,10 @@
|
||||
from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint
|
||||
|
||||
|
||||
class TrnResultDataBioLot(JskultBatchEntrypoint):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def execute(self):
|
||||
# TODO: ここで生物由来ロット分解処理を実行する
|
||||
pass
|
||||
10
ecs/jskult-batch/src/batch/update_business_day.py
Normal file
10
ecs/jskult-batch/src/batch/update_business_day.py
Normal file
@ -0,0 +1,10 @@
|
||||
from src.batch.jskult_batch_entrypoint import JskultBatchEntrypoint
|
||||
|
||||
|
||||
class UpdateBusinessDay(JskultBatchEntrypoint):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
|
||||
def execute(self):
|
||||
# TODO: ここで日付更新処理を実行する
|
||||
pass
|
||||
0
ecs/jskult-batch/src/db/__init__.py
Normal file
0
ecs/jskult-batch/src/db/__init__.py
Normal file
195
ecs/jskult-batch/src/db/database.py
Normal file
195
ecs/jskult-batch/src/db/database.py
Normal file
@ -0,0 +1,195 @@
|
||||
from sqlalchemy import (Connection, CursorResult, Engine, QueuePool,
|
||||
create_engine, text)
|
||||
from sqlalchemy.engine.url import URL
|
||||
from tenacity import retry, stop_after_attempt, wait_exponential
|
||||
|
||||
from src.error.exceptions import DBException
|
||||
from src.logging.get_logger import get_logger
|
||||
from src.system_var import environment
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class Database:
|
||||
"""データベース操作クラス"""
|
||||
__connection: Connection = None
|
||||
__transactional_engine: Engine = None
|
||||
__autocommit_engine: Engine = None
|
||||
__host: str = None
|
||||
__port: str = None
|
||||
__username: str = None
|
||||
__password: str = None
|
||||
__schema: str = None
|
||||
__autocommit: bool = None
|
||||
__connection_string: str = None
|
||||
|
||||
def __init__(self, username: str, password: str, host: str, port: int, schema: str, autocommit: bool = False) -> None:
|
||||
"""このクラスの新たなインスタンスを初期化します
|
||||
|
||||
Args:
|
||||
username (str): DBユーザー名
|
||||
password (str): DBパスワード
|
||||
host (str): DBホスト名
|
||||
port (int): DBポート
|
||||
schema (str): DBスキーマ名
|
||||
autocommit(bool): 自動コミットモードで接続するかどうか(Trueの場合、トランザクションの有無に限らず即座にコミットされる). Defaults to False.
|
||||
"""
|
||||
self.__username = username
|
||||
self.__password = password
|
||||
self.__host = host
|
||||
self.__port = int(port)
|
||||
self.__schema = schema
|
||||
self.__autocommit = autocommit
|
||||
|
||||
self.__connection_string = URL.create(
|
||||
drivername='mysql+pymysql',
|
||||
username=self.__username,
|
||||
password=self.__password,
|
||||
host=self.__host,
|
||||
port=self.__port,
|
||||
database=self.__schema,
|
||||
query={"charset": "utf8mb4", "local_infile": "1"},
|
||||
)
|
||||
|
||||
self.__transactional_engine = create_engine(
|
||||
self.__connection_string,
|
||||
pool_timeout=5,
|
||||
poolclass=QueuePool
|
||||
)
|
||||
|
||||
self.__autocommit_engine = self.__transactional_engine.execution_options(isolation_level='AUTOCOMMIT')
|
||||
|
||||
@classmethod
|
||||
def get_instance(cls, autocommit=False):
|
||||
"""インスタンスを取得します
|
||||
|
||||
Args:
|
||||
autocommit (bool, optional): 自動コミットモードで接続するかどうか(Trueの場合、トランザクションの有無に限らず即座にコミットされる). Defaults to False.
|
||||
Returns:
|
||||
Database: DB操作クラスインスタンス
|
||||
"""
|
||||
return cls(
|
||||
username=environment.DB_USERNAME,
|
||||
password=environment.DB_PASSWORD,
|
||||
host=environment.DB_HOST,
|
||||
port=environment.DB_PORT,
|
||||
schema=environment.DB_SCHEMA,
|
||||
autocommit=autocommit
|
||||
)
|
||||
|
||||
@retry(
|
||||
wait=wait_exponential(
|
||||
multiplier=environment.DB_CONNECTION_RETRY_INTERVAL_INIT,
|
||||
min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS,
|
||||
max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS
|
||||
),
|
||||
stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT),
|
||||
retry_error_cls=DBException
|
||||
)
|
||||
def connect(self):
|
||||
"""
|
||||
DBに接続します。接続に失敗した場合、リトライします。\n
|
||||
インスタンスのautocommitがTrueの場合、自動コミットモードで接続する。（明示的なトランザクションも無視される）
|
||||
Raises:
|
||||
DBException: 接続失敗
|
||||
"""
|
||||
try:
|
||||
self.__connection = (
|
||||
self.__autocommit_engine.connect() if self.__autocommit is True
|
||||
else self.__transactional_engine.connect())
|
||||
except Exception as e:
|
||||
raise DBException(e)
|
||||
|
||||
def execute_select(self, select_query: str, parameters=None) -> list[dict]:
|
||||
"""SELECTクエリを実行します。
|
||||
|
||||
Args:
|
||||
select_query (str): SELECT文
|
||||
parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None.
|
||||
|
||||
Raises:
|
||||
DBException: DBエラー
|
||||
|
||||
Returns:
|
||||
list[dict]: カラム名: 値の辞書リスト
|
||||
"""
|
||||
if self.__connection is None:
|
||||
raise DBException('DBに接続していません')
|
||||
|
||||
result = None
|
||||
try:
|
||||
# トランザクションが開始している場合は、トランザクションを引き継ぐ
|
||||
if self.__connection.in_transaction():
|
||||
result = self.__connection.execute(text(select_query), parameters)
|
||||
else:
|
||||
# トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。
|
||||
result = self.__execute_with_transaction(select_query, parameters)
|
||||
except Exception as e:
|
||||
raise DBException(f'SQL Error: {e}')
|
||||
|
||||
result_rows = result.mappings().all()
|
||||
return result_rows
|
||||
|
||||
def execute(self, query: str, parameters=None) -> CursorResult:
|
||||
"""SQLクエリを実行します。
|
||||
|
||||
Args:
|
||||
query (str): SQL文
|
||||
parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None.
|
||||
|
||||
Raises:
|
||||
DBException: DBエラー
|
||||
|
||||
Returns:
|
||||
CursorResult: 取得結果
|
||||
"""
|
||||
if self.__connection is None:
|
||||
raise DBException('DBに接続していません')
|
||||
|
||||
result = None
|
||||
try:
|
||||
# トランザクションが開始している場合は、トランザクションを引き継ぐ
|
||||
if self.__connection.in_transaction():
|
||||
result = self.__connection.execute(text(query), parameters)
|
||||
else:
|
||||
# トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。
|
||||
result = self.__execute_with_transaction(query, parameters)
|
||||
except Exception as e:
|
||||
raise DBException(f'SQL Error: {e}')
|
||||
|
||||
return result
|
||||
|
||||
def begin(self):
|
||||
"""トランザクションを開始します。"""
|
||||
if not self.__connection.in_transaction():
|
||||
self.__connection.begin()
|
||||
|
||||
def commit(self):
|
||||
"""トランザクションをコミットします"""
|
||||
if self.__connection.in_transaction():
|
||||
self.__connection.commit()
|
||||
|
||||
def rollback(self):
|
||||
"""トランザクションをロールバックします"""
|
||||
if self.__connection.in_transaction():
|
||||
self.__connection.rollback()
|
||||
|
||||
def disconnect(self):
|
||||
"""DB接続を切断します。"""
|
||||
if self.__connection is not None:
|
||||
self.__connection.close()
|
||||
self.__connection = None
|
||||
|
||||
def to_jst(self):
|
||||
self.execute('SET time_zone = "+9:00"')
|
||||
|
||||
def __execute_with_transaction(self, query: str, parameters: dict):
|
||||
# トランザクションを開始してクエリを実行する
|
||||
with self.__connection.begin():
|
||||
try:
|
||||
result = self.__connection.execute(text(query), parameters=parameters)
|
||||
except Exception as e:
|
||||
self.__connection.rollback()
|
||||
raise e
|
||||
# ここでコミットされる
|
||||
return result
|
||||
0
ecs/jskult-batch/src/error/__init__.py
Normal file
0
ecs/jskult-batch/src/error/__init__.py
Normal file
10
ecs/jskult-batch/src/error/exceptions.py
Normal file
10
ecs/jskult-batch/src/error/exceptions.py
Normal file
@ -0,0 +1,10 @@
|
||||
class MeDaCaException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class DBException(MeDaCaException):
|
||||
pass
|
||||
|
||||
|
||||
class BatchOperationException(MeDaCaException):
|
||||
pass
|
||||
37
ecs/jskult-batch/src/logging/get_logger.py
Normal file
37
ecs/jskult-batch/src/logging/get_logger.py
Normal file
@ -0,0 +1,37 @@
|
||||
import logging
|
||||
|
||||
from src.system_var.environment import LOG_LEVEL
|
||||
|
||||
# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する
|
||||
for name in ["boto3", "botocore", "s3transfer", "urllib3"]:
|
||||
logging.getLogger(name).setLevel(logging.WARNING)
|
||||
|
||||
|
||||
def get_logger(log_name: str) -> logging.Logger:
|
||||
"""一意のログ出力モジュールを取得します。
|
||||
|
||||
Args:
|
||||
log_name (str): ロガー名
|
||||
|
||||
Returns:
|
||||
_type_: _description_
|
||||
"""
|
||||
logger = logging.getLogger(log_name)
|
||||
level = logging.getLevelName(LOG_LEVEL)
|
||||
if not isinstance(level, int):
|
||||
level = logging.INFO
|
||||
logger.setLevel(level)
|
||||
|
||||
if not logger.hasHandlers():
|
||||
handler = logging.StreamHandler()
|
||||
logger.addHandler(handler)
|
||||
|
||||
formatter = logging.Formatter(
|
||||
'%(name)s\t[%(levelname)s]\t%(asctime)s\t%(message)s',
|
||||
'%Y-%m-%d %H:%M:%S'
|
||||
)
|
||||
|
||||
for handler in logger.handlers:
|
||||
handler.setFormatter(formatter)
|
||||
|
||||
return logger
|
||||
39
ecs/jskult-batch/src/manager/jskult_batch_run_manager.py
Normal file
39
ecs/jskult-batch/src/manager/jskult_batch_run_manager.py
Normal file
@ -0,0 +1,39 @@
|
||||
import os
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import boto3
|
||||
|
||||
TABLE_NAME = 'mbj-newdwh2021-staging-jskult-batch-run-manage'
|
||||
|
||||
dynamodb = boto3.client('dynamodb')
|
||||
|
||||
|
||||
class JskultBatchRunManager:
|
||||
def __init__(self, execution_id: str):
|
||||
self._execution_id: str = execution_id
|
||||
|
||||
# バッチ処理ステータスをsuccessで登録および更新を行う
|
||||
def batch_success(self):
|
||||
self._put_dynamodb_record('success')
|
||||
|
||||
# バッチ処理ステータスをfailedで登録および更新を行う
|
||||
def batch_failed(self):
|
||||
self._put_dynamodb_record('failed')
|
||||
|
||||
# バッチ処理ステータスをretryで登録および更新を行う
|
||||
def batch_retry(self):
|
||||
self._put_dynamodb_record('retry')
|
||||
|
||||
def _put_dynamodb_record(self, record: str):
|
||||
# バッチ実行管理テーブルの登録、更新（upsert)
|
||||
now = int(datetime.now(timezone.utc).timestamp() * 1000)
|
||||
|
||||
options = {
|
||||
'TableName': TABLE_NAME,
|
||||
'Item': {
|
||||
'execution_id': {'S': self._execution_id},
|
||||
'status': {'S': record},
|
||||
'createdAt': {'N': str(now)},
|
||||
},
|
||||
}
|
||||
dynamodb.put_item(**options)
|
||||
320
ecs/jskult-batch/src/manager/jskult_batch_status_manager.py
Normal file
320
ecs/jskult-batch/src/manager/jskult_batch_status_manager.py
Normal file
@ -0,0 +1,320 @@
|
||||
import os
|
||||
from src.error.exceptions import MaxRunCountReachedException
|
||||
from src.db.database import Database
|
||||
|
||||
# 実消化&アルトマーク_バッチステータス管理テーブルを管理するクラス
|
||||
|
||||
|
||||
class JskultBatchStatusManager:
|
||||
def __init__(self, process_name: str, process_type: str, max_run_count_flg: int, receive_file_count: int):
|
||||
|
||||
# 処理名
|
||||
self._process_name: str = process_name
|
||||
|
||||
# 管理区分
|
||||
self._process_type: str = process_type
|
||||
|
||||
# 最大起動回数
|
||||
self._max_run_count_flg: str = max_run_count_flg
|
||||
|
||||
# 受信ファイル数
|
||||
self._receive_file_count: str = receive_file_count
|
||||
|
||||
# DB接続モジュール(バッチ)のget_instanceを呼び出し設定
|
||||
self._db = Database.get_instance()
|
||||
|
||||
# 処理ステータスの登録および更新を行う
|
||||
|
||||
def set_process_status(self, process_status: str):
|
||||
try:
|
||||
# DB接続開始
|
||||
self._db.connect()
|
||||
self._db.begin()
|
||||
self._db.to_jst()
|
||||
self._db.execute(
|
||||
f"""
|
||||
CALL
|
||||
internal07.upsert_jskult_batch_status_manage(
|
||||
{self._process_name},
|
||||
{self._process_type},
|
||||
{process_status},
|
||||
NULL,
|
||||
NULL
|
||||
);"""
|
||||
)
|
||||
self._db.commit()
|
||||
|
||||
except Exception as e:
|
||||
|
||||
# Exceptionをcatchした場合はrollback
|
||||
print(self._db)
|
||||
self._db.rollback()
|
||||
raise e
|
||||
|
||||
# 後続処理を実行してよいか判定する
|
||||
def can_run_post_process(self):
|
||||
|
||||
# SELECTの結果からレコード数を取得
|
||||
record = self._db.execute_select(
|
||||
f"""
|
||||
SELECT
|
||||
COUNT(*)
|
||||
FROM
|
||||
internal07.jskult_batch_status_manage
|
||||
WHERE
|
||||
process_name = {self._process_name}
|
||||
AND
|
||||
process_date = src07.get_syor_date();
|
||||
"""
|
||||
)
|
||||
|
||||
record_count = record[0]['count']
|
||||
|
||||
if record_count == 0:
|
||||
raise ValueError("レコードの取得が出来ませんでした。")
|
||||
|
||||
# 起動回数のインクリメント
|
||||
self._increment_run_count()
|
||||
|
||||
# データ取込が完了していた場合
|
||||
if self._is_done_data_import():
|
||||
return True
|
||||
|
||||
# 最大起動回数に到達していない場合
|
||||
if not self._is_max_run_count_reached:
|
||||
return False
|
||||
|
||||
# 最大起動回数フラグを立てる
|
||||
self._activate_max_run_count_flg()
|
||||
|
||||
return True
|
||||
|
||||
# 日付テーブル更新を実行してよいか判定する
|
||||
def can_run_business_day_update(self):
|
||||
|
||||
# SELECTの結果からレコード数を取得
|
||||
record = self._db.execute_select(
|
||||
f"""
|
||||
SELECT
|
||||
COUNT(*)
|
||||
FROM
|
||||
internal07.jskult_batch_status_manage
|
||||
WHERE
|
||||
process_name = {self._process_name}
|
||||
AND
|
||||
process_date = src07.get_syor_date();
|
||||
"""
|
||||
)
|
||||
|
||||
record_count = record[0]['count']
|
||||
|
||||
if record_count == 0:
|
||||
raise ValueError("レコードの取得が出来ませんでした。")
|
||||
|
||||
# 起動回数のインクリメント
|
||||
self._increment_run_count()
|
||||
|
||||
# 後続処理が完了していた場合
|
||||
if self._is_done_post_process():
|
||||
return True
|
||||
|
||||
# 最大起動回数に到達していない場合
|
||||
if not self._is_max_run_count_reached():
|
||||
return False
|
||||
|
||||
# 最大起動回数フラグを立てる
|
||||
self._activate_max_run_count_flg()
|
||||
|
||||
# 最大起動回数に到達した場合にメッセージをスロー
|
||||
raise MaxRunCountReachedException("最大起動回数に到達しました")
|
||||
|
||||
# アルトマークデータ連携があったかを確認する
|
||||
|
||||
def is_done_ultmarc_import(self):
|
||||
|
||||
# SELECTの結果からレコード数を取得
|
||||
record_count = self._db.execute_select(
|
||||
f"""
|
||||
SELECT
|
||||
COUNT(*)
|
||||
FROM
|
||||
internal07.jskult_batch_status_manage
|
||||
WHERE
|
||||
process_name = 'jskult-batch-ultmarc-io'
|
||||
AND
|
||||
process_date = src07.get_syor_date();
|
||||
"""
|
||||
)
|
||||
# アルトマークデータ連携が無かった場合
|
||||
if record_count.pop() == 0:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
# 起動回数をインクリメントする
|
||||
def _increment_run_count(self):
|
||||
try:
|
||||
# DB接続開始
|
||||
self._db.connect()
|
||||
self._db.begin()
|
||||
self._db.to_jst()
|
||||
# SELECTの結果からレコードを取得
|
||||
record = self._db.execute_select(
|
||||
f"""
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
internal07.jskult_batch_status_manage
|
||||
WHERE
|
||||
process_name = {self._process_name}
|
||||
AND
|
||||
process_date = src07.get_syor_date();
|
||||
"""
|
||||
)
|
||||
|
||||
run_count += record[0]['run_count']
|
||||
|
||||
self._db.execute(
|
||||
f"""
|
||||
CALL
|
||||
upsert_jskult_batch_status_manage(
|
||||
{self._process_name},
|
||||
{self._process_type},
|
||||
NULL,
|
||||
{run_count},
|
||||
NULL);
|
||||
"""
|
||||
)
|
||||
|
||||
self._db.commit()
|
||||
|
||||
except Exception as e:
|
||||
|
||||
# Exceptionをcatchした場合はrollbakc
|
||||
self._db.rollback()
|
||||
raise e
|
||||
|
||||
# データ取込処理が完了しているかを判定する
|
||||
def _is_done_data_import(self):
|
||||
|
||||
# SELECTの結果からレコード数を取得
|
||||
record = self._db.execute_select(
|
||||
f"""
|
||||
SELECT
|
||||
COUNT(*)
|
||||
FROM
|
||||
internal07.jskult_batch_status_manage
|
||||
WHERE
|
||||
process_type = {self._process_type}
|
||||
AND
|
||||
process_status = 'done'
|
||||
AND
|
||||
process_date = src07.get_syor_date();
|
||||
"""
|
||||
)
|
||||
|
||||
record_count = record[0]['count']
|
||||
|
||||
# データ取込数が一致している場合
|
||||
if (self._receive_file_count == record_count):
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
# 後続処理のすべての処理が完了しているかを判定する
|
||||
def _is_done_post_process(self):
|
||||
|
||||
# 生物由来データロット分解のチェック
|
||||
if not self._is_done_process("jskult-batch-trn-result-data-bio-lot"):
|
||||
return False
|
||||
|
||||
# メルク施設マスタ作成のチェック
|
||||
if not self._is_done_process("jskult-batch-mst-inst"):
|
||||
return False
|
||||
|
||||
# DCF削除新規マスタ作成のチェック
|
||||
if not self._is_done_process("jskult-batch-dcf-inst-merge-io"):
|
||||
return False
|
||||
|
||||
# 全ての後続処理が完了している場合Trueを返す
|
||||
return True
|
||||
|
||||
# データ取込処理が完了しているかを判定する
|
||||
|
||||
def _is_done_process(self):
|
||||
|
||||
# SELECTの結果からレコード数を取得
|
||||
record = self._db.execute_select(
|
||||
f"""
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
internal07.jskult_batch_status_manage
|
||||
WHERE
|
||||
process_name = {self._process_name}
|
||||
AND
|
||||
process_status = 'done'
|
||||
AND
|
||||
process_date = src07.get_syor_date();
|
||||
"""
|
||||
)
|
||||
|
||||
record_count = record[0]['count']
|
||||
|
||||
if (record_count == 0):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
# 起動回数が最大回数に到達しているか判定する
|
||||
|
||||
def _is_max_run_count_reached(self):
|
||||
|
||||
# SELECTの結果からレコードを取得
|
||||
record = self._db.execute_select(
|
||||
f"""
|
||||
SELECT
|
||||
*
|
||||
FROM
|
||||
internal07.jskult_batch_status_manage
|
||||
WHERE
|
||||
process_name = {self._process_name}
|
||||
AND
|
||||
process_date = src07.get_syor_date();
|
||||
"""
|
||||
)
|
||||
|
||||
run_count = record[0]['run_count']
|
||||
|
||||
# 取得した起動回数とフィールド変数の最大起動回数が一致するか確認
|
||||
if run_count == self._max_run_count_flg:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _activate_max_run_count_flg(self):
|
||||
try:
|
||||
# DB接続開始
|
||||
self._db.connect()
|
||||
self._db.begin()
|
||||
self._db.to_jst()
|
||||
|
||||
# 最大起動回数フラグにフラグを立てる
|
||||
self._db.execute(
|
||||
f"""
|
||||
CALL
|
||||
upsert_jskult_batch_status_manage(
|
||||
{self._process_name},
|
||||
{self._process_type},
|
||||
NULL,
|
||||
NULL,
|
||||
1);
|
||||
"""
|
||||
)
|
||||
|
||||
self._db.commit()
|
||||
|
||||
except Exception as e:
|
||||
|
||||
self._db.rollback()
|
||||
raise e
|
||||
122
ecs/jskult-batch/src/manager/jskult_hdke_tbl_manager.py
Normal file
122
ecs/jskult-batch/src/manager/jskult_hdke_tbl_manager.py
Normal file
@ -0,0 +1,122 @@
|
||||
from datetime import datetime
|
||||
|
||||
from src.db.database import Database
|
||||
from src.error.exceptions import BatchOperationException, DBException
|
||||
from src.system_var import constants
|
||||
|
||||
|
||||
class JskultHdkeTblManager:
|
||||
_db: Database
|
||||
|
||||
def __init__(self):
|
||||
self._db = Database.get_instance()
|
||||
|
||||
def get_batch_statuses(self) -> tuple[str, str, str]:
|
||||
"""日次バッチ処理中フラグ、dump取得状況区分、処理日を取得する
|
||||
|
||||
Raises:
|
||||
BatchOperationException: DB操作の何らかのエラー
|
||||
|
||||
Returns:
|
||||
tuple[str, str, str]: [0]日次バッチ処理中フラグ、[1]dump取得状況区分、[2]処理日
|
||||
"""
|
||||
sql = 'SELECT bch_actf, dump_sts_kbn, src07.get_syor_date() AS syor_date FROM src07.hdke_tbl'
|
||||
try:
|
||||
self._db.connect()
|
||||
hdke_tbl_result = self._db.execute_select(sql)
|
||||
except DBException as e:
|
||||
raise BatchOperationException(e)
|
||||
finally:
|
||||
self._db.disconnect()
|
||||
|
||||
if len(hdke_tbl_result) == 0:
|
||||
raise BatchOperationException('日付テーブルが取得できませんでした')
|
||||
|
||||
# 必ず1件取れる
|
||||
hdke_tbl_record = hdke_tbl_result[0]
|
||||
batch_processing_flag = hdke_tbl_record['bch_actf']
|
||||
dump_status_kbn = hdke_tbl_record['dump_sts_kbn']
|
||||
syor_date = hdke_tbl_record['syor_date']
|
||||
# 処理日を文字列に変換する
|
||||
syor_date_str = datetime.strftime(syor_date, '%Y/%m/%d')
|
||||
|
||||
return batch_processing_flag, dump_status_kbn, syor_date_str
|
||||
|
||||
def update_batch_process_start(self):
|
||||
"""バッチ処理中フラグを処理中に更新する
|
||||
|
||||
Raises:
|
||||
BatchOperationException: DB操作の何らかのエラー
|
||||
"""
|
||||
|
||||
sql = """\
|
||||
UPDATE src07.hdke_tbl
|
||||
SET
|
||||
bch_actf = :start,
|
||||
updater = CURRENT_USER(),
|
||||
update_date = NOW()
|
||||
"""
|
||||
try:
|
||||
self._db.connect()
|
||||
self._db.to_jst()
|
||||
self._db.execute(
|
||||
sql, {'start': constants.BATCH_ACTF_BATCH_START})
|
||||
except DBException as e:
|
||||
raise BatchOperationException(e)
|
||||
finally:
|
||||
self._db.disconnect()
|
||||
|
||||
return
|
||||
|
||||
def update_batch_process_complete(self) -> None:
|
||||
"""バッチ正常終了処理時の更新処理
|
||||
|
||||
Raises:
|
||||
BatchOperationException: DB操作の何らかのエラー
|
||||
"""
|
||||
|
||||
sql = """\
|
||||
UPDATE src07.hdke_tbl
|
||||
SET
|
||||
bch_actf = :batch_complete,
|
||||
dump_sts_kbn = :dump_unprocessed,
|
||||
syor_date = DATE_FORMAT((src07.get_syor_date() + interval 1 day), '%Y%m%d'), -- +1日
|
||||
updater = CURRENT_USER(),
|
||||
update_date = NOW()
|
||||
"""
|
||||
try:
|
||||
self._db.connect()
|
||||
self._db.to_jst()
|
||||
self._db.execute(sql, {
|
||||
'batch_complete': constants.BATCH_ACTF_BATCH_UNPROCESSED,
|
||||
'dump_unprocessed': constants.DUMP_STATUS_KBN_UNPROCESSED
|
||||
})
|
||||
except DBException as e:
|
||||
raise BatchOperationException(e)
|
||||
finally:
|
||||
self._db.disconnect()
|
||||
|
||||
def can_run_process(self) -> bool:
|
||||
"""バッチ処理を起動してよいかを判定する
|
||||
|
||||
Raises:
|
||||
BatchOperationException: DB操作の何らかのエラー
|
||||
|
||||
Returns:
|
||||
bool: バッチ処理を実行して良い場合はTrue
|
||||
"""
|
||||
try:
|
||||
# 日次バッチ処置中フラグ、dump処理状態区分を取得
|
||||
batch_processing_flag, dump_status_kbn, _ = self.get_batch_statuses()
|
||||
except DBException as e:
|
||||
raise BatchOperationException(e)
|
||||
finally:
|
||||
self._db.disconnect()
|
||||
# 日次バッチ処理中ではない場合、後続の処理は行わない
|
||||
if batch_processing_flag != constants.BATCH_ACTF_BATCH_START:
|
||||
return False
|
||||
# dump取得が正常終了していない場合、後続の処理は行わない
|
||||
if dump_status_kbn != constants.DUMP_STATUS_KBN_COMPLETE:
|
||||
return False
|
||||
|
||||
return True
|
||||
0
ecs/jskult-batch/src/system_var/__init__.py
Normal file
0
ecs/jskult-batch/src/system_var/__init__.py
Normal file
17
ecs/jskult-batch/src/system_var/constants.py
Normal file
17
ecs/jskult-batch/src/system_var/constants.py
Normal file
@ -0,0 +1,17 @@
|
||||
# バッチ正常終了コード
|
||||
BATCH_EXIT_CODE_SUCCESS = 0
|
||||
|
||||
# バッチ処理中フラグ：未処理
|
||||
BATCH_ACTF_BATCH_UNPROCESSED = '0'
|
||||
# バッチ処理中フラグ：処理中
|
||||
BATCH_ACTF_BATCH_IN_PROCESSING = '1'
|
||||
# dump取得状態区分：未処理
|
||||
DUMP_STATUS_KBN_UNPROCESSED = '0'
|
||||
# dump取得状態区分：dump取得正常終了
|
||||
DUMP_STATUS_KBN_COMPLETE = '2'
|
||||
|
||||
# カレンダーファイルのコメントシンボル
|
||||
CALENDAR_COMMENT_SYMBOL = '#'
|
||||
|
||||
# 月曜日(datetime.weekday()で月曜日を表す数字)
|
||||
WEEKDAY_MONDAY = 0
|
||||
18
ecs/jskult-batch/src/system_var/environment.py
Normal file
18
ecs/jskult-batch/src/system_var/environment.py
Normal file
@ -0,0 +1,18 @@
|
||||
import os
|
||||
|
||||
# Database
|
||||
DB_HOST = os.environ['DB_HOST']
|
||||
DB_PORT = int(os.environ['DB_PORT'])
|
||||
DB_USERNAME = os.environ['DB_USERNAME']
|
||||
DB_PASSWORD = os.environ['DB_PASSWORD']
|
||||
DB_SCHEMA = os.environ['DB_SCHEMA']
|
||||
|
||||
# 処理名
|
||||
PROCESS_NAME = os.environ['PROCESS_NAME']
|
||||
|
||||
# 初期値がある環境変数
|
||||
LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO')
|
||||
DB_CONNECTION_MAX_RETRY_ATTEMPT = int(os.environ.get('DB_CONNECTION_MAX_RETRY_ATTEMPT', 4))
|
||||
DB_CONNECTION_RETRY_INTERVAL_INIT = int(os.environ.get('DB_CONNECTION_RETRY_INTERVAL', 5))
|
||||
DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MIN_SECONDS', 5))
|
||||
DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MAX_SECONDS', 50))
|
||||
@ -0,0 +1,83 @@
|
||||
-- A5M2で実行時に[SQL] - [スラッシュ(/)のみの行でSQLを区切る]に変えてから実行する
|
||||
-- $$から始まる文字は後からREPLACEする文字を示す独自ルール
|
||||
-- 当プロシージャは、同一セッション内での並列処理を実行することができない
|
||||
-- 実行者の権限でストアドプロシージャを実行するために、「SQL SECURITY INVOKER」を付与している
|
||||
CREATE PROCEDURE `internal07`.`upsert_jskult_batch_status_manage`(
|
||||
process_name varchar(100),
|
||||
process_type varchar(50),
|
||||
process_status varchar(50),
|
||||
total_run_count INT,
|
||||
max_run_count_flg TINYINT(1)
|
||||
)
|
||||
SQL SECURITY INVOKER
|
||||
BEGIN
|
||||
|
||||
-- エラー処理
|
||||
DECLARE EXIT HANDLER FOR SQLEXCEPTION
|
||||
BEGIN
|
||||
GET DIAGNOSTICS CONDITION 1
|
||||
@error_state = RETURNED_SQLSTATE, @error_msg = MESSAGE_TEXT;
|
||||
ROLLBACK;
|
||||
SET @error_msg = (
|
||||
CASE
|
||||
WHEN LENGTH(@error_msg) > 128 THEN CONCAT(SUBSTRING(@error_msg, 1, 125), '...')
|
||||
ELSE @error_msg
|
||||
END
|
||||
);
|
||||
SIGNAL SQLSTATE '45000'
|
||||
SET MYSQL_ERRNO = @error_state, MESSAGE_TEXT = @error_msg;
|
||||
END;
|
||||
|
||||
-- UPSERT STATEMENT設定
|
||||
SET @upsert_statement =
|
||||
'INSERT INTO internal07.jskult_batch_status_manage(
|
||||
process_name, process_date, process_type, process_status,
|
||||
total_run_count,max_run_count_flg,
|
||||
ins_user, ins_date, upd_user, upd_date
|
||||
)
|
||||
VALUES(
|
||||
?, src07.get_syor_date(),?, ?,
|
||||
IFNULL(?, 0),
|
||||
IFNULL(?, 0),
|
||||
CURRENT_USER(), CURRENT_TIMESTAMP(), NULL, NULL
|
||||
)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
process_type = CASE WHEN ? is NULL THEN process_type ELSE ? END,
|
||||
process_status = CASE WHEN ? is NULL THEN process_status ELSE ? END,
|
||||
total_run_count = CASE WHEN ? is NULL THEN total_run_count ELSE ? END,
|
||||
max_run_count_flg = CASE WHEN ? is NULL THEN max_run_count_flg ELSE ? END,
|
||||
upd_user = CURRENT_USER(),
|
||||
upd_date = CURRENT_TIMESTAMP();';
|
||||
|
||||
SET @process_name = process_name;
|
||||
SET @process_type = process_type;
|
||||
SET @process_status = process_status;
|
||||
SET @total_run_count = total_run_count;
|
||||
SET @max_run_count_flg = max_run_count_flg;
|
||||
|
||||
-- UPSERT実行
|
||||
PREPARE stmt FROM @upsert_statement;
|
||||
EXECUTE stmt USING
|
||||
-- INSERT用処理名
|
||||
@process_name,
|
||||
-- INSERT用管理区分
|
||||
@process_type,
|
||||
-- INSERT用処理ステータス
|
||||
@process_status,
|
||||
-- INSERT用起動回数
|
||||
@total_run_count,
|
||||
-- INSERT用最大起動回数フラグ
|
||||
@max_run_count_flg,
|
||||
|
||||
-- UPDATE用管理区分
|
||||
@process_type, @process_type,
|
||||
-- UPDATE用処理ステータス
|
||||
@process_status, @process_status,
|
||||
-- UPDATE用起動回数
|
||||
@total_run_count, @total_run_count,
|
||||
-- UPDATE用最大起動回数フラグ
|
||||
@max_run_count_flg, @max_run_count_flg;
|
||||
DEALLOCATE PREPARE stmt;
|
||||
|
||||
|
||||
END;
|
||||
Loading…
x
Reference in New Issue
Block a user