diff --git a/.vscode/launch.json b/.vscode/launch.json index 753ba49f..cde61101 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -9,7 +9,7 @@ "type": "python", "request": "launch", // windowsだと\区切りかも - "program": "ecs\\dataimport\\dataimport\\controller.py", + "program": "ecs/jskult-batch-archive-jsk-data/entrypoint.py", "console": "integratedTerminal", "justMyCode": true, "envFile": "${workspaceFolder}/.env" diff --git a/ecs/jskult-batch-archive-jsk-data/.dockerignore b/ecs/jskult-batch-archive-jsk-data/.dockerignore new file mode 100644 index 00000000..8b9da402 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/.dockerignore @@ -0,0 +1,12 @@ +tests/* +.coverage +.env +.env.example +.report/* +.vscode/* +.pytest_cache/* +*/__pychache__/* +Dockerfile +pytest.ini +README.md +*.sql diff --git a/ecs/jskult-batch-archive-jsk-data/.env.example b/ecs/jskult-batch-archive-jsk-data/.env.example new file mode 100644 index 00000000..c217d53f --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/.env.example @@ -0,0 +1,15 @@ +DB_HOST=**************** +DB_PORT=**************** +DB_USERNAME=**************** +DB_PASSWORD=**************** +DB_SCHEMA=**************** + +LOG_LEVEL=INFO + +JSKULT_CONFIG_BUCKET=********************** +JSKULT_ARCHIVE_BUCKET=**************** + +DB_CONNECTION_MAX_RETRY_ATTEMPT=4 +DB_CONNECTION_RETRY_INTERVAL_INIT=5 +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=5 +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=50 \ No newline at end of file diff --git a/ecs/jskult-batch-archive-jsk-data/.gitignore b/ecs/jskult-batch-archive-jsk-data/.gitignore new file mode 100644 index 00000000..bd0b37f8 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/.gitignore @@ -0,0 +1,10 @@ +.vscode/settings.json +.env + +# python +__pycache__ + +# python test +.pytest_cache +.coverage +.report/ \ No newline at end of file diff --git a/ecs/jskult-batch-archive-jsk-data/.vscode/launch.json b/ecs/jskult-batch-archive-jsk-data/.vscode/launch.json new file mode 100644 index 00000000..5a9a4867 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // IntelliSense を使用して利用可能な属性を学べます。 + // 既存の属性の説明をホバーして表示します。 + // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "(DEBUG)archive", + "type": "python", + "request": "launch", + "program": "entrypoint.py", + "console": "integratedTerminal", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/ecs/jskult-batch-archive-jsk-data/.vscode/recommended_settings.json b/ecs/jskult-batch-archive-jsk-data/.vscode/recommended_settings.json new file mode 100644 index 00000000..2fde8732 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/.vscode/recommended_settings.json @@ -0,0 +1,31 @@ +{ + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + // 自身の環境に合わせて変えてください + "python.defaultInterpreterPath": "", + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": [ + "--max-line-length=200", + "--ignore=F541" + ], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Path": "autopep8", + "python.formatting.autopep8Args": [ + "--max-line-length", "200", + "--ignore=F541" + ], + "python.testing.pytestArgs": [ + "tests/batch/" + ], + + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} diff --git a/ecs/jskult-batch-archive-jsk-data/Dockerfile b/ecs/jskult-batch-archive-jsk-data/Dockerfile new file mode 100644 index 00000000..fc0fde90 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.12-slim-bookworm + +ENV TZ="Asia/Tokyo" +# pythonの標準出力をバッファリングしないフラグ +ENV PYTHONUNBUFFERED=1 +# pythonのバイトコードを生成しないフラグ +ENV PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /usr/src/app +COPY Pipfile Pipfile.lock ./ +RUN \ + apt update -y && \ + pip install pipenv --no-cache-dir && \ + pipenv install --system --deploy && \ + pip uninstall -y pipenv virtualenv-clone virtualenv + +COPY src ./src +COPY entrypoint.py entrypoint.py + +CMD ["python", "entrypoint.py"] diff --git a/ecs/jskult-batch-archive-jsk-data/Pipfile b/ecs/jskult-batch-archive-jsk-data/Pipfile new file mode 100644 index 00000000..2b56d8c0 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/Pipfile @@ -0,0 +1,29 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[scripts] +"test:ultmarc" = "pytest tests/batch/ultmarc/" +"test:ultmarc:cov" = "pytest --cov=src/batch/ultmarc/ --cov-branch --cov-report=term-missing tests/batch/ultmarc/" +"test:vjsk" = "pytest tests/batch/vjsk/" +"test:vjsk:cov" = "pytest --cov=src/batch/vjsk/ --cov-branch --cov-report=term-missing tests/batch/vjsk/" + +[packages] +boto3 = "*" +PyMySQL = "*" +sqlalchemy = "*" +tenacity = "*" + +[dev-packages] +autopep8 = "*" +flake8 = "*" +pytest = "*" +pytest-cov = "*" +boto3 = "*" + +[requires] +python_version = "3.12" + +[pipenv] +allow_prereleases = true diff --git a/ecs/jskult-batch-archive-jsk-data/Pipfile.lock b/ecs/jskult-batch-archive-jsk-data/Pipfile.lock new file mode 100644 index 00000000..7179f5b9 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/Pipfile.lock @@ -0,0 +1,447 @@ +{ + "_meta": { + "hash": { + "sha256": "aa2d1d97600fea225b7d249dae0d065190d00fdadbf85b20773e0c1d9862f5c1" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.12" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "boto3": { + "hashes": [ + "sha256:70ab8364f1f6f0a7e0eaf97f62fbdacf9c1e4cc1de330faf1c146ef9ab01e7d0", + "sha256:bcf73aca469add09e165b8793be18e7578db8d2604d82505ab13dc2495bad982" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==1.38.23" + }, + "botocore": { + "hashes": [ + "sha256:29685c91050a870c3809238dc5da1ac65a48a3a20b4bca46b6057dcb6b39c72a", + "sha256:a7f818672f10d7a080c2c4558428011c3e0abc1039a047d27ac76ec846158457" + ], + "markers": "python_version >= '3.9'", + "version": "==1.38.23" + }, + "greenlet": { + "hashes": [ + "sha256:00cd814b8959b95a546e47e8d589610534cfb71f19802ea8a2ad99d95d702057", + "sha256:02a98600899ca1ca5d3a2590974c9e3ec259503b2d6ba6527605fcd74e08e207", + "sha256:02f5972ff02c9cf615357c17ab713737cccfd0eaf69b951084a9fd43f39833d3", + "sha256:055916fafad3e3388d27dd68517478933a97edc2fc54ae79d3bec827de2c64c4", + "sha256:0a16fb934fcabfdfacf21d79e6fed81809d8cd97bc1be9d9c89f0e4567143d7b", + "sha256:1592a615b598643dbfd566bac8467f06c8c8ab6e56f069e573832ed1d5d528cc", + "sha256:1919cbdc1c53ef739c94cf2985056bcc0838c1f217b57647cbf4578576c63825", + "sha256:1e4747712c4365ef6765708f948acc9c10350719ca0545e362c24ab973017370", + "sha256:1e76106b6fc55fa3d6fe1c527f95ee65e324a13b62e243f77b48317346559708", + "sha256:1f72667cc341c95184f1c68f957cb2d4fc31eef81646e8e59358a10ce6689457", + "sha256:2593283bf81ca37d27d110956b79e8723f9aa50c4bcdc29d3c0543d4743d2763", + "sha256:2dc5c43bb65ec3669452af0ab10729e8fdc17f87a1f2ad7ec65d4aaaefabf6bf", + "sha256:3091bc45e6b0c73f225374fefa1536cd91b1e987377b12ef5b19129b07d93ebe", + "sha256:354f67445f5bed6604e493a06a9a49ad65675d3d03477d38a4db4a427e9aad0e", + "sha256:3885f85b61798f4192d544aac7b25a04ece5fe2704670b4ab73c2d2c14ab740d", + "sha256:3ab7194ee290302ca15449f601036007873028712e92ca15fc76597a0aeb4c59", + "sha256:3aeca9848d08ce5eb653cf16e15bb25beeab36e53eb71cc32569f5f3afb2a3aa", + "sha256:44671c29da26539a5f142257eaba5110f71887c24d40df3ac87f1117df589e0e", + "sha256:45f9f4853fb4cc46783085261c9ec4706628f3b57de3e68bae03e8f8b3c0de51", + "sha256:4bd139e4943547ce3a56ef4b8b1b9479f9e40bb47e72cc906f0f66b9d0d5cab3", + "sha256:4fefc7aa68b34b9224490dfda2e70ccf2131368493add64b4ef2d372955c207e", + "sha256:6629311595e3fe7304039c67f00d145cd1d38cf723bb5b99cc987b23c1433d61", + "sha256:6fadd183186db360b61cb34e81117a096bff91c072929cd1b529eb20dd46e6c5", + "sha256:71566302219b17ca354eb274dfd29b8da3c268e41b646f330e324e3967546a74", + "sha256:7409796591d879425997a518138889d8d17e63ada7c99edc0d7a1c22007d4907", + "sha256:752f0e79785e11180ebd2e726c8a88109ded3e2301d40abced2543aa5d164275", + "sha256:7791dcb496ec53d60c7f1c78eaa156c21f402dda38542a00afc3e20cae0f480f", + "sha256:782743700ab75716650b5238a4759f840bb2dcf7bff56917e9ffdf9f1f23ec59", + "sha256:7c9896249fbef2c615853b890ee854f22c671560226c9221cfd27c995db97e5c", + "sha256:85f3e248507125bf4af607a26fd6cb8578776197bd4b66e35229cdf5acf1dfbf", + "sha256:89c69e9a10670eb7a66b8cef6354c24671ba241f46152dd3eed447f79c29fb5b", + "sha256:8cb8553ee954536500d88a1a2f58fcb867e45125e600e80f586ade399b3f8819", + "sha256:9ae572c996ae4b5e122331e12bbb971ea49c08cc7c232d1bd43150800a2d6c65", + "sha256:9c7b15fb9b88d9ee07e076f5a683027bc3befd5bb5d25954bb633c385d8b737e", + "sha256:9ea5231428af34226c05f927e16fc7f6fa5e39e3ad3cd24ffa48ba53a47f4240", + "sha256:a31ead8411a027c2c4759113cf2bd473690517494f3d6e4bf67064589afcd3c5", + "sha256:a8fa80665b1a29faf76800173ff5325095f3e66a78e62999929809907aca5659", + "sha256:ad053d34421a2debba45aa3cc39acf454acbcd025b3fc1a9f8a0dee237abd485", + "sha256:b24c7844c0a0afc3ccbeb0b807adeefb7eff2b5599229ecedddcfeb0ef333bec", + "sha256:b50a8c5c162469c3209e5ec92ee4f95c8231b11db6a04db09bbe338176723bb8", + "sha256:ba30e88607fb6990544d84caf3c706c4b48f629e18853fc6a646f82db9629418", + "sha256:bf3fc9145141250907730886b031681dfcc0de1c158f3cc51c092223c0f381ce", + "sha256:c23ea227847c9dbe0b3910f5c0dd95658b607137614eb821e6cbaecd60d81cc6", + "sha256:c3cc1a3ed00ecfea8932477f729a9f616ad7347a5e55d50929efa50a86cb7be7", + "sha256:c49e9f7c6f625507ed83a7485366b46cbe325717c60837f7244fc99ba16ba9d6", + "sha256:d0cb7d47199001de7658c213419358aa8937df767936506db0db7ce1a71f4a2f", + "sha256:d8009ae46259e31bc73dc183e402f548e980c96f33a6ef58cc2e7865db012e13", + "sha256:da956d534a6d1b9841f95ad0f18ace637668f680b1339ca4dcfb2c1837880a0b", + "sha256:dcb9cebbf3f62cb1e5afacae90761ccce0effb3adaa32339a0670fe7805d8068", + "sha256:decb0658ec19e5c1f519faa9a160c0fc85a41a7e6654b3ce1b44b939f8bf1325", + "sha256:df4d1509efd4977e6a844ac96d8be0b9e5aa5d5c77aa27ca9f4d3f92d3fcf330", + "sha256:eeb27bece45c0c2a5842ac4c5a1b5c2ceaefe5711078eed4e8043159fa05c834", + "sha256:efcdfb9df109e8a3b475c016f60438fcd4be68cd13a365d42b35914cdab4bb2b", + "sha256:fd9fb7c941280e2c837b603850efc93c999ae58aae2b40765ed682a6907ebbc5", + "sha256:fe46d4f8e94e637634d54477b0cfabcf93c53f29eedcbdeecaf2af32029b4421" + ], + "markers": "python_version >= '3.9'", + "version": "==3.2.2" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "pymysql": { + "hashes": [ + "sha256:4de15da4c61dc132f4fb9ab763063e693d521a80fd0e87943b9a453dd4c19d6c", + "sha256:e127611aaf2b417403c60bf4dc570124aeb4a57f5f37b8e95ae399a42f904cd0" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==1.1.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.9.0.post0" + }, + "s3transfer": { + "hashes": [ + "sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be", + "sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177" + ], + "markers": "python_version >= '3.9'", + "version": "==0.13.0" + }, + "six": { + "hashes": [ + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.17.0" + }, + "sqlalchemy": { + "hashes": [ + "sha256:023b3ee6169969beea3bb72312e44d8b7c27c75b347942d943cf49397b7edeb5", + "sha256:03968a349db483936c249f4d9cd14ff2c296adfa1290b660ba6516f973139582", + "sha256:05132c906066142103b83d9c250b60508af556982a385d96c4eaa9fb9720ac2b", + "sha256:087b6b52de812741c27231b5a3586384d60c353fbd0e2f81405a814b5591dc8b", + "sha256:0b3dbf1e7e9bc95f4bac5e2fb6d3fb2f083254c3fdd20a1789af965caf2d2348", + "sha256:118c16cd3f1b00c76d69343e38602006c9cfb9998fa4f798606d28d63f23beda", + "sha256:1936af879e3db023601196a1684d28e12f19ccf93af01bf3280a3262c4b6b4e5", + "sha256:1e3f196a0c59b0cae9a0cd332eb1a4bda4696e863f4f1cf84ab0347992c548c2", + "sha256:23a8825495d8b195c4aa9ff1c430c28f2c821e8c5e2d98089228af887e5d7e29", + "sha256:293cd444d82b18da48c9f71cd7005844dbbd06ca19be1ccf6779154439eec0b8", + "sha256:32f9dc8c44acdee06c8fc6440db9eae8b4af8b01e4b1aee7bdd7241c22edff4f", + "sha256:34ea30ab3ec98355235972dadc497bb659cc75f8292b760394824fab9cf39826", + "sha256:3d3549fc3e40667ec7199033a4e40a2f669898a00a7b18a931d3efb4c7900504", + "sha256:41836fe661cc98abfae476e14ba1906220f92c4e528771a8a3ae6a151242d2ae", + "sha256:4d44522480e0bf34c3d63167b8cfa7289c1c54264c2950cc5fc26e7850967e45", + "sha256:4eeb195cdedaf17aab6b247894ff2734dcead6c08f748e617bfe05bd5a218443", + "sha256:4f67766965996e63bb46cfbf2ce5355fc32d9dd3b8ad7e536a920ff9ee422e23", + "sha256:57df5dc6fdb5ed1a88a1ed2195fd31927e705cad62dedd86b46972752a80f576", + "sha256:598d9ebc1e796431bbd068e41e4de4dc34312b7aa3292571bb3674a0cb415dd1", + "sha256:5b14e97886199c1f52c14629c11d90c11fbb09e9334fa7bb5f6d068d9ced0ce0", + "sha256:5e22575d169529ac3e0a120cf050ec9daa94b6a9597993d1702884f6954a7d71", + "sha256:60c578c45c949f909a4026b7807044e7e564adf793537fc762b2489d522f3d11", + "sha256:6145afea51ff0af7f2564a05fa95eb46f542919e6523729663a5d285ecb3cf5e", + "sha256:6375cd674fe82d7aa9816d1cb96ec592bac1726c11e0cafbf40eeee9a4516b5f", + "sha256:6854175807af57bdb6425e47adbce7d20a4d79bbfd6f6d6519cd10bb7109a7f8", + "sha256:6ab60a5089a8f02009f127806f777fca82581c49e127f08413a66056bd9166dd", + "sha256:725875a63abf7c399d4548e686debb65cdc2549e1825437096a0af1f7e374814", + "sha256:7492967c3386df69f80cf67efd665c0f667cee67032090fe01d7d74b0e19bb08", + "sha256:81965cc20848ab06583506ef54e37cf15c83c7e619df2ad16807c03100745dea", + "sha256:81c24e0c0fde47a9723c81d5806569cddef103aebbf79dbc9fcbb617153dea30", + "sha256:81eedafa609917040d39aa9332e25881a8e7a0862495fcdf2023a9667209deda", + "sha256:81f413674d85cfd0dfcd6512e10e0f33c19c21860342a4890c3a2b59479929f9", + "sha256:8280856dd7c6a68ab3a164b4a4b1c51f7691f6d04af4d4ca23d6ecf2261b7923", + "sha256:82ca366a844eb551daff9d2e6e7a9e5e76d2612c8564f58db6c19a726869c1df", + "sha256:8b4af17bda11e907c51d10686eda89049f9ce5669b08fbe71a29747f1e876036", + "sha256:90144d3b0c8b139408da50196c5cad2a6909b51b23df1f0538411cd23ffa45d3", + "sha256:906e6b0d7d452e9a98e5ab8507c0da791856b2380fdee61b765632bb8698026f", + "sha256:90c11ceb9a1f482c752a71f203a81858625d8df5746d787a4786bca4ffdf71c6", + "sha256:911cc493ebd60de5f285bcae0491a60b4f2a9f0f5c270edd1c4dbaef7a38fc04", + "sha256:9a420a91913092d1e20c86a2f5f1fc85c1a8924dbcaf5e0586df8aceb09c9cc2", + "sha256:9f8c9fdd15a55d9465e590a402f42082705d66b05afc3ffd2d2eb3c6ba919560", + "sha256:a104c5694dfd2d864a6f91b0956eb5d5883234119cb40010115fd45a16da5e70", + "sha256:a373a400f3e9bac95ba2a06372c4fd1412a7cee53c37fc6c05f829bf672b8769", + "sha256:a62448526dd9ed3e3beedc93df9bb6b55a436ed1474db31a2af13b313a70a7e1", + "sha256:a8808d5cf866c781150d36a3c8eb3adccfa41a8105d031bf27e92c251e3969d6", + "sha256:b1f09b6821406ea1f94053f346f28f8215e293344209129a9c0fcc3578598d7b", + "sha256:b2ac41acfc8d965fb0c464eb8f44995770239668956dc4cdf502d1b1ffe0d747", + "sha256:b46fa6eae1cd1c20e6e6f44e19984d438b6b2d8616d21d783d150df714f44078", + "sha256:b50eab9994d64f4a823ff99a0ed28a6903224ddbe7fef56a6dd865eec9243440", + "sha256:bfc9064f6658a3d1cadeaa0ba07570b83ce6801a1314985bf98ec9b95d74e15f", + "sha256:c0b0e5e1b5d9f3586601048dd68f392dc0cc99a59bb5faf18aab057ce00d00b2", + "sha256:c153265408d18de4cc5ded1941dcd8315894572cddd3c58df5d5b5705b3fa28d", + "sha256:d4ae769b9c1c7757e4ccce94b0641bc203bbdf43ba7a2413ab2523d8d047d8dc", + "sha256:dc56c9788617b8964ad02e8fcfeed4001c1f8ba91a9e1f31483c0dffb207002a", + "sha256:dd5ec3aa6ae6e4d5b5de9357d2133c07be1aff6405b136dad753a16afb6717dd", + "sha256:edba70118c4be3c2b1f90754d308d0b79c6fe2c0fdc52d8ddf603916f83f4db9", + "sha256:ff8e80c4c4932c10493ff97028decfdb622de69cae87e0f127a7ebe32b4069c6" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==2.0.41" + }, + "tenacity": { + "hashes": [ + "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb", + "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==9.1.2" + }, + "typing-extensions": { + "hashes": [ + "sha256:6cd49c8b914bb3869a16ed9d1001e3d0ff1d84fae4838076fe3b361ab8b32b65", + "sha256:90196079d79b4658568e177f50c24c327b73a85e664c0af9f3937e2015b65956" + ], + "markers": "python_version >= '3.9'", + "version": "==4.14.0rc1" + }, + "urllib3": { + "hashes": [ + "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466", + "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813" + ], + "markers": "python_version >= '3.9'", + "version": "==2.4.0" + } + }, + "develop": { + "autopep8": { + "hashes": [ + "sha256:89440a4f969197b69a995e4ce0661b031f455a9f776d2c5ba3dbd83466931758", + "sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==2.3.2" + }, + "boto3": { + "hashes": [ + "sha256:70ab8364f1f6f0a7e0eaf97f62fbdacf9c1e4cc1de330faf1c146ef9ab01e7d0", + "sha256:bcf73aca469add09e165b8793be18e7578db8d2604d82505ab13dc2495bad982" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==1.38.23" + }, + "botocore": { + "hashes": [ + "sha256:29685c91050a870c3809238dc5da1ac65a48a3a20b4bca46b6057dcb6b39c72a", + "sha256:a7f818672f10d7a080c2c4558428011c3e0abc1039a047d27ac76ec846158457" + ], + "markers": "python_version >= '3.9'", + "version": "==1.38.23" + }, + "coverage": { + "extras": [ + "toml" + ], + "hashes": [ + "sha256:00f2e2f2e37f47e5f54423aeefd6c32a7dbcedc033fcd3928a4f4948e8b96af7", + "sha256:05364b9cc82f138cc86128dc4e2e1251c2981a2218bfcd556fe6b0fbaa3501be", + "sha256:0774df1e093acb6c9e4d58bce7f86656aeed6c132a16e2337692c12786b32404", + "sha256:07a989c867986c2a75f158f03fdb413128aad29aca9d4dbce5fc755672d96f11", + "sha256:0bdc8bf760459a4a4187b452213e04d039990211f98644c7292adf1e471162b5", + "sha256:0e49824808d4375ede9dd84e9961a59c47f9113039f1a525e6be170aa4f5c34d", + "sha256:145b07bea229821d51811bf15eeab346c236d523838eda395ea969d120d13347", + "sha256:159b81df53a5fcbc7d45dae3adad554fdbde9829a994e15227b3f9d816d00b36", + "sha256:1676628065a498943bd3f64f099bb573e08cf1bc6088bbe33cf4424e0876f4b3", + "sha256:1aec326ed237e5880bfe69ad41616d333712c7937bcefc1343145e972938f9b3", + "sha256:1e1448bb72b387755e1ff3ef1268a06617afd94188164960dba8d0245a46004b", + "sha256:1efa4166ba75ccefd647f2d78b64f53f14fb82622bc94c5a5cb0a622f50f1c9e", + "sha256:26a4636ddb666971345541b59899e969f3b301143dd86b0ddbb570bd591f1e85", + "sha256:2bd0a0a5054be160777a7920b731a0570284db5142abaaf81bcbb282b8d99279", + "sha256:2c08b05ee8d7861e45dc5a2cc4195c8c66dca5ac613144eb6ebeaff2d502e73d", + "sha256:2db10dedeb619a771ef0e2949ccba7b75e33905de959c2643a4607bef2f3fb3a", + "sha256:2f9bc608fbafaee40eb60a9a53dbfb90f53cc66d3d32c2849dc27cf5638a21e3", + "sha256:34759ee2c65362163699cc917bdb2a54114dd06d19bab860725f94ef45a3d9b7", + "sha256:3da9b771c98977a13fbc3830f6caa85cae6c9c83911d24cb2d218e9394259c57", + "sha256:3f5673888d3676d0a745c3d0e16da338c5eea300cb1f4ada9c872981265e76d8", + "sha256:4000a31c34932e7e4fa0381a3d6deb43dc0c8f458e3e7ea6502e6238e10be625", + "sha256:43ff5033d657cd51f83015c3b7a443287250dc14e69910577c3e03bd2e06f27b", + "sha256:46d532db4e5ff3979ce47d18e2fe8ecad283eeb7367726da0e5ef88e4fe64740", + "sha256:496948261eaac5ac9cf43f5d0a9f6eb7a6d4cb3bedb2c5d294138142f5c18f2a", + "sha256:4c26c2396674816deaeae7ded0e2b42c26537280f8fe313335858ffff35019be", + "sha256:5040536cf9b13fb033f76bcb5e1e5cb3b57c4807fef37db9e0ed129c6a094257", + "sha256:546e537d9e24efc765c9c891328f30f826e3e4808e31f5d0f87c4ba12bbd1622", + "sha256:5e818796f71702d7a13e50c70de2a1924f729228580bcba1607cccf32eea46e6", + "sha256:5feb7f2c3e6ea94d3b877def0270dff0947b8d8c04cfa34a17be0a4dc1836879", + "sha256:641988828bc18a6368fe72355df5f1703e44411adbe49bba5644b941ce6f2e3a", + "sha256:670a13249b957bb9050fab12d86acef7bf8f6a879b9d1a883799276e0d4c674a", + "sha256:6782a12bf76fa61ad9350d5a6ef5f3f020b57f5e6305cbc663803f2ebd0f270a", + "sha256:684ca9f58119b8e26bef860db33524ae0365601492e86ba0b71d513f525e7050", + "sha256:6e6c86888fd076d9e0fe848af0a2142bf606044dc5ceee0aa9eddb56e26895a0", + "sha256:726f32ee3713f7359696331a18daf0c3b3a70bb0ae71141b9d3c52be7c595e32", + "sha256:76090fab50610798cc05241bf83b603477c40ee87acd358b66196ab0ca44ffa1", + "sha256:8165584ddedb49204c4e18da083913bdf6a982bfb558632a79bdaadcdafd0d48", + "sha256:820157de3a589e992689ffcda8639fbabb313b323d26388d02e154164c57b07f", + "sha256:8369a7c8ef66bded2b6484053749ff220dbf83cba84f3398c84c51a6f748a008", + "sha256:86a323a275e9e44cdf228af9b71c5030861d4d2610886ab920d9945672a81223", + "sha256:876cbfd0b09ce09d81585d266c07a32657beb3eaec896f39484b631555be0fe2", + "sha256:8966a821e2083c74d88cca5b7dcccc0a3a888a596a04c0b9668a891de3a0cc53", + "sha256:8ab4a51cb39dc1933ba627e0875046d150e88478dbe22ce145a68393e9652975", + "sha256:8e1a26e7e50076e35f7afafde570ca2b4d7900a491174ca357d29dece5aacee7", + "sha256:94316e13f0981cbbba132c1f9f365cac1d26716aaac130866ca812006f662199", + "sha256:9a990f6510b3292686713bfef26d0049cd63b9c7bb17e0864f133cbfd2e6167f", + "sha256:9fe449ee461a3b0c7105690419d0b0aba1232f4ff6d120a9e241e58a556733f7", + "sha256:a886d531373a1f6ff9fad2a2ba4a045b68467b779ae729ee0b3b10ac20033b27", + "sha256:ab9b09a2349f58e73f8ebc06fac546dd623e23b063e5398343c5270072e3201c", + "sha256:b039ffddc99ad65d5078ef300e0c7eed08c270dc26570440e3ef18beb816c1ca", + "sha256:b069938961dfad881dc2f8d02b47645cd2f455d3809ba92a8a687bf513839787", + "sha256:b99058eef42e6a8dcd135afb068b3d53aff3921ce699e127602efff9956457a9", + "sha256:bd8ec21e1443fd7a447881332f7ce9d35b8fbd2849e761bb290b584535636b0a", + "sha256:bf8111cddd0f2b54d34e96613e7fbdd59a673f0cf5574b61134ae75b6f5a33b8", + "sha256:c9392773cffeb8d7e042a7b15b82a414011e9d2b5fdbbd3f7e6a6b17d5e21b20", + "sha256:cb86337a4fcdd0e598ff2caeb513ac604d2f3da6d53df2c8e368e07ee38e277d", + "sha256:da23ce9a3d356d0affe9c7036030b5c8f14556bd970c9b224f9c8205505e3b99", + "sha256:dc67994df9bcd7e0150a47ef41278b9e0a0ea187caba72414b71dc590b99a108", + "sha256:de77c3ba8bb686d1c411e78ee1b97e6e0b963fb98b1637658dd9ad2c875cf9d7", + "sha256:e2f6fe3654468d061942591aef56686131335b7a8325684eda85dacdf311356c", + "sha256:e6ea7dba4e92926b7b5f0990634b78ea02f208d04af520c73a7c876d5a8d36cb", + "sha256:e6fcbbd35a96192d042c691c9e0c49ef54bd7ed865846a3c9d624c30bb67ce46", + "sha256:ea561010914ec1c26ab4188aef8b1567272ef6de096312716f90e5baa79ef8ca", + "sha256:eacd2de0d30871eff893bab0b67840a96445edcb3c8fd915e6b11ac4b2f3fa6d", + "sha256:ec455eedf3ba0bbdf8f5a570012617eb305c63cb9f03428d39bf544cb2b94837", + "sha256:ef2f22795a7aca99fc3c84393a55a53dd18ab8c93fb431004e4d8f0774150f54", + "sha256:fd51355ab8a372d89fb0e6a31719e825cf8df8b6724bee942fb5b92c3f016ba3" + ], + "markers": "python_version >= '3.9'", + "version": "==7.8.2" + }, + "flake8": { + "hashes": [ + "sha256:93b92ba5bdb60754a6da14fa3b93a9361fd00a59632ada61fd7b130436c40343", + "sha256:fa558ae3f6f7dbf2b4f22663e5343b6b6023620461f8d4ff2019ef4b5ee70426" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==7.2.0" + }, + "iniconfig": { + "hashes": [ + "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", + "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760" + ], + "markers": "python_version >= '3.8'", + "version": "==2.1.0" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "mccabe": { + "hashes": [ + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "packaging": { + "hashes": [ + "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", + "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f" + ], + "markers": "python_version >= '3.8'", + "version": "==25.0" + }, + "pluggy": { + "hashes": [ + "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", + "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746" + ], + "markers": "python_version >= '3.9'", + "version": "==1.6.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:35863c5974a271c7a726ed228a14a4f6daf49df369d8c50cd9a6f58a5e143ba9", + "sha256:c8415bf09abe81d9c7f872502a6eee881fbe85d8763dd5b9924bb0a01d67efae" + ], + "markers": "python_version >= '3.9'", + "version": "==2.13.0" + }, + "pyflakes": { + "hashes": [ + "sha256:5039c8339cbb1944045f4ee5466908906180f13cc99cc9949348d10f82a5c32a", + "sha256:6dfd61d87b97fba5dcfaaf781171ac16be16453be6d816147989e7f6e6a9576b" + ], + "markers": "python_version >= '3.9'", + "version": "==3.3.2" + }, + "pytest": { + "hashes": [ + "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", + "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845" + ], + "index": "pypi", + "markers": "python_version >= '3.8'", + "version": "==8.3.5" + }, + "pytest-cov": { + "hashes": [ + "sha256:46935f7aaefba760e716c2ebfbe1c216240b9592966e7da99ea8292d4d3e2a0a", + "sha256:bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde" + ], + "index": "pypi", + "markers": "python_version >= '3.9'", + "version": "==6.1.1" + }, + "python-dateutil": { + "hashes": [ + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.9.0.post0" + }, + "s3transfer": { + "hashes": [ + "sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be", + "sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177" + ], + "markers": "python_version >= '3.9'", + "version": "==0.13.0" + }, + "six": { + "hashes": [ + "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", + "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.17.0" + }, + "urllib3": { + "hashes": [ + "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466", + "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813" + ], + "markers": "python_version >= '3.9'", + "version": "==2.4.0" + } + } +} diff --git a/ecs/jskult-batch-archive-jsk-data/README.md b/ecs/jskult-batch-archive-jsk-data/README.md new file mode 100644 index 00000000..b9a9340c --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/README.md @@ -0,0 +1,85 @@ +# 実消化&アルトマーク 日次バッチ + +## 概要 + +実消化過去データアーカイブ処理。 + +## 環境情報 + +- Python 3.12 +- MySQL 8.23 +- VSCode + +## 環境構築 + +- Python の構築 + + - Merck_NewDWH 開発 2021 の Wiki、[Python 環境構築](https://nds-tyo.backlog.com/alias/wiki/1874930)を参照 + - 「Pipenv の導入」までを行っておくこと + - 構築完了後、プロジェクト配下で以下のコマンドを実行し、Python の仮想環境を作成する + - `pipenv install --dev --python ` + - この手順で出力される仮想環境のパスは、後述する VSCode の設定手順で使用するため、控えておく + +- MySQL の環境構築 + - Windows の場合、以下のリンクからダウンロードする + - + - Docker を利用する場合、「newsdwh-tools」リポジトリの MySQL 設定を使用すると便利 + - 「crm-table-to-ddl」フォルダ内で以下のコマンドを実行すると + - `docker-compose up -d` + - Docker の構築手順は、[Docker のセットアップ手順](https://nds-tyo.backlog.com/alias/wiki/1754332)を参照のこと + - データを投入する + - 立ち上げたデータベースに「src05」スキーマを作成する + - [ローカル開発用データ](https://ndstokyo.sharepoint.com/:f:/r/sites/merck-new-dwh-team/Shared%20Documents/03.NewDWH%E6%A7%8B%E7%AF%89%E3%83%95%E3%82%A7%E3%83%BC%E3%82%BA3/02.%E9%96%8B%E7%99%BA/90.%E9%96%8B%E7%99%BA%E5%85%B1%E6%9C%89/%E3%83%AD%E3%83%BC%E3%82%AB%E3%83%AB%E9%96%8B%E7%99%BA%E7%94%A8%E3%83%87%E3%83%BC%E3%82%BF?csf=1&web=1&e=VVcRUs)をダウンロードし、mysql コマンドを使用して復元する + - `mysql -h <ホスト名> -P <ポート> -u <ユーザー名> -p src05 < src05_dump.sql` +- 環境変数の設定 + - 「.env.example」ファイルをコピーし、「.env」ファイルを作成する + - 環境変数を設定する。設定内容は PRJ メンバーより共有を受けてください +- VSCode の設定 + - 「.vscode/recommended_settings.json」ファイルをコピーし、「settings.json」ファイルを作成する + - 「python.defaultInterpreterPath」を、Python の構築手順で作成した仮想環境のパスに変更する + +## 実行 + +- VSCode 上で「F5」キーを押下すると、バッチ処理が起動する。 +- 「entrypoint.py」が、バッチ処理のエントリーポイント。 +- 実際の処理は、「src/jobctrl_daily.py」で行っている。 + + +### テスト準備 + +- VSCodeで以下の拡張機能をインストールする + - Python + - Python Test Explorer for Visual Studio Code + - Test Explorer UI +- VSCode 上でショートカット「ctrl」+「shift」+「P」でコマンドパレットを開く +- コマンドパレットの検索窓に「Python」と入力し、「Python: テストを構成する」を押下する +- 現在のワークスペースを選び、「pytest」を選択する +- 「tests」フォルダを選択する +- バックグランドで、pytest モジュールのインストールが始まれば成功 + + +## フォルダ構成 + +```text +. +├── Pipfile -- Pythonモジュールの依存関係を管理するファイル +├── Dockerfile -- Dockerイメージを作成するためのファイル +├── Pipfile -- Pythonモジュールの依存関係を管理するファイル +├── Pipfile.lock -- Pythonモジュールの依存関係バージョン固定用ファイル +├── README.md -- 当ファイル +├── entrypoint.py -- バッチ処理のエントリーポイントになるpythonファイル +└── src -- ソースコードの保管場所 + ├── aws -- AWS関連処理 + │ └── s3.py -- S3クライアントとバケット処理 + ├── batch -- バッチ処理関連ソース置き場 + │ ├── archive_jsk_data.py -- 実消化過去データアーカイブ処理 + │ └── jskult_archive_manager.py -- アーカイブ管理テーブル操作処理 + ├── db + │ └── database.py -- データベース操作共通処理 + ├── error + │ └── exceptions.py -- カスタム例外 + ├── logging + │ └── get_logger.py -- ログ出力の共通処理 + └── system_var + ├── constants.py -- 定数 + └── environment.py -- 環境変数 diff --git a/ecs/jskult-batch-archive-jsk-data/entrypoint.py b/ecs/jskult-batch-archive-jsk-data/entrypoint.py new file mode 100644 index 00000000..ad075788 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/entrypoint.py @@ -0,0 +1,10 @@ +"""実消化過去データアーカイブ処理ののエントリーポイント""" +from src.batch import archive_jsk_data + +if __name__ == '__main__': + # try: + exit(archive_jsk_data.exec()) + # except Exception: + # エラーが起きても、正常系のコードで返す。 + # エラーが起きた事実はbatch_process内でログを出す。 + exit(0) diff --git a/ecs/jskult-batch-archive-jsk-data/pytest.ini b/ecs/jskult-batch-archive-jsk-data/pytest.ini new file mode 100644 index 00000000..5dbe2661 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +log_format = %(levelname)s %(asctime)s %(message)s +log_date_format = %Y-%m-%d %H:%M:%S diff --git a/ecs/jskult-batch-archive-jsk-data/src/__init__.py b/ecs/jskult-batch-archive-jsk-data/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-archive-jsk-data/src/aws/__init__.py b/ecs/jskult-batch-archive-jsk-data/src/aws/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-archive-jsk-data/src/aws/s3.py b/ecs/jskult-batch-archive-jsk-data/src/aws/s3.py new file mode 100644 index 00000000..a6e0074a --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/src/aws/s3.py @@ -0,0 +1,62 @@ +import gzip +import os +import os.path as path +import shutil +import tempfile +import boto3 +from src.system_var import environment + + +class S3Client: + __s3_client = boto3.client('s3') + _bucket_name: str + + def list_objects(self, bucket_name: str, folder_name: str): + response = self.__s3_client.list_objects_v2(Bucket=bucket_name, Prefix=folder_name) + if response['KeyCount'] == 0: + return [] + contents = response['Contents'] + # 末尾がスラッシュで終わるものはフォルダとみなしてスキップする + objects = [{'filename': content['Key'], 'size': content['Size']} + for content in contents if not content['Key'].endswith('/')] + return objects + + def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: + copy_source = {'Bucket': src_bucket, 'Key': src_key} + self.__s3_client.copy(copy_source, dest_bucket, dest_key) + return + + def download_file(self, bucket_name: str, file_key: str, file): + self.__s3_client.download_fileobj( + Bucket=bucket_name, + Key=file_key, + Fileobj=file + ) + return + + def upload_file(self, local_file_path: str, bucket_name: str, file_key: str): + self.__s3_client.upload_file( + local_file_path, + Bucket=bucket_name, + Key=file_key + ) + + def delete_file(self, bucket_name: str, file_key: str): + self.__s3_client.delete_object( + Bucket=bucket_name, + Key=file_key + ) + +class S3Bucket(): + _s3_client = S3Client() + _bucket_name: str = None + +class JskultArchiveBucket(S3Bucket): + _bucket_name = environment.JSKULT_ARCHIVE_BUCKET + + def upload_archive_zip_file(self, archive_zip: str, archive_zip_path: str, send_folder: str): + # S3バケットにファイルを移動 + archive_zip_name = f'{send_folder}/{archive_zip}' + s3_client = S3Client() + s3_client.upload_file(archive_zip_path, self._bucket_name, archive_zip_name) + return f"{self._bucket_name}/{archive_zip_name}" \ No newline at end of file diff --git a/ecs/jskult-batch-archive-jsk-data/src/batch/archive_jsk_data.py b/ecs/jskult-batch-archive-jsk-data/src/batch/archive_jsk_data.py new file mode 100644 index 00000000..d9693c98 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/src/batch/archive_jsk_data.py @@ -0,0 +1,62 @@ +from src.logging.get_logger import get_logger +from src.batch.jskult_archive_manager import JskultArchiveManager +from src.aws.s3 import JskultArchiveBucket +import os.path as path +from datetime import timedelta +import tempfile +import csv +import zipfile + + +logger = get_logger("実消化_過去データアーカイブ処理") + +def exec(): + try: + logger.info("処理開始:実消化_過去データアーカイブ処理") + jskult_archive_manager = JskultArchiveManager() + # アーカイブ管理テーブルから対象テーブル、条件項目、条件年月、実行間隔（月）、前回条件年月、保存先を取得 + jskult_archive_manage_data_list = jskult_archive_manager.get_archive_manage() + + # 取得したレコード分繰り返す + for jskult_archive_manage_data in jskult_archive_manage_data_list: + # 対象テーブルで条件項目が条件年月以前のデータを取得 + archive_data = jskult_archive_manager.get_archive_data(jskult_archive_manage_data["target_table"], jskult_archive_manage_data["filter_column"], jskult_archive_manage_data["filter_date"]) + # 取得データが0件の場合、スキップする + if not archive_data: + logger.info(f"アーカイブ対象データがありませんでした。対象テーブル：{jskult_archive_manage_data['target_table']} 条件年月：{jskult_archive_manage_data['filter_date']}") + continue + + # 一時フォルダ作成 + with tempfile.TemporaryDirectory() as temporary_dir: + # 取得したデータをCSVに出力 + day_after_prev_filter_date = jskult_archive_manage_data["prev_filter_date"] + timedelta(days=1) + file_name = f'{jskult_archive_manage_data["target_table"]}_{day_after_prev_filter_date.strftime('%Y%m%d')}_{jskult_archive_manage_data["filter_date"].strftime('%Y%m%d')}' + csv_file_path = path.join(temporary_dir, f"{file_name}.csv") + headers = archive_data[0].keys() + with open(csv_file_path, 'w', newline='') as file: + writer = csv.DictWriter(file, fieldnames=headers, quoting=csv.QUOTE_ALL) + writer.writeheader() + writer.writerows(archive_data) + logger.info(f"CSVファイル作成に成功しました。{file_name}.csv") + + # 作成したCSVをzip形式に圧縮 + zip_file_path = path.join(temporary_dir, f"{file_name}.zip") + with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) as zipf: + zipf.write(csv_file_path) + logger.info(f"zip形式への圧縮に成功しました。{file_name}.zip") + + # 圧縮したCSVを保存先へアップロード + archive_bucket = JskultArchiveBucket() + upload_file_path = archive_bucket.upload_archive_zip_file(f"{file_name}.zip", zip_file_path, jskult_archive_manage_data["archive_storage"]) + logger.info(f"{upload_file_path}へのアップロードに成功しました。") + + # アーカイブしたデータをDBから削除 + jskult_archive_manager.delete_archive_data(jskult_archive_manage_data["target_table"], jskult_archive_manage_data["filter_column"], jskult_archive_manage_data["filter_date"]) + logger.info(f"アーカイブしたデータのDBから削除に成功しました。対象テーブル：{jskult_archive_manage_data['target_table']} 条件年月：{jskult_archive_manage_data['filter_date']}") + + # 次回に向けてアーカイブ管理テーブルを更新する + jskult_archive_manager.update_archive_manage(jskult_archive_manage_data["target_table"]) + logger.info(f"アーカイブ管理テーブルの更新に成功しました。対象テーブル：{jskult_archive_manage_data['target_table']}") + logger.info("処理終了:実消化_過去データアーカイブ処理") + except Exception as e: + logger.info(f"異常終了:実消化_過去データアーカイブ処理 {e}") \ No newline at end of file diff --git a/ecs/jskult-batch-archive-jsk-data/src/batch/common/__init__.py b/ecs/jskult-batch-archive-jsk-data/src/batch/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-archive-jsk-data/src/batch/jskult_archive_manager.py b/ecs/jskult-batch-archive-jsk-data/src/batch/jskult_archive_manager.py new file mode 100644 index 00000000..033be013 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/src/batch/jskult_archive_manager.py @@ -0,0 +1,103 @@ +from src.db.database import Database +from src.logging.get_logger import get_logger +logger = get_logger("アーカイブ管理テーブル操作") + +class JskultArchiveManager: + _db : Database = None + def __init__(self): + self._db = Database.get_instance() + + def get_archive_manage(self): + try: + logger.info("処理開始：アーカイブ管理テーブル取得") + sql = """ + select + target_table + , filter_column + , filter_date + , run_interval_months + , prev_filter_date + , archive_storage + from + internal07.jskult_archive_manage; + """ + self._db.connect() + jskult_archive_manage_data = self._db.execute_select(sql) + logger.info("処理終了：アーカイブ管理テーブル取得") + return jskult_archive_manage_data + except Exception as e: + logger.info("異常終了：アーカイブ管理テーブル取得") + raise + finally: + self._db.disconnect() + + def get_archive_data(self,target_table:str, filter_column:str, filter_date:str): + try: + logger.info("処理開始：アーカイブデータ取得") + sql = f""" + select + * + from + src07.{target_table} + where + str_to_date({filter_column},'%Y%m%d') <= '{filter_date}'; + """ + self._db.connect() + target_table_data = self._db.execute_select(sql) + logger.info("処理開始：アーカイブデータ終了") + return target_table_data + except Exception as e: + logger.info("異常終了：アーカイブ管理テーブル取得") + raise + finally: + self._db.disconnect() + + def delete_archive_data(self, target_table:str, filter_column:str, filter_date:str): + try: + logger.info("処理開始：アーカイブ後データ削除") + sql = f""" + delete from + src07.{target_table} + where + str_to_date({filter_column},'%Y%m%d') <= '{filter_date}'; + """ + self._db.connect() + self._db.begin() + self._db.execute(sql) + self._db.commit() + logger.info("処理終了：アーカイブ後データ削除") + return + except: + self._db.rollback() + logger.info("異常終了：アーカイブ後データ削除") + raise + finally: + self._db.disconnect() + + def update_archive_manage(self, target_table:str): + try: + logger.info("処理開始：アーカイブ管理テーブル条件年月更新") + sql = f""" + update internal07.jskult_archive_manage + set + prev_filter_date = filter_date + , filter_date = LAST_DAY( + DATE_ADD(filter_date, INTERVAL run_interval_months MONTH) + ) + , upd_user = CURRENT_USER () + , upd_date = NOW() + where + target_table = '{target_table}'; + """ + self._db.connect() + self._db.begin() + self._db.execute(sql) + self._db.commit() + logger.info("処理終了：アーカイブ管理テーブル条件年月更新") + return + except: + self._db.rollback() + logger.info("異常終了：アーカイブ管理テーブル条件年月更新") + raise + finally: + self._db.disconnect() diff --git a/ecs/jskult-batch-archive-jsk-data/src/db/__init__.py b/ecs/jskult-batch-archive-jsk-data/src/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-archive-jsk-data/src/db/database.py b/ecs/jskult-batch-archive-jsk-data/src/db/database.py new file mode 100644 index 00000000..5ddaba4e --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/src/db/database.py @@ -0,0 +1,195 @@ +from sqlalchemy import (Connection, CursorResult, Engine, QueuePool, + create_engine, text) +from sqlalchemy.engine.url import URL +from tenacity import retry, stop_after_attempt, wait_exponential + +from src.error.exceptions import DBException +from src.logging.get_logger import get_logger +from src.system_var import environment + +logger = get_logger(__name__) + + +class Database: + """データベース操作クラス""" + __connection: Connection = None + __transactional_engine: Engine = None + __autocommit_engine: Engine = None + __host: str = None + __port: str = None + __username: str = None + __password: str = None + __schema: str = None + __autocommit: bool = None + __connection_string: str = None + + def __init__(self, username: str, password: str, host: str, port: int, schema: str, autocommit: bool = False) -> None: + """このクラスの新たなインスタンスを初期化します + + Args: + username (str): DBユーザー名 + password (str): DBパスワード + host (str): DBホスト名 + port (int): DBポート + schema (str): DBスキーマ名 + autocommit(bool): 自動コミットモードで接続するかどうか(Trueの場合、トランザクションの有無に限らず即座にコミットされる). Defaults to False. + """ + self.__username = username + self.__password = password + self.__host = host + self.__port = int(port) + self.__schema = schema + self.__autocommit = autocommit + + self.__connection_string = URL.create( + drivername='mysql+pymysql', + username=self.__username, + password=self.__password, + host=self.__host, + port=self.__port, + database=self.__schema, + query={"charset": "utf8mb4", "local_infile": "1"}, + ) + + self.__transactional_engine = create_engine( + self.__connection_string, + pool_timeout=5, + poolclass=QueuePool + ) + + self.__autocommit_engine = self.__transactional_engine.execution_options(isolation_level='AUTOCOMMIT') + + @classmethod + def get_instance(cls, autocommit=False): + """インスタンスを取得します + + Args: + autocommit (bool, optional): 自動コミットモードで接続するかどうか(Trueの場合、トランザクションの有無に限らず即座にコミットされる). Defaults to False. + Returns: + Database: DB操作クラスインスタンス + """ + return cls( + username=environment.DB_USERNAME, + password=environment.DB_PASSWORD, + host=environment.DB_HOST, + port=environment.DB_PORT, + schema=environment.DB_SCHEMA, + autocommit=autocommit + ) + + @retry( + wait=wait_exponential( + multiplier=environment.DB_CONNECTION_RETRY_INTERVAL_INIT, + min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, + max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS + ), + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), + retry_error_cls=DBException + ) + def connect(self): + """ + DBに接続します。接続に失敗した場合、リトライします。\n + インスタンスのautocommitがTrueの場合、自動コミットモードで接続する。（明示的なトランザクションも無視される） + Raises: + DBException: 接続失敗 + """ + try: + self.__connection = ( + self.__autocommit_engine.connect() if self.__autocommit is True + else self.__transactional_engine.connect()) + except Exception as e: + raise DBException(e) + + def execute_select(self, select_query: str, parameters=None) -> list[dict]: + """SELECTクエリを実行します。 + + Args: + select_query (str): SELECT文 + parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None. + + Raises: + DBException: DBエラー + + Returns: + list[dict]: カラム名: 値の辞書リスト + """ + if self.__connection is None: + raise DBException('DBに接続していません') + + result = None + try: + # トランザクションが開始している場合は、トランザクションを引き継ぐ + if self.__connection.in_transaction(): + result = self.__connection.execute(text(select_query), parameters) + else: + # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。 + result = self.__execute_with_transaction(select_query, parameters) + except Exception as e: + raise DBException(f'SQL Error: {e}') + + result_rows = result.mappings().all() + return result_rows + + def execute(self, query: str, parameters=None) -> CursorResult: + """SQLクエリを実行します。 + + Args: + query (str): SQL文 + parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None. + + Raises: + DBException: DBエラー + + Returns: + CursorResult: 取得結果 + """ + if self.__connection is None: + raise DBException('DBに接続していません') + + result = None + try: + # トランザクションが開始している場合は、トランザクションを引き継ぐ + if self.__connection.in_transaction(): + result = self.__connection.execute(text(query), parameters) + else: + # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。 + result = self.__execute_with_transaction(query, parameters) + except Exception as e: + raise DBException(f'SQL Error: {e}') + + return result + + def begin(self): + """トランザクションを開始します。""" + if not self.__connection.in_transaction(): + self.__connection.begin() + + def commit(self): + """トランザクションをコミットします""" + if self.__connection.in_transaction(): + self.__connection.commit() + + def rollback(self): + """トランザクションをロールバックします""" + if self.__connection.in_transaction(): + self.__connection.rollback() + + def disconnect(self): + """DB接続を切断します。""" + if self.__connection is not None: + self.__connection.close() + self.__connection = None + + def to_jst(self): + self.execute('SET time_zone = "+9:00"') + + def __execute_with_transaction(self, query: str, parameters: dict): + # トランザクションを開始してクエリを実行する + with self.__connection.begin(): + try: + result = self.__connection.execute(text(query), parameters=parameters) + except Exception as e: + self.__connection.rollback() + raise e + # ここでコミットされる + return result diff --git a/ecs/jskult-batch-archive-jsk-data/src/error/__init__.py b/ecs/jskult-batch-archive-jsk-data/src/error/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-archive-jsk-data/src/error/exceptions.py b/ecs/jskult-batch-archive-jsk-data/src/error/exceptions.py new file mode 100644 index 00000000..2db060ff --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/src/error/exceptions.py @@ -0,0 +1,6 @@ +class MeDaCaException(Exception): + pass + + +class DBException(MeDaCaException): + pass diff --git a/ecs/jskult-batch-archive-jsk-data/src/logging/get_logger.py b/ecs/jskult-batch-archive-jsk-data/src/logging/get_logger.py new file mode 100644 index 00000000..f36f1199 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/src/logging/get_logger.py @@ -0,0 +1,37 @@ +import logging + +from src.system_var.environment import LOG_LEVEL + +# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する +for name in ["boto3", "botocore", "s3transfer", "urllib3"]: + logging.getLogger(name).setLevel(logging.WARNING) + + +def get_logger(log_name: str) -> logging.Logger: + """一意のログ出力モジュールを取得します。 + + Args: + log_name (str): ロガー名 + + Returns: + _type_: _description_ + """ + logger = logging.getLogger(log_name) + level = logging.getLevelName(LOG_LEVEL) + if not isinstance(level, int): + level = logging.INFO + logger.setLevel(level) + + if not logger.hasHandlers(): + handler = logging.StreamHandler() + logger.addHandler(handler) + + formatter = logging.Formatter( + '%(name)s\t[%(levelname)s]\t%(asctime)s\t%(message)s', + '%Y-%m-%d %H:%M:%S' + ) + + for handler in logger.handlers: + handler.setFormatter(formatter) + + return logger diff --git a/ecs/jskult-batch-archive-jsk-data/src/system_var/__init__.py b/ecs/jskult-batch-archive-jsk-data/src/system_var/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-archive-jsk-data/src/system_var/constants.py b/ecs/jskult-batch-archive-jsk-data/src/system_var/constants.py new file mode 100644 index 00000000..8a555af3 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/src/system_var/constants.py @@ -0,0 +1,2 @@ +# バッチ正常終了コード +BATCH_EXIT_CODE_SUCCESS = 0 diff --git a/ecs/jskult-batch-archive-jsk-data/src/system_var/environment.py b/ecs/jskult-batch-archive-jsk-data/src/system_var/environment.py new file mode 100644 index 00000000..249aa4f0 --- /dev/null +++ b/ecs/jskult-batch-archive-jsk-data/src/system_var/environment.py @@ -0,0 +1,18 @@ +import os + +# Database +DB_HOST = os.environ['DB_HOST'] +DB_PORT = int(os.environ['DB_PORT']) +DB_USERNAME = os.environ['DB_USERNAME'] +DB_PASSWORD = os.environ['DB_PASSWORD'] +DB_SCHEMA = os.environ['DB_SCHEMA'] + +# AWS +JSKULT_ARCHIVE_BUCKET = os.environ['JSKULT_ARCHIVE_BUCKET'] + +# 初期値がある環境変数 +LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') +DB_CONNECTION_MAX_RETRY_ATTEMPT = int(os.environ.get('DB_CONNECTION_MAX_RETRY_ATTEMPT', 4)) +DB_CONNECTION_RETRY_INTERVAL_INIT = int(os.environ.get('DB_CONNECTION_RETRY_INTERVAL', 5)) +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MIN_SECONDS', 5)) +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MAX_SECONDS', 50))