Merge pull request #482 feature-NEWDWH2021-1902 into develop

.vscode/launch.json

@@ -9,7 +9,7 @@
             "type": "python",
             "request": "launch",
             // windowsだと\区切りかも
-            "program": "ecs\\dataimport\\dataimport\\controller.py",
+            "program": "ecs/jskult-batch-archive-jsk-data/entrypoint.py",
             "console": "integratedTerminal",
             "justMyCode": true,
             "envFile": "${workspaceFolder}/.env"

ecs/jskult-batch-archive-jsk-data/.dockerignore

@@ -0,0 +1,12 @@
+tests/*
+.coverage
+.env
+.env.example
+.report/*
+.vscode/*
+.pytest_cache/*
+*/__pychache__/*
+Dockerfile
+pytest.ini
+README.md
+*.sql

ecs/jskult-batch-archive-jsk-data/.env.example

@@ -0,0 +1,15 @@
+DB_HOST=****************
+DB_PORT=****************
+DB_USERNAME=****************
+DB_PASSWORD=****************
+DB_SCHEMA=****************
+
+LOG_LEVEL=INFO
+
+JSKULT_CONFIG_BUCKET=**********************
+JSKULT_ARCHIVE_BUCKET=****************
+
+DB_CONNECTION_MAX_RETRY_ATTEMPT=4
+DB_CONNECTION_RETRY_INTERVAL_INIT=5
+DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=5
+DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=50

ecs/jskult-batch-archive-jsk-data/.gitignore

@@ -0,0 +1,10 @@
+.vscode/settings.json
+.env
+
+# python
+__pycache__
+
+# python test
+.pytest_cache
+.coverage
+.report/

ecs/jskult-batch-archive-jsk-data/.vscode/launch.json

@@ -0,0 +1,16 @@
+{
+    // IntelliSense を使用して利用可能な属性を学べます。
+    // 既存の属性の説明をホバーして表示します。
+    // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "(DEBUG)archive",
+            "type": "python",
+            "request": "launch",
+            "program": "entrypoint.py",
+            "console": "integratedTerminal",
+            "justMyCode": true
+        }
+    ]
+}

ecs/jskult-batch-archive-jsk-data/.vscode/recommended_settings.json

@@ -0,0 +1,31 @@
+{
+  "[python]": {
+    "editor.defaultFormatter": null,
+    "editor.formatOnSave": true,
+    "editor.codeActionsOnSave": {
+      "source.organizeImports": true
+    }
+  },
+  // 自身の環境に合わせて変えてください
+  "python.defaultInterpreterPath": "<pythonインタプリターのパス>",
+  "python.linting.lintOnSave": true,
+  "python.linting.enabled": true,
+  "python.linting.pylintEnabled": false,
+  "python.linting.flake8Enabled": true,
+  "python.linting.flake8Args": [
+    "--max-line-length=200",
+    "--ignore=F541"
+  ],
+  "python.formatting.provider": "autopep8",
+  "python.formatting.autopep8Path": "autopep8",
+  "python.formatting.autopep8Args": [
+    "--max-line-length", "200",
+    "--ignore=F541"
+  ],
+  "python.testing.pytestArgs": [
+    "tests/batch/"
+  ],
+
+  "python.testing.unittestEnabled": false,
+  "python.testing.pytestEnabled": true
+}

ecs/jskult-batch-archive-jsk-data/Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.12-slim-bookworm
+
+ENV TZ="Asia/Tokyo"
+# pythonの標準出力をバッファリングしないフラグ
+ENV PYTHONUNBUFFERED=1
+# pythonのバイトコードを生成しないフラグ
+ENV PYTHONDONTWRITEBYTECODE=1
+
+WORKDIR /usr/src/app
+COPY Pipfile Pipfile.lock ./
+RUN \
+    apt update -y && \
+    pip install pipenv --no-cache-dir && \
+    pipenv install --system --deploy && \
+    pip uninstall -y pipenv virtualenv-clone virtualenv
+
+COPY src ./src
+COPY entrypoint.py entrypoint.py 
+
+CMD ["python", "entrypoint.py"]

ecs/jskult-batch-archive-jsk-data/Pipfile

@@ -0,0 +1,23 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+boto3 = "*"
+PyMySQL = "*"
+sqlalchemy = "*"
+tenacity = "*"
+
+[dev-packages]
+autopep8 = "*"
+flake8 = "*"
+pytest = "*"
+pytest-cov = "*"
+boto3 = "*"
+
+[requires]
+python_version = "3.12"
+
+[pipenv]
+allow_prereleases = true

ecs/jskult-batch-archive-jsk-data/Pipfile.lock

@@ -0,0 +1,447 @@
+{
+    "_meta": {
+        "hash": {
+            "sha256": "aa2d1d97600fea225b7d249dae0d065190d00fdadbf85b20773e0c1d9862f5c1"
+        },
+        "pipfile-spec": 6,
+        "requires": {
+            "python_version": "3.12"
+        },
+        "sources": [
+            {
+                "name": "pypi",
+                "url": "https://pypi.org/simple",
+                "verify_ssl": true
+            }
+        ]
+    },
+    "default": {
+        "boto3": {
+            "hashes": [
+                "sha256:70ab8364f1f6f0a7e0eaf97f62fbdacf9c1e4cc1de330faf1c146ef9ab01e7d0",
+                "sha256:bcf73aca469add09e165b8793be18e7578db8d2604d82505ab13dc2495bad982"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.9'",
+            "version": "==1.38.23"
+        },
+        "botocore": {
+            "hashes": [
+                "sha256:29685c91050a870c3809238dc5da1ac65a48a3a20b4bca46b6057dcb6b39c72a",
+                "sha256:a7f818672f10d7a080c2c4558428011c3e0abc1039a047d27ac76ec846158457"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==1.38.23"
+        },
+        "greenlet": {
+            "hashes": [
+                "sha256:00cd814b8959b95a546e47e8d589610534cfb71f19802ea8a2ad99d95d702057",
+                "sha256:02a98600899ca1ca5d3a2590974c9e3ec259503b2d6ba6527605fcd74e08e207",
+                "sha256:02f5972ff02c9cf615357c17ab713737cccfd0eaf69b951084a9fd43f39833d3",
+                "sha256:055916fafad3e3388d27dd68517478933a97edc2fc54ae79d3bec827de2c64c4",
+                "sha256:0a16fb934fcabfdfacf21d79e6fed81809d8cd97bc1be9d9c89f0e4567143d7b",
+                "sha256:1592a615b598643dbfd566bac8467f06c8c8ab6e56f069e573832ed1d5d528cc",
+                "sha256:1919cbdc1c53ef739c94cf2985056bcc0838c1f217b57647cbf4578576c63825",
+                "sha256:1e4747712c4365ef6765708f948acc9c10350719ca0545e362c24ab973017370",
+                "sha256:1e76106b6fc55fa3d6fe1c527f95ee65e324a13b62e243f77b48317346559708",
+                "sha256:1f72667cc341c95184f1c68f957cb2d4fc31eef81646e8e59358a10ce6689457",
+                "sha256:2593283bf81ca37d27d110956b79e8723f9aa50c4bcdc29d3c0543d4743d2763",
+                "sha256:2dc5c43bb65ec3669452af0ab10729e8fdc17f87a1f2ad7ec65d4aaaefabf6bf",
+                "sha256:3091bc45e6b0c73f225374fefa1536cd91b1e987377b12ef5b19129b07d93ebe",
+                "sha256:354f67445f5bed6604e493a06a9a49ad65675d3d03477d38a4db4a427e9aad0e",
+                "sha256:3885f85b61798f4192d544aac7b25a04ece5fe2704670b4ab73c2d2c14ab740d",
+                "sha256:3ab7194ee290302ca15449f601036007873028712e92ca15fc76597a0aeb4c59",
+                "sha256:3aeca9848d08ce5eb653cf16e15bb25beeab36e53eb71cc32569f5f3afb2a3aa",
+                "sha256:44671c29da26539a5f142257eaba5110f71887c24d40df3ac87f1117df589e0e",
+                "sha256:45f9f4853fb4cc46783085261c9ec4706628f3b57de3e68bae03e8f8b3c0de51",
+                "sha256:4bd139e4943547ce3a56ef4b8b1b9479f9e40bb47e72cc906f0f66b9d0d5cab3",
+                "sha256:4fefc7aa68b34b9224490dfda2e70ccf2131368493add64b4ef2d372955c207e",
+                "sha256:6629311595e3fe7304039c67f00d145cd1d38cf723bb5b99cc987b23c1433d61",
+                "sha256:6fadd183186db360b61cb34e81117a096bff91c072929cd1b529eb20dd46e6c5",
+                "sha256:71566302219b17ca354eb274dfd29b8da3c268e41b646f330e324e3967546a74",
+                "sha256:7409796591d879425997a518138889d8d17e63ada7c99edc0d7a1c22007d4907",
+                "sha256:752f0e79785e11180ebd2e726c8a88109ded3e2301d40abced2543aa5d164275",
+                "sha256:7791dcb496ec53d60c7f1c78eaa156c21f402dda38542a00afc3e20cae0f480f",
+                "sha256:782743700ab75716650b5238a4759f840bb2dcf7bff56917e9ffdf9f1f23ec59",
+                "sha256:7c9896249fbef2c615853b890ee854f22c671560226c9221cfd27c995db97e5c",
+                "sha256:85f3e248507125bf4af607a26fd6cb8578776197bd4b66e35229cdf5acf1dfbf",
+                "sha256:89c69e9a10670eb7a66b8cef6354c24671ba241f46152dd3eed447f79c29fb5b",
+                "sha256:8cb8553ee954536500d88a1a2f58fcb867e45125e600e80f586ade399b3f8819",
+                "sha256:9ae572c996ae4b5e122331e12bbb971ea49c08cc7c232d1bd43150800a2d6c65",
+                "sha256:9c7b15fb9b88d9ee07e076f5a683027bc3befd5bb5d25954bb633c385d8b737e",
+                "sha256:9ea5231428af34226c05f927e16fc7f6fa5e39e3ad3cd24ffa48ba53a47f4240",
+                "sha256:a31ead8411a027c2c4759113cf2bd473690517494f3d6e4bf67064589afcd3c5",
+                "sha256:a8fa80665b1a29faf76800173ff5325095f3e66a78e62999929809907aca5659",
+                "sha256:ad053d34421a2debba45aa3cc39acf454acbcd025b3fc1a9f8a0dee237abd485",
+                "sha256:b24c7844c0a0afc3ccbeb0b807adeefb7eff2b5599229ecedddcfeb0ef333bec",
+                "sha256:b50a8c5c162469c3209e5ec92ee4f95c8231b11db6a04db09bbe338176723bb8",
+                "sha256:ba30e88607fb6990544d84caf3c706c4b48f629e18853fc6a646f82db9629418",
+                "sha256:bf3fc9145141250907730886b031681dfcc0de1c158f3cc51c092223c0f381ce",
+                "sha256:c23ea227847c9dbe0b3910f5c0dd95658b607137614eb821e6cbaecd60d81cc6",
+                "sha256:c3cc1a3ed00ecfea8932477f729a9f616ad7347a5e55d50929efa50a86cb7be7",
+                "sha256:c49e9f7c6f625507ed83a7485366b46cbe325717c60837f7244fc99ba16ba9d6",
+                "sha256:d0cb7d47199001de7658c213419358aa8937df767936506db0db7ce1a71f4a2f",
+                "sha256:d8009ae46259e31bc73dc183e402f548e980c96f33a6ef58cc2e7865db012e13",
+                "sha256:da956d534a6d1b9841f95ad0f18ace637668f680b1339ca4dcfb2c1837880a0b",
+                "sha256:dcb9cebbf3f62cb1e5afacae90761ccce0effb3adaa32339a0670fe7805d8068",
+                "sha256:decb0658ec19e5c1f519faa9a160c0fc85a41a7e6654b3ce1b44b939f8bf1325",
+                "sha256:df4d1509efd4977e6a844ac96d8be0b9e5aa5d5c77aa27ca9f4d3f92d3fcf330",
+                "sha256:eeb27bece45c0c2a5842ac4c5a1b5c2ceaefe5711078eed4e8043159fa05c834",
+                "sha256:efcdfb9df109e8a3b475c016f60438fcd4be68cd13a365d42b35914cdab4bb2b",
+                "sha256:fd9fb7c941280e2c837b603850efc93c999ae58aae2b40765ed682a6907ebbc5",
+                "sha256:fe46d4f8e94e637634d54477b0cfabcf93c53f29eedcbdeecaf2af32029b4421"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==3.2.2"
+        },
+        "jmespath": {
+            "hashes": [
+                "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980",
+                "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.0.1"
+        },
+        "pymysql": {
+            "hashes": [
+                "sha256:4de15da4c61dc132f4fb9ab763063e693d521a80fd0e87943b9a453dd4c19d6c",
+                "sha256:e127611aaf2b417403c60bf4dc570124aeb4a57f5f37b8e95ae399a42f904cd0"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.7'",
+            "version": "==1.1.1"
+        },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",
+                "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==2.9.0.post0"
+        },
+        "s3transfer": {
+            "hashes": [
+                "sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be",
+                "sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==0.13.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274",
+                "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==1.17.0"
+        },
+        "sqlalchemy": {
+            "hashes": [
+                "sha256:023b3ee6169969beea3bb72312e44d8b7c27c75b347942d943cf49397b7edeb5",
+                "sha256:03968a349db483936c249f4d9cd14ff2c296adfa1290b660ba6516f973139582",
+                "sha256:05132c906066142103b83d9c250b60508af556982a385d96c4eaa9fb9720ac2b",
+                "sha256:087b6b52de812741c27231b5a3586384d60c353fbd0e2f81405a814b5591dc8b",
+                "sha256:0b3dbf1e7e9bc95f4bac5e2fb6d3fb2f083254c3fdd20a1789af965caf2d2348",
+                "sha256:118c16cd3f1b00c76d69343e38602006c9cfb9998fa4f798606d28d63f23beda",
+                "sha256:1936af879e3db023601196a1684d28e12f19ccf93af01bf3280a3262c4b6b4e5",
+                "sha256:1e3f196a0c59b0cae9a0cd332eb1a4bda4696e863f4f1cf84ab0347992c548c2",
+                "sha256:23a8825495d8b195c4aa9ff1c430c28f2c821e8c5e2d98089228af887e5d7e29",
+                "sha256:293cd444d82b18da48c9f71cd7005844dbbd06ca19be1ccf6779154439eec0b8",
+                "sha256:32f9dc8c44acdee06c8fc6440db9eae8b4af8b01e4b1aee7bdd7241c22edff4f",
+                "sha256:34ea30ab3ec98355235972dadc497bb659cc75f8292b760394824fab9cf39826",
+                "sha256:3d3549fc3e40667ec7199033a4e40a2f669898a00a7b18a931d3efb4c7900504",
+                "sha256:41836fe661cc98abfae476e14ba1906220f92c4e528771a8a3ae6a151242d2ae",
+                "sha256:4d44522480e0bf34c3d63167b8cfa7289c1c54264c2950cc5fc26e7850967e45",
+                "sha256:4eeb195cdedaf17aab6b247894ff2734dcead6c08f748e617bfe05bd5a218443",
+                "sha256:4f67766965996e63bb46cfbf2ce5355fc32d9dd3b8ad7e536a920ff9ee422e23",
+                "sha256:57df5dc6fdb5ed1a88a1ed2195fd31927e705cad62dedd86b46972752a80f576",
+                "sha256:598d9ebc1e796431bbd068e41e4de4dc34312b7aa3292571bb3674a0cb415dd1",
+                "sha256:5b14e97886199c1f52c14629c11d90c11fbb09e9334fa7bb5f6d068d9ced0ce0",
+                "sha256:5e22575d169529ac3e0a120cf050ec9daa94b6a9597993d1702884f6954a7d71",
+                "sha256:60c578c45c949f909a4026b7807044e7e564adf793537fc762b2489d522f3d11",
+                "sha256:6145afea51ff0af7f2564a05fa95eb46f542919e6523729663a5d285ecb3cf5e",
+                "sha256:6375cd674fe82d7aa9816d1cb96ec592bac1726c11e0cafbf40eeee9a4516b5f",
+                "sha256:6854175807af57bdb6425e47adbce7d20a4d79bbfd6f6d6519cd10bb7109a7f8",
+                "sha256:6ab60a5089a8f02009f127806f777fca82581c49e127f08413a66056bd9166dd",
+                "sha256:725875a63abf7c399d4548e686debb65cdc2549e1825437096a0af1f7e374814",
+                "sha256:7492967c3386df69f80cf67efd665c0f667cee67032090fe01d7d74b0e19bb08",
+                "sha256:81965cc20848ab06583506ef54e37cf15c83c7e619df2ad16807c03100745dea",
+                "sha256:81c24e0c0fde47a9723c81d5806569cddef103aebbf79dbc9fcbb617153dea30",
+                "sha256:81eedafa609917040d39aa9332e25881a8e7a0862495fcdf2023a9667209deda",
+                "sha256:81f413674d85cfd0dfcd6512e10e0f33c19c21860342a4890c3a2b59479929f9",
+                "sha256:8280856dd7c6a68ab3a164b4a4b1c51f7691f6d04af4d4ca23d6ecf2261b7923",
+                "sha256:82ca366a844eb551daff9d2e6e7a9e5e76d2612c8564f58db6c19a726869c1df",
+                "sha256:8b4af17bda11e907c51d10686eda89049f9ce5669b08fbe71a29747f1e876036",
+                "sha256:90144d3b0c8b139408da50196c5cad2a6909b51b23df1f0538411cd23ffa45d3",
+                "sha256:906e6b0d7d452e9a98e5ab8507c0da791856b2380fdee61b765632bb8698026f",
+                "sha256:90c11ceb9a1f482c752a71f203a81858625d8df5746d787a4786bca4ffdf71c6",
+                "sha256:911cc493ebd60de5f285bcae0491a60b4f2a9f0f5c270edd1c4dbaef7a38fc04",
+                "sha256:9a420a91913092d1e20c86a2f5f1fc85c1a8924dbcaf5e0586df8aceb09c9cc2",
+                "sha256:9f8c9fdd15a55d9465e590a402f42082705d66b05afc3ffd2d2eb3c6ba919560",
+                "sha256:a104c5694dfd2d864a6f91b0956eb5d5883234119cb40010115fd45a16da5e70",
+                "sha256:a373a400f3e9bac95ba2a06372c4fd1412a7cee53c37fc6c05f829bf672b8769",
+                "sha256:a62448526dd9ed3e3beedc93df9bb6b55a436ed1474db31a2af13b313a70a7e1",
+                "sha256:a8808d5cf866c781150d36a3c8eb3adccfa41a8105d031bf27e92c251e3969d6",
+                "sha256:b1f09b6821406ea1f94053f346f28f8215e293344209129a9c0fcc3578598d7b",
+                "sha256:b2ac41acfc8d965fb0c464eb8f44995770239668956dc4cdf502d1b1ffe0d747",
+                "sha256:b46fa6eae1cd1c20e6e6f44e19984d438b6b2d8616d21d783d150df714f44078",
+                "sha256:b50eab9994d64f4a823ff99a0ed28a6903224ddbe7fef56a6dd865eec9243440",
+                "sha256:bfc9064f6658a3d1cadeaa0ba07570b83ce6801a1314985bf98ec9b95d74e15f",
+                "sha256:c0b0e5e1b5d9f3586601048dd68f392dc0cc99a59bb5faf18aab057ce00d00b2",
+                "sha256:c153265408d18de4cc5ded1941dcd8315894572cddd3c58df5d5b5705b3fa28d",
+                "sha256:d4ae769b9c1c7757e4ccce94b0641bc203bbdf43ba7a2413ab2523d8d047d8dc",
+                "sha256:dc56c9788617b8964ad02e8fcfeed4001c1f8ba91a9e1f31483c0dffb207002a",
+                "sha256:dd5ec3aa6ae6e4d5b5de9357d2133c07be1aff6405b136dad753a16afb6717dd",
+                "sha256:edba70118c4be3c2b1f90754d308d0b79c6fe2c0fdc52d8ddf603916f83f4db9",
+                "sha256:ff8e80c4c4932c10493ff97028decfdb622de69cae87e0f127a7ebe32b4069c6"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.7'",
+            "version": "==2.0.41"
+        },
+        "tenacity": {
+            "hashes": [
+                "sha256:1169d376c297e7de388d18b4481760d478b0e99a777cad3a9c86e556f4b697cb",
+                "sha256:f77bf36710d8b73a50b2dd155c97b870017ad21afe6ab300326b0371b3b05138"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.9'",
+            "version": "==9.1.2"
+        },
+        "typing-extensions": {
+            "hashes": [
+                "sha256:6cd49c8b914bb3869a16ed9d1001e3d0ff1d84fae4838076fe3b361ab8b32b65",
+                "sha256:90196079d79b4658568e177f50c24c327b73a85e664c0af9f3937e2015b65956"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==4.14.0rc1"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466",
+                "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==2.4.0"
+        }
+    },
+    "develop": {
+        "autopep8": {
+            "hashes": [
+                "sha256:89440a4f969197b69a995e4ce0661b031f455a9f776d2c5ba3dbd83466931758",
+                "sha256:ce8ad498672c845a0c3de2629c15b635ec2b05ef8177a6e7c91c74f3e9b51128"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.9'",
+            "version": "==2.3.2"
+        },
+        "boto3": {
+            "hashes": [
+                "sha256:70ab8364f1f6f0a7e0eaf97f62fbdacf9c1e4cc1de330faf1c146ef9ab01e7d0",
+                "sha256:bcf73aca469add09e165b8793be18e7578db8d2604d82505ab13dc2495bad982"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.9'",
+            "version": "==1.38.23"
+        },
+        "botocore": {
+            "hashes": [
+                "sha256:29685c91050a870c3809238dc5da1ac65a48a3a20b4bca46b6057dcb6b39c72a",
+                "sha256:a7f818672f10d7a080c2c4558428011c3e0abc1039a047d27ac76ec846158457"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==1.38.23"
+        },
+        "coverage": {
+            "extras": [
+                "toml"
+            ],
+            "hashes": [
+                "sha256:00f2e2f2e37f47e5f54423aeefd6c32a7dbcedc033fcd3928a4f4948e8b96af7",
+                "sha256:05364b9cc82f138cc86128dc4e2e1251c2981a2218bfcd556fe6b0fbaa3501be",
+                "sha256:0774df1e093acb6c9e4d58bce7f86656aeed6c132a16e2337692c12786b32404",
+                "sha256:07a989c867986c2a75f158f03fdb413128aad29aca9d4dbce5fc755672d96f11",
+                "sha256:0bdc8bf760459a4a4187b452213e04d039990211f98644c7292adf1e471162b5",
+                "sha256:0e49824808d4375ede9dd84e9961a59c47f9113039f1a525e6be170aa4f5c34d",
+                "sha256:145b07bea229821d51811bf15eeab346c236d523838eda395ea969d120d13347",
+                "sha256:159b81df53a5fcbc7d45dae3adad554fdbde9829a994e15227b3f9d816d00b36",
+                "sha256:1676628065a498943bd3f64f099bb573e08cf1bc6088bbe33cf4424e0876f4b3",
+                "sha256:1aec326ed237e5880bfe69ad41616d333712c7937bcefc1343145e972938f9b3",
+                "sha256:1e1448bb72b387755e1ff3ef1268a06617afd94188164960dba8d0245a46004b",
+                "sha256:1efa4166ba75ccefd647f2d78b64f53f14fb82622bc94c5a5cb0a622f50f1c9e",
+                "sha256:26a4636ddb666971345541b59899e969f3b301143dd86b0ddbb570bd591f1e85",
+                "sha256:2bd0a0a5054be160777a7920b731a0570284db5142abaaf81bcbb282b8d99279",
+                "sha256:2c08b05ee8d7861e45dc5a2cc4195c8c66dca5ac613144eb6ebeaff2d502e73d",
+                "sha256:2db10dedeb619a771ef0e2949ccba7b75e33905de959c2643a4607bef2f3fb3a",
+                "sha256:2f9bc608fbafaee40eb60a9a53dbfb90f53cc66d3d32c2849dc27cf5638a21e3",
+                "sha256:34759ee2c65362163699cc917bdb2a54114dd06d19bab860725f94ef45a3d9b7",
+                "sha256:3da9b771c98977a13fbc3830f6caa85cae6c9c83911d24cb2d218e9394259c57",
+                "sha256:3f5673888d3676d0a745c3d0e16da338c5eea300cb1f4ada9c872981265e76d8",
+                "sha256:4000a31c34932e7e4fa0381a3d6deb43dc0c8f458e3e7ea6502e6238e10be625",
+                "sha256:43ff5033d657cd51f83015c3b7a443287250dc14e69910577c3e03bd2e06f27b",
+                "sha256:46d532db4e5ff3979ce47d18e2fe8ecad283eeb7367726da0e5ef88e4fe64740",
+                "sha256:496948261eaac5ac9cf43f5d0a9f6eb7a6d4cb3bedb2c5d294138142f5c18f2a",
+                "sha256:4c26c2396674816deaeae7ded0e2b42c26537280f8fe313335858ffff35019be",
+                "sha256:5040536cf9b13fb033f76bcb5e1e5cb3b57c4807fef37db9e0ed129c6a094257",
+                "sha256:546e537d9e24efc765c9c891328f30f826e3e4808e31f5d0f87c4ba12bbd1622",
+                "sha256:5e818796f71702d7a13e50c70de2a1924f729228580bcba1607cccf32eea46e6",
+                "sha256:5feb7f2c3e6ea94d3b877def0270dff0947b8d8c04cfa34a17be0a4dc1836879",
+                "sha256:641988828bc18a6368fe72355df5f1703e44411adbe49bba5644b941ce6f2e3a",
+                "sha256:670a13249b957bb9050fab12d86acef7bf8f6a879b9d1a883799276e0d4c674a",
+                "sha256:6782a12bf76fa61ad9350d5a6ef5f3f020b57f5e6305cbc663803f2ebd0f270a",
+                "sha256:684ca9f58119b8e26bef860db33524ae0365601492e86ba0b71d513f525e7050",
+                "sha256:6e6c86888fd076d9e0fe848af0a2142bf606044dc5ceee0aa9eddb56e26895a0",
+                "sha256:726f32ee3713f7359696331a18daf0c3b3a70bb0ae71141b9d3c52be7c595e32",
+                "sha256:76090fab50610798cc05241bf83b603477c40ee87acd358b66196ab0ca44ffa1",
+                "sha256:8165584ddedb49204c4e18da083913bdf6a982bfb558632a79bdaadcdafd0d48",
+                "sha256:820157de3a589e992689ffcda8639fbabb313b323d26388d02e154164c57b07f",
+                "sha256:8369a7c8ef66bded2b6484053749ff220dbf83cba84f3398c84c51a6f748a008",
+                "sha256:86a323a275e9e44cdf228af9b71c5030861d4d2610886ab920d9945672a81223",
+                "sha256:876cbfd0b09ce09d81585d266c07a32657beb3eaec896f39484b631555be0fe2",
+                "sha256:8966a821e2083c74d88cca5b7dcccc0a3a888a596a04c0b9668a891de3a0cc53",
+                "sha256:8ab4a51cb39dc1933ba627e0875046d150e88478dbe22ce145a68393e9652975",
+                "sha256:8e1a26e7e50076e35f7afafde570ca2b4d7900a491174ca357d29dece5aacee7",
+                "sha256:94316e13f0981cbbba132c1f9f365cac1d26716aaac130866ca812006f662199",
+                "sha256:9a990f6510b3292686713bfef26d0049cd63b9c7bb17e0864f133cbfd2e6167f",
+                "sha256:9fe449ee461a3b0c7105690419d0b0aba1232f4ff6d120a9e241e58a556733f7",
+                "sha256:a886d531373a1f6ff9fad2a2ba4a045b68467b779ae729ee0b3b10ac20033b27",
+                "sha256:ab9b09a2349f58e73f8ebc06fac546dd623e23b063e5398343c5270072e3201c",
+                "sha256:b039ffddc99ad65d5078ef300e0c7eed08c270dc26570440e3ef18beb816c1ca",
+                "sha256:b069938961dfad881dc2f8d02b47645cd2f455d3809ba92a8a687bf513839787",
+                "sha256:b99058eef42e6a8dcd135afb068b3d53aff3921ce699e127602efff9956457a9",
+                "sha256:bd8ec21e1443fd7a447881332f7ce9d35b8fbd2849e761bb290b584535636b0a",
+                "sha256:bf8111cddd0f2b54d34e96613e7fbdd59a673f0cf5574b61134ae75b6f5a33b8",
+                "sha256:c9392773cffeb8d7e042a7b15b82a414011e9d2b5fdbbd3f7e6a6b17d5e21b20",
+                "sha256:cb86337a4fcdd0e598ff2caeb513ac604d2f3da6d53df2c8e368e07ee38e277d",
+                "sha256:da23ce9a3d356d0affe9c7036030b5c8f14556bd970c9b224f9c8205505e3b99",
+                "sha256:dc67994df9bcd7e0150a47ef41278b9e0a0ea187caba72414b71dc590b99a108",
+                "sha256:de77c3ba8bb686d1c411e78ee1b97e6e0b963fb98b1637658dd9ad2c875cf9d7",
+                "sha256:e2f6fe3654468d061942591aef56686131335b7a8325684eda85dacdf311356c",
+                "sha256:e6ea7dba4e92926b7b5f0990634b78ea02f208d04af520c73a7c876d5a8d36cb",
+                "sha256:e6fcbbd35a96192d042c691c9e0c49ef54bd7ed865846a3c9d624c30bb67ce46",
+                "sha256:ea561010914ec1c26ab4188aef8b1567272ef6de096312716f90e5baa79ef8ca",
+                "sha256:eacd2de0d30871eff893bab0b67840a96445edcb3c8fd915e6b11ac4b2f3fa6d",
+                "sha256:ec455eedf3ba0bbdf8f5a570012617eb305c63cb9f03428d39bf544cb2b94837",
+                "sha256:ef2f22795a7aca99fc3c84393a55a53dd18ab8c93fb431004e4d8f0774150f54",
+                "sha256:fd51355ab8a372d89fb0e6a31719e825cf8df8b6724bee942fb5b92c3f016ba3"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==7.8.2"
+        },
+        "flake8": {
+            "hashes": [
+                "sha256:93b92ba5bdb60754a6da14fa3b93a9361fd00a59632ada61fd7b130436c40343",
+                "sha256:fa558ae3f6f7dbf2b4f22663e5343b6b6023620461f8d4ff2019ef4b5ee70426"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.9'",
+            "version": "==7.2.0"
+        },
+        "iniconfig": {
+            "hashes": [
+                "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7",
+                "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==2.1.0"
+        },
+        "jmespath": {
+            "hashes": [
+                "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980",
+                "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"
+            ],
+            "markers": "python_version >= '3.7'",
+            "version": "==1.0.1"
+        },
+        "mccabe": {
+            "hashes": [
+                "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325",
+                "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==0.7.0"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484",
+                "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"
+            ],
+            "markers": "python_version >= '3.8'",
+            "version": "==25.0"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3",
+                "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==1.6.0"
+        },
+        "pycodestyle": {
+            "hashes": [
+                "sha256:35863c5974a271c7a726ed228a14a4f6daf49df369d8c50cd9a6f58a5e143ba9",
+                "sha256:c8415bf09abe81d9c7f872502a6eee881fbe85d8763dd5b9924bb0a01d67efae"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==2.13.0"
+        },
+        "pyflakes": {
+            "hashes": [
+                "sha256:5039c8339cbb1944045f4ee5466908906180f13cc99cc9949348d10f82a5c32a",
+                "sha256:6dfd61d87b97fba5dcfaaf781171ac16be16453be6d816147989e7f6e6a9576b"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==3.3.2"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820",
+                "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.8'",
+            "version": "==8.3.5"
+        },
+        "pytest-cov": {
+            "hashes": [
+                "sha256:46935f7aaefba760e716c2ebfbe1c216240b9592966e7da99ea8292d4d3e2a0a",
+                "sha256:bddf29ed2d0ab6f4df17b4c55b0a657287db8684af9c42ea546b21b1041b3dde"
+            ],
+            "index": "pypi",
+            "markers": "python_version >= '3.9'",
+            "version": "==6.1.1"
+        },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3",
+                "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==2.9.0.post0"
+        },
+        "s3transfer": {
+            "hashes": [
+                "sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be",
+                "sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==0.13.0"
+        },
+        "six": {
+            "hashes": [
+                "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274",
+                "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==1.17.0"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466",
+                "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813"
+            ],
+            "markers": "python_version >= '3.9'",
+            "version": "==2.4.0"
+        }
+    }
+}

ecs/jskult-batch-archive-jsk-data/README.md

@@ -0,0 +1,72 @@
+# 実消化過去データアーカイブ処理
+
+## 概要
+
+実消化過去データアーカイブ処理。
+
+## 環境情報
+
+- Python 3.12
+- MySQL 8.23
+- VSCode
+
+## 環境構築
+
+- Python の構築
+
+  - Merck_NewDWH 開発 2021 の Wiki、[Python 環境構築](https://nds-tyo.backlog.com/alias/wiki/1874930)を参照
+    - 「Pipenv の導入」までを行っておくこと
+  - 構築完了後、プロジェクト配下で以下のコマンドを実行し、Python の仮想環境を作成する
+    - `pipenv install --dev --python <pyenvでインストールしたpythonバージョン>`
+    - この手順で出力される仮想環境のパスは、後述する VSCode の設定手順で使用するため、控えておく
+
+- MySQL の環境構築
+  - Windows の場合、以下のリンクからダウンロードする
+    - <https://dev.mysql.com/downloads/installer/>
+  - Docker を利用する場合、「newsdwh-tools」リポジトリの MySQL 設定を使用すると便利
+    - 「crm-table-to-ddl」フォルダ内で以下のコマンドを実行すると
+      - `docker-compose up -d`
+      - Docker の構築手順は、[Docker のセットアップ手順](https://nds-tyo.backlog.com/alias/wiki/1754332)を参照のこと
+  - データを投入する
+    - 立ち上げたデータベースに「src05」スキーマを作成する
+    - [ローカル開発用データ](https://ndstokyo.sharepoint.com/:f:/r/sites/merck-new-dwh-team/Shared%20Documents/03.NewDWH%E6%A7%8B%E7%AF%89%E3%83%95%E3%82%A7%E3%83%BC%E3%82%BA3/02.%E9%96%8B%E7%99%BA/90.%E9%96%8B%E7%99%BA%E5%85%B1%E6%9C%89/%E3%83%AD%E3%83%BC%E3%82%AB%E3%83%AB%E9%96%8B%E7%99%BA%E7%94%A8%E3%83%87%E3%83%BC%E3%82%BF?csf=1&web=1&e=VVcRUs)をダウンロードし、mysql コマンドを使用して復元する
+      - `mysql -h <ホスト名> -P <ポート> -u <ユーザー名> -p src05 < src05_dump.sql`
+- 環境変数の設定
+  - 「.env.example」ファイルをコピーし、「.env」ファイルを作成する
+  - 環境変数を設定する。設定内容は PRJ メンバーより共有を受けてください
+- VSCode の設定
+  - 「.vscode/recommended_settings.json」ファイルをコピーし、「settings.json」ファイルを作成する
+  - 「python.defaultInterpreterPath」を、Python の構築手順で作成した仮想環境のパスに変更する
+
+## 実行
+
+- VSCode 上で「F5」キーを押下すると、バッチ処理が起動する。
+- 「entrypoint.py」が、バッチ処理のエントリーポイント。
+- 実際の処理は、「src/jobctrl_daily.py」で行っている。
+
+
+## フォルダ構成
+
+```text
+.
+├── Pipfile                                                     -- Pythonモジュールの依存関係を管理するファイル
+├── Dockerfile                                                  -- Dockerイメージを作成するためのファイル
+├── Pipfile                                                     -- Pythonモジュールの依存関係を管理するファイル
+├── Pipfile.lock                                                -- Pythonモジュールの依存関係バージョン固定用ファイル
+├── README.md                                                   -- 当ファイル
+├── entrypoint.py                                               -- バッチ処理のエントリーポイントになるpythonファイル
+└── src                                                         -- ソースコードの保管場所
+    ├── aws                                                     -- AWS関連処理
+    │   └── s3.py                                               -- S3クライアントとバケット処理
+    ├── batch                                                   -- バッチ処理関連ソース置き場
+    │   ├── archive_jsk_data.py                                 -- 実消化過去データアーカイブ処理
+    │   └── jskult_archive_manager.py                           -- アーカイブ管理テーブル操作処理
+    ├── db
+    │   └── database.py                                         -- データベース操作共通処理
+    ├── error
+    │   └── exceptions.py                                       -- カスタム例外
+    ├── logging
+    │   └── get_logger.py                                       -- ログ出力の共通処理
+    └── system_var
+        ├── constants.py                                        -- 定数
+        └── environment.py                                      -- 環境変数

ecs/jskult-batch-archive-jsk-data/entrypoint.py

@@ -0,0 +1,10 @@
+"""実消化過去データアーカイブ処理ののエントリーポイント"""
+from src.batch import archive_jsk_data
+
+if __name__ == '__main__':
+    # try:
+        exit(archive_jsk_data.exec())
+    # except Exception:
+        # エラーが起きても、正常系のコードで返す。
+        # エラーが起きた事実はbatch_process内でログを出す。
+        exit(0)

ecs/jskult-batch-archive-jsk-data/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+log_format = %(levelname)s %(asctime)s %(message)s
+log_date_format = %Y-%m-%d %H:%M:%S

ecs/jskult-batch-archive-jsk-data/src/aws/s3.py

@@ -0,0 +1,61 @@
+import boto3
+from src.system_var import environment
+
+
+class S3Client:
+    __s3_client = boto3.client('s3')
+    _bucket_name: str
+
+    def list_objects(self, bucket_name: str, folder_name: str):
+        response = self.__s3_client.list_objects_v2(
+            Bucket=bucket_name, Prefix=folder_name)
+        if response['KeyCount'] == 0:
+            return []
+        contents = response['Contents']
+        # 末尾がスラッシュで終わるものはフォルダとみなしてスキップする
+        objects = [{'filename': content['Key'], 'size': content['Size']}
+                   for content in contents if not content['Key'].endswith('/')]
+        return objects
+
+    def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None:
+        copy_source = {'Bucket': src_bucket, 'Key': src_key}
+        self.__s3_client.copy(copy_source, dest_bucket, dest_key)
+        return
+
+    def download_file(self, bucket_name: str, file_key: str, file):
+        self.__s3_client.download_fileobj(
+            Bucket=bucket_name,
+            Key=file_key,
+            Fileobj=file
+        )
+        return
+
+    def upload_file(self, local_file_path: str, bucket_name: str, file_key: str):
+        self.__s3_client.upload_file(
+            local_file_path,
+            Bucket=bucket_name,
+            Key=file_key
+        )
+
+    def delete_file(self, bucket_name: str, file_key: str):
+        self.__s3_client.delete_object(
+            Bucket=bucket_name,
+            Key=file_key
+        )
+
+
+class S3Bucket():
+    _s3_client = S3Client()
+    _bucket_name: str = None
+
+
+class JskultArchiveBucket(S3Bucket):
+    _bucket_name = environment.JSKULT_ARCHIVE_BUCKET
+
+    def upload_archive_zip_file(self, archive_zip: str, archive_zip_path: str, send_folder: str):
+        # S3バケットにファイルを移動
+        archive_zip_name = f'{send_folder}/{archive_zip}'
+        s3_client = S3Client()
+        s3_client.upload_file(
+            archive_zip_path, self._bucket_name, archive_zip_name)
+        return f"{self._bucket_name}/{archive_zip_name}"

ecs/jskult-batch-archive-jsk-data/src/batch/archive_jsk_data.py

@@ -0,0 +1,75 @@
+import csv
+import os.path as path
+import tempfile
+import zipfile
+from datetime import timedelta
+
+from src.aws.s3 import JskultArchiveBucket
+from src.batch.jskult_archive_manager import JskultArchiveManager
+from src.logging.get_logger import get_logger
+
+logger = get_logger("実消化_過去データアーカイブ処理")
+
+
+def exec():
+    """実消化_過去データアーカイブ処理"""
+    try:
+        logger.info("処理開始:実消化_過去データアーカイブ処理")
+        jskult_archive_manager = JskultArchiveManager()
+        # アーカイブ管理テーブルから対象テーブル、条件項目、条件年月、実行間隔（月）、前回条件年月、保存先を取得
+        jskult_archive_manage_data_list = jskult_archive_manager.get_archive_manage()
+
+        # 取得したレコード分繰り返す
+        for jskult_archive_manage_data in jskult_archive_manage_data_list:
+            # 対象テーブルで条件項目が条件年月以前のデータを取得
+            archive_data = jskult_archive_manager.get_archive_data(
+                jskult_archive_manage_data["target_table"], jskult_archive_manage_data["filter_column"], jskult_archive_manage_data["filter_date"])
+            # 取得データが0件の場合、スキップする
+            if not archive_data:
+                logger.info(
+                    f"アーカイブ対象データがありませんでした。対象テーブル：{jskult_archive_manage_data['target_table']} 条件年月：{jskult_archive_manage_data['filter_date']}")
+                continue
+
+            # 一時フォルダ作成
+            with tempfile.TemporaryDirectory() as temporary_dir:
+                # 取得したデータをCSVに出力
+                day_after_prev_filter_date = jskult_archive_manage_data["prev_filter_date"] + timedelta(
+                    days=1)
+                file_name = f'{jskult_archive_manage_data["target_table"]}_{day_after_prev_filter_date.strftime('%Y%m%d')}_{jskult_archive_manage_data["filter_date"].strftime('%Y%m%d')}'
+                csv_file_path = path.join(temporary_dir, f"{file_name}.csv")
+                headers = archive_data[0].keys()
+                with open(csv_file_path, 'w', newline='') as file:
+                    writer = csv.DictWriter(
+                        file, fieldnames=headers, quoting=csv.QUOTE_ALL)
+                    writer.writeheader()
+                    writer.writerows(archive_data)
+                    logger.info(f"CSVファイル作成に成功しました。{file_name}.csv")
+
+                # 作成したCSVをzip形式に圧縮
+                zip_file_path = path.join(temporary_dir, f"{file_name}.zip")
+                with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+                    zipf.write(csv_file_path)
+                logger.info(f"zip形式への圧縮に成功しました。{file_name}.zip")
+
+                # 圧縮したCSVを保存先へアップロード
+                archive_bucket = JskultArchiveBucket()
+                upload_file_path = archive_bucket.upload_archive_zip_file(
+                    f"{file_name}.zip", zip_file_path, jskult_archive_manage_data["archive_storage"])
+                logger.info(f"{upload_file_path}へのアップロードに成功しました。")
+
+                # アーカイブしたデータをDBから削除
+                jskult_archive_manager.delete_archive_data(
+                    jskult_archive_manage_data["target_table"],
+                    jskult_archive_manage_data["filter_column"],
+                    jskult_archive_manage_data["filter_date"])
+                logger.info(
+                    f"アーカイブしたデータのDBから削除に成功しました。対象テーブル：{jskult_archive_manage_data['target_table']} 条件年月：{jskult_archive_manage_data['filter_date']}")
+
+                # 次回に向けてアーカイブ管理テーブルを更新する
+                jskult_archive_manager.update_archive_manage(
+                    jskult_archive_manage_data["target_table"])
+                logger.info(
+                    f"アーカイブ管理テーブルの更新に成功しました。対象テーブル：{jskult_archive_manage_data['target_table']}")
+        logger.info("処理終了:実消化_過去データアーカイブ処理")
+    except Exception as e:
+        logger.exception(f"異常終了:実消化_過去データアーカイブ処理 {e}")

ecs/jskult-batch-archive-jsk-data/src/batch/jskult_archive_manager.py

@@ -0,0 +1,112 @@
+from src.db.database import Database
+from src.logging.get_logger import get_logger
+
+logger = get_logger("アーカイブ管理テーブル操作")
+
+
+class JskultArchiveManager:
+    _db: Database = None
+
+    def __init__(self):
+        self._db = Database.get_instance()
+
+    def get_archive_manage(self):
+        """対象テーブル、対象項目、対象年月、実行間隔（月）、前回対象年月、保存先を取得"""
+        try:
+            logger.info("処理開始：get_archive_manage")
+            sql = """
+                select
+                    target_table
+                    , filter_column
+                    , filter_date
+                    , run_interval_months
+                    , prev_filter_date
+                    , archive_storage
+                from
+                    internal07.jskult_archive_manage;
+            """
+            self._db.connect()
+            jskult_archive_manage_data = self._db.execute_select(sql)
+            logger.info("処理終了：get_archive_manage")
+            return jskult_archive_manage_data
+        except Exception as e:
+            logger.info("異常終了：get_archive_manage")
+            raise
+        finally:
+            self._db.disconnect()
+
+    def get_archive_data(self,target_table:str, filter_column:str, filter_date:str):
+        """アーカイブするデータを取得"""
+        try:
+            logger.info("処理開始：get_archive_data")
+            sql = f"""
+                select
+                    *
+                from
+                    src07.{target_table}
+                where
+                    str_to_date({filter_column},'%Y%m%d') <= :filter_date;
+            """
+            self._db.connect()
+            parameter_dict = {'filter_date' : filter_date}
+            target_table_data = self._db.execute_select(sql, parameter_dict)
+            logger.info("処理終了：get_archive_data")
+            return target_table_data
+        except Exception as e:
+            logger.info("異常終了：get_archive_data")
+            raise
+        finally:
+            self._db.disconnect()
+
+    def delete_archive_data(self, target_table:str, filter_column:str, filter_date:str):
+        """アーカイブしたデータを削除"""
+        try:
+            logger.info("処理開始：delete_archive_data")
+            sql = f"""
+                delete from
+                    src07.{target_table}
+                where
+                    str_to_date({filter_column},'%Y%m%d') <= :filter_date;
+            """
+            self._db.connect()
+            self._db.begin()
+            parameter_dict = {'filter_date' : filter_date}
+            self._db.execute(sql, parameter_dict)
+            self._db.commit()
+            logger.info("処理終了：delete_archive_data")
+            return
+        except Exception as e:
+            self._db.rollback()
+            logger.info("異常終了：delete_archive_data")
+            raise
+        finally:
+            self._db.disconnect()
+
+    def update_archive_manage(self, target_table:str):
+        """アーカイブ管理テーブルの指定した対象テーブルのレコードを更新する"""
+        try:
+            logger.info("処理開始：update_archive_manage")
+            sql = f"""
+                update internal07.jskult_archive_manage
+                set
+                    prev_filter_date = filter_date
+                    , filter_date = LAST_DAY(
+                        DATE_ADD(filter_date, INTERVAL run_interval_months MONTH)
+                    ) 
+                    , upd_user = CURRENT_USER()
+                    , upd_date = NOW() 
+                where
+                    target_table = '{target_table}';
+            """
+            self._db.connect()
+            self._db.begin()
+            self._db.execute(sql)
+            self._db.commit()
+            logger.info("処理終了：update_archive_manage")
+            return
+        except Exception as e:
+            self._db.rollback()
+            logger.info("異常終了：update_archive_manage")
+            raise
+        finally:
+            self._db.disconnect()

ecs/jskult-batch-archive-jsk-data/src/db/database.py

@@ -0,0 +1,198 @@
+from sqlalchemy import (Connection, CursorResult, Engine, QueuePool,
+                        create_engine, text)
+from sqlalchemy.engine.url import URL
+from src.error.exceptions import DBException
+from src.logging.get_logger import get_logger
+from src.system_var import environment
+from tenacity import retry, stop_after_attempt, wait_exponential
+
+logger = get_logger(__name__)
+
+
+class Database:
+    """データベース操作クラス"""
+    __connection: Connection = None
+    __transactional_engine: Engine = None
+    __autocommit_engine: Engine = None
+    __host: str = None
+    __port: str = None
+    __username: str = None
+    __password: str = None
+    __schema: str = None
+    __autocommit: bool = None
+    __connection_string: str = None
+
+    def __init__(self, username: str, password: str, host: str, port: int, schema: str, autocommit: bool = False) -> None:
+        """このクラスの新たなインスタンスを初期化します
+
+        Args:
+            username (str): DBユーザー名
+            password (str): DBパスワード
+            host (str): DBホスト名
+            port (int): DBポート
+            schema (str): DBスキーマ名
+            autocommit(bool): 自動コミットモードで接続するかどうか(Trueの場合、トランザクションの有無に限らず即座にコミットされる). Defaults to False.
+        """
+        self.__username = username
+        self.__password = password
+        self.__host = host
+        self.__port = int(port)
+        self.__schema = schema
+        self.__autocommit = autocommit
+
+        self.__connection_string = URL.create(
+            drivername='mysql+pymysql',
+            username=self.__username,
+            password=self.__password,
+            host=self.__host,
+            port=self.__port,
+            database=self.__schema,
+            query={"charset": "utf8mb4", "local_infile": "1"},
+        )
+
+        self.__transactional_engine = create_engine(
+            self.__connection_string,
+            pool_timeout=5,
+            poolclass=QueuePool
+        )
+
+        self.__autocommit_engine = self.__transactional_engine.execution_options(
+            isolation_level='AUTOCOMMIT')
+
+    @classmethod
+    def get_instance(cls, autocommit=False):
+        """インスタンスを取得します
+
+        Args:
+            autocommit (bool, optional): 自動コミットモードで接続するかどうか(Trueの場合、トランザクションの有無に限らず即座にコミットされる). Defaults to False.
+        Returns:
+            Database: DB操作クラスインスタンス
+        """
+        return cls(
+            username=environment.DB_USERNAME,
+            password=environment.DB_PASSWORD,
+            host=environment.DB_HOST,
+            port=environment.DB_PORT,
+            schema=environment.DB_SCHEMA,
+            autocommit=autocommit
+        )
+
+    @retry(
+        wait=wait_exponential(
+            multiplier=environment.DB_CONNECTION_RETRY_INTERVAL_INIT,
+            min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS,
+            max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS
+        ),
+        stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT),
+        retry_error_cls=DBException
+    )
+    def connect(self):
+        """
+        DBに接続します。接続に失敗した場合、リトライします。\n
+        インスタンスのautocommitがTrueの場合、自動コミットモードで接続する。（明示的なトランザクションも無視される）
+        Raises:
+            DBException: 接続失敗
+        """
+        try:
+            self.__connection = (
+                self.__autocommit_engine.connect() if self.__autocommit is True
+                else self.__transactional_engine.connect())
+        except Exception as e:
+            raise DBException(e)
+
+    def execute_select(self, select_query: str, parameters=None) -> list[dict]:
+        """SELECTクエリを実行します。
+
+        Args:
+            select_query (str): SELECT文
+            parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None.
+
+        Raises:
+            DBException: DBエラー
+
+        Returns:
+            list[dict]: カラム名: 値の辞書リスト
+        """
+        if self.__connection is None:
+            raise DBException('DBに接続していません')
+
+        result = None
+        try:
+            # トランザクションが開始している場合は、トランザクションを引き継ぐ
+            if self.__connection.in_transaction():
+                result = self.__connection.execute(
+                    text(select_query), parameters)
+            else:
+                # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。
+                result = self.__execute_with_transaction(
+                    select_query, parameters)
+        except Exception as e:
+            raise DBException(f'SQL Error: {e}')
+
+        result_rows = result.mappings().all()
+        return result_rows
+
+    def execute(self, query: str, parameters=None) -> CursorResult:
+        """SQLクエリを実行します。
+
+        Args:
+            query (str): SQL文
+            parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None.
+
+        Raises:
+            DBException: DBエラー
+
+        Returns:
+            CursorResult: 取得結果
+        """
+        if self.__connection is None:
+            raise DBException('DBに接続していません')
+
+        result = None
+        try:
+            # トランザクションが開始している場合は、トランザクションを引き継ぐ
+            if self.__connection.in_transaction():
+                result = self.__connection.execute(text(query), parameters)
+            else:
+                # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。
+                result = self.__execute_with_transaction(query, parameters)
+        except Exception as e:
+            raise DBException(f'SQL Error: {e}')
+
+        return result
+
+    def begin(self):
+        """トランザクションを開始します。"""
+        if not self.__connection.in_transaction():
+            self.__connection.begin()
+
+    def commit(self):
+        """トランザクションをコミットします"""
+        if self.__connection.in_transaction():
+            self.__connection.commit()
+
+    def rollback(self):
+        """トランザクションをロールバックします"""
+        if self.__connection.in_transaction():
+            self.__connection.rollback()
+
+    def disconnect(self):
+        """DB接続を切断します。"""
+        if self.__connection is not None:
+            self.__connection.close()
+            self.__connection = None
+
+    def to_jst(self):
+        self.execute('SET time_zone = "+9:00"')
+
+    def __execute_with_transaction(self, query: str, parameters: dict):
+        # トランザクションを開始してクエリを実行する
+        with self.__connection.begin():
+            try:
+                result = self.__connection.execute(
+                    text(query), parameters=parameters)
+            except Exception as e:
+                self.__connection.rollback()
+                raise e
+        # ここでコミットされる
+        return result

ecs/jskult-batch-archive-jsk-data/src/error/exceptions.py

@@ -0,0 +1,6 @@
+class MeDaCaException(Exception):
+    pass
+
+
+class DBException(MeDaCaException):
+    pass

ecs/jskult-batch-archive-jsk-data/src/logging/get_logger.py

@@ -0,0 +1,37 @@
+import logging
+
+from src.system_var.environment import LOG_LEVEL
+
+# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する
+for name in ["boto3", "botocore", "s3transfer", "urllib3"]:
+    logging.getLogger(name).setLevel(logging.WARNING)
+
+
+def get_logger(log_name: str) -> logging.Logger:
+    """一意のログ出力モジュールを取得します。
+
+    Args:
+        log_name (str): ロガー名
+
+    Returns:
+        _type_: _description_
+    """
+    logger = logging.getLogger(log_name)
+    level = logging.getLevelName(LOG_LEVEL)
+    if not isinstance(level, int):
+        level = logging.INFO
+    logger.setLevel(level)
+
+    if not logger.hasHandlers():
+        handler = logging.StreamHandler()
+        logger.addHandler(handler)
+
+        formatter = logging.Formatter(
+            '%(name)s\t[%(levelname)s]\t%(asctime)s\t%(message)s',
+            '%Y-%m-%d %H:%M:%S'
+        )
+
+        for handler in logger.handlers:
+            handler.setFormatter(formatter)
+
+    return logger

ecs/jskult-batch-archive-jsk-data/src/system_var/constants.py

@@ -0,0 +1,2 @@
+# バッチ正常終了コード
+BATCH_EXIT_CODE_SUCCESS = 0

ecs/jskult-batch-archive-jsk-data/src/system_var/environment.py

@@ -0,0 +1,22 @@
+import os
+
+# Database
+DB_HOST = os.environ['DB_HOST']
+DB_PORT = int(os.environ['DB_PORT'])
+DB_USERNAME = os.environ['DB_USERNAME']
+DB_PASSWORD = os.environ['DB_PASSWORD']
+DB_SCHEMA = os.environ['DB_SCHEMA']
+
+# AWS
+JSKULT_ARCHIVE_BUCKET = os.environ['JSKULT_ARCHIVE_BUCKET']
+
+# 初期値がある環境変数
+LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO')
+DB_CONNECTION_MAX_RETRY_ATTEMPT = int(
+    os.environ.get('DB_CONNECTION_MAX_RETRY_ATTEMPT', 4))
+DB_CONNECTION_RETRY_INTERVAL_INIT = int(
+    os.environ.get('DB_CONNECTION_RETRY_INTERVAL', 5))
+DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS = int(
+    os.environ.get('DB_CONNECTION_RETRY_MIN_SECONDS', 5))
+DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS = int(
+    os.environ.get('DB_CONNECTION_RETRY_MAX_SECONDS', 50))