From 359b8a4085b38da5b13e4e1278ac9740f06ec3e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E6=9C=A8=E8=A6=81?= Date: Wed, 26 Jul 2023 18:52:38 +0900 Subject: [PATCH 1/5] =?UTF-8?q?feat:=20=E5=8D=B8=E8=B2=A9=E5=A3=B2?= =?UTF-8?q?=E5=AE=9F=E7=B8=BE=E5=85=A8=E4=BB=B6=E6=B4=97=E6=9B=BF(?= =?UTF-8?q?=E4=BB=AE=E4=BD=9C=E6=88=90)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-laundering/.dockerignore | 12 + ecs/jskult-batch-laundering/.env.example | 20 ++ ecs/jskult-batch-laundering/.gitignore | 10 + .../.vscode/launch.json | 16 ++ .../.vscode/recommended_settings.json | 31 +++ ecs/jskult-batch-laundering/Dockerfile | 20 ++ ecs/jskult-batch-laundering/Pipfile | 20 ++ ecs/jskult-batch-laundering/Pipfile.lock | 262 ++++++++++++++++++ ecs/jskult-batch-laundering/README.md | 48 ++++ ecs/jskult-batch-laundering/entrypoint.py | 10 + ecs/jskult-batch-laundering/src/__init__.py | 0 .../src/aws/__init__.py | 0 ecs/jskult-batch-laundering/src/aws/s3.py | 95 +++++++ .../src/batch/batch_functions.py | 51 ++++ .../src/batch/common/__init__.py | 0 .../src/batch/common/batch_context.py | 29 ++ .../src/batch/common/calendar_file.py | 32 +++ .../src/batch/laundering/__init__.py | 0 .../create_inst_merge_for_laundering.py | 31 +++ .../laundering/sales_results_laundering.py | 153 ++++++++++ .../src/db/__init__.py | 0 .../src/db/database.py | 179 ++++++++++++ .../src/error/__init__.py | 0 .../src/error/exceptions.py | 10 + .../src/jobctrl_laundering.py | 23 ++ .../src/logging/get_logger.py | 37 +++ .../src/system_var/__init__.py | 0 .../src/system_var/constants.py | 8 + .../src/system_var/environment.py | 25 ++ .../src/time/elapsed_time.py | 22 ++ 30 files changed, 1144 insertions(+) create mode 100644 ecs/jskult-batch-laundering/.dockerignore create mode 100644 ecs/jskult-batch-laundering/.env.example create mode 100644 ecs/jskult-batch-laundering/.gitignore create mode 100644 ecs/jskult-batch-laundering/.vscode/launch.json create mode 100644 ecs/jskult-batch-laundering/.vscode/recommended_settings.json create mode 100644 ecs/jskult-batch-laundering/Dockerfile create mode 100644 ecs/jskult-batch-laundering/Pipfile create mode 100644 ecs/jskult-batch-laundering/Pipfile.lock create mode 100644 ecs/jskult-batch-laundering/README.md create mode 100644 ecs/jskult-batch-laundering/entrypoint.py create mode 100644 ecs/jskult-batch-laundering/src/__init__.py create mode 100644 ecs/jskult-batch-laundering/src/aws/__init__.py create mode 100644 ecs/jskult-batch-laundering/src/aws/s3.py create mode 100644 ecs/jskult-batch-laundering/src/batch/batch_functions.py create mode 100644 ecs/jskult-batch-laundering/src/batch/common/__init__.py create mode 100644 ecs/jskult-batch-laundering/src/batch/common/batch_context.py create mode 100644 ecs/jskult-batch-laundering/src/batch/common/calendar_file.py create mode 100644 ecs/jskult-batch-laundering/src/batch/laundering/__init__.py create mode 100644 ecs/jskult-batch-laundering/src/batch/laundering/create_inst_merge_for_laundering.py create mode 100644 ecs/jskult-batch-laundering/src/batch/laundering/sales_results_laundering.py create mode 100644 ecs/jskult-batch-laundering/src/db/__init__.py create mode 100644 ecs/jskult-batch-laundering/src/db/database.py create mode 100644 ecs/jskult-batch-laundering/src/error/__init__.py create mode 100644 ecs/jskult-batch-laundering/src/error/exceptions.py create mode 100644 ecs/jskult-batch-laundering/src/jobctrl_laundering.py create mode 100644 ecs/jskult-batch-laundering/src/logging/get_logger.py create mode 100644 ecs/jskult-batch-laundering/src/system_var/__init__.py create mode 100644 ecs/jskult-batch-laundering/src/system_var/constants.py create mode 100644 ecs/jskult-batch-laundering/src/system_var/environment.py create mode 100644 ecs/jskult-batch-laundering/src/time/elapsed_time.py diff --git a/ecs/jskult-batch-laundering/.dockerignore b/ecs/jskult-batch-laundering/.dockerignore new file mode 100644 index 00000000..8b9da402 --- /dev/null +++ b/ecs/jskult-batch-laundering/.dockerignore @@ -0,0 +1,12 @@ +tests/* +.coverage +.env +.env.example +.report/* +.vscode/* +.pytest_cache/* +*/__pychache__/* +Dockerfile +pytest.ini +README.md +*.sql diff --git a/ecs/jskult-batch-laundering/.env.example b/ecs/jskult-batch-laundering/.env.example new file mode 100644 index 00000000..56ae6677 --- /dev/null +++ b/ecs/jskult-batch-laundering/.env.example @@ -0,0 +1,20 @@ +DB_HOST=************ +DB_PORT=3306 +DB_USERNAME=************ +DB_PASSWORD=************ +DB_SCHEMA=src05 + +JSKULT_BACKUP_BUCKET=mbj-newdwh2021-staging-backup-jskult +JSKULT_CONFIG_BUCKET=mbj-newdwh2021-staging-config + +LOG_LEVEL=INFO +JSKULT_CONFIG_CALENDAR_FOLDER=jskult/calendar +DB_CONNECTION_MAX_RETRY_ATTEMPT=************ +DB_CONNECTION_RETRY_INTERVAL_INIT=************ +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=************ +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=************ + +# 連携データ抽出期間 +SALES_LAUNDERING_EXTRACT_DATE_PERIOD=7 +# 洗替対象テーブル名 +SALES_LAUNDERING_TARGET_TABLE_NAME=sales_lau_all diff --git a/ecs/jskult-batch-laundering/.gitignore b/ecs/jskult-batch-laundering/.gitignore new file mode 100644 index 00000000..bd0b37f8 --- /dev/null +++ b/ecs/jskult-batch-laundering/.gitignore @@ -0,0 +1,10 @@ +.vscode/settings.json +.env + +# python +__pycache__ + +# python test +.pytest_cache +.coverage +.report/ \ No newline at end of file diff --git a/ecs/jskult-batch-laundering/.vscode/launch.json b/ecs/jskult-batch-laundering/.vscode/launch.json new file mode 100644 index 00000000..ac80ce4f --- /dev/null +++ b/ecs/jskult-batch-laundering/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // IntelliSense を使用して利用可能な属性を学べます。 + // 既存の属性の説明をホバーして表示します。 + // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "(DEBUG)jskult batch laundering", + "type": "python", + "request": "launch", + "program": "entrypoint.py", + "console": "integratedTerminal", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/ecs/jskult-batch-laundering/.vscode/recommended_settings.json b/ecs/jskult-batch-laundering/.vscode/recommended_settings.json new file mode 100644 index 00000000..b5e79d73 --- /dev/null +++ b/ecs/jskult-batch-laundering/.vscode/recommended_settings.json @@ -0,0 +1,31 @@ +{ + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + // 自身の環境に合わせて変えてください + "python.defaultInterpreterPath": "", + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": [ + "--max-line-length=200", + "--ignore=F541" + ], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Path": "autopep8", + "python.formatting.autopep8Args": [ + "--max-line-length", "200", + "--ignore=F541" + ], + "python.testing.pytestArgs": [ + "tests/batch/ultmarc" + ], + + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} diff --git a/ecs/jskult-batch-laundering/Dockerfile b/ecs/jskult-batch-laundering/Dockerfile new file mode 100644 index 00000000..dd891d48 --- /dev/null +++ b/ecs/jskult-batch-laundering/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.9 + +ENV TZ="Asia/Tokyo" + +WORKDIR /usr/src/app +COPY Pipfile Pipfile.lock ./ +RUN \ + apt update -y && \ + # パッケージのセキュリティアップデートのみを適用するコマンド + apt install -y unattended-upgrades && \ + unattended-upgrades && \ + pip install --upgrade pip wheel setuptools && \ + pip install pipenv --no-cache-dir && \ + pipenv install --system --deploy && \ + pip uninstall -y pipenv virtualenv-clone virtualenv + +COPY src ./src +COPY entrypoint.py entrypoint.py + +CMD ["python", "entrypoint.py"] diff --git a/ecs/jskult-batch-laundering/Pipfile b/ecs/jskult-batch-laundering/Pipfile new file mode 100644 index 00000000..fe0fdc38 --- /dev/null +++ b/ecs/jskult-batch-laundering/Pipfile @@ -0,0 +1,20 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +boto3 = "*" +sqlalchemy = "*" +tenacity = "*" +pymysql = "*" + +[dev-packages] +autopep8 = "*" +flake8 = "*" + +[requires] +python_version = "3.9" + +[pipenv] +allow_prereleases = true diff --git a/ecs/jskult-batch-laundering/Pipfile.lock b/ecs/jskult-batch-laundering/Pipfile.lock new file mode 100644 index 00000000..10d05022 --- /dev/null +++ b/ecs/jskult-batch-laundering/Pipfile.lock @@ -0,0 +1,262 @@ +{ + "_meta": { + "hash": { + "sha256": "a2be870e254760b62220c10400b05fa66d24b2cc1bcd6f21044735e320a62e53" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "boto3": { + "hashes": [ + "sha256:908f9c277325d68963dfcfce963a05336f0eb19505fc239c0ab9d01f4cba0296", + "sha256:e1e535e9fb23977252f13652ed2fa9b4f2d59a53b04a5f2fad3ee415b6a3b2b0" + ], + "index": "pypi", + "version": "==1.27.0" + }, + "botocore": { + "hashes": [ + "sha256:b9cb5b78a289f0615a48d85066f01869029aa41b95993f2c0c55003df037c23f", + "sha256:cac1333f41ec98e6f75bbba3f2c74b9e76aa3847469ecea6e7773a0af0049bee" + ], + "markers": "python_version >= '3.7'", + "version": "==1.30.0" + }, + "greenlet": { + "hashes": [ + "sha256:0a9dfcadc1d79696e90ccb1275c30ad4ec5fd3d1ab3ae6671286fac78ef33435", + "sha256:0f313771cb8ee0a04dfdf586b7d4076180d80c94be09049daeea018089b5b957", + "sha256:17503397bf6cbb5e364217143b6150c540020c51a3f6b08f9a20cd67c25e2ca8", + "sha256:180ec55cb127bc745669eddc9793ffab6e0cf7311e67e1592f183d6ca00d88c1", + "sha256:1b3f3568478bc21b85968e8038c4f98f4bf0039a692791bc324b5e0d1522f4b1", + "sha256:1bd4ea36f0aeb14ca335e0c9594a5aaefa1ac4e2db7d86ba38f0be96166b3102", + "sha256:21ebcb570e0d8501457d6a2695a44c5af3b6c2143dc6644ec73574beba067c90", + "sha256:24071eee113d75fedebaeb86264d94f04b5a24e311c5ba3e8003c07d00112a7e", + "sha256:270432cfdd6a50016b8259b3bbf398a3f7c06a06f2c68c7b93e49f53bc193bcf", + "sha256:271ed380389d2f7e4c1545b6e0837986e62504ab561edbaff05da9c9f3f98f96", + "sha256:2840187a94e258445e62ff1545e34f0b1a14aef4d0078e5c88246688d2b6515e", + "sha256:2cda110faee67613fed221f90467003f477088ef1cc84c8fc88537785a5b4de9", + "sha256:2e160a65cc6023a237be870f2072513747d512a1d018efa083acce0b673cccc0", + "sha256:2fcf7af83516db35af3d0ed5d182dea8585eddd891977adff1b74212f4bfd2fd", + "sha256:36cebce1f30964d5672fd956860e7e7b69772da69658d5743cb676b442eeff36", + "sha256:42bfe67824a9b53e73f568f982f0d1d4c7ac0f587d2e702a23f8a7b505d7b7c2", + "sha256:450a7e52a515402fd110ba807f1a7d464424bfa703be4effbcb97e1dfbfcc621", + "sha256:463d63ca5d8c236788284a9a44b9715372a64d5318a6b5eee36815df1ea0ba3d", + "sha256:4d0c0ffd732466ff324ced144fad55ed5deca36f6036c1d8f04cec69b084c9d6", + "sha256:4ff2a765f4861fc018827eab4df1992f7508d06c62de5d2fe8a6ac2233d4f1d0", + "sha256:53abf19b7dc62795c67b8d0a3d8ef866db166b21017632fff2624cf8fbf3481c", + "sha256:5552d7be37d878e9b6359bbffa0512d857bb9703616a4c0656b49c10739d5971", + "sha256:585810056a8adacd3152945ebfcd25deb58335d41f16ae4e0f3d768918957f9a", + "sha256:5942b1d6ba447cff1ec23a21ec525dde2288f00464950bc647f4e0f03bd537d1", + "sha256:5c355c99be5bb23e85d899b059a4f22fdf8a0741c57e7029425ee63eb436f689", + "sha256:5f61df4fe07864561f49b45c8bd4d2c42e3f03d2872ed05c844902a58b875028", + "sha256:665942d3a954c3e4c976581715f57fb3b86f4cf6bae3ac30b133f8ff777ac6c7", + "sha256:68368e908f14887fb202a81960bfbe3a02d97e6d3fa62b821556463084ffb131", + "sha256:6aac94ff957b5dea0216af71ab59c602e1b947b394e4f5e878a5a65643090038", + "sha256:889934aa8d72b6bfc46babd1dc4b817a56c97ec0f4a10ae7551fb60ab1f96fae", + "sha256:a00550757fca1b9cbc479f8eb1cf3514dbc0103b3f76eae46341c26ddcca67a9", + "sha256:a4a2d6ed0515c05afd5cc435361ced0baabd9ba4536ddfe8ad9a95bcb702c8ce", + "sha256:a8dd92fd76a61af2abc8ccad0c6c6069b3c4ebd4727ecc9a7c33aae37651c8c7", + "sha256:ab81f9ff3e3c2ca65e824454214c10985a846cd9bee5f4d04e15cd875d9fe13b", + "sha256:ac10196b8cde7a082e4e371ff171407270d3337c8d57ed43030094eb01d9c95c", + "sha256:b767930af686551dc96a5eb70af3736709d547ffa275c11a5e820bfb3ae61d8d", + "sha256:b9a1f4d256b81f59ba87bb7a29b9b38b1c018e052dba60a543cb0ddb5062d159", + "sha256:ba94c08321b5d345100fc64eb1ab235f42faf9aabba805cface55ebe677f1c2c", + "sha256:bab71f73001cd15723c4e2ca398f2f48e0a3f584c619eefddb1525e8986e06eb", + "sha256:bce5cf2b0f0b29680396c5c98ab39a011bd70f2dfa8b8a6811a69ee6d920cf9f", + "sha256:c02e514c72e745e49a3ae7e672a1018ba9b68460c21e0361054e956e5d595bc6", + "sha256:c3fb459ced6c5e3b2a895f23f1400f93e9b24d85c30fbe2d637d4f7706a1116b", + "sha256:cd31ab223e43ac64fd23f8f5dad249addadac2a459f040546200acbf7e84e353", + "sha256:ce70aa089ec589b5d5fab388af9f8c9f9dfe8fe4ad844820a92eb240d8628ddf", + "sha256:d47b2e1ad1429da9aa459ef189fbcd8a74ec28a16bc4c3f5f3cf3f88e36535eb", + "sha256:d61bad421c1f496f9fb6114dbd7c30a1dac0e9ff90e9be06f4472cbd8f7a1704", + "sha256:d7ba2e5cb119eddbc10874b41047ad99525e39e397f7aef500e6da0d6f46ab91", + "sha256:dde0ab052c7a1deee8d13d72c37f2afecee30ebdf6eb139790157eaddf04dd61", + "sha256:df34b52aa50a38d7a79f3abc9fda7e400791447aa0400ed895f275f6d8b0bb1f", + "sha256:e0fc20e6e6b298861035a5fc5dcf9fbaa0546318e8bda81112591861a7dcc28f", + "sha256:e20d5e8dc76b73db9280464d6e81bea05e51a99f4d4dd29c5f78dc79f294a5d3", + "sha256:e31d1a33dc9006b278f72cb0aacfe397606c2693aa2fdc0c2f2dcddbad9e0b53", + "sha256:e3a99f890f2cc5535e1b3a90049c6ca9ff9da9ec251cc130c8d269997f9d32ee", + "sha256:e7b192c3df761d0fdd17c2d42d41c28460f124f5922e8bd524018f1d35610682", + "sha256:ed0f4fad4c3656e34d20323a789b6a2d210a6bb82647d9c86dded372f55c58a1", + "sha256:f34ec09702be907727fd479046193725441aaaf7ed4636ca042734f469bb7451", + "sha256:f3530c0ec1fc98c43d5b7061781a8c55bd0db44f789f8152e19d9526cbed6021", + "sha256:f5672082576d0e9f52fa0fa732ff57254d65faeb4a471bc339fe54b58b3e79d2", + "sha256:ffb9f8969789771e95d3c982a36be81f0adfaa7302a1d56e29f168ca15e284b8" + ], + "markers": "platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", + "version": "==3.0.0a1" + }, + "jmespath": { + "hashes": [ + "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", + "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" + ], + "markers": "python_version >= '3.7'", + "version": "==1.0.1" + }, + "pymysql": { + "hashes": [ + "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96", + "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7" + ], + "index": "pypi", + "version": "==1.1.0" + }, + "python-dateutil": { + "hashes": [ + "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", + "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.2" + }, + "s3transfer": { + "hashes": [ + "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346", + "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9" + ], + "markers": "python_version >= '3.7'", + "version": "==0.6.1" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.16.0" + }, + "sqlalchemy": { + "hashes": [ + "sha256:04383f1e3452f6739084184e427e9d5cb4e68ddc765d52157bf5ef30d5eca14f", + "sha256:125f9f7e62ddf8b590c069729080ffe18b68a20d9882eb0947f72e06274601d7", + "sha256:1822620c89779b85f7c23d535c8e04b79c517739ae07aaed48c81e591ed5498e", + "sha256:21583808d37f126a647652c90332ac1d3a102edf3c94bcc3319edcc0ea2300cc", + "sha256:218fb20c01e95004f50a3062bf4c447dcb360cab8274232f31947e254f118298", + "sha256:2269b1f9b8be47e52b70936069a25a3771eff53367aa5cc59bb94f28a6412e13", + "sha256:234678ed6576531b8e4be255b980f20368bf07241a2e67b84e6b0fe679edb9c4", + "sha256:28da17059ecde53e2d10ba813d38db942b9f6344360b2958b25872d5cb729d35", + "sha256:2c6ff5767d954f6091113fedcaaf49cdec2197ae4c5301fe83d5ae4393c82f33", + "sha256:36a87e26fe8fa8c466fae461a8fcb780d0a1cbf8206900759fc6fe874475a3ce", + "sha256:394ac3adf3676fad76d4b8fcecddf747627f17f0738dc94bac15f303d05b03d4", + "sha256:40a3dc52b2b16f08b5c16b9ee7646329e4b3411e9280e5e8d57b19eaa51cbef4", + "sha256:48111d56afea5699bab72c38ec95561796b81befff9e13d1dd5ce251ab25f51d", + "sha256:48b40dc2895841ea89d89df9eb3ac69e2950a659db20a369acf4259f68e6dc1f", + "sha256:513411d73503a6fc5804f01fae3b3d44f267c1b3a06cfeac02e9286a7330e857", + "sha256:51736cfb607cf4e8fafb693906f9bc4e5ee55be0b096d44bd7f20cd8489b8571", + "sha256:5f40e3a7d0a464f1c8593f2991e5520b2f5b26da24e88000bbd4423f86103d4f", + "sha256:6150560fcffc6aee5ec9a97419ac768c7a9f56baf7a7eb59cb4b1b6a4d463ad9", + "sha256:724355973297bbe547f3eb98b46ade65a67a3d5a6303f17ab59a2dc6fb938943", + "sha256:74ddcafb6488f382854a7da851c404c394be3729bb3d91b02ad86c5458140eff", + "sha256:7830e01b02d440c27f2a5be68296e74ccb55e6a5b5962ffafd360b98930b2e5e", + "sha256:7f31d4e7ca1dd8ca5a27fd5eaa0f9e2732fe769ff7dd35bf7bba179597e4df07", + "sha256:8741d3d401383e54b2aada37cbd10f55c5d444b360eae3a82f74a2be568a7710", + "sha256:910d45bf3673f0e4ef13858674bd23cfdafdc8368b45b948bf511797dbbb401d", + "sha256:aa995b21f853864996e4056d9fde479bcecf8b7bff4beb3555eebbbba815f35d", + "sha256:af7e2ba75bf84b64adb331918188dda634689a2abb151bc1a583e488363fd2f8", + "sha256:b0eaf82cc844f6b46defe15ad243ea00d1e39ed3859df61130c263dc7204da6e", + "sha256:b114a16bc03dfe20b625062e456affd7b9938286e05a3f904a025b9aacc29dd4", + "sha256:b47be4c6281a86670ea5cfbbbe6c3a65366a8742f5bc8b986f790533c60b5ddb", + "sha256:ba03518e64d86f000dc24ab3d3a1aa876bcbaa8aa15662ac2df5e81537fa3394", + "sha256:cc9c2630c423ac4973492821b2969f5fe99d9736f3025da670095668fbfcd4d5", + "sha256:cf07ff9920cb3ca9d73525dfd4f36ddf9e1a83734ea8b4f724edfd9a2c6e82d9", + "sha256:cf175d26f6787cce30fe6c04303ca0aeeb0ad40eeb22e3391f24b32ec432a1e1", + "sha256:d0aeb3afaa19f187a70fa592fbe3c20a056b57662691fd3abf60f016aa5c1848", + "sha256:e186e9e95fb5d993b075c33fe4f38a22105f7ce11cecb5c17b5618181e356702", + "sha256:e2d5c3596254cf1a96474b98e7ce20041c74c008b0f101c1cb4f8261cb77c6d3", + "sha256:e3189432db2f5753b4fde1aa90a61c69976f4e7e31d1cf4611bfe3514ed07478", + "sha256:e3a6b2788f193756076061626679c5c5a6d600ddf8324f986bc72004c3e9d92e", + "sha256:ead58cae2a089eee1b0569060999cb5f2b2462109498a0937cc230a7556945a1", + "sha256:f2f389f77c68dc22cb51f026619291c4a38aeb4b7ecb5f998fd145b2d81ca513", + "sha256:f593170fc09c5abb1205a738290b39532f7380094dc151805009a07ae0e85330" + ], + "index": "pypi", + "version": "==2.0.17" + }, + "tenacity": { + "hashes": [ + "sha256:2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0", + "sha256:43af037822bd0029025877f3b2d97cc4d7bb0c2991000a3d59d71517c5c969e0" + ], + "index": "pypi", + "version": "==8.2.2" + }, + "typing-extensions": { + "hashes": [ + "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", + "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" + ], + "markers": "python_version >= '3.7'", + "version": "==4.7.1" + }, + "urllib3": { + "hashes": [ + "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f", + "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", + "version": "==1.26.16" + } + }, + "develop": { + "autopep8": { + "hashes": [ + "sha256:86e9303b5e5c8160872b2f5ef611161b2893e9bfe8ccc7e2f76385947d57a2f1", + "sha256:f9849cdd62108cb739dbcdbfb7fdcc9a30d1b63c4cc3e1c1f893b5360941b61c" + ], + "index": "pypi", + "version": "==2.0.2" + }, + "flake8": { + "hashes": [ + "sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7", + "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181" + ], + "index": "pypi", + "version": "==6.0.0" + }, + "mccabe": { + "hashes": [ + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053", + "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610" + ], + "markers": "python_version >= '3.6'", + "version": "==2.10.0" + }, + "pyflakes": { + "hashes": [ + "sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf", + "sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd" + ], + "markers": "python_version >= '3.6'", + "version": "==3.0.1" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version < '3.11'", + "version": "==2.0.1" + } + } +} diff --git a/ecs/jskult-batch-laundering/README.md b/ecs/jskult-batch-laundering/README.md new file mode 100644 index 00000000..f6e7c2da --- /dev/null +++ b/ecs/jskult-batch-laundering/README.md @@ -0,0 +1,48 @@ +# 実消化&アルトマーク 月次バッチ + +## 概要 + +実消化&アルトマークの月次バッチ処理。 + +## 環境情報 + +- Python 3.9 +- MySQL 8.23 +- VSCode + +## 環境構築 + +- Python の構築 + + - Merck_NewDWH 開発 2021 の Wiki、[Python 環境構築](https://nds-tyo.backlog.com/alias/wiki/1874930)を参照 + - 「Pipenv の導入」までを行っておくこと + - 構築完了後、プロジェクト配下で以下のコマンドを実行し、Python の仮想環境を作成する + - `pipenv install --dev --python ` + - この手順で出力される仮想環境のパスは、後述する VSCode の設定手順で使用するため、控えておく + +- MySQL の環境構築 + - Windows の場合、以下のリンクからダウンロードする + - + - Docker を利用する場合、「newsdwh-tools」リポジトリの MySQL 設定を使用すると便利 + - 「crm-table-to-ddl」フォルダ内で以下のコマンドを実行すると + - `docker-compose up -d` + - Docker の構築手順は、[Docker のセットアップ手順](https://nds-tyo.backlog.com/alias/wiki/1754332)を参照のこと + - データを投入する + - 立ち上げたデータベースに「src05」スキーマを作成する + - [ローカル開発用データ](https://ndstokyo.sharepoint.com/:f:/r/sites/merck-new-dwh-team/Shared%20Documents/03.NewDWH%E6%A7%8B%E7%AF%89%E3%83%95%E3%82%A7%E3%83%BC%E3%82%BA3/02.%E9%96%8B%E7%99%BA/90.%E9%96%8B%E7%99%BA%E5%85%B1%E6%9C%89/%E3%83%AD%E3%83%BC%E3%82%AB%E3%83%AB%E9%96%8B%E7%99%BA%E7%94%A8%E3%83%87%E3%83%BC%E3%82%BF?csf=1&web=1&e=VVcRUs)をダウンロードし、mysql コマンドを使用して復元する + - `mysql -h <ホスト名> -P <ポート> -u <ユーザー名> -p src05 < src05_dump.sql` +- 環境変数の設定 + - 「.env.example」ファイルをコピーし、「.env」ファイルを作成する + - 環境変数を設定する。設定内容は PRJ メンバーより共有を受けてください +- VSCode の設定 + - 「.vscode/recommended_settings.json」ファイルをコピーし、「settings.json」ファイルを作成する + - 「python.defaultInterpreterPath」を、Python の構築手順で作成した仮想環境のパスに変更する + +## 実行 + +- VSCode 上で「F5」キーを押下すると、バッチ処理が起動する。 +- 「entrypoint.py」が、バッチ処理のエントリーポイント。 +- 実際の処理は、「src/jobctrl_monthly.py」で行っている。 + + +## フォルダ構成（工事中） diff --git a/ecs/jskult-batch-laundering/entrypoint.py b/ecs/jskult-batch-laundering/entrypoint.py new file mode 100644 index 00000000..624e61da --- /dev/null +++ b/ecs/jskult-batch-laundering/entrypoint.py @@ -0,0 +1,10 @@ +"""実消化＆アルトマーク実績洗替処理のエントリーポイント""" +from src import jobctrl_laundering + +if __name__ == '__main__': + try: + exit(jobctrl_laundering.exec()) + except Exception: + # エラーが起きても、正常系のコードで返す。 + # エラーが起きた事実はbatch_process内でログを出す。 + exit(0) diff --git a/ecs/jskult-batch-laundering/src/__init__.py b/ecs/jskult-batch-laundering/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-laundering/src/aws/__init__.py b/ecs/jskult-batch-laundering/src/aws/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-laundering/src/aws/s3.py b/ecs/jskult-batch-laundering/src/aws/s3.py new file mode 100644 index 00000000..17ed0fe9 --- /dev/null +++ b/ecs/jskult-batch-laundering/src/aws/s3.py @@ -0,0 +1,95 @@ +import os.path as path +import tempfile + +import boto3 + +from src.system_var import environment + + +class S3Client: + __s3_client = boto3.client('s3') + _bucket_name: str + + def list_objects(self, bucket_name: str, folder_name: str): + response = self.__s3_client.list_objects_v2(Bucket=bucket_name, Prefix=folder_name) + if response['KeyCount'] == 0: + return [] + contents = response['Contents'] + # 末尾がスラッシュで終わるものはフォルダとみなしてスキップする + objects = [{'filename': content['Key'], 'size': content['Size']} for content in contents if not content['Key'].endswith('/')] + return objects + + def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: + copy_source = {'Bucket': src_bucket, 'Key': src_key} + self.__s3_client.copy(copy_source, dest_bucket, dest_key) + return + + def download_file(self, bucket_name: str, file_key: str, file): + self.__s3_client.download_fileobj( + Bucket=bucket_name, + Key=file_key, + Fileobj=file + ) + return + + def upload_file(self, local_file_path: str, bucket_name: str, file_key: str): + self.__s3_client.upload_file( + local_file_path, + Bucket=bucket_name, + Key=file_key + ) + + def delete_file(self, bucket_name: str, file_key: str): + self.__s3_client.delete_object( + Bucket=bucket_name, + Key=file_key + ) + + +class S3Bucket(): + _s3_client = S3Client() + _bucket_name: str = None + + +class ConfigBucket(S3Bucket): + _bucket_name = environment.JSKULT_CONFIG_BUCKET + + def download_arisj_output_day_list(self): + # 一時ファイルとして保存する + temporary_dir = tempfile.mkdtemp() + temporary_file_path = path.join(temporary_dir, environment.JSKULT_CONFIG_CALENDAR_ARISJ_OUTPUT_DAY_LIST_FILE_NAME) + arisj_output_day_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_ARISJ_OUTPUT_DAY_LIST_FILE_NAME}' + with open(temporary_file_path, mode='wb') as f: + self._s3_client.download_file(self._bucket_name, arisj_output_day_list_key, f) + f.seek(0) + return temporary_file_path + + +class ArisjBucket(S3Bucket): + _bucket_name = environment.ARISJ_DATA_BUCKET + _folder = environment.ARISJ_BACKUP_FOLDER + + def upload_arisj_csv_file(self, arisj_create_csv: str, csv_file_path: str): + # s3にCSVファイルをUPする + Bucket = environment.ARISJ_DATA_BUCKET + folder = environment.ARISJ_DATA_FOLDER + csv_file_name = f'{folder}/{arisj_create_csv}' + s3_client = S3Client() + s3_client.upload_file(csv_file_path, Bucket, csv_file_name) + return + + def backup_arisj_csv_file(self, dat_file_key: str, datetime_key: str): + # バックアップバケットにコピー + arisj_backup_bucket = ArisjBackupBucket() + folder = environment.ARISJ_DATA_FOLDER + dat_file_key = f'{folder}/{dat_file_key}' + backup_key = f'{arisj_backup_bucket._folder}/{datetime_key}/{dat_file_key.replace(f"{self._folder}/", "")}' + self._s3_client.copy(self._bucket_name, dat_file_key, arisj_backup_bucket._bucket_name, backup_key) + + +class JskUltBackupBucket(S3Bucket): + _bucket_name = environment.JSKULT_BACKUP_BUCKET + + +class ArisjBackupBucket(JskUltBackupBucket): + _folder = environment.ARISJ_BACKUP_FOLDER diff --git a/ecs/jskult-batch-laundering/src/batch/batch_functions.py b/ecs/jskult-batch-laundering/src/batch/batch_functions.py new file mode 100644 index 00000000..40cf84f2 --- /dev/null +++ b/ecs/jskult-batch-laundering/src/batch/batch_functions.py @@ -0,0 +1,51 @@ +"""バッチ処理の共通関数""" +import logging +import textwrap +from datetime import datetime + +from src.db.database import Database +from src.error.exceptions import BatchOperationException, DBException + + +def get_batch_statuses() -> tuple[str, str]: + """日付テーブルから、以下を取得して返す。 + - バッチ処理中フラグ + - 処理日（YYYY/MM/DD） + + Raises: + BatchOperationException: 日付テーブルが取得できないとき、何らかのエラーが発生したとき + + Returns: + tuple[str, str]: [0]バッチ処理中フラグ，[1]処理日 + """ + db = Database.get_instance() + sql = 'SELECT bch_actf, src05.get_syor_date() AS syor_date FROM src05.hdke_tbl' + try: + db.connect() + hdke_tbl_result = db.execute_select(sql) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + if len(hdke_tbl_result) == 0: + raise BatchOperationException('日付テーブルが取得できませんでした') + + # 必ず1件取れる + hdke_tbl_record = hdke_tbl_result[0] + batch_processing_flag = hdke_tbl_record['bch_actf'] + syor_date = hdke_tbl_record['syor_date'] + # 処理日を文字列に変換する + syor_date_str = datetime.strftime(syor_date, '%Y/%m/%d') + + return batch_processing_flag, syor_date_str + + +def logging_sql(logger: logging.Logger, sql: str) -> None: + """SQL文をデバッグログで出力する + + Args: + logger (logging.Logger): ロガー + sql (str): SQL文 + """ + logger.debug(f'\n{"-" * 15}\n{textwrap.dedent(sql)[1:-1]}\n{"-" * 15}') diff --git a/ecs/jskult-batch-laundering/src/batch/common/__init__.py b/ecs/jskult-batch-laundering/src/batch/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-laundering/src/batch/common/batch_context.py b/ecs/jskult-batch-laundering/src/batch/common/batch_context.py new file mode 100644 index 00000000..8c8c12fb --- /dev/null +++ b/ecs/jskult-batch-laundering/src/batch/common/batch_context.py @@ -0,0 +1,29 @@ +class BatchContext: + __instance = None + __syor_date: str # 処理日(yyyy/mm/dd形式) + __is_arisj_output_day: bool # 月次バッチ起動日フラグ + + def __init__(self) -> None: + self.__is_arisj_output_day = False + + @classmethod + def get_instance(cls): + if cls.__instance is None: + cls.__instance = cls() + return cls.__instance + + @property + def syor_date(self): + return self.__syor_date + + @syor_date.setter + def syor_date(self, syor_date_str: str): + self.__syor_date = syor_date_str + + @property + def is_arisj_output_day(self): + return self.__is_arisj_output_day + + @is_arisj_output_day.setter + def is_arisj_output_day(self, flag: bool): + self.__is_arisj_output_day = flag diff --git a/ecs/jskult-batch-laundering/src/batch/common/calendar_file.py b/ecs/jskult-batch-laundering/src/batch/common/calendar_file.py new file mode 100644 index 00000000..b456f03c --- /dev/null +++ b/ecs/jskult-batch-laundering/src/batch/common/calendar_file.py @@ -0,0 +1,32 @@ +from src.system_var import constants + + +class CalendarFile: + """カレンダーファイル""" + + __calendar_file_lines: list[str] + + def __init__(self, calendar_file_path): + with open(calendar_file_path) as f: + self.__calendar_file_lines: list[str] = f.readlines() + + def compare_date(self, date_str: str) -> bool: + """与えられた日付がカレンダーファイル内に含まれているかどうか + カレンダーファイル内の日付はyyyy/mm/ddで書かれている前提 + コメント（#）が含まれている行は無視される + + Args: + date_str (str): yyyy/mm/dd文字列 + + Returns: + bool: 含まれていればTrue + """ + for calendar_date in self.__calendar_file_lines: + # コメント行が含まれている場合はスキップ + if constants.CALENDAR_COMMENT_SYMBOL in calendar_date: + continue + + if date_str in calendar_date: + return True + + return False diff --git a/ecs/jskult-batch-laundering/src/batch/laundering/__init__.py b/ecs/jskult-batch-laundering/src/batch/laundering/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-laundering/src/batch/laundering/create_inst_merge_for_laundering.py b/ecs/jskult-batch-laundering/src/batch/laundering/create_inst_merge_for_laundering.py new file mode 100644 index 00000000..2c9d7353 --- /dev/null +++ b/ecs/jskult-batch-laundering/src/batch/laundering/create_inst_merge_for_laundering.py @@ -0,0 +1,31 @@ +from src.db.database import Database +from src.error.exceptions import BatchOperationException +from src.logging.get_logger import get_logger + +logger = get_logger('洗替用マスタ作成') + + +def exec(): + db = Database.get_instance() + + try: + db.connect() + + logger.debug('処理開始') + call_v_inst_merge_t_create(db) + call_inst_merge_t_create(db) + logger.debug('処理終了') + except Exception as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + +def call_v_inst_merge_t_create(db: Database): + db.execute('CALL src05.v_inst_merge_t_create()') + return + + +def call_inst_merge_t_create(db: Database): + db.execute('CALL src05.inst_merge_t_create()') + return diff --git a/ecs/jskult-batch-laundering/src/batch/laundering/sales_results_laundering.py b/ecs/jskult-batch-laundering/src/batch/laundering/sales_results_laundering.py new file mode 100644 index 00000000..ac2c79f7 --- /dev/null +++ b/ecs/jskult-batch-laundering/src/batch/laundering/sales_results_laundering.py @@ -0,0 +1,153 @@ +from src.batch.batch_functions import logging_sql +from src.db.database import Database +from src.error.exceptions import BatchOperationException +from src.logging.get_logger import get_logger +from src.system_var import environment + + +logger = get_logger('卸販売実績全件洗替') + + +def exec(): + db = Database.get_instance() + try: + db.connect() + logger.debug('処理開始') + # 卸販売実績テーブル（洗替後）作成 + _call_sales_lau_upsert(db) + # １：卸組織洗替 + _call_whs_org_laundering(db) + # ３：HCO施設コードの洗替 + _update_sales_lau_from_vop_hco_merge_v(db) + # ４：メルク施設コードの洗替 + _update_mst_inst_laundering(db) + logger.debug('処理終了') + except Exception as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + +def _call_sales_lau_upsert(db: Database): + # 卸販売実績テーブル（洗替後）作成 + logger.info('sales_lau_upsert(プロシージャ―) 開始') + db.execute(f""" + CALL src05.sales_lau_upsert( + '{environment.SALES_LAUNDERING_TARGET_TABLE_NAME}', + (src05.get_syor_date() - interval {environment.SALES_LAUNDERING_EXTRACT_DATE_PERIOD} day), + src05.get_syor_date() + ) + """) + logger.info('sales_lau_upsert(プロシージャ―) 終了') + return + + +def _call_whs_org_laundering(db: Database): + # 卸組織洗替 + logger.info('whs_org_laundering(プロシージャ―) 開始') + db.execute(f""" + CALL src05.whs_org_laundering( + '{environment.SALES_LAUNDERING_TARGET_TABLE_NAME}' + ) + """) + logger.info('whs_org_laundering(プロシージャ―) 終了') + return + + +def _update_sales_lau_from_vop_hco_merge_v(db: Database): + # HCO施設コードの洗替 + if _count_v_inst_merge_t(db) == 0: + logger.info('V施設統合マスタ(洗替処理一時テーブル)にデータは存在しません') + return + + _call_v_inst_merge_laundering(db) + return + + +def _count_v_inst_merge_t(db: Database) -> int: + # V施設統合マスタ(洗替処理一時テーブル)のデータ件数の取得 + try: + sql = """ + SELECT + COUNT(v_inst_cd) AS cnt + FROM + internal05.v_inst_merge_t + """ + result = db.execute_select(sql) + logging_sql(logger, sql) + logger.info('V施設統合マスタ(洗替処理一時テーブル)のデータ件数の取得 成功') + except Exception as e: + logger.debug('V施設統合マスタ(洗替処理一時テーブル)のデータ件数の取得 失敗') + raise e + + return result[0]['cnt'] + + +def _call_v_inst_merge_laundering(db: Database): + # HCO施設コードの洗替(プロシージャ―の呼び出し) + logger.info('v_inst_merge_laundering(プロシージャ―) 開始') + db.execute(f""" + CALL src05.v_inst_merge_laundering( + '{environment.SALES_LAUNDERING_TARGET_TABLE_NAME}' + ) + """) + logger.info('v_inst_merge_laundering(プロシージャ―) 終了') + return + + +def _update_mst_inst_laundering(db: Database): + # メルク施設コードの洗替 + _call_hco_to_mdb_laundering(db) + _update_sales_lau_from_dcf_inst_merge(db) + + +def _call_hco_to_mdb_laundering(db: Database): + # A:医療機関のデータはMDB変換表からHCO⇒DCFへ変換 + logger.info('hco_to_mdb_laundering(プロシージャ―) 開始') + db.execute(f""" + CALL src05.hco_to_mdb_laundering( + '{environment.SALES_LAUNDERING_TARGET_TABLE_NAME}' + ) + """) + logger.info('hco_to_mdb_laundering(プロシージャ―) 終了') + return + + +def _update_sales_lau_from_dcf_inst_merge(db: Database): + # B:DCF施設統合マスタがある場合は、コードを変換し、住所等をSETする + if _count_inst_merge_t(db) == 0: + logger.info('アルトマーク施設統合マスタ(洗替処理一時テーブル)にデータは存在しません') + return + _call_inst_merge_laundering(db) + return + + +def _count_inst_merge_t(db: Database) -> int: + # アルトマーク施設統合マスタ(洗替処理一時テーブル)のデータ件数の取得 + try: + sql = """ + SELECT + COUNT(dcf_dsf_inst_cd) AS cnt + FROM + internal05.inst_merge_t + """ + result = db.execute_select(sql) + logging_sql(logger, sql) + logger.info('アルトマーク施設統合マスタ(洗替処理一時テーブル)のデータ件数の取得 成功') + except Exception as e: + logger.debug('アルトマーク施設統合マスタ(洗替処理一時テーブル)のデータ件数の取得 失敗') + raise e + + return result[0]['cnt'] + + +def _call_inst_merge_laundering(db: Database): + # B:DCF施設統合マスタがある場合は、コードを変換し、住所等をSETする(プロシージャ―の呼び出し) + logger.info('inst_merge_laundering(プロシージャ―) 開始') + db.execute(f""" + CALL src05.inst_merge_laundering( + '{environment.SALES_LAUNDERING_TARGET_TABLE_NAME}' + ) + """) + logger.info('inst_merge_laundering(プロシージャ―) 終了') + return diff --git a/ecs/jskult-batch-laundering/src/db/__init__.py b/ecs/jskult-batch-laundering/src/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-laundering/src/db/database.py b/ecs/jskult-batch-laundering/src/db/database.py new file mode 100644 index 00000000..d2171fb5 --- /dev/null +++ b/ecs/jskult-batch-laundering/src/db/database.py @@ -0,0 +1,179 @@ +from sqlalchemy import (Connection, CursorResult, Engine, QueuePool, + create_engine, text) +from sqlalchemy.engine.url import URL +from src.error.exceptions import DBException +from src.logging.get_logger import get_logger +from src.system_var import environment +from tenacity import retry, stop_after_attempt, wait_exponential + +logger = get_logger(__name__) + + +class Database: + """データベース操作クラス""" + __connection: Connection = None + __engine: Engine = None + __host: str = None + __port: str = None + __username: str = None + __password: str = None + __schema: str = None + __connection_string: str = None + + def __init__(self, username: str, password: str, host: str, port: int, schema: str) -> None: + """このクラスの新たなインスタンスを初期化します + + Args: + username (str): DBユーザー名 + password (str): DBパスワード + host (str): DBホスト名 + port (int): DBポート + schema (str): DBスキーマ名 + """ + self.__username = username + self.__password = password + self.__host = host + self.__port = int(port) + self.__schema = schema + + self.__connection_string = URL.create( + drivername='mysql+pymysql', + username=self.__username, + password=self.__password, + host=self.__host, + port=self.__port, + database=self.__schema, + query={"charset": "utf8mb4"} + ) + + self.__engine = create_engine( + self.__connection_string, + pool_timeout=5, + poolclass=QueuePool + ) + + @classmethod + def get_instance(cls): + """インスタンスを取得します + + Returns: + Database: DB操作クラスインスタンス + """ + return cls( + username=environment.DB_USERNAME, + password=environment.DB_PASSWORD, + host=environment.DB_HOST, + port=environment.DB_PORT, + schema=environment.DB_SCHEMA + ) + + @retry( + wait=wait_exponential( + multiplier=environment.DB_CONNECTION_RETRY_INTERVAL_INIT, + min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, + max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS + ), + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), + retry_error_cls=DBException + ) + def connect(self): + """ + DBに接続します。接続に失敗した場合、リトライします。 + Raises: + DBException: 接続失敗 + """ + try: + self.__connection = self.__engine.connect() + except Exception as e: + raise DBException(e) + + def execute_select(self, select_query: str, parameters=None) -> list[dict]: + """SELECTクエリを実行します。 + + Args: + select_query (str): SELECT文 + parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None. + + Raises: + DBException: DBエラー + + Returns: + list[dict]: カラム名: 値の辞書リスト + """ + if self.__connection is None: + raise DBException('DBに接続していません') + + result = None + try: + # トランザクションが開始している場合は、トランザクションを引き継ぐ + if self.__connection.in_transaction(): + result = self.__connection.execute(text(select_query), parameters) + else: + # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。 + result = self.__execute_with_transaction(select_query, parameters) + except Exception as e: + raise DBException(f'SQL Error: {e}') + + result_rows = result.mappings().all() + return result_rows + + def execute(self, query: str, parameters=None) -> CursorResult: + """SQLクエリを実行します。 + + Args: + query (str): SQL文 + parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None. + + Raises: + DBException: DBエラー + + Returns: + CursorResult: 取得結果 + """ + if self.__connection is None: + raise DBException('DBに接続していません') + + result = None + try: + # トランザクションが開始している場合は、トランザクションを引き継ぐ + if self.__connection.in_transaction(): + result = self.__connection.execute(text(query), parameters) + else: + # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。 + result = self.__execute_with_transaction(query, parameters) + except Exception as e: + raise DBException(f'SQL Error: {e}') + + return result + + def begin(self): + """トランザクションを開始します。""" + if not self.__connection.in_transaction(): + self.__connection.begin() + + def commit(self): + """トランザクションをコミットします""" + if self.__connection.in_transaction(): + self.__connection.commit() + + def rollback(self): + """トランザクションをロールバックします""" + if self.__connection.in_transaction(): + self.__connection.rollback() + + def disconnect(self): + """DB接続を切断します。""" + if self.__connection is not None: + self.__connection.close() + self.__connection = None + + def __execute_with_transaction(self, query: str, parameters: dict): + # トランザクションを開始してクエリを実行する + with self.__connection.begin(): + try: + result = self.__connection.execute(text(query), parameters=parameters) + except Exception as e: + self.__connection.rollback() + raise e + # ここでコミットされる + return result diff --git a/ecs/jskult-batch-laundering/src/error/__init__.py b/ecs/jskult-batch-laundering/src/error/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-laundering/src/error/exceptions.py b/ecs/jskult-batch-laundering/src/error/exceptions.py new file mode 100644 index 00000000..055c24f6 --- /dev/null +++ b/ecs/jskult-batch-laundering/src/error/exceptions.py @@ -0,0 +1,10 @@ +class MeDaCaException(Exception): + pass + + +class DBException(MeDaCaException): + pass + + +class BatchOperationException(MeDaCaException): + pass diff --git a/ecs/jskult-batch-laundering/src/jobctrl_laundering.py b/ecs/jskult-batch-laundering/src/jobctrl_laundering.py new file mode 100644 index 00000000..699ff6e5 --- /dev/null +++ b/ecs/jskult-batch-laundering/src/jobctrl_laundering.py @@ -0,0 +1,23 @@ +"""実消化＆アルトマーク実績洗替処理""" +from src.batch.batch_functions import get_batch_statuses +from src.batch.common.batch_context import BatchContext +from src.error.exceptions import BatchOperationException +from src.logging.get_logger import get_logger +from src.system_var import constants +from src.batch.laundering import (sales_results_laundering, + create_inst_merge_for_laundering) + +logger = get_logger('実消化＆アルトマーク実績洗替処理') + + +def exec(): + try: + logger.debug('処理開始') + # 洗替用マスタ作成 + create_inst_merge_for_laundering.exec() + # 卸販売実績全件洗替 + sales_results_laundering.exec() + logger.debug('処理終了') + except BatchOperationException as e: + logger.exception(f'（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS diff --git a/ecs/jskult-batch-laundering/src/logging/get_logger.py b/ecs/jskult-batch-laundering/src/logging/get_logger.py new file mode 100644 index 00000000..f36f1199 --- /dev/null +++ b/ecs/jskult-batch-laundering/src/logging/get_logger.py @@ -0,0 +1,37 @@ +import logging + +from src.system_var.environment import LOG_LEVEL + +# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する +for name in ["boto3", "botocore", "s3transfer", "urllib3"]: + logging.getLogger(name).setLevel(logging.WARNING) + + +def get_logger(log_name: str) -> logging.Logger: + """一意のログ出力モジュールを取得します。 + + Args: + log_name (str): ロガー名 + + Returns: + _type_: _description_ + """ + logger = logging.getLogger(log_name) + level = logging.getLevelName(LOG_LEVEL) + if not isinstance(level, int): + level = logging.INFO + logger.setLevel(level) + + if not logger.hasHandlers(): + handler = logging.StreamHandler() + logger.addHandler(handler) + + formatter = logging.Formatter( + '%(name)s\t[%(levelname)s]\t%(asctime)s\t%(message)s', + '%Y-%m-%d %H:%M:%S' + ) + + for handler in logger.handlers: + handler.setFormatter(formatter) + + return logger diff --git a/ecs/jskult-batch-laundering/src/system_var/__init__.py b/ecs/jskult-batch-laundering/src/system_var/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-batch-laundering/src/system_var/constants.py b/ecs/jskult-batch-laundering/src/system_var/constants.py new file mode 100644 index 00000000..aaa8a3c0 --- /dev/null +++ b/ecs/jskult-batch-laundering/src/system_var/constants.py @@ -0,0 +1,8 @@ +# バッチ正常終了コード +BATCH_EXIT_CODE_SUCCESS = 0 + +# バッチ処理中フラグ：処理中 +BATCH_ACTF_BATCH_IN_PROCESSING = '1' + +# カレンダーファイルのコメントシンボル +CALENDAR_COMMENT_SYMBOL = '#' diff --git a/ecs/jskult-batch-laundering/src/system_var/environment.py b/ecs/jskult-batch-laundering/src/system_var/environment.py new file mode 100644 index 00000000..6755de11 --- /dev/null +++ b/ecs/jskult-batch-laundering/src/system_var/environment.py @@ -0,0 +1,25 @@ +import os + +# Database +DB_HOST = os.environ['DB_HOST'] +DB_PORT = int(os.environ['DB_PORT']) +DB_USERNAME = os.environ['DB_USERNAME'] +DB_PASSWORD = os.environ['DB_PASSWORD'] +DB_SCHEMA = os.environ['DB_SCHEMA'] + +# AWS +JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] +JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] +JSKULT_CONFIG_CALENDAR_FOLDER = os.environ['JSKULT_CONFIG_CALENDAR_FOLDER'] + +# 初期値がある環境変数 +LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') +DB_CONNECTION_MAX_RETRY_ATTEMPT = int(os.environ.get('DB_CONNECTION_MAX_RETRY_ATTEMPT', 4)) +DB_CONNECTION_RETRY_INTERVAL_INIT = int(os.environ.get('DB_CONNECTION_RETRY_INTERVAL', 5)) +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MIN_SECONDS', 5)) +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MAX_SECONDS', 50)) + +# 連携データ抽出期間 +SALES_LAUNDERING_EXTRACT_DATE_PERIOD = int(os.environ['SALES_LAUNDERING_EXTRACT_DATE_PERIOD']) +# 洗替対象テーブル名 +SALES_LAUNDERING_TARGET_TABLE_NAME = os.environ['SALES_LAUNDERING_TARGET_TABLE_NAME'] diff --git a/ecs/jskult-batch-laundering/src/time/elapsed_time.py b/ecs/jskult-batch-laundering/src/time/elapsed_time.py new file mode 100644 index 00000000..c1432e91 --- /dev/null +++ b/ecs/jskult-batch-laundering/src/time/elapsed_time.py @@ -0,0 +1,22 @@ +import time + + +class ElapsedTime: + """処理実行時間計測クラス""" + def __init__(self) -> None: + """このクラスの新たなインスタンスを初期化します。""" + self.__start = time.perf_counter() + + @property + def of(self): + """インスタンス化してからの経過時間をhh:mm:ssの形式にフォーマットして返す + Returns: + str: 時分秒形式の経過時間 + """ + elapsed_time = time.perf_counter() - self.__start + h, rem = divmod(elapsed_time, 3600) + m, s = divmod(rem, 60) + h_str = f'{h:02.0f} hour ' if h > 0.0 else '' + m_str = f'{m:02.0f} min ' if m > 0.0 else '' + s_str = f'{s:06.02f} sec' if s > 0.0 else '' + return f"{h_str}{m_str}{s_str}" From ea78a31ec5ed4d2460e92da0acfe3422f452f6f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E6=9C=A8=E8=A6=81?= Date: Thu, 27 Jul 2023 10:31:59 +0900 Subject: [PATCH 2/5] =?UTF-8?q?feat:=20=E5=85=A8=E4=BB=B6=E5=8D=B8?= =?UTF-8?q?=E8=B2=A9=E5=A3=B2=E5=AE=9F=E7=B8=BE=E6=B4=97=E6=9B=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/batch_functions.py | 83 +++++++++++++++++-- .../src/batch/common/calendar_file.py | 32 ------- .../src/jobctrl_laundering.py | 26 ++++-- .../src/system_var/constants.py | 4 +- 4 files changed, 98 insertions(+), 47 deletions(-) delete mode 100644 ecs/jskult-batch-laundering/src/batch/common/calendar_file.py diff --git a/ecs/jskult-batch-laundering/src/batch/batch_functions.py b/ecs/jskult-batch-laundering/src/batch/batch_functions.py index 40cf84f2..87231e17 100644 --- a/ecs/jskult-batch-laundering/src/batch/batch_functions.py +++ b/ecs/jskult-batch-laundering/src/batch/batch_functions.py @@ -1,25 +1,25 @@ """バッチ処理の共通関数""" import logging import textwrap -from datetime import datetime from src.db.database import Database from src.error.exceptions import BatchOperationException, DBException +from src.system_var import constants def get_batch_statuses() -> tuple[str, str]: """日付テーブルから、以下を取得して返す。 - バッチ処理中フラグ - - 処理日（YYYY/MM/DD） + - dump取得状況区分 Raises: BatchOperationException: 日付テーブルが取得できないとき、何らかのエラーが発生したとき Returns: - tuple[str, str]: [0]バッチ処理中フラグ，[1]処理日 + tuple[str, str]: [0]バッチ処理中フラグ，[1]dump取得状況区分 """ db = Database.get_instance() - sql = 'SELECT bch_actf, src05.get_syor_date() AS syor_date FROM src05.hdke_tbl' + sql = 'SELECT bch_actf, dump_sts_kbn FROM src05.hdke_tbl' try: db.connect() hdke_tbl_result = db.execute_select(sql) @@ -34,11 +34,78 @@ def get_batch_statuses() -> tuple[str, str]: # 必ず1件取れる hdke_tbl_record = hdke_tbl_result[0] batch_processing_flag = hdke_tbl_record['bch_actf'] - syor_date = hdke_tbl_record['syor_date'] - # 処理日を文字列に変換する - syor_date_str = datetime.strftime(syor_date, '%Y/%m/%d') + dump_status_kbn = hdke_tbl_record['dump_sts_kbn'] - return batch_processing_flag, syor_date_str + return batch_processing_flag, dump_status_kbn + + +def update_dump_status_kbn_in_processing() -> None: + """dump取得状況区分を処理中に更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = 'UPDATE src05.hdke_tbl SET dump_sts_kbn = :in_processing' + try: + db.connect() + db.execute(sql, {'in_processing': constants.BATCH_ACTF_BATCH_IN_PROCESSING}) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + return + + +def update_dump_status_kbn_error() -> None: + """dump取得状況区分をエラーに更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = """\ + UPDATE src05.hdke_tbl + SET + dump_sts_kbn = :dump_unprocessed + """ + try: + db.connect() + db.execute(sql, { + 'dump_unprocessed': constants.DUMP_STATUS_KBN_ERROR + }) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + return + + +def update_dump_status_kbn_complete() -> None: + """dump取得状況区分を正常終了に更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = """\ + UPDATE src05.hdke_tbl + SET + dump_sts_kbn = :dump_unprocessed + """ + try: + db.connect() + db.execute(sql, { + 'dump_unprocessed': constants.DUMP_STATUS_KBN_COMPLETE + }) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + return def logging_sql(logger: logging.Logger, sql: str) -> None: diff --git a/ecs/jskult-batch-laundering/src/batch/common/calendar_file.py b/ecs/jskult-batch-laundering/src/batch/common/calendar_file.py deleted file mode 100644 index b456f03c..00000000 --- a/ecs/jskult-batch-laundering/src/batch/common/calendar_file.py +++ /dev/null @@ -1,32 +0,0 @@ -from src.system_var import constants - - -class CalendarFile: - """カレンダーファイル""" - - __calendar_file_lines: list[str] - - def __init__(self, calendar_file_path): - with open(calendar_file_path) as f: - self.__calendar_file_lines: list[str] = f.readlines() - - def compare_date(self, date_str: str) -> bool: - """与えられた日付がカレンダーファイル内に含まれているかどうか - カレンダーファイル内の日付はyyyy/mm/ddで書かれている前提 - コメント（#）が含まれている行は無視される - - Args: - date_str (str): yyyy/mm/dd文字列 - - Returns: - bool: 含まれていればTrue - """ - for calendar_date in self.__calendar_file_lines: - # コメント行が含まれている場合はスキップ - if constants.CALENDAR_COMMENT_SYMBOL in calendar_date: - continue - - if date_str in calendar_date: - return True - - return False diff --git a/ecs/jskult-batch-laundering/src/jobctrl_laundering.py b/ecs/jskult-batch-laundering/src/jobctrl_laundering.py index 699ff6e5..c0735162 100644 --- a/ecs/jskult-batch-laundering/src/jobctrl_laundering.py +++ b/ecs/jskult-batch-laundering/src/jobctrl_laundering.py @@ -1,6 +1,5 @@ """実消化＆アルトマーク実績洗替処理""" from src.batch.batch_functions import get_batch_statuses -from src.batch.common.batch_context import BatchContext from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger from src.system_var import constants @@ -12,12 +11,29 @@ logger = get_logger('実消化＆アルトマーク実績洗替処理') def exec(): try: - logger.debug('処理開始') + logger.info('週次バッチ：開始') + try: + # 日次バッチ処置中フラグ、dump処理状態区分、処理日を取得 + batch_processing_flag, dump_status_kbn = get_batch_statuses() + except BatchOperationException as e: + logger.exception(f'日付テーブル取得（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # 日次バッチ処理中の場合、後続の処理は行わない + if batch_processing_flag == constants.BATCH_ACTF_BATCH_IN_PROCESSING: + logger.warning('日次バッチ処理中のため、実消化＆アルトマーク実績洗替処理を終了します。') + return constants.BATCH_EXIT_CODE_SUCCESS + + # dump取得が正常終了していない場合、後続の処理は行わない + if dump_status_kbn != constants.DUMP_STATUS_KBN_UNPROCESSED: + logger.warning('dump取得が正常終了していないため、実消化＆アルトマーク実績洗替処理を終了します。') + return constants.BATCH_EXIT_CODE_SUCCESS + # 洗替用マスタ作成 create_inst_merge_for_laundering.exec() # 卸販売実績全件洗替 sales_results_laundering.exec() - logger.debug('処理終了') - except BatchOperationException as e: - logger.exception(f'（異常終了）{e}') + logger.info('週次バッチ：終了') + except Exception as e: + logger.exception(f'実消化＆アルトマーク実績洗替処理中に想定外のエラーが発生しました {e}') return constants.BATCH_EXIT_CODE_SUCCESS diff --git a/ecs/jskult-batch-laundering/src/system_var/constants.py b/ecs/jskult-batch-laundering/src/system_var/constants.py index aaa8a3c0..ed3ea806 100644 --- a/ecs/jskult-batch-laundering/src/system_var/constants.py +++ b/ecs/jskult-batch-laundering/src/system_var/constants.py @@ -4,5 +4,5 @@ BATCH_EXIT_CODE_SUCCESS = 0 # バッチ処理中フラグ：処理中 BATCH_ACTF_BATCH_IN_PROCESSING = '1' -# カレンダーファイルのコメントシンボル -CALENDAR_COMMENT_SYMBOL = '#' +# dump取得状態区分：未処理 +DUMP_STATUS_KBN_UNPROCESSED = '0' From 0661493b2f41e64ba91ba9b85943f2531294d657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E6=9C=A8=E8=A6=81?= Date: Fri, 28 Jul 2023 14:34:25 +0900 Subject: [PATCH 3/5] =?UTF-8?q?feat:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-laundering/.env.example | 4 - ecs/jskult-batch-laundering/Pipfile | 1 - ecs/jskult-batch-laundering/Pipfile.lock | 142 ++++++------------ ecs/jskult-batch-laundering/README.md | 33 +++- ecs/jskult-batch-laundering/entrypoint.py | 2 +- .../src/aws/__init__.py | 0 ecs/jskult-batch-laundering/src/aws/s3.py | 95 ------------ .../src/batch/batch_functions.py | 70 --------- .../src/batch/common/__init__.py | 0 .../src/batch/common/batch_context.py | 29 ---- .../src/jobctrl_laundering.py | 10 +- .../src/system_var/environment.py | 5 - .../src/time/elapsed_time.py | 22 --- 13 files changed, 78 insertions(+), 335 deletions(-) delete mode 100644 ecs/jskult-batch-laundering/src/aws/__init__.py delete mode 100644 ecs/jskult-batch-laundering/src/aws/s3.py delete mode 100644 ecs/jskult-batch-laundering/src/batch/common/__init__.py delete mode 100644 ecs/jskult-batch-laundering/src/batch/common/batch_context.py delete mode 100644 ecs/jskult-batch-laundering/src/time/elapsed_time.py diff --git a/ecs/jskult-batch-laundering/.env.example b/ecs/jskult-batch-laundering/.env.example index 56ae6677..055fde91 100644 --- a/ecs/jskult-batch-laundering/.env.example +++ b/ecs/jskult-batch-laundering/.env.example @@ -4,11 +4,7 @@ DB_USERNAME=************ DB_PASSWORD=************ DB_SCHEMA=src05 -JSKULT_BACKUP_BUCKET=mbj-newdwh2021-staging-backup-jskult -JSKULT_CONFIG_BUCKET=mbj-newdwh2021-staging-config - LOG_LEVEL=INFO -JSKULT_CONFIG_CALENDAR_FOLDER=jskult/calendar DB_CONNECTION_MAX_RETRY_ATTEMPT=************ DB_CONNECTION_RETRY_INTERVAL_INIT=************ DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=************ diff --git a/ecs/jskult-batch-laundering/Pipfile b/ecs/jskult-batch-laundering/Pipfile index fe0fdc38..65b30a51 100644 --- a/ecs/jskult-batch-laundering/Pipfile +++ b/ecs/jskult-batch-laundering/Pipfile @@ -4,7 +4,6 @@ verify_ssl = true name = "pypi" [packages] -boto3 = "*" sqlalchemy = "*" tenacity = "*" pymysql = "*" diff --git a/ecs/jskult-batch-laundering/Pipfile.lock b/ecs/jskult-batch-laundering/Pipfile.lock index 10d05022..0399cd6e 100644 --- a/ecs/jskult-batch-laundering/Pipfile.lock +++ b/ecs/jskult-batch-laundering/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a2be870e254760b62220c10400b05fa66d24b2cc1bcd6f21044735e320a62e53" + "sha256": "e2e2efd7ebd6ad719931983f277105dd94c4ebb6363f7c00e75dd8ca05523713" }, "pipfile-spec": 6, "requires": { @@ -16,22 +16,6 @@ ] }, "default": { - "boto3": { - "hashes": [ - "sha256:908f9c277325d68963dfcfce963a05336f0eb19505fc239c0ab9d01f4cba0296", - "sha256:e1e535e9fb23977252f13652ed2fa9b4f2d59a53b04a5f2fad3ee415b6a3b2b0" - ], - "index": "pypi", - "version": "==1.27.0" - }, - "botocore": { - "hashes": [ - "sha256:b9cb5b78a289f0615a48d85066f01869029aa41b95993f2c0c55003df037c23f", - "sha256:cac1333f41ec98e6f75bbba3f2c74b9e76aa3847469ecea6e7773a0af0049bee" - ], - "markers": "python_version >= '3.7'", - "version": "==1.30.0" - }, "greenlet": { "hashes": [ "sha256:0a9dfcadc1d79696e90ccb1275c30ad4ec5fd3d1ab3ae6671286fac78ef33435", @@ -97,14 +81,6 @@ "markers": "platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", "version": "==3.0.0a1" }, - "jmespath": { - "hashes": [ - "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", - "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" - ], - "markers": "python_version >= '3.7'", - "version": "==1.0.1" - }, "pymysql": { "hashes": [ "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96", @@ -113,76 +89,52 @@ "index": "pypi", "version": "==1.1.0" }, - "python-dateutil": { - "hashes": [ - "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", - "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.8.2" - }, - "s3transfer": { - "hashes": [ - "sha256:3c0da2d074bf35d6870ef157158641178a4204a6e689e82546083e31e0311346", - "sha256:640bb492711f4c0c0905e1f62b6aaeb771881935ad27884852411f8e9cacbca9" - ], - "markers": "python_version >= '3.7'", - "version": "==0.6.1" - }, - "six": { - "hashes": [ - "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", - "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==1.16.0" - }, "sqlalchemy": { "hashes": [ - "sha256:04383f1e3452f6739084184e427e9d5cb4e68ddc765d52157bf5ef30d5eca14f", - "sha256:125f9f7e62ddf8b590c069729080ffe18b68a20d9882eb0947f72e06274601d7", - "sha256:1822620c89779b85f7c23d535c8e04b79c517739ae07aaed48c81e591ed5498e", - "sha256:21583808d37f126a647652c90332ac1d3a102edf3c94bcc3319edcc0ea2300cc", - "sha256:218fb20c01e95004f50a3062bf4c447dcb360cab8274232f31947e254f118298", - "sha256:2269b1f9b8be47e52b70936069a25a3771eff53367aa5cc59bb94f28a6412e13", - "sha256:234678ed6576531b8e4be255b980f20368bf07241a2e67b84e6b0fe679edb9c4", - "sha256:28da17059ecde53e2d10ba813d38db942b9f6344360b2958b25872d5cb729d35", - "sha256:2c6ff5767d954f6091113fedcaaf49cdec2197ae4c5301fe83d5ae4393c82f33", - "sha256:36a87e26fe8fa8c466fae461a8fcb780d0a1cbf8206900759fc6fe874475a3ce", - "sha256:394ac3adf3676fad76d4b8fcecddf747627f17f0738dc94bac15f303d05b03d4", - "sha256:40a3dc52b2b16f08b5c16b9ee7646329e4b3411e9280e5e8d57b19eaa51cbef4", - "sha256:48111d56afea5699bab72c38ec95561796b81befff9e13d1dd5ce251ab25f51d", - "sha256:48b40dc2895841ea89d89df9eb3ac69e2950a659db20a369acf4259f68e6dc1f", - "sha256:513411d73503a6fc5804f01fae3b3d44f267c1b3a06cfeac02e9286a7330e857", - "sha256:51736cfb607cf4e8fafb693906f9bc4e5ee55be0b096d44bd7f20cd8489b8571", - "sha256:5f40e3a7d0a464f1c8593f2991e5520b2f5b26da24e88000bbd4423f86103d4f", - "sha256:6150560fcffc6aee5ec9a97419ac768c7a9f56baf7a7eb59cb4b1b6a4d463ad9", - "sha256:724355973297bbe547f3eb98b46ade65a67a3d5a6303f17ab59a2dc6fb938943", - "sha256:74ddcafb6488f382854a7da851c404c394be3729bb3d91b02ad86c5458140eff", - "sha256:7830e01b02d440c27f2a5be68296e74ccb55e6a5b5962ffafd360b98930b2e5e", - "sha256:7f31d4e7ca1dd8ca5a27fd5eaa0f9e2732fe769ff7dd35bf7bba179597e4df07", - "sha256:8741d3d401383e54b2aada37cbd10f55c5d444b360eae3a82f74a2be568a7710", - "sha256:910d45bf3673f0e4ef13858674bd23cfdafdc8368b45b948bf511797dbbb401d", - "sha256:aa995b21f853864996e4056d9fde479bcecf8b7bff4beb3555eebbbba815f35d", - "sha256:af7e2ba75bf84b64adb331918188dda634689a2abb151bc1a583e488363fd2f8", - "sha256:b0eaf82cc844f6b46defe15ad243ea00d1e39ed3859df61130c263dc7204da6e", - "sha256:b114a16bc03dfe20b625062e456affd7b9938286e05a3f904a025b9aacc29dd4", - "sha256:b47be4c6281a86670ea5cfbbbe6c3a65366a8742f5bc8b986f790533c60b5ddb", - "sha256:ba03518e64d86f000dc24ab3d3a1aa876bcbaa8aa15662ac2df5e81537fa3394", - "sha256:cc9c2630c423ac4973492821b2969f5fe99d9736f3025da670095668fbfcd4d5", - "sha256:cf07ff9920cb3ca9d73525dfd4f36ddf9e1a83734ea8b4f724edfd9a2c6e82d9", - "sha256:cf175d26f6787cce30fe6c04303ca0aeeb0ad40eeb22e3391f24b32ec432a1e1", - "sha256:d0aeb3afaa19f187a70fa592fbe3c20a056b57662691fd3abf60f016aa5c1848", - "sha256:e186e9e95fb5d993b075c33fe4f38a22105f7ce11cecb5c17b5618181e356702", - "sha256:e2d5c3596254cf1a96474b98e7ce20041c74c008b0f101c1cb4f8261cb77c6d3", - "sha256:e3189432db2f5753b4fde1aa90a61c69976f4e7e31d1cf4611bfe3514ed07478", - "sha256:e3a6b2788f193756076061626679c5c5a6d600ddf8324f986bc72004c3e9d92e", - "sha256:ead58cae2a089eee1b0569060999cb5f2b2462109498a0937cc230a7556945a1", - "sha256:f2f389f77c68dc22cb51f026619291c4a38aeb4b7ecb5f998fd145b2d81ca513", - "sha256:f593170fc09c5abb1205a738290b39532f7380094dc151805009a07ae0e85330" + "sha256:024d2f67fb3ec697555e48caeb7147cfe2c08065a4f1a52d93c3d44fc8e6ad1c", + "sha256:0bf0fd65b50a330261ec7fe3d091dfc1c577483c96a9fa1e4323e932961aa1b5", + "sha256:16a310f5bc75a5b2ce7cb656d0e76eb13440b8354f927ff15cbaddd2523ee2d1", + "sha256:1d90ccc15ba1baa345796a8fb1965223ca7ded2d235ccbef80a47b85cea2d71a", + "sha256:22bafb1da60c24514c141a7ff852b52f9f573fb933b1e6b5263f0daa28ce6db9", + "sha256:2c69ce70047b801d2aba3e5ff3cba32014558966109fecab0c39d16c18510f15", + "sha256:2e7b69d9ced4b53310a87117824b23c509c6fc1f692aa7272d47561347e133b6", + "sha256:314145c1389b021a9ad5aa3a18bac6f5d939f9087d7fc5443be28cba19d2c972", + "sha256:3afa8a21a9046917b3a12ffe016ba7ebe7a55a6fc0c7d950beb303c735c3c3ad", + "sha256:430614f18443b58ceb9dedec323ecddc0abb2b34e79d03503b5a7579cd73a531", + "sha256:43699eb3f80920cc39a380c159ae21c8a8924fe071bccb68fc509e099420b148", + "sha256:539010665c90e60c4a1650afe4ab49ca100c74e6aef882466f1de6471d414be7", + "sha256:57d100a421d9ab4874f51285c059003292433c648df6abe6c9c904e5bd5b0828", + "sha256:5831138f0cc06b43edf5f99541c64adf0ab0d41f9a4471fd63b54ae18399e4de", + "sha256:584f66e5e1979a7a00f4935015840be627e31ca29ad13f49a6e51e97a3fb8cae", + "sha256:5d6afc41ca0ecf373366fd8e10aee2797128d3ae45eb8467b19da4899bcd1ee0", + "sha256:61ada5831db36d897e28eb95f0f81814525e0d7927fb51145526c4e63174920b", + "sha256:6b54d1ad7a162857bb7c8ef689049c7cd9eae2f38864fc096d62ae10bc100c7d", + "sha256:7351c05db355da112e056a7b731253cbeffab9dfdb3be1e895368513c7d70106", + "sha256:77a14fa20264af73ddcdb1e2b9c5a829b8cc6b8304d0f093271980e36c200a3f", + "sha256:851a37898a8a39783aab603c7348eb5b20d83c76a14766a43f56e6ad422d1ec8", + "sha256:89bc2b374ebee1a02fd2eae6fd0570b5ad897ee514e0f84c5c137c942772aa0c", + "sha256:8e712cfd2e07b801bc6b60fdf64853bc2bd0af33ca8fa46166a23fe11ce0dbb0", + "sha256:8f9eb4575bfa5afc4b066528302bf12083da3175f71b64a43a7c0badda2be365", + "sha256:8fc05b59142445a4efb9c1fd75c334b431d35c304b0e33f4fa0ff1ea4890f92e", + "sha256:96f0463573469579d32ad0c91929548d78314ef95c210a8115346271beeeaaa2", + "sha256:9deaae357edc2091a9ed5d25e9ee8bba98bcfae454b3911adeaf159c2e9ca9e3", + "sha256:a752b7a9aceb0ba173955d4f780c64ee15a1a991f1c52d307d6215c6c73b3a4c", + "sha256:ae7473a67cd82a41decfea58c0eac581209a0aa30f8bc9190926fbf628bb17f7", + "sha256:b15afbf5aa76f2241184c1d3b61af1a72ba31ce4161013d7cb5c4c2fca04fd6e", + "sha256:c896d4e6ab2eba2afa1d56be3d0b936c56d4666e789bfc59d6ae76e9fcf46145", + "sha256:cb4e688f6784427e5f9479d1a13617f573de8f7d4aa713ba82813bcd16e259d1", + "sha256:cda283700c984e699e8ef0fcc5c61f00c9d14b6f65a4f2767c97242513fcdd84", + "sha256:cf7b5e3856cbf1876da4e9d9715546fa26b6e0ba1a682d5ed2fc3ca4c7c3ec5b", + "sha256:d6894708eeb81f6d8193e996257223b6bb4041cb05a17cd5cf373ed836ef87a2", + "sha256:d8f2afd1aafded7362b397581772c670f20ea84d0a780b93a1a1529da7c3d369", + "sha256:dd4d410a76c3762511ae075d50f379ae09551d92525aa5bb307f8343bf7c2c12", + "sha256:eb60699de43ba1a1f77363f563bb2c652f7748127ba3a774f7cf2c7804aa0d3d", + "sha256:f469f15068cd8351826df4080ffe4cc6377c5bf7d29b5a07b0e717dddb4c7ea2", + "sha256:f82c310ddf97b04e1392c33cf9a70909e0ae10a7e2ddc1d64495e3abdc5d19fb", + "sha256:fa51ce4aea583b0c6b426f4b0563d3535c1c75986c4373a0987d84d22376585b" ], "index": "pypi", - "version": "==2.0.17" + "version": "==2.0.19" }, "tenacity": { "hashes": [ @@ -199,14 +151,6 @@ ], "markers": "python_version >= '3.7'", "version": "==4.7.1" - }, - "urllib3": { - "hashes": [ - "sha256:8d36afa7616d8ab714608411b4a3b13e58f463aee519024578e062e141dce20f", - "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.26.16" } }, "develop": { diff --git a/ecs/jskult-batch-laundering/README.md b/ecs/jskult-batch-laundering/README.md index f6e7c2da..b520a3ca 100644 --- a/ecs/jskult-batch-laundering/README.md +++ b/ecs/jskult-batch-laundering/README.md @@ -1,8 +1,8 @@ -# 実消化&アルトマーク 月次バッチ +# 実績全件洗替処理 ## 概要 -実消化&アルトマークの月次バッチ処理。 +実績全件洗替処理の週次バッチ処理。 ## 環境情報 @@ -42,7 +42,32 @@ - VSCode 上で「F5」キーを押下すると、バッチ処理が起動する。 - 「entrypoint.py」が、バッチ処理のエントリーポイント。 -- 実際の処理は、「src/jobctrl_monthly.py」で行っている。 +- 実際の処理は、「src/jobctrl_laundering.py」で行っている。 -## フォルダ構成（工事中） +## フォルダ構成 +├── .env.example -- ローカル実行用の環境変数のサンプル値。 +├── .dockerignore -- docker build時のコンテキストに含めるファイルの抑制リスト +├── .gitignore -- Git差分管理除外リスト +├── Dockerfile -- Dockerイメージを作成するためのファイル +├── Pipfile -- Pythonモジュールの依存関係を管理するファイル +├── Pipfile.lock -- Pythonモジュールの依存関係バージョン固定用ファイル +├── README.md -- 当ファイル +├── entrypoint.py -- バッチ処理のエントリーポイントになるpythonファイル +└── src -- ソースコードの保管場所 + ├── batch -- バッチ処理関連ソース置き場 + │ ├── batch_functions.py -- バッチ処理共通関数置き場 + │ └── laundering -- 実績全件洗替関連処理 + │ ├── create_inst_merge_for_laundering.py -- 洗替用マスタ作成処理 + │ └── sales_results_laundering.py -- 卸販売実績全件洗替処理 + ├── db + │ └── database.py -- データベース操作共通処理 + ├── error + │ └── exceptions.py -- カスタム例外 + ├── jobctrl_laundering.py -- 実績全件洗替処理のエントリーポイント。「entrypoint.py」 から呼ばれる。 + ├── logging + │ └── get_logger.py -- ログ出力の共通処理 + └── system_var + ├── constants.py -- 定数ファイル + └── environment.py -- 環境変数ファイル + diff --git a/ecs/jskult-batch-laundering/entrypoint.py b/ecs/jskult-batch-laundering/entrypoint.py index 624e61da..a17fe8a8 100644 --- a/ecs/jskult-batch-laundering/entrypoint.py +++ b/ecs/jskult-batch-laundering/entrypoint.py @@ -1,4 +1,4 @@ -"""実消化＆アルトマーク実績洗替処理のエントリーポイント""" +"""実績全件洗替処理のエントリーポイント""" from src import jobctrl_laundering if __name__ == '__main__': diff --git a/ecs/jskult-batch-laundering/src/aws/__init__.py b/ecs/jskult-batch-laundering/src/aws/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/ecs/jskult-batch-laundering/src/aws/s3.py b/ecs/jskult-batch-laundering/src/aws/s3.py deleted file mode 100644 index 17ed0fe9..00000000 --- a/ecs/jskult-batch-laundering/src/aws/s3.py +++ /dev/null @@ -1,95 +0,0 @@ -import os.path as path -import tempfile - -import boto3 - -from src.system_var import environment - - -class S3Client: - __s3_client = boto3.client('s3') - _bucket_name: str - - def list_objects(self, bucket_name: str, folder_name: str): - response = self.__s3_client.list_objects_v2(Bucket=bucket_name, Prefix=folder_name) - if response['KeyCount'] == 0: - return [] - contents = response['Contents'] - # 末尾がスラッシュで終わるものはフォルダとみなしてスキップする - objects = [{'filename': content['Key'], 'size': content['Size']} for content in contents if not content['Key'].endswith('/')] - return objects - - def copy(self, src_bucket: str, src_key: str, dest_bucket: str, dest_key: str) -> None: - copy_source = {'Bucket': src_bucket, 'Key': src_key} - self.__s3_client.copy(copy_source, dest_bucket, dest_key) - return - - def download_file(self, bucket_name: str, file_key: str, file): - self.__s3_client.download_fileobj( - Bucket=bucket_name, - Key=file_key, - Fileobj=file - ) - return - - def upload_file(self, local_file_path: str, bucket_name: str, file_key: str): - self.__s3_client.upload_file( - local_file_path, - Bucket=bucket_name, - Key=file_key - ) - - def delete_file(self, bucket_name: str, file_key: str): - self.__s3_client.delete_object( - Bucket=bucket_name, - Key=file_key - ) - - -class S3Bucket(): - _s3_client = S3Client() - _bucket_name: str = None - - -class ConfigBucket(S3Bucket): - _bucket_name = environment.JSKULT_CONFIG_BUCKET - - def download_arisj_output_day_list(self): - # 一時ファイルとして保存する - temporary_dir = tempfile.mkdtemp() - temporary_file_path = path.join(temporary_dir, environment.JSKULT_CONFIG_CALENDAR_ARISJ_OUTPUT_DAY_LIST_FILE_NAME) - arisj_output_day_list_key = f'{environment.JSKULT_CONFIG_CALENDAR_FOLDER}/{environment.JSKULT_CONFIG_CALENDAR_ARISJ_OUTPUT_DAY_LIST_FILE_NAME}' - with open(temporary_file_path, mode='wb') as f: - self._s3_client.download_file(self._bucket_name, arisj_output_day_list_key, f) - f.seek(0) - return temporary_file_path - - -class ArisjBucket(S3Bucket): - _bucket_name = environment.ARISJ_DATA_BUCKET - _folder = environment.ARISJ_BACKUP_FOLDER - - def upload_arisj_csv_file(self, arisj_create_csv: str, csv_file_path: str): - # s3にCSVファイルをUPする - Bucket = environment.ARISJ_DATA_BUCKET - folder = environment.ARISJ_DATA_FOLDER - csv_file_name = f'{folder}/{arisj_create_csv}' - s3_client = S3Client() - s3_client.upload_file(csv_file_path, Bucket, csv_file_name) - return - - def backup_arisj_csv_file(self, dat_file_key: str, datetime_key: str): - # バックアップバケットにコピー - arisj_backup_bucket = ArisjBackupBucket() - folder = environment.ARISJ_DATA_FOLDER - dat_file_key = f'{folder}/{dat_file_key}' - backup_key = f'{arisj_backup_bucket._folder}/{datetime_key}/{dat_file_key.replace(f"{self._folder}/", "")}' - self._s3_client.copy(self._bucket_name, dat_file_key, arisj_backup_bucket._bucket_name, backup_key) - - -class JskUltBackupBucket(S3Bucket): - _bucket_name = environment.JSKULT_BACKUP_BUCKET - - -class ArisjBackupBucket(JskUltBackupBucket): - _folder = environment.ARISJ_BACKUP_FOLDER diff --git a/ecs/jskult-batch-laundering/src/batch/batch_functions.py b/ecs/jskult-batch-laundering/src/batch/batch_functions.py index 87231e17..3028d84b 100644 --- a/ecs/jskult-batch-laundering/src/batch/batch_functions.py +++ b/ecs/jskult-batch-laundering/src/batch/batch_functions.py @@ -4,7 +4,6 @@ import textwrap from src.db.database import Database from src.error.exceptions import BatchOperationException, DBException -from src.system_var import constants def get_batch_statuses() -> tuple[str, str]: @@ -39,75 +38,6 @@ def get_batch_statuses() -> tuple[str, str]: return batch_processing_flag, dump_status_kbn -def update_dump_status_kbn_in_processing() -> None: - """dump取得状況区分を処理中に更新する - - Raises: - BatchOperationException: DB操作の何らかのエラー - """ - db = Database.get_instance() - sql = 'UPDATE src05.hdke_tbl SET dump_sts_kbn = :in_processing' - try: - db.connect() - db.execute(sql, {'in_processing': constants.BATCH_ACTF_BATCH_IN_PROCESSING}) - except DBException as e: - raise BatchOperationException(e) - finally: - db.disconnect() - - return - - -def update_dump_status_kbn_error() -> None: - """dump取得状況区分をエラーに更新する - - Raises: - BatchOperationException: DB操作の何らかのエラー - """ - db = Database.get_instance() - sql = """\ - UPDATE src05.hdke_tbl - SET - dump_sts_kbn = :dump_unprocessed - """ - try: - db.connect() - db.execute(sql, { - 'dump_unprocessed': constants.DUMP_STATUS_KBN_ERROR - }) - except DBException as e: - raise BatchOperationException(e) - finally: - db.disconnect() - - return - - -def update_dump_status_kbn_complete() -> None: - """dump取得状況区分を正常終了に更新する - - Raises: - BatchOperationException: DB操作の何らかのエラー - """ - db = Database.get_instance() - sql = """\ - UPDATE src05.hdke_tbl - SET - dump_sts_kbn = :dump_unprocessed - """ - try: - db.connect() - db.execute(sql, { - 'dump_unprocessed': constants.DUMP_STATUS_KBN_COMPLETE - }) - except DBException as e: - raise BatchOperationException(e) - finally: - db.disconnect() - - return - - def logging_sql(logger: logging.Logger, sql: str) -> None: """SQL文をデバッグログで出力する diff --git a/ecs/jskult-batch-laundering/src/batch/common/__init__.py b/ecs/jskult-batch-laundering/src/batch/common/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/ecs/jskult-batch-laundering/src/batch/common/batch_context.py b/ecs/jskult-batch-laundering/src/batch/common/batch_context.py deleted file mode 100644 index 8c8c12fb..00000000 --- a/ecs/jskult-batch-laundering/src/batch/common/batch_context.py +++ /dev/null @@ -1,29 +0,0 @@ -class BatchContext: - __instance = None - __syor_date: str # 処理日(yyyy/mm/dd形式) - __is_arisj_output_day: bool # 月次バッチ起動日フラグ - - def __init__(self) -> None: - self.__is_arisj_output_day = False - - @classmethod - def get_instance(cls): - if cls.__instance is None: - cls.__instance = cls() - return cls.__instance - - @property - def syor_date(self): - return self.__syor_date - - @syor_date.setter - def syor_date(self, syor_date_str: str): - self.__syor_date = syor_date_str - - @property - def is_arisj_output_day(self): - return self.__is_arisj_output_day - - @is_arisj_output_day.setter - def is_arisj_output_day(self, flag: bool): - self.__is_arisj_output_day = flag diff --git a/ecs/jskult-batch-laundering/src/jobctrl_laundering.py b/ecs/jskult-batch-laundering/src/jobctrl_laundering.py index c0735162..e8b05870 100644 --- a/ecs/jskult-batch-laundering/src/jobctrl_laundering.py +++ b/ecs/jskult-batch-laundering/src/jobctrl_laundering.py @@ -1,4 +1,4 @@ -"""実消化＆アルトマーク実績洗替処理""" +"""実消化＆アルトマーク実績全件洗替処理""" from src.batch.batch_functions import get_batch_statuses from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger @@ -6,7 +6,7 @@ from src.system_var import constants from src.batch.laundering import (sales_results_laundering, create_inst_merge_for_laundering) -logger = get_logger('実消化＆アルトマーク実績洗替処理') +logger = get_logger('実消化＆アルトマーク実績全件洗替処理') def exec(): @@ -21,12 +21,12 @@ def exec(): # 日次バッチ処理中の場合、後続の処理は行わない if batch_processing_flag == constants.BATCH_ACTF_BATCH_IN_PROCESSING: - logger.warning('日次バッチ処理中のため、実消化＆アルトマーク実績洗替処理を終了します。') + logger.warning('日次バッチ処理中のため、実消化＆アルトマーク実績全件洗替処理を終了します。') return constants.BATCH_EXIT_CODE_SUCCESS # dump取得が正常終了していない場合、後続の処理は行わない if dump_status_kbn != constants.DUMP_STATUS_KBN_UNPROCESSED: - logger.warning('dump取得が正常終了していないため、実消化＆アルトマーク実績洗替処理を終了します。') + logger.warning('dump取得が正常終了していないため、実消化＆アルトマーク実績全件洗替処理を終了します。') return constants.BATCH_EXIT_CODE_SUCCESS # 洗替用マスタ作成 @@ -35,5 +35,5 @@ def exec(): sales_results_laundering.exec() logger.info('週次バッチ：終了') except Exception as e: - logger.exception(f'実消化＆アルトマーク実績洗替処理中に想定外のエラーが発生しました {e}') + logger.exception(f'実消化＆アルトマーク実績全件洗替処理中に想定外のエラーが発生しました {e}') return constants.BATCH_EXIT_CODE_SUCCESS diff --git a/ecs/jskult-batch-laundering/src/system_var/environment.py b/ecs/jskult-batch-laundering/src/system_var/environment.py index 6755de11..0e08f06b 100644 --- a/ecs/jskult-batch-laundering/src/system_var/environment.py +++ b/ecs/jskult-batch-laundering/src/system_var/environment.py @@ -7,11 +7,6 @@ DB_USERNAME = os.environ['DB_USERNAME'] DB_PASSWORD = os.environ['DB_PASSWORD'] DB_SCHEMA = os.environ['DB_SCHEMA'] -# AWS -JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] -JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] -JSKULT_CONFIG_CALENDAR_FOLDER = os.environ['JSKULT_CONFIG_CALENDAR_FOLDER'] - # 初期値がある環境変数 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') DB_CONNECTION_MAX_RETRY_ATTEMPT = int(os.environ.get('DB_CONNECTION_MAX_RETRY_ATTEMPT', 4)) diff --git a/ecs/jskult-batch-laundering/src/time/elapsed_time.py b/ecs/jskult-batch-laundering/src/time/elapsed_time.py deleted file mode 100644 index c1432e91..00000000 --- a/ecs/jskult-batch-laundering/src/time/elapsed_time.py +++ /dev/null @@ -1,22 +0,0 @@ -import time - - -class ElapsedTime: - """処理実行時間計測クラス""" - def __init__(self) -> None: - """このクラスの新たなインスタンスを初期化します。""" - self.__start = time.perf_counter() - - @property - def of(self): - """インスタンス化してからの経過時間をhh:mm:ssの形式にフォーマットして返す - Returns: - str: 時分秒形式の経過時間 - """ - elapsed_time = time.perf_counter() - self.__start - h, rem = divmod(elapsed_time, 3600) - m, s = divmod(rem, 60) - h_str = f'{h:02.0f} hour ' if h > 0.0 else '' - m_str = f'{m:02.0f} min ' if m > 0.0 else '' - s_str = f'{s:06.02f} sec' if s > 0.0 else '' - return f"{h_str}{m_str}{s_str}" From 08da261cff1f59a03104948a114eb4fd5db9319a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E6=9C=A8=E8=A6=81?= Date: Fri, 28 Jul 2023 16:56:52 +0900 Subject: [PATCH 4/5] =?UTF-8?q?feat:=20=E3=83=AC=E3=83=93=E3=83=A5?= =?UTF-8?q?=E3=83=BC=E6=8C=87=E6=91=98=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-laundering/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/jskult-batch-laundering/README.md b/ecs/jskult-batch-laundering/README.md index b520a3ca..723e97fb 100644 --- a/ecs/jskult-batch-laundering/README.md +++ b/ecs/jskult-batch-laundering/README.md @@ -1,4 +1,4 @@ -# 実績全件洗替処理 +# 実消化＆アルトマーク実績全件洗替処理 ## 概要 From 09b87ecfd53c3215ef25d96805f931c1f056fecd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E6=9C=A8=E8=A6=81?= Date: Mon, 31 Jul 2023 14:47:40 +0900 Subject: [PATCH 5/5] =?UTF-8?q?feat:=20=E5=8D=98=E4=BD=93=E8=A9=A6?= =?UTF-8?q?=E9=A8=93=E3=83=AC=E3=83=93=E3=83=A5=E3=83=BC=E5=AF=BE=E5=BF=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-laundering/src/jobctrl_laundering.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/jskult-batch-laundering/src/jobctrl_laundering.py b/ecs/jskult-batch-laundering/src/jobctrl_laundering.py index e8b05870..b06589e4 100644 --- a/ecs/jskult-batch-laundering/src/jobctrl_laundering.py +++ b/ecs/jskult-batch-laundering/src/jobctrl_laundering.py @@ -6,7 +6,7 @@ from src.system_var import constants from src.batch.laundering import (sales_results_laundering, create_inst_merge_for_laundering) -logger = get_logger('実消化＆アルトマーク実績全件洗替処理') +logger = get_logger('卸販売実績全件洗替') def exec():