From 8caa5ab14d35af49cc900968251bbff6920e400e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Mon, 10 Jul 2023 11:54:06 +0900 Subject: [PATCH 01/16] =?UTF-8?q?=E4=BB=AE=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/.dockerignore | 12 + ecs/jskult-dbdump/.env.example | 16 ++ ecs/jskult-dbdump/.gitignore | 10 + ecs/jskult-dbdump/.vscode/launch.json | 17 ++ .../.vscode/recommended_settings.json | 31 +++ ecs/jskult-dbdump/Dockerfile | 28 +++ ecs/jskult-dbdump/Pipfile | 19 ++ ecs/jskult-dbdump/Pipfile.lock | 206 ++++++++++++++++++ ecs/jskult-dbdump/README.md | 48 ++++ .../backup_rds_src05_20230710091352.gz | 0 ecs/jskult-dbdump/entrypoint.py | 10 + ecs/jskult-dbdump/my.cnf | 4 + ecs/jskult-dbdump/src/__init__.py | 0 .../src/batch/batch_functions.py | 118 ++++++++++ .../src/batch/common/__init__.py | 0 .../src/batch/common/batch_context.py | 29 +++ ecs/jskult-dbdump/src/db/__init__.py | 0 ecs/jskult-dbdump/src/db/database.py | 178 +++++++++++++++ ecs/jskult-dbdump/src/error/__init__.py | 0 ecs/jskult-dbdump/src/error/exceptions.py | 10 + ecs/jskult-dbdump/src/jobctrl_dbdump.py | 109 +++++++++ ecs/jskult-dbdump/src/logging/get_logger.py | 37 ++++ ecs/jskult-dbdump/src/system_var/__init__.py | 0 ecs/jskult-dbdump/src/system_var/constants.py | 15 ++ .../src/system_var/environment.py | 21 ++ 25 files changed, 918 insertions(+) create mode 100644 ecs/jskult-dbdump/.dockerignore create mode 100644 ecs/jskult-dbdump/.env.example create mode 100644 ecs/jskult-dbdump/.gitignore create mode 100644 ecs/jskult-dbdump/.vscode/launch.json create mode 100644 ecs/jskult-dbdump/.vscode/recommended_settings.json create mode 100644 ecs/jskult-dbdump/Dockerfile create mode 100644 ecs/jskult-dbdump/Pipfile create mode 100644 ecs/jskult-dbdump/Pipfile.lock create mode 100644 ecs/jskult-dbdump/README.md create mode 100644 ecs/jskult-dbdump/backup_rds_src05_20230710091352.gz create mode 100644 ecs/jskult-dbdump/entrypoint.py create mode 100644 ecs/jskult-dbdump/my.cnf create mode 100644 ecs/jskult-dbdump/src/__init__.py create mode 100644 ecs/jskult-dbdump/src/batch/batch_functions.py create mode 100644 ecs/jskult-dbdump/src/batch/common/__init__.py create mode 100644 ecs/jskult-dbdump/src/batch/common/batch_context.py create mode 100644 ecs/jskult-dbdump/src/db/__init__.py create mode 100644 ecs/jskult-dbdump/src/db/database.py create mode 100644 ecs/jskult-dbdump/src/error/__init__.py create mode 100644 ecs/jskult-dbdump/src/error/exceptions.py create mode 100644 ecs/jskult-dbdump/src/jobctrl_dbdump.py create mode 100644 ecs/jskult-dbdump/src/logging/get_logger.py create mode 100644 ecs/jskult-dbdump/src/system_var/__init__.py create mode 100644 ecs/jskult-dbdump/src/system_var/constants.py create mode 100644 ecs/jskult-dbdump/src/system_var/environment.py diff --git a/ecs/jskult-dbdump/.dockerignore b/ecs/jskult-dbdump/.dockerignore new file mode 100644 index 00000000..8b9da402 --- /dev/null +++ b/ecs/jskult-dbdump/.dockerignore @@ -0,0 +1,12 @@ +tests/* +.coverage +.env +.env.example +.report/* +.vscode/* +.pytest_cache/* +*/__pychache__/* +Dockerfile +pytest.ini +README.md +*.sql diff --git a/ecs/jskult-dbdump/.env.example b/ecs/jskult-dbdump/.env.example new file mode 100644 index 00000000..b331b25e --- /dev/null +++ b/ecs/jskult-dbdump/.env.example @@ -0,0 +1,16 @@ +DB_HOST=************ +DB_PORT=3306 +DB_USERNAME=************ +DB_PASSWORD=************ +DB_SCHEMA=src05 + +JSKULT_BACKUP_BUCKET=mbj-newdwh2021-staging-backup-jskult +JSKULT_CONFIG_BUCKET=mbj-newdwh2021-staging-config + +DUMP_BACKUP_FOLDER=dump + +LOG_LEVEL=INFO +DB_CONNECTION_MAX_RETRY_ATTEMPT=************ +DB_CONNECTION_RETRY_INTERVAL_INIT=************ +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=************ +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=************ diff --git a/ecs/jskult-dbdump/.gitignore b/ecs/jskult-dbdump/.gitignore new file mode 100644 index 00000000..bd0b37f8 --- /dev/null +++ b/ecs/jskult-dbdump/.gitignore @@ -0,0 +1,10 @@ +.vscode/settings.json +.env + +# python +__pycache__ + +# python test +.pytest_cache +.coverage +.report/ \ No newline at end of file diff --git a/ecs/jskult-dbdump/.vscode/launch.json b/ecs/jskult-dbdump/.vscode/launch.json new file mode 100644 index 00000000..a0178c26 --- /dev/null +++ b/ecs/jskult-dbdump/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // IntelliSense を使用して利用可能な属性を学べます。 + // 既存の属性の説明をホバーして表示します。 + // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "(DEBUG)jskult batch dbdump", + "type": "python", + "request": "launch", + "program": "entrypoint.py", + "console": "integratedTerminal", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/ecs/jskult-dbdump/.vscode/recommended_settings.json b/ecs/jskult-dbdump/.vscode/recommended_settings.json new file mode 100644 index 00000000..b5e79d73 --- /dev/null +++ b/ecs/jskult-dbdump/.vscode/recommended_settings.json @@ -0,0 +1,31 @@ +{ + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + // 自身の環境に合わせて変えてください + "python.defaultInterpreterPath": "", + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": [ + "--max-line-length=200", + "--ignore=F541" + ], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Path": "autopep8", + "python.formatting.autopep8Args": [ + "--max-line-length", "200", + "--ignore=F541" + ], + "python.testing.pytestArgs": [ + "tests/batch/ultmarc" + ], + + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} diff --git a/ecs/jskult-dbdump/Dockerfile b/ecs/jskult-dbdump/Dockerfile new file mode 100644 index 00000000..f87a7215 --- /dev/null +++ b/ecs/jskult-dbdump/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.9 + +ENV TZ="Asia/Tokyo" + +WORKDIR /usr/src/app +COPY Pipfile Pipfile.lock ./ +RUN \ + apt update -y && \ + # パッケージのセキュリティアップデートのみを適用するコマンド + apt install -y unattended-upgrades && \ + unattended-upgrades && \ + pip install --upgrade pip wheel setuptools && \ + pip install pipenv --no-cache-dir && \ + pipenv install --system --deploy && \ + pip uninstall -y pipenv virtualenv-clone virtualenv + +# mysql-clientと必要なパッケージインストール +RUN apt install -y less vim curl unzip sudo default-mysql-client + +# aws cli v2 のインストール +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +RUN unzip awscliv2.zip +RUN sudo ./aws/install + +COPY src ./src +COPY entrypoint.py entrypoint.py + +CMD ["python", "entrypoint.py"] diff --git a/ecs/jskult-dbdump/Pipfile b/ecs/jskult-dbdump/Pipfile new file mode 100644 index 00000000..65b30a51 --- /dev/null +++ b/ecs/jskult-dbdump/Pipfile @@ -0,0 +1,19 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +sqlalchemy = "*" +tenacity = "*" +pymysql = "*" + +[dev-packages] +autopep8 = "*" +flake8 = "*" + +[requires] +python_version = "3.9" + +[pipenv] +allow_prereleases = true diff --git a/ecs/jskult-dbdump/Pipfile.lock b/ecs/jskult-dbdump/Pipfile.lock new file mode 100644 index 00000000..58713cc6 --- /dev/null +++ b/ecs/jskult-dbdump/Pipfile.lock @@ -0,0 +1,206 @@ +{ + "_meta": { + "hash": { + "sha256": "e2e2efd7ebd6ad719931983f277105dd94c4ebb6363f7c00e75dd8ca05523713" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "greenlet": { + "hashes": [ + "sha256:0a9dfcadc1d79696e90ccb1275c30ad4ec5fd3d1ab3ae6671286fac78ef33435", + "sha256:0f313771cb8ee0a04dfdf586b7d4076180d80c94be09049daeea018089b5b957", + "sha256:17503397bf6cbb5e364217143b6150c540020c51a3f6b08f9a20cd67c25e2ca8", + "sha256:180ec55cb127bc745669eddc9793ffab6e0cf7311e67e1592f183d6ca00d88c1", + "sha256:1b3f3568478bc21b85968e8038c4f98f4bf0039a692791bc324b5e0d1522f4b1", + "sha256:1bd4ea36f0aeb14ca335e0c9594a5aaefa1ac4e2db7d86ba38f0be96166b3102", + "sha256:21ebcb570e0d8501457d6a2695a44c5af3b6c2143dc6644ec73574beba067c90", + "sha256:24071eee113d75fedebaeb86264d94f04b5a24e311c5ba3e8003c07d00112a7e", + "sha256:270432cfdd6a50016b8259b3bbf398a3f7c06a06f2c68c7b93e49f53bc193bcf", + "sha256:271ed380389d2f7e4c1545b6e0837986e62504ab561edbaff05da9c9f3f98f96", + "sha256:2840187a94e258445e62ff1545e34f0b1a14aef4d0078e5c88246688d2b6515e", + "sha256:2cda110faee67613fed221f90467003f477088ef1cc84c8fc88537785a5b4de9", + "sha256:2e160a65cc6023a237be870f2072513747d512a1d018efa083acce0b673cccc0", + "sha256:2fcf7af83516db35af3d0ed5d182dea8585eddd891977adff1b74212f4bfd2fd", + "sha256:36cebce1f30964d5672fd956860e7e7b69772da69658d5743cb676b442eeff36", + "sha256:42bfe67824a9b53e73f568f982f0d1d4c7ac0f587d2e702a23f8a7b505d7b7c2", + "sha256:450a7e52a515402fd110ba807f1a7d464424bfa703be4effbcb97e1dfbfcc621", + "sha256:463d63ca5d8c236788284a9a44b9715372a64d5318a6b5eee36815df1ea0ba3d", + "sha256:4d0c0ffd732466ff324ced144fad55ed5deca36f6036c1d8f04cec69b084c9d6", + "sha256:4ff2a765f4861fc018827eab4df1992f7508d06c62de5d2fe8a6ac2233d4f1d0", + "sha256:53abf19b7dc62795c67b8d0a3d8ef866db166b21017632fff2624cf8fbf3481c", + "sha256:5552d7be37d878e9b6359bbffa0512d857bb9703616a4c0656b49c10739d5971", + "sha256:585810056a8adacd3152945ebfcd25deb58335d41f16ae4e0f3d768918957f9a", + "sha256:5942b1d6ba447cff1ec23a21ec525dde2288f00464950bc647f4e0f03bd537d1", + "sha256:5c355c99be5bb23e85d899b059a4f22fdf8a0741c57e7029425ee63eb436f689", + "sha256:5f61df4fe07864561f49b45c8bd4d2c42e3f03d2872ed05c844902a58b875028", + "sha256:665942d3a954c3e4c976581715f57fb3b86f4cf6bae3ac30b133f8ff777ac6c7", + "sha256:68368e908f14887fb202a81960bfbe3a02d97e6d3fa62b821556463084ffb131", + "sha256:6aac94ff957b5dea0216af71ab59c602e1b947b394e4f5e878a5a65643090038", + "sha256:889934aa8d72b6bfc46babd1dc4b817a56c97ec0f4a10ae7551fb60ab1f96fae", + "sha256:a00550757fca1b9cbc479f8eb1cf3514dbc0103b3f76eae46341c26ddcca67a9", + "sha256:a4a2d6ed0515c05afd5cc435361ced0baabd9ba4536ddfe8ad9a95bcb702c8ce", + "sha256:a8dd92fd76a61af2abc8ccad0c6c6069b3c4ebd4727ecc9a7c33aae37651c8c7", + "sha256:ab81f9ff3e3c2ca65e824454214c10985a846cd9bee5f4d04e15cd875d9fe13b", + "sha256:ac10196b8cde7a082e4e371ff171407270d3337c8d57ed43030094eb01d9c95c", + "sha256:b767930af686551dc96a5eb70af3736709d547ffa275c11a5e820bfb3ae61d8d", + "sha256:b9a1f4d256b81f59ba87bb7a29b9b38b1c018e052dba60a543cb0ddb5062d159", + "sha256:ba94c08321b5d345100fc64eb1ab235f42faf9aabba805cface55ebe677f1c2c", + "sha256:bab71f73001cd15723c4e2ca398f2f48e0a3f584c619eefddb1525e8986e06eb", + "sha256:bce5cf2b0f0b29680396c5c98ab39a011bd70f2dfa8b8a6811a69ee6d920cf9f", + "sha256:c02e514c72e745e49a3ae7e672a1018ba9b68460c21e0361054e956e5d595bc6", + "sha256:c3fb459ced6c5e3b2a895f23f1400f93e9b24d85c30fbe2d637d4f7706a1116b", + "sha256:cd31ab223e43ac64fd23f8f5dad249addadac2a459f040546200acbf7e84e353", + "sha256:ce70aa089ec589b5d5fab388af9f8c9f9dfe8fe4ad844820a92eb240d8628ddf", + "sha256:d47b2e1ad1429da9aa459ef189fbcd8a74ec28a16bc4c3f5f3cf3f88e36535eb", + "sha256:d61bad421c1f496f9fb6114dbd7c30a1dac0e9ff90e9be06f4472cbd8f7a1704", + "sha256:d7ba2e5cb119eddbc10874b41047ad99525e39e397f7aef500e6da0d6f46ab91", + "sha256:dde0ab052c7a1deee8d13d72c37f2afecee30ebdf6eb139790157eaddf04dd61", + "sha256:df34b52aa50a38d7a79f3abc9fda7e400791447aa0400ed895f275f6d8b0bb1f", + "sha256:e0fc20e6e6b298861035a5fc5dcf9fbaa0546318e8bda81112591861a7dcc28f", + "sha256:e20d5e8dc76b73db9280464d6e81bea05e51a99f4d4dd29c5f78dc79f294a5d3", + "sha256:e31d1a33dc9006b278f72cb0aacfe397606c2693aa2fdc0c2f2dcddbad9e0b53", + "sha256:e3a99f890f2cc5535e1b3a90049c6ca9ff9da9ec251cc130c8d269997f9d32ee", + "sha256:e7b192c3df761d0fdd17c2d42d41c28460f124f5922e8bd524018f1d35610682", + "sha256:ed0f4fad4c3656e34d20323a789b6a2d210a6bb82647d9c86dded372f55c58a1", + "sha256:f34ec09702be907727fd479046193725441aaaf7ed4636ca042734f469bb7451", + "sha256:f3530c0ec1fc98c43d5b7061781a8c55bd0db44f789f8152e19d9526cbed6021", + "sha256:f5672082576d0e9f52fa0fa732ff57254d65faeb4a471bc339fe54b58b3e79d2", + "sha256:ffb9f8969789771e95d3c982a36be81f0adfaa7302a1d56e29f168ca15e284b8" + ], + "markers": "platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", + "version": "==3.0.0a1" + }, + "pymysql": { + "hashes": [ + "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96", + "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7" + ], + "index": "pypi", + "version": "==1.1.0" + }, + "sqlalchemy": { + "hashes": [ + "sha256:00aa050faf24ce5f2af643e2b86822fa1d7149649995f11bc1e769bbfbf9010b", + "sha256:09397a18733fa2a4c7680b746094f980060666ee549deafdb5e102a99ce4619b", + "sha256:0f7fdcce52cd882b559a57b484efc92e108efeeee89fab6b623aba1ac68aad2e", + "sha256:10514adc41fc8f5922728fbac13d401a1aefcf037f009e64ca3b92464e33bf0e", + "sha256:10e001a84f820fea2640e4500e12322b03afc31d8f4f6b813b44813b2a7c7e0d", + "sha256:194f2d5a7cb3739875c4d25b3fe288ab0b3dc33f7c857ba2845830c8c51170a0", + "sha256:1aac42a21a7fa6c9665392c840b295962992ddf40aecf0a88073bc5c76728117", + "sha256:1fb792051db66e09c200e7bc3bda3b1eb18a5b8eb153d2cedb2b14b56a68b8cb", + "sha256:2756485f49e7df5c2208bdc64263d19d23eba70666f14ad12d6d8278a2fff65f", + "sha256:2b791577c546b6bbd7b43953565fcb0a2fec63643ad605353dd48afbc3c48317", + "sha256:420bc6d06d4ae7fb6921524334689eebcbea7bf2005efef070a8562cc9527a37", + "sha256:45b07470571bda5ee7f5ec471271bbde97267cc8403fce05e280c36ea73f4754", + "sha256:4ebc542d2289c0b016d6945fd07a7e2e23f4abc41e731ac8ad18a9e0c2fd0ec2", + "sha256:556dc18e39b6edb76239acfd1c010e37395a54c7fde8c57481c15819a3ffb13e", + "sha256:589aba9a35869695b319ed76c6f673d896cd01a7ff78054be1596df7ad9b096f", + "sha256:5c95e3e7cc6285bf7ff263eabb0d3bfe3def9a1ff98124083d45e5ece72f4579", + "sha256:5dd574a37be388512c72fe0d7318cb8e31743a9b2699847a025e0c08c5bf579d", + "sha256:67fbb40db3985c0cfb942fe8853ad94a5e9702d2987dec03abadc2f3b6a24afb", + "sha256:6852cd34d96835e4c9091c1e6087325efb5b607b75fd9f7075616197d1c4688a", + "sha256:69ae0e9509c43474e33152abe1385b8954922544616426bf793481e1a37e094f", + "sha256:6c5bae4c288bda92a7550fe8de9e068c0a7cd56b1c5d888aae5b40f0e13b40bd", + "sha256:774bd401e7993452ba0596e741c0c4d6d22f882dd2a798993859181dbffadc62", + "sha256:79228a7b90d95957354f37b9d46f2cc8926262ae17b0d3ed8f36c892f2a37e06", + "sha256:7b8cba5a25e95041e3413d91f9e50616bcfaec95afa038ce7dc02efefe576745", + "sha256:7db97eabd440327c35b751d5ebf78a107f505586485159bcc87660da8bb1fdca", + "sha256:7ddd6d35c598af872f9a0a5bce7f7c4a1841684a72dab3302e3df7f17d1b5249", + "sha256:82edf3a6090554a83942cec79151d6b5eb96e63d143e80e4cf6671e5d772f6be", + "sha256:8b7b3ebfa9416c8eafaffa65216e229480c495e305a06ba176dcac32710744e6", + "sha256:8da677135eff43502b7afab5a1e641edfb2dc734ba7fc146e9b1b86817a728e2", + "sha256:908c850b98cac1e203ababd4ba76868d19ae0d7172cdc75d3f1b7829b16837d2", + "sha256:9da4ee8f711e077633730955c8f3cd2485c9abf5ea0f80aac23221a3224b9a8c", + "sha256:a6f1d8256d06f58e6ece150fbe05c63c7f9510df99ee8ac37423f5476a2cebb4", + "sha256:afb322ca05e2603deedbcd2e9910f11a3fd2f42bdeafe63018e5641945c7491c", + "sha256:b52c6741073de5a744d27329f9803938dcad5c9fee7e61690c705f72973f4175", + "sha256:ba633b51835036ff0f402c21f3ff567c565a22ff0a5732b060a68f4660e2a38f", + "sha256:bfa1a0f83bdf8061db8d17c2029454722043f1e4dd1b3d3d3120d1b54e75825a", + "sha256:bffd6cd47c2e68970039c0d3e355c9ed761d3ca727b204e63cd294cad0e3df90", + "sha256:d7a2c1e711ce59ac9d0bba780318bcd102d2958bb423209f24c6354d8c4da930", + "sha256:da46beef0ce882546d92b7b2e8deb9e04dbb8fec72945a8eb28b347ca46bc15a", + "sha256:ebdd2418ab4e2e26d572d9a1c03877f8514a9b7436729525aa571862507b3fea", + "sha256:fc44e50f9d5e96af1a561faa36863f9191f27364a4df3eb70bca66e9370480b6" + ], + "index": "pypi", + "version": "==2.0.18" + }, + "tenacity": { + "hashes": [ + "sha256:2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0", + "sha256:43af037822bd0029025877f3b2d97cc4d7bb0c2991000a3d59d71517c5c969e0" + ], + "index": "pypi", + "version": "==8.2.2" + }, + "typing-extensions": { + "hashes": [ + "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", + "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" + ], + "markers": "python_version >= '3.7'", + "version": "==4.7.1" + } + }, + "develop": { + "autopep8": { + "hashes": [ + "sha256:86e9303b5e5c8160872b2f5ef611161b2893e9bfe8ccc7e2f76385947d57a2f1", + "sha256:f9849cdd62108cb739dbcdbfb7fdcc9a30d1b63c4cc3e1c1f893b5360941b61c" + ], + "index": "pypi", + "version": "==2.0.2" + }, + "flake8": { + "hashes": [ + "sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7", + "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181" + ], + "index": "pypi", + "version": "==6.0.0" + }, + "mccabe": { + "hashes": [ + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053", + "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610" + ], + "markers": "python_version >= '3.6'", + "version": "==2.10.0" + }, + "pyflakes": { + "hashes": [ + "sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf", + "sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd" + ], + "markers": "python_version >= '3.6'", + "version": "==3.0.1" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version < '3.11'", + "version": "==2.0.1" + } + } +} diff --git a/ecs/jskult-dbdump/README.md b/ecs/jskult-dbdump/README.md new file mode 100644 index 00000000..f6e7c2da --- /dev/null +++ b/ecs/jskult-dbdump/README.md @@ -0,0 +1,48 @@ +# 実消化&アルトマーク 月次バッチ + +## 概要 + +実消化&アルトマークの月次バッチ処理。 + +## 環境情報 + +- Python 3.9 +- MySQL 8.23 +- VSCode + +## 環境構築 + +- Python の構築 + + - Merck_NewDWH 開発 2021 の Wiki、[Python 環境構築](https://nds-tyo.backlog.com/alias/wiki/1874930)を参照 + - 「Pipenv の導入」までを行っておくこと + - 構築完了後、プロジェクト配下で以下のコマンドを実行し、Python の仮想環境を作成する + - `pipenv install --dev --python ` + - この手順で出力される仮想環境のパスは、後述する VSCode の設定手順で使用するため、控えておく + +- MySQL の環境構築 + - Windows の場合、以下のリンクからダウンロードする + - + - Docker を利用する場合、「newsdwh-tools」リポジトリの MySQL 設定を使用すると便利 + - 「crm-table-to-ddl」フォルダ内で以下のコマンドを実行すると + - `docker-compose up -d` + - Docker の構築手順は、[Docker のセットアップ手順](https://nds-tyo.backlog.com/alias/wiki/1754332)を参照のこと + - データを投入する + - 立ち上げたデータベースに「src05」スキーマを作成する + - [ローカル開発用データ](https://ndstokyo.sharepoint.com/:f:/r/sites/merck-new-dwh-team/Shared%20Documents/03.NewDWH%E6%A7%8B%E7%AF%89%E3%83%95%E3%82%A7%E3%83%BC%E3%82%BA3/02.%E9%96%8B%E7%99%BA/90.%E9%96%8B%E7%99%BA%E5%85%B1%E6%9C%89/%E3%83%AD%E3%83%BC%E3%82%AB%E3%83%AB%E9%96%8B%E7%99%BA%E7%94%A8%E3%83%87%E3%83%BC%E3%82%BF?csf=1&web=1&e=VVcRUs)をダウンロードし、mysql コマンドを使用して復元する + - `mysql -h <ホスト名> -P <ポート> -u <ユーザー名> -p src05 < src05_dump.sql` +- 環境変数の設定 + - 「.env.example」ファイルをコピーし、「.env」ファイルを作成する + - 環境変数を設定する。設定内容は PRJ メンバーより共有を受けてください +- VSCode の設定 + - 「.vscode/recommended_settings.json」ファイルをコピーし、「settings.json」ファイルを作成する + - 「python.defaultInterpreterPath」を、Python の構築手順で作成した仮想環境のパスに変更する + +## 実行 + +- VSCode 上で「F5」キーを押下すると、バッチ処理が起動する。 +- 「entrypoint.py」が、バッチ処理のエントリーポイント。 +- 実際の処理は、「src/jobctrl_monthly.py」で行っている。 + + +## フォルダ構成（工事中） diff --git a/ecs/jskult-dbdump/backup_rds_src05_20230710091352.gz b/ecs/jskult-dbdump/backup_rds_src05_20230710091352.gz new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/entrypoint.py b/ecs/jskult-dbdump/entrypoint.py new file mode 100644 index 00000000..9a13c457 --- /dev/null +++ b/ecs/jskult-dbdump/entrypoint.py @@ -0,0 +1,10 @@ +"""実消化&アルトマーク 月次バッチのエントリーポイント""" +from src import jobctrl_dbdump + +if __name__ == '__main__': + try: + exit(jobctrl_dbdump.exec()) + except Exception: + # エラーが起きても、正常系のコードで返す。 + # エラーが起きた事実はbatch_process内でログを出す。 + exit(0) diff --git a/ecs/jskult-dbdump/my.cnf b/ecs/jskult-dbdump/my.cnf new file mode 100644 index 00000000..8bdb2c20 --- /dev/null +++ b/ecs/jskult-dbdump/my.cnf @@ -0,0 +1,4 @@ +[client] +user=root +password=admin +host=localhost \ No newline at end of file diff --git a/ecs/jskult-dbdump/src/__init__.py b/ecs/jskult-dbdump/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/batch/batch_functions.py b/ecs/jskult-dbdump/src/batch/batch_functions.py new file mode 100644 index 00000000..da329291 --- /dev/null +++ b/ecs/jskult-dbdump/src/batch/batch_functions.py @@ -0,0 +1,118 @@ +"""バッチ処理の共通関数""" +import logging +import textwrap + +from src.db.database import Database +from src.error.exceptions import BatchOperationException, DBException +from src.system_var import constants + + +def get_batch_statuses() -> tuple[str, str]: + """日付テーブルから、以下を取得して返す。 + - バッチ処理中フラグ + - dump取得状況区分 + + Raises: + BatchOperationException: 日付テーブルが取得できないとき、何らかのエラーが発生したとき + + Returns: + tuple[str, str]: [0]バッチ処理中フラグ，[1]dump取得状況区分 + """ + db = Database.get_instance() + sql = 'SELECT bch_actf, dump_sts_kbn FROM src05.hdke_tbl' + try: + db.connect() + hdke_tbl_result = db.execute_select(sql) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + if len(hdke_tbl_result) == 0: + raise BatchOperationException('日付テーブルが取得できませんでした') + + # 必ず1件取れる + hdke_tbl_record = hdke_tbl_result[0] + batch_processing_flag = hdke_tbl_record['bch_actf'] + dump_status_kbn = hdke_tbl_record['dump_sts_kbn'] + + return batch_processing_flag, dump_status_kbn + + +def update_dump_status_kbn_in_processing() -> None: + """dump取得状況区分を処理中に更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = 'UPDATE src05.hdke_tbl SET dump_sts_kbn = :in_processing' + try: + db.connect() + db.execute(sql, {'in_processing': constants.BATCH_ACTF_BATCH_IN_PROCESSING}) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + return + + +def update_dump_status_kbn_in_error() -> None: + """dump取得状況区分をエラーに更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = """\ + UPDATE src05.hdke_tbl + SET + dump_sts_kbn = :dump_unprocessed + """ + try: + db.connect() + db.execute(sql, { + 'dump_unprocessed': constants.DUMP_STATUS_KBN_ERROR + }) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + return + + +def update_dump_status_kbn_in_complete() -> None: + """dump取得状況区分を正常終了に更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = """\ + UPDATE src05.hdke_tbl + SET + dump_sts_kbn = :dump_unprocessed + """ + try: + db.connect() + db.execute(sql, { + 'dump_unprocessed': constants.DUMP_STATUS_KBN_COMPLETE + }) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + return + + +def logging_sql(logger: logging.Logger, sql: str) -> None: + """SQL文をデバッグログで出力する + + Args: + logger (logging.Logger): ロガー + sql (str): SQL文 + """ + logger.debug(f'\n{"-" * 15}\n{textwrap.dedent(sql)[1:-1]}\n{"-" * 15}') diff --git a/ecs/jskult-dbdump/src/batch/common/__init__.py b/ecs/jskult-dbdump/src/batch/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/batch/common/batch_context.py b/ecs/jskult-dbdump/src/batch/common/batch_context.py new file mode 100644 index 00000000..8c8c12fb --- /dev/null +++ b/ecs/jskult-dbdump/src/batch/common/batch_context.py @@ -0,0 +1,29 @@ +class BatchContext: + __instance = None + __syor_date: str # 処理日(yyyy/mm/dd形式) + __is_arisj_output_day: bool # 月次バッチ起動日フラグ + + def __init__(self) -> None: + self.__is_arisj_output_day = False + + @classmethod + def get_instance(cls): + if cls.__instance is None: + cls.__instance = cls() + return cls.__instance + + @property + def syor_date(self): + return self.__syor_date + + @syor_date.setter + def syor_date(self, syor_date_str: str): + self.__syor_date = syor_date_str + + @property + def is_arisj_output_day(self): + return self.__is_arisj_output_day + + @is_arisj_output_day.setter + def is_arisj_output_day(self, flag: bool): + self.__is_arisj_output_day = flag diff --git a/ecs/jskult-dbdump/src/db/__init__.py b/ecs/jskult-dbdump/src/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/db/database.py b/ecs/jskult-dbdump/src/db/database.py new file mode 100644 index 00000000..f67a21b9 --- /dev/null +++ b/ecs/jskult-dbdump/src/db/database.py @@ -0,0 +1,178 @@ +from sqlalchemy import (Connection, CursorResult, Engine, QueuePool, + create_engine, text) +from sqlalchemy.engine.url import URL +from tenacity import retry, stop_after_attempt, wait_exponential + +from src.error.exceptions import DBException +from src.logging.get_logger import get_logger +from src.system_var import environment + +logger = get_logger(__name__) + + +class Database: + """データベース操作クラス""" + __connection: Connection = None + __engine: Engine = None + __host: str = None + __port: str = None + __username: str = None + __password: str = None + __schema: str = None + __connection_string: str = None + + def __init__(self, username: str, password: str, host: str, port: int, schema: str) -> None: + """このクラスの新たなインスタンスを初期化します + + Args: + username (str): DBユーザー名 + password (str): DBパスワード + host (str): DBホスト名 + port (int): DBポート + schema (str): DBスキーマ名 + """ + self.__username = username + self.__password = password + self.__host = host + self.__port = int(port) + self.__schema = schema + + self.__connection_string = URL.create( + drivername='mysql+pymysql', + username=self.__username, + password=self.__password, + host=self.__host, + port=self.__port, + database=self.__schema, + query={"charset": "utf8mb4"} + ) + + self.__engine = create_engine( + self.__connection_string, + pool_timeout=5, + poolclass=QueuePool + ) + + @classmethod + def get_instance(cls): + """インスタンスを取得します + + Returns: + Database: DB操作クラスインスタンス + """ + return cls( + username=environment.DB_USERNAME, + password=environment.DB_PASSWORD, + host=environment.DB_HOST, + port=environment.DB_PORT, + schema=environment.DB_SCHEMA + ) + + @retry( + wait=wait_exponential( + multiplier=environment.DB_CONNECTION_RETRY_INTERVAL_INIT, + min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, + max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS + ), + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) + def connect(self): + """ + DBに接続します。接続に失敗した場合、リトライします。 + Raises: + DBException: 接続失敗 + """ + try: + self.__connection = self.__engine.connect() + except Exception as e: + raise DBException(e) + + def execute_select(self, select_query: str, parameters=None) -> list[dict]: + """SELECTクエリを実行します。 + + Args: + select_query (str): SELECT文 + parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None. + + Raises: + DBException: DBエラー + + Returns: + list[dict]: カラム名: 値の辞書リスト + """ + if self.__connection is None: + raise DBException('DBに接続していません') + + result = None + try: + # トランザクションが開始している場合は、トランザクションを引き継ぐ + if self.__connection.in_transaction(): + result = self.__connection.execute(text(select_query), parameters) + else: + # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。 + result = self.__execute_with_transaction(select_query, parameters) + except Exception as e: + raise DBException(f'SQL Error: {e}') + + result_rows = result.mappings().all() + return result_rows + + def execute(self, query: str, parameters=None) -> CursorResult: + """SQLクエリを実行します。 + + Args: + query (str): SQL文 + parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None. + + Raises: + DBException: DBエラー + + Returns: + CursorResult: 取得結果 + """ + if self.__connection is None: + raise DBException('DBに接続していません') + + result = None + try: + # トランザクションが開始している場合は、トランザクションを引き継ぐ + if self.__connection.in_transaction(): + result = self.__connection.execute(text(query), parameters) + else: + # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。 + result = self.__execute_with_transaction(query, parameters) + except Exception as e: + raise DBException(f'SQL Error: {e}') + + return result + + def begin(self): + """トランザクションを開始します。""" + if not self.__connection.in_transaction(): + self.__connection.begin() + + def commit(self): + """トランザクションをコミットします""" + if self.__connection.in_transaction(): + self.__connection.commit() + + def rollback(self): + """トランザクションをロールバックします""" + if self.__connection.in_transaction(): + self.__connection.rollback() + + def disconnect(self): + """DB接続を切断します。""" + if self.__connection is not None: + self.__connection.close() + self.__connection = None + + def __execute_with_transaction(self, query: str, parameters: dict): + # トランザクションを開始してクエリを実行する + with self.__connection.begin(): + try: + result = self.__connection.execute(text(query), parameters=parameters) + except Exception as e: + self.__connection.rollback() + raise e + # ここでコミットされる + return result diff --git a/ecs/jskult-dbdump/src/error/__init__.py b/ecs/jskult-dbdump/src/error/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/error/exceptions.py b/ecs/jskult-dbdump/src/error/exceptions.py new file mode 100644 index 00000000..055c24f6 --- /dev/null +++ b/ecs/jskult-dbdump/src/error/exceptions.py @@ -0,0 +1,10 @@ +class MeDaCaException(Exception): + pass + + +class DBException(MeDaCaException): + pass + + +class BatchOperationException(MeDaCaException): + pass diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py new file mode 100644 index 00000000..3ae7bdb6 --- /dev/null +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -0,0 +1,109 @@ +"""日次バッチ処理前DBダンプ取得""" + +import datetime +import textwrap +import subprocess +import os +from src.system_var import environment +from src.error.exceptions import BatchOperationException +from src.logging.get_logger import get_logger +from src.system_var import constants +from src.batch.batch_functions import ( + get_batch_statuses, update_dump_status_kbn_in_processing, + update_dump_status_kbn_in_complete, update_dump_status_kbn_in_error) +import gzip + +logger = get_logger('日次バッチ処理前DBダンプ取得') + + +def exec(): + try: + logger.info('日次バッチ処理前DBダンプ取得：開始') + try: + # 日次バッチ処置中フラグ、dump処理状態区分を取得 + batch_processing_flag, dump_status_kbn = get_batch_statuses() + except BatchOperationException as e: + logger.exception(f'日次ジョブ取得エラー（異常終了）\n{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # 日次バッチ処理中の場合、処理は行わない + if batch_processing_flag == constants.BATCH_ACTF_BATCH_IN_PROCESSING: + logger.error('日次ジョブ処理中エラー（異常終了）') + return constants.BATCH_EXIT_CODE_SUCCESS + + # dump処理状態区分が処理中またはエラーの場合、処理は行わない + if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED or dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: + logger.error('dump処理状態区エラー（異常終了）') + return constants.BATCH_EXIT_CODE_SUCCESS + + # dump処理状態区分を処理中に更新 + try: + update_dump_status_kbn_in_processing() + except BatchOperationException as e: + logger.exception(f'dump処理状態区分更新(未処理→処理中) エラー（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # MySQL接続情報を作成する + my_cnf_file_content = f""" + [client] + user={environment.DB_USERNAME} + password={environment.DB_PASSWORD} + host={environment.DB_HOST} + """ + # my.cnfファイルのパス + my_cnf_path = os.path.join('.', 'my.cnf') + + # my.cnfファイルを生成する + with open(my_cnf_path, 'w') as f: + f.write(textwrap.dedent(my_cnf_file_content)[1:-1]) + + dt_now = datetime.datetime.now() + converted_value = dt_now.strftime('%Y%m%d%H%M%S') + file_name = f'backup_rds_src05_{converted_value}.gz' + s3_file_name = f's3://{environment.JSKULT_BACKUP_BUCKET}/{environment.DUMP_BACKUP_FOLDER}/{dt_now.year}/{dt_now.strftime("%m")}/{dt_now.strftime("%d")}/{file_name}' + + mysqldump_cmd = [ + 'mysqldump', + '--user={mysql_user}'.format(mysql_user=environment.DB_USERNAME), + '--password={db_pw}'.format(db_pw=environment.DB_PASSWORD), + '--host={db_host}'.format(db_host=environment.DB_HOST), + '--port={db_port}'.format(db_port=environment.DB_PORT), + '{db_name}'.format(db_name=environment.DB_SCHEMA) + ] + + # mysqldumpコマンドを実行してデータを標準出力に出力 + mysqldump_output = subprocess.check_output(mysqldump_cmd) + + # gzipで圧縮 + compressed_data = gzip.compress(mysqldump_output) + + # AWS CLIを使ってS3にアップロード + aws_cli_cmd = [ + 'aws', + 's3', + 'cp', + '-', + s3_file_name + ] + subprocess.run(aws_cli_cmd, input=compressed_data) + + # dump処理状態区分を正常終了に更新 + try: + update_dump_status_kbn_in_complete() + except BatchOperationException as e: + logger.exception(f'dump処理状態区分更新(処理中→正常終了) エラー（異常終了）\n{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # 正常終了を保守ユーザーに通知 + logger.info('[NOTICE]日次バッチ処理前DBダンプ取得：終了（正常終了）') + return constants.BATCH_EXIT_CODE_SUCCESS + + except Exception as e: + # dump処理状態区分をエラーに更新 + try: + update_dump_status_kbn_in_error() + except BatchOperationException as e: + logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）\n{e}') + raise constants.BATCH_EXIT_CODE_SUCCESS + logger.exception(f'日次バッチ処理前DBダンプ取得中に想定外のエラーが発生しました \n{e}') + raise e diff --git a/ecs/jskult-dbdump/src/logging/get_logger.py b/ecs/jskult-dbdump/src/logging/get_logger.py new file mode 100644 index 00000000..f36f1199 --- /dev/null +++ b/ecs/jskult-dbdump/src/logging/get_logger.py @@ -0,0 +1,37 @@ +import logging + +from src.system_var.environment import LOG_LEVEL + +# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する +for name in ["boto3", "botocore", "s3transfer", "urllib3"]: + logging.getLogger(name).setLevel(logging.WARNING) + + +def get_logger(log_name: str) -> logging.Logger: + """一意のログ出力モジュールを取得します。 + + Args: + log_name (str): ロガー名 + + Returns: + _type_: _description_ + """ + logger = logging.getLogger(log_name) + level = logging.getLevelName(LOG_LEVEL) + if not isinstance(level, int): + level = logging.INFO + logger.setLevel(level) + + if not logger.hasHandlers(): + handler = logging.StreamHandler() + logger.addHandler(handler) + + formatter = logging.Formatter( + '%(name)s\t[%(levelname)s]\t%(asctime)s\t%(message)s', + '%Y-%m-%d %H:%M:%S' + ) + + for handler in logger.handlers: + handler.setFormatter(formatter) + + return logger diff --git a/ecs/jskult-dbdump/src/system_var/__init__.py b/ecs/jskult-dbdump/src/system_var/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/system_var/constants.py b/ecs/jskult-dbdump/src/system_var/constants.py new file mode 100644 index 00000000..993e5a7d --- /dev/null +++ b/ecs/jskult-dbdump/src/system_var/constants.py @@ -0,0 +1,15 @@ +# バッチ正常終了コード +BATCH_EXIT_CODE_SUCCESS = 0 + +# バッチ処理中フラグ：未処理 +BATCH_ACTF_BATCH_UNPROCESSED = '0' +# バッチ処理中フラグ：処理中 +BATCH_ACTF_BATCH_IN_PROCESSING = '1' +# dump取得状態区分：未処理 +DUMP_STATUS_KBN_UNPROCESSED = '0' +# dump取得状態区分：処理中 +DUMP_STATUS_KBN_PROCESSED = '1' +# dump取得状態区分：正常終了 +DUMP_STATUS_KBN_COMPLETE = '2' +# dump取得状態区分：エラー +DUMP_STATUS_KBN_ERROR = '9' diff --git a/ecs/jskult-dbdump/src/system_var/environment.py b/ecs/jskult-dbdump/src/system_var/environment.py new file mode 100644 index 00000000..123935a2 --- /dev/null +++ b/ecs/jskult-dbdump/src/system_var/environment.py @@ -0,0 +1,21 @@ +import os + +# Database +DB_HOST = os.environ['DB_HOST'] +DB_PORT = int(os.environ['DB_PORT']) +DB_USERNAME = os.environ['DB_USERNAME'] +DB_PASSWORD = os.environ['DB_PASSWORD'] +DB_SCHEMA = os.environ['DB_SCHEMA'] + +# AWS +JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] +JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] + +# 初期値がある環境変数 +LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') +DB_CONNECTION_MAX_RETRY_ATTEMPT = int(os.environ.get('DB_CONNECTION_MAX_RETRY_ATTEMPT', 4)) +DB_CONNECTION_RETRY_INTERVAL_INIT = int(os.environ.get('DB_CONNECTION_RETRY_INTERVAL', 5)) +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MIN_SECONDS', 5)) +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MAX_SECONDS', 50)) + +DUMP_BACKUP_FOLDER = os.environ['DUMP_BACKUP_FOLDER'] From cf02c6158804df78fd0c5da095c8e71a0088ba41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Mon, 10 Jul 2023 13:52:14 +0900 Subject: [PATCH 02/16] =?UTF-8?q?=E4=BB=AE=E5=AE=8C=E6=88=90=EF=BC=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/.env.example | 1 - .../backup_rds_src05_20230710091352.gz | 0 ecs/jskult-dbdump/src/jobctrl_dbdump.py | 18 +++++++++++++++++- .../src/system_var/environment.py | 1 - 4 files changed, 17 insertions(+), 3 deletions(-) delete mode 100644 ecs/jskult-dbdump/backup_rds_src05_20230710091352.gz diff --git a/ecs/jskult-dbdump/.env.example b/ecs/jskult-dbdump/.env.example index b331b25e..6db2d461 100644 --- a/ecs/jskult-dbdump/.env.example +++ b/ecs/jskult-dbdump/.env.example @@ -5,7 +5,6 @@ DB_PASSWORD=************ DB_SCHEMA=src05 JSKULT_BACKUP_BUCKET=mbj-newdwh2021-staging-backup-jskult -JSKULT_CONFIG_BUCKET=mbj-newdwh2021-staging-config DUMP_BACKUP_FOLDER=dump diff --git a/ecs/jskult-dbdump/backup_rds_src05_20230710091352.gz b/ecs/jskult-dbdump/backup_rds_src05_20230710091352.gz deleted file mode 100644 index e69de29b..00000000 diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 3ae7bdb6..42be4e51 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -57,6 +57,23 @@ def exec(): with open(my_cnf_path, 'w') as f: f.write(textwrap.dedent(my_cnf_file_content)[1:-1]) + #*************************************************************** + # # MySQLコマンドを実行する + # command = ['mysqldump', f'--defaults-file={my_cnf_path}', '-P', f"'{environment.DB_PORT}'", environment.DB_SCHEMA] + # mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # # gzipコマンドを実行して圧縮する + # gzip_process = subprocess.Popen(['gzip'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # # aws s3 cpコマンドを実行してアップロードする + # s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_name], stdin=gzip_process.stdout ,stderr=subprocess.PIPE) + # # 出力を取得する + # mysqldump_process.stdout.close() + # _, error = s3_cp_process.communicate() + + # if s3_cp_process.returncode != 0: + # print("Error: ", error.decode("utf-8")) + # raise Exception('Error') + #*************************************************************** + dt_now = datetime.datetime.now() converted_value = dt_now.strftime('%Y%m%d%H%M%S') file_name = f'backup_rds_src05_{converted_value}.gz' @@ -70,7 +87,6 @@ def exec(): '--port={db_port}'.format(db_port=environment.DB_PORT), '{db_name}'.format(db_name=environment.DB_SCHEMA) ] - # mysqldumpコマンドを実行してデータを標準出力に出力 mysqldump_output = subprocess.check_output(mysqldump_cmd) diff --git a/ecs/jskult-dbdump/src/system_var/environment.py b/ecs/jskult-dbdump/src/system_var/environment.py index 123935a2..46f08c5d 100644 --- a/ecs/jskult-dbdump/src/system_var/environment.py +++ b/ecs/jskult-dbdump/src/system_var/environment.py @@ -9,7 +9,6 @@ DB_SCHEMA = os.environ['DB_SCHEMA'] # AWS JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] -JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] # 初期値がある環境変数 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') From 59548a1fa4392ba99b4e721697552b79e64c63a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Mon, 10 Jul 2023 19:44:30 +0900 Subject: [PATCH 03/16] =?UTF-8?q?=E3=83=91=E3=83=A9=E3=83=A1=E3=83=BC?= =?UTF-8?q?=E3=82=BF=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 42be4e51..477a1388 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -57,23 +57,6 @@ def exec(): with open(my_cnf_path, 'w') as f: f.write(textwrap.dedent(my_cnf_file_content)[1:-1]) - #*************************************************************** - # # MySQLコマンドを実行する - # command = ['mysqldump', f'--defaults-file={my_cnf_path}', '-P', f"'{environment.DB_PORT}'", environment.DB_SCHEMA] - # mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # # gzipコマンドを実行して圧縮する - # gzip_process = subprocess.Popen(['gzip'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # # aws s3 cpコマンドを実行してアップロードする - # s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_name], stdin=gzip_process.stdout ,stderr=subprocess.PIPE) - # # 出力を取得する - # mysqldump_process.stdout.close() - # _, error = s3_cp_process.communicate() - - # if s3_cp_process.returncode != 0: - # print("Error: ", error.decode("utf-8")) - # raise Exception('Error') - #*************************************************************** - dt_now = datetime.datetime.now() converted_value = dt_now.strftime('%Y%m%d%H%M%S') file_name = f'backup_rds_src05_{converted_value}.gz' @@ -85,6 +68,10 @@ def exec(): '--password={db_pw}'.format(db_pw=environment.DB_PASSWORD), '--host={db_host}'.format(db_host=environment.DB_HOST), '--port={db_port}'.format(db_port=environment.DB_PORT), + '--no-tablespaces', + '--skip-column-statistics', + '--single-transaction', + '--set-gtid-purged=OFF', '{db_name}'.format(db_name=environment.DB_SCHEMA) ] # mysqldumpコマンドを実行してデータを標準出力に出力 From 47d164a9a254ba6146b86b149f99d931bc4d8df0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Tue, 11 Jul 2023 11:08:09 +0900 Subject: [PATCH 04/16] =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 477a1388..1f821456 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -33,7 +33,7 @@ def exec(): # dump処理状態区分が処理中またはエラーの場合、処理は行わない if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED or dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: - logger.error('dump処理状態区エラー（異常終了）') + logger.error('dump処理状態区分エラー（異常終了）') return constants.BATCH_EXIT_CODE_SUCCESS # dump処理状態区分を処理中に更新 From c7c5eade7b311aea55d45b1f25f950ed2cafae0f Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 14:42:28 +0900 Subject: [PATCH 05/16] =?UTF-8?q?fix:=20my.cnf=E3=81=AFgit=E8=BF=BD?= =?UTF-8?q?=E8=B7=A1=E5=AF=BE=E8=B1=A1=E5=A4=96=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/.gitignore | 1 + ecs/jskult-dbdump/my.cnf | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 ecs/jskult-dbdump/my.cnf diff --git a/ecs/jskult-dbdump/.gitignore b/ecs/jskult-dbdump/.gitignore index bd0b37f8..cf44449e 100644 --- a/ecs/jskult-dbdump/.gitignore +++ b/ecs/jskult-dbdump/.gitignore @@ -1,5 +1,6 @@ .vscode/settings.json .env +my.cnf # python __pycache__ diff --git a/ecs/jskult-dbdump/my.cnf b/ecs/jskult-dbdump/my.cnf deleted file mode 100644 index 8bdb2c20..00000000 --- a/ecs/jskult-dbdump/my.cnf +++ /dev/null @@ -1,4 +0,0 @@ -[client] -user=root -password=admin -host=localhost \ No newline at end of file From 268282486378eb456d639ac8d6510b0e70014aca Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 14:46:46 +0900 Subject: [PATCH 06/16] =?UTF-8?q?feat:=20Docker=E3=82=A4=E3=83=A1=E3=83=BC?= =?UTF-8?q?=E3=82=B8=E3=81=A7mysql-client=208.x=E3=82=92=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/Dockerfile | 38 ++++++++++++++-------- ecs/jskult-dbdump/mysql_dpkg_selection.txt | 3 ++ 2 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 ecs/jskult-dbdump/mysql_dpkg_selection.txt diff --git a/ecs/jskult-dbdump/Dockerfile b/ecs/jskult-dbdump/Dockerfile index f87a7215..1d1a865e 100644 --- a/ecs/jskult-dbdump/Dockerfile +++ b/ecs/jskult-dbdump/Dockerfile @@ -1,28 +1,40 @@ -FROM python:3.9 +FROM python:3.9-bullseye ENV TZ="Asia/Tokyo" WORKDIR /usr/src/app COPY Pipfile Pipfile.lock ./ +# apt mysqlパッケージのdpkg次のコマンド注入用 +COPY mysql_dpkg_selection.txt ./ +# 必要なパッケージインストール +RUN apt update && apt install -y less vim curl wget gzip unzip sudo lsb-release + +# # mysqlをインストール +RUN \ + wget https://dev.mysql.com/get/mysql-apt-config_0.8.25-1_all.deb && \ + dpkg -i mysql-apt-config_0.8.25-1_all.deb < mysql_dpkg_selection.txt && \ + apt update && \ + apt install -y mysql-client + +# aws cli v2 のインストール +RUN \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + sudo ./aws/install + +# python関連のライブラリインストール RUN \ - apt update -y && \ - # パッケージのセキュリティアップデートのみを適用するコマンド - apt install -y unattended-upgrades && \ - unattended-upgrades && \ pip install --upgrade pip wheel setuptools && \ pip install pipenv --no-cache-dir && \ pipenv install --system --deploy && \ pip uninstall -y pipenv virtualenv-clone virtualenv -# mysql-clientと必要なパッケージインストール -RUN apt install -y less vim curl unzip sudo default-mysql-client - -# aws cli v2 のインストール -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -RUN unzip awscliv2.zip -RUN sudo ./aws/install +# パッケージのセキュリティアップデートのみを適用するコマンドを実行 +RUN \ + apt install -y unattended-upgrades && \ + unattended-upgrades COPY src ./src -COPY entrypoint.py entrypoint.py +COPY entrypoint.py entrypoint.py CMD ["python", "entrypoint.py"] diff --git a/ecs/jskult-dbdump/mysql_dpkg_selection.txt b/ecs/jskult-dbdump/mysql_dpkg_selection.txt new file mode 100644 index 00000000..d3cb46f9 --- /dev/null +++ b/ecs/jskult-dbdump/mysql_dpkg_selection.txt @@ -0,0 +1,3 @@ +1 +1 +4 \ No newline at end of file From dbfe62e30b7588024c8cf23527cabde4854230d6 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 16:31:14 +0900 Subject: [PATCH 07/16] =?UTF-8?q?feat:=20=E3=83=AD=E3=83=BC=E3=82=AB?= =?UTF-8?q?=E3=83=AB=E3=81=AE=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8?= =?UTF-8?q?=E3=82=92=E7=B5=8C=E7=94=B1=E3=81=9B=E3=81=9A=E3=80=81=E3=83=91?= =?UTF-8?q?=E3=82=A4=E3=83=97=E3=83=A9=E3=82=A4=E3=83=B3=E3=81=AE=E3=81=BF?= =?UTF-8?q?=E3=81=A7=E5=AE=9F=E8=A1=8C=E3=80=82=E3=82=A8=E3=83=A9=E3=83=BC?= =?UTF-8?q?=E3=83=8F=E3=83=B3=E3=83=89=E3=83=AA=E3=83=B3=E3=82=B0=E3=81=AF?= =?UTF-8?q?=E5=AE=9F=E8=A3=85=E3=81=A7=E3=81=8D=E3=81=A6=E3=81=84=E3=81=AA?= =?UTF-8?q?=E3=81=84=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 80 ++++++++++++------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 1f821456..dcc02211 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -1,17 +1,17 @@ """日次バッチ処理前DBダンプ取得""" import datetime -import textwrap -import subprocess import os -from src.system_var import environment +import subprocess +import textwrap + +from src.batch.batch_functions import (get_batch_statuses, + update_dump_status_kbn_in_complete, + update_dump_status_kbn_in_error, + update_dump_status_kbn_in_processing) from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger -from src.system_var import constants -from src.batch.batch_functions import ( - get_batch_statuses, update_dump_status_kbn_in_processing, - update_dump_status_kbn_in_complete, update_dump_status_kbn_in_error) -import gzip +from src.system_var import constants, environment logger = get_logger('日次バッチ処理前DBダンプ取得') @@ -23,7 +23,7 @@ def exec(): # 日次バッチ処置中フラグ、dump処理状態区分を取得 batch_processing_flag, dump_status_kbn = get_batch_statuses() except BatchOperationException as e: - logger.exception(f'日次ジョブ取得エラー（異常終了）\n{e}') + logger.exception(f'日次ジョブ取得エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS # 日次バッチ処理中の場合、処理は行わない @@ -40,7 +40,7 @@ def exec(): try: update_dump_status_kbn_in_processing() except BatchOperationException as e: - logger.exception(f'dump処理状態区分更新(未処理→処理中) エラー（異常終了）{e}') + logger.exception(f'dump処理状態区分更新(未処理→処理中) エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS # MySQL接続情報を作成する @@ -51,7 +51,7 @@ def exec(): host={environment.DB_HOST} """ # my.cnfファイルのパス - my_cnf_path = os.path.join('.', 'my.cnf') + my_cnf_path = os.path.join('my.cnf') # my.cnfファイルを生成する with open(my_cnf_path, 'w') as f: @@ -62,39 +62,37 @@ def exec(): file_name = f'backup_rds_src05_{converted_value}.gz' s3_file_name = f's3://{environment.JSKULT_BACKUP_BUCKET}/{environment.DUMP_BACKUP_FOLDER}/{dt_now.year}/{dt_now.strftime("%m")}/{dt_now.strftime("%d")}/{file_name}' - mysqldump_cmd = [ - 'mysqldump', - '--user={mysql_user}'.format(mysql_user=environment.DB_USERNAME), - '--password={db_pw}'.format(db_pw=environment.DB_PASSWORD), - '--host={db_host}'.format(db_host=environment.DB_HOST), - '--port={db_port}'.format(db_port=environment.DB_PORT), - '--no-tablespaces', - '--skip-column-statistics', - '--single-transaction', - '--set-gtid-purged=OFF', - '{db_name}'.format(db_name=environment.DB_SCHEMA) + # 'mysqldump --login-path=dwhadmin --no-tablespaces --skip-column-statistics --single-transaction --set-gtid-purged=OFF dwh > /data/mountdwh/backup/%s/before/' + # mysqldumpコマンドを実行する + command = [ + 'mysqldump', + f'--defaults-file={my_cnf_path}', + '-P', + f"{environment.DB_PORT}", + '--no-tablespaces', + '--skip-column-statistics', + '--single-transaction', + '--set-gtid-purged=OFF', + environment.DB_SCHEMA + # 'src05' ] - # mysqldumpコマンドを実行してデータを標準出力に出力 - mysqldump_output = subprocess.check_output(mysqldump_cmd) - - # gzipで圧縮 - compressed_data = gzip.compress(mysqldump_output) - - # AWS CLIを使ってS3にアップロード - aws_cli_cmd = [ - 'aws', - 's3', - 'cp', - '-', - s3_file_name - ] - subprocess.run(aws_cli_cmd, input=compressed_data) - + mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # gzipコマンドを実行してdump結果を圧縮する + gzip_process = subprocess.Popen(['gzip'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # aws s3 cpコマンドを実行してアップロードする + s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_name], stdin=gzip_process.stdout, stderr=subprocess.PIPE) + # 出力を取得する + mysqldump_process.stdout.close() + gzip_process.stdout.close() + _, error = s3_cp_process.communicate() + if s3_cp_process.returncode != 0: + print("Error: ", error.decode("utf-8")) + raise Exception('Error') # dump処理状態区分を正常終了に更新 try: update_dump_status_kbn_in_complete() except BatchOperationException as e: - logger.exception(f'dump処理状態区分更新(処理中→正常終了) エラー（異常終了）\n{e}') + logger.exception(f'dump処理状態区分更新(処理中→正常終了) エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS # 正常終了を保守ユーザーに通知 @@ -106,7 +104,7 @@ def exec(): try: update_dump_status_kbn_in_error() except BatchOperationException as e: - logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）\n{e}') + logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）:{e}') raise constants.BATCH_EXIT_CODE_SUCCESS - logger.exception(f'日次バッチ処理前DBダンプ取得中に想定外のエラーが発生しました \n{e}') + logger.exception(f'日次バッチ処理前DBダンプ取得中に想定外のエラーが発生しました :{e}') raise e From 0b19ba8e04bdd7cc790be65595b77eabac3c52e4 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 17:03:13 +0900 Subject: [PATCH 08/16] =?UTF-8?q?feat:=20=E5=90=84=E3=82=B3=E3=83=9E?= =?UTF-8?q?=E3=83=B3=E3=83=89=E3=81=94=E3=81=A8=E3=81=AB=E3=82=A8=E3=83=A9?= =?UTF-8?q?=E3=83=BC=E3=83=8F=E3=83=B3=E3=83=89=E3=83=AA=E3=83=B3=E3=82=B0?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 27 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index dcc02211..dc688f69 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -62,8 +62,7 @@ def exec(): file_name = f'backup_rds_src05_{converted_value}.gz' s3_file_name = f's3://{environment.JSKULT_BACKUP_BUCKET}/{environment.DUMP_BACKUP_FOLDER}/{dt_now.year}/{dt_now.strftime("%m")}/{dt_now.strftime("%d")}/{file_name}' - # 'mysqldump --login-path=dwhadmin --no-tablespaces --skip-column-statistics --single-transaction --set-gtid-purged=OFF dwh > /data/mountdwh/backup/%s/before/' - # mysqldumpコマンドを実行する + # mysqldumpコマンドを実行し、dumpを取得する command = [ 'mysqldump', f'--defaults-file={my_cnf_path}', @@ -81,22 +80,32 @@ def exec(): gzip_process = subprocess.Popen(['gzip'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # aws s3 cpコマンドを実行してアップロードする s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_name], stdin=gzip_process.stdout, stderr=subprocess.PIPE) - # 出力を取得する + # mysqldumpの標準出力をgzipに接続したため、標準出力をクローズする mysqldump_process.stdout.close() + # gzipの標準出力をaws s3 cpに接続したため、標準出力をクローズする gzip_process.stdout.close() + + # パイプラインを実行し、エラーハンドリング + _, error = mysqldump_process.communicate() + if mysqldump_process.returncode != 0: + raise BatchOperationException(error.decode('utf-8')) + + _, error = gzip_process.communicate() + if gzip_process.returncode != 0: + raise BatchOperationException(error.decode('utf-8')) + _, error = s3_cp_process.communicate() if s3_cp_process.returncode != 0: - print("Error: ", error.decode("utf-8")) - raise Exception('Error') - # dump処理状態区分を正常終了に更新 + raise BatchOperationException(error.decode('utf-8')) + + # # dump処理状態区分を正常終了に更新 try: update_dump_status_kbn_in_complete() except BatchOperationException as e: logger.exception(f'dump処理状態区分更新(処理中→正常終了) エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS - # 正常終了を保守ユーザーに通知 - logger.info('[NOTICE]日次バッチ処理前DBダンプ取得：終了（正常終了）') + logger.info('日次バッチ処理前DBダンプ取得：終了（正常終了）') return constants.BATCH_EXIT_CODE_SUCCESS except Exception as e: @@ -105,6 +114,6 @@ def exec(): update_dump_status_kbn_in_error() except BatchOperationException as e: logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）:{e}') - raise constants.BATCH_EXIT_CODE_SUCCESS + raise e logger.exception(f'日次バッチ処理前DBダンプ取得中に想定外のエラーが発生しました :{e}') raise e From a108f20a21860a6f6ce1c16569ee2795dab57a7d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 17:06:39 +0900 Subject: [PATCH 09/16] =?UTF-8?q?feat:=20DB=E6=8E=A5=E7=B6=9A=E3=83=AA?= =?UTF-8?q?=E3=83=88=E3=83=A9=E3=82=A4=E6=99=82=E3=81=ABRetry=E3=82=A8?= =?UTF-8?q?=E3=83=A9=E3=83=BC=E3=81=8C=E8=BF=94=E3=81=95=E3=82=8C=E3=80=81?= =?UTF-8?q?=E6=8D=95=E7=B8=9B=E3=81=A7=E3=81=8D=E3=81=A6=E3=81=84=E3=81=AA?= =?UTF-8?q?=E3=81=8B=E3=81=A3=E3=81=9F=E3=81=9F=E3=82=81=E3=80=81=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/db/database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ecs/jskult-dbdump/src/db/database.py b/ecs/jskult-dbdump/src/db/database.py index f67a21b9..67f5c5a5 100644 --- a/ecs/jskult-dbdump/src/db/database.py +++ b/ecs/jskult-dbdump/src/db/database.py @@ -74,7 +74,9 @@ class Database: min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS ), - stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), + retry_error_cls=DBException + ) def connect(self): """ DBに接続します。接続に失敗した場合、リトライします。 From c87bee4e9ad40213fefc6f376f86c8ddcd2cbb8a Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 17:44:30 +0900 Subject: [PATCH 10/16] =?UTF-8?q?feat:=20=E3=82=A8=E3=83=A9=E3=83=BC?= =?UTF-8?q?=E3=83=8F=E3=83=B3=E3=83=89=E3=83=AA=E3=83=B3=E3=82=B0=E3=81=AE?= =?UTF-8?q?=E3=82=84=E3=82=8A=E6=96=B9=E3=82=92=E7=B5=B1=E4=B8=80=E3=80=82?= =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/Dockerfile | 4 ++-- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ecs/jskult-dbdump/Dockerfile b/ecs/jskult-dbdump/Dockerfile index 1d1a865e..9c6838cb 100644 --- a/ecs/jskult-dbdump/Dockerfile +++ b/ecs/jskult-dbdump/Dockerfile @@ -4,12 +4,12 @@ ENV TZ="Asia/Tokyo" WORKDIR /usr/src/app COPY Pipfile Pipfile.lock ./ -# apt mysqlパッケージのdpkg次のコマンド注入用 +# mysql-apt-config をdpkgでインストールする際に標準出力に渡す文字列ファイルをコピー COPY mysql_dpkg_selection.txt ./ # 必要なパッケージインストール RUN apt update && apt install -y less vim curl wget gzip unzip sudo lsb-release -# # mysqlをインストール +# mysqlをインストール RUN \ wget https://dev.mysql.com/get/mysql-apt-config_0.8.25-1_all.deb && \ dpkg -i mysql-apt-config_0.8.25-1_all.deb < mysql_dpkg_selection.txt && \ diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index dc688f69..964c34e8 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -88,17 +88,20 @@ def exec(): # パイプラインを実行し、エラーハンドリング _, error = mysqldump_process.communicate() if mysqldump_process.returncode != 0: - raise BatchOperationException(error.decode('utf-8')) + logger.error(f'`mysqldump`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS _, error = gzip_process.communicate() if gzip_process.returncode != 0: - raise BatchOperationException(error.decode('utf-8')) + logger.error(f'`gzip`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS _, error = s3_cp_process.communicate() if s3_cp_process.returncode != 0: - raise BatchOperationException(error.decode('utf-8')) + logger.error(f'`aws s3 cp`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS - # # dump処理状態区分を正常終了に更新 + # dump処理状態区分を正常終了に更新 try: update_dump_status_kbn_in_complete() except BatchOperationException as e: From 1ae009115d6258593741b9eb9102e2cec1007087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Tue, 11 Jul 2023 18:02:52 +0900 Subject: [PATCH 11/16] =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E3=83=AD?= =?UTF-8?q?=E3=82=B0=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 964c34e8..02b8da84 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -23,17 +23,17 @@ def exec(): # 日次バッチ処置中フラグ、dump処理状態区分を取得 batch_processing_flag, dump_status_kbn = get_batch_statuses() except BatchOperationException as e: - logger.exception(f'日次ジョブ取得エラー（異常終了）:{e}') + logger.exception(f'日付テーブル取得エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS # 日次バッチ処理中の場合、処理は行わない if batch_processing_flag == constants.BATCH_ACTF_BATCH_IN_PROCESSING: - logger.error('日次ジョブ処理中エラー（異常終了）') + logger.error('日次バッチ処理中の為、処理を終了') return constants.BATCH_EXIT_CODE_SUCCESS # dump処理状態区分が処理中またはエラーの場合、処理は行わない if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED or dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: - logger.error('dump処理状態区分エラー（異常終了）') + logger.error(f'dump処理状態区分が実行不可な状態の為、処理を終了 dump処理状態区分={dump_status_kbn}') return constants.BATCH_EXIT_CODE_SUCCESS # dump処理状態区分を処理中に更新 @@ -73,7 +73,6 @@ def exec(): '--single-transaction', '--set-gtid-purged=OFF', environment.DB_SCHEMA - # 'src05' ] mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # gzipコマンドを実行してdump結果を圧縮する @@ -117,6 +116,6 @@ def exec(): update_dump_status_kbn_in_error() except BatchOperationException as e: logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）:{e}') - raise e + return constants.BATCH_EXIT_CODE_SUCCESS logger.exception(f'日次バッチ処理前DBダンプ取得中に想定外のエラーが発生しました :{e}') - raise e + return constants.BATCH_EXIT_CODE_SUCCESS From c29044873096a87ef03d0546c70274ada88d4299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 12 Jul 2023 10:48:07 +0900 Subject: [PATCH 12/16] =?UTF-8?q?=E6=8C=87=E6=91=98=E4=BA=8B=E9=A0=85?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/README.md | 6 ++-- ecs/jskult-dbdump/entrypoint.py | 2 +- .../src/batch/batch_functions.py | 4 +-- .../src/batch/common/batch_context.py | 29 ------------------- ecs/jskult-dbdump/src/db/database.py | 7 +---- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 13 +++++---- .../src/system_var/environment.py | 2 +- 7 files changed, 15 insertions(+), 48 deletions(-) delete mode 100644 ecs/jskult-dbdump/src/batch/common/batch_context.py diff --git a/ecs/jskult-dbdump/README.md b/ecs/jskult-dbdump/README.md index f6e7c2da..7d8f0ccd 100644 --- a/ecs/jskult-dbdump/README.md +++ b/ecs/jskult-dbdump/README.md @@ -1,8 +1,8 @@ -# 実消化&アルトマーク 月次バッチ +# 日次バッチ処理前DBダンプ取得　 ## 概要 -実消化&アルトマークの月次バッチ処理。 +日次バッチ処理前DBダンプ取得処理。 ## 環境情報 @@ -42,7 +42,7 @@ - VSCode 上で「F5」キーを押下すると、バッチ処理が起動する。 - 「entrypoint.py」が、バッチ処理のエントリーポイント。 -- 実際の処理は、「src/jobctrl_monthly.py」で行っている。 +- 実際の処理は、「src/jobctrl_dbdump.py」で行っている。 ## フォルダ構成（工事中） diff --git a/ecs/jskult-dbdump/entrypoint.py b/ecs/jskult-dbdump/entrypoint.py index 9a13c457..543cb134 100644 --- a/ecs/jskult-dbdump/entrypoint.py +++ b/ecs/jskult-dbdump/entrypoint.py @@ -1,4 +1,4 @@ -"""実消化&アルトマーク 月次バッチのエントリーポイント""" +"""実消化&アルトマーク 日次バッチ処理前DBダンプ取得のエントリーポイント""" from src import jobctrl_dbdump if __name__ == '__main__': diff --git a/ecs/jskult-dbdump/src/batch/batch_functions.py b/ecs/jskult-dbdump/src/batch/batch_functions.py index da329291..87231e17 100644 --- a/ecs/jskult-dbdump/src/batch/batch_functions.py +++ b/ecs/jskult-dbdump/src/batch/batch_functions.py @@ -58,7 +58,7 @@ def update_dump_status_kbn_in_processing() -> None: return -def update_dump_status_kbn_in_error() -> None: +def update_dump_status_kbn_error() -> None: """dump取得状況区分をエラーに更新する Raises: @@ -83,7 +83,7 @@ def update_dump_status_kbn_in_error() -> None: return -def update_dump_status_kbn_in_complete() -> None: +def update_dump_status_kbn_complete() -> None: """dump取得状況区分を正常終了に更新する Raises: diff --git a/ecs/jskult-dbdump/src/batch/common/batch_context.py b/ecs/jskult-dbdump/src/batch/common/batch_context.py deleted file mode 100644 index 8c8c12fb..00000000 --- a/ecs/jskult-dbdump/src/batch/common/batch_context.py +++ /dev/null @@ -1,29 +0,0 @@ -class BatchContext: - __instance = None - __syor_date: str # 処理日(yyyy/mm/dd形式) - __is_arisj_output_day: bool # 月次バッチ起動日フラグ - - def __init__(self) -> None: - self.__is_arisj_output_day = False - - @classmethod - def get_instance(cls): - if cls.__instance is None: - cls.__instance = cls() - return cls.__instance - - @property - def syor_date(self): - return self.__syor_date - - @syor_date.setter - def syor_date(self, syor_date_str: str): - self.__syor_date = syor_date_str - - @property - def is_arisj_output_day(self): - return self.__is_arisj_output_day - - @is_arisj_output_day.setter - def is_arisj_output_day(self, flag: bool): - self.__is_arisj_output_day = flag diff --git a/ecs/jskult-dbdump/src/db/database.py b/ecs/jskult-dbdump/src/db/database.py index 67f5c5a5..ee127d51 100644 --- a/ecs/jskult-dbdump/src/db/database.py +++ b/ecs/jskult-dbdump/src/db/database.py @@ -4,11 +4,8 @@ from sqlalchemy.engine.url import URL from tenacity import retry, stop_after_attempt, wait_exponential from src.error.exceptions import DBException -from src.logging.get_logger import get_logger from src.system_var import environment -logger = get_logger(__name__) - class Database: """データベース操作クラス""" @@ -74,9 +71,7 @@ class Database: min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS ), - stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), - retry_error_cls=DBException - ) + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) def connect(self): """ DBに接続します。接続に失敗した場合、リトライします。 diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 02b8da84..7eba902a 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -6,8 +6,8 @@ import subprocess import textwrap from src.batch.batch_functions import (get_batch_statuses, - update_dump_status_kbn_in_complete, - update_dump_status_kbn_in_error, + update_dump_status_kbn_complete, + update_dump_status_kbn_error, update_dump_status_kbn_in_processing) from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger @@ -28,12 +28,12 @@ def exec(): # 日次バッチ処理中の場合、処理は行わない if batch_processing_flag == constants.BATCH_ACTF_BATCH_IN_PROCESSING: - logger.error('日次バッチ処理中の為、処理を終了') + logger.error('日次バッチ処理中の為、日次バッチ処理前DBダンプ取得を終了します。') return constants.BATCH_EXIT_CODE_SUCCESS # dump処理状態区分が処理中またはエラーの場合、処理は行わない if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED or dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: - logger.error(f'dump処理状態区分が実行不可な状態の為、処理を終了 dump処理状態区分={dump_status_kbn}') + logger.error(f'ダンプ取得が実行不可能な状態の為、日次バッチ処理前DBダンプ取得を終了します。 dump処理状態区分={dump_status_kbn}') return constants.BATCH_EXIT_CODE_SUCCESS # dump処理状態区分を処理中に更新 @@ -74,6 +74,7 @@ def exec(): '--set-gtid-purged=OFF', environment.DB_SCHEMA ] + mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # gzipコマンドを実行してdump結果を圧縮する gzip_process = subprocess.Popen(['gzip'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -102,7 +103,7 @@ def exec(): # dump処理状態区分を正常終了に更新 try: - update_dump_status_kbn_in_complete() + update_dump_status_kbn_complete() except BatchOperationException as e: logger.exception(f'dump処理状態区分更新(処理中→正常終了) エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS @@ -113,7 +114,7 @@ def exec(): except Exception as e: # dump処理状態区分をエラーに更新 try: - update_dump_status_kbn_in_error() + update_dump_status_kbn_error() except BatchOperationException as e: logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS diff --git a/ecs/jskult-dbdump/src/system_var/environment.py b/ecs/jskult-dbdump/src/system_var/environment.py index 46f08c5d..ff510581 100644 --- a/ecs/jskult-dbdump/src/system_var/environment.py +++ b/ecs/jskult-dbdump/src/system_var/environment.py @@ -9,6 +9,7 @@ DB_SCHEMA = os.environ['DB_SCHEMA'] # AWS JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] +DUMP_BACKUP_FOLDER = os.environ['DUMP_BACKUP_FOLDER'] # 初期値がある環境変数 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') @@ -17,4 +18,3 @@ DB_CONNECTION_RETRY_INTERVAL_INIT = int(os.environ.get('DB_CONNECTION_RETRY_INTE DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MIN_SECONDS', 5)) DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MAX_SECONDS', 50)) -DUMP_BACKUP_FOLDER = os.environ['DUMP_BACKUP_FOLDER'] From 3c3178fe70becf2c581ff65c56f336d3df53debb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 12 Jul 2023 13:15:37 +0900 Subject: [PATCH 13/16] =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E3=81=AE?= =?UTF-8?q?=E6=96=87=E8=A8=80=E3=82=92=E3=82=8F=E3=81=91=E3=81=BE=E3=81=97?= =?UTF-8?q?=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 7eba902a..eebbaf3e 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -31,8 +31,13 @@ def exec(): logger.error('日次バッチ処理中の為、日次バッチ処理前DBダンプ取得を終了します。') return constants.BATCH_EXIT_CODE_SUCCESS - # dump処理状態区分が処理中またはエラーの場合、処理は行わない - if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED or dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: + # dump処理状態区分が処理中の場合、処理は行わない + if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED: + logger.error(f'ダンプ取得中の為、日次バッチ処理前DBダンプ取得を終了します。 dump処理状態区分={dump_status_kbn}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # dump処理状態区分がエラーの場合、処理は行わない + if dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: logger.error(f'ダンプ取得が実行不可能な状態の為、日次バッチ処理前DBダンプ取得を終了します。 dump処理状態区分={dump_status_kbn}') return constants.BATCH_EXIT_CODE_SUCCESS From e0797fae1f293f0f88edef160f08e8c1bcd68513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 12 Jul 2023 14:26:43 +0900 Subject: [PATCH 14/16] =?UTF-8?q?=E9=96=93=E9=81=95=E3=81=88=E3=81=A6?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=81=97=E3=81=A6=E3=81=97=E3=81=BE=E3=81=A3?= =?UTF-8?q?=E3=81=9F=E3=81=AE=E3=82=92=E6=88=BB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/db/database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ecs/jskult-dbdump/src/db/database.py b/ecs/jskult-dbdump/src/db/database.py index ee127d51..b675381c 100644 --- a/ecs/jskult-dbdump/src/db/database.py +++ b/ecs/jskult-dbdump/src/db/database.py @@ -71,7 +71,9 @@ class Database: min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS ), - stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), + retry_error_cls=DBException + ) def connect(self): """ DBに接続します。接続に失敗した場合、リトライします。 From 5ebd71b03d47b05b211c0fdb6004c6d65e5c8ce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 12 Jul 2023 14:48:10 +0900 Subject: [PATCH 15/16] =?UTF-8?q?gzip=E3=82=B3=E3=83=9E=E3=83=B3=E3=83=89?= =?UTF-8?q?=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index eebbaf3e..c93b2a0f 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -82,7 +82,7 @@ def exec(): mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # gzipコマンドを実行してdump結果を圧縮する - gzip_process = subprocess.Popen(['gzip'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + gzip_process = subprocess.Popen(['gzip', '-c'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # aws s3 cpコマンドを実行してアップロードする s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_name], stdin=gzip_process.stdout, stderr=subprocess.PIPE) # mysqldumpの標準出力をgzipに接続したため、標準出力をクローズする From b3759b4ae9144a227881559837f7772df88483aa Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 13 Jul 2023 10:29:01 +0900 Subject: [PATCH 16/16] =?UTF-8?q?feat:=20dump=E5=8F=96=E5=BE=97=E6=99=82?= =?UTF-8?q?=E3=80=81=E5=87=BA=E5=8A=9B=E5=85=88=E3=81=AE=E3=83=91=E3=82=B9?= =?UTF-8?q?=E3=82=92=E3=83=AD=E3=82=B0=E5=87=BA=E5=8A=9B=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/Dockerfile | 2 +- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ecs/jskult-dbdump/Dockerfile b/ecs/jskult-dbdump/Dockerfile index 9c6838cb..3c410b72 100644 --- a/ecs/jskult-dbdump/Dockerfile +++ b/ecs/jskult-dbdump/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-bullseye +FROM python:3.9 ENV TZ="Asia/Tokyo" diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index c93b2a0f..3dac256f 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -64,8 +64,8 @@ def exec(): dt_now = datetime.datetime.now() converted_value = dt_now.strftime('%Y%m%d%H%M%S') - file_name = f'backup_rds_src05_{converted_value}.gz' - s3_file_name = f's3://{environment.JSKULT_BACKUP_BUCKET}/{environment.DUMP_BACKUP_FOLDER}/{dt_now.year}/{dt_now.strftime("%m")}/{dt_now.strftime("%d")}/{file_name}' + dump_file_name = f'backup_rds_src05_{converted_value}.gz' + s3_file_path = f's3://{environment.JSKULT_BACKUP_BUCKET}/{environment.DUMP_BACKUP_FOLDER}/{dt_now.year}/{dt_now.strftime("%m")}/{dt_now.strftime("%d")}/{dump_file_name}' # mysqldumpコマンドを実行し、dumpを取得する command = [ @@ -79,12 +79,12 @@ def exec(): '--set-gtid-purged=OFF', environment.DB_SCHEMA ] - + mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # gzipコマンドを実行してdump結果を圧縮する gzip_process = subprocess.Popen(['gzip', '-c'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # aws s3 cpコマンドを実行してアップロードする - s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_name], stdin=gzip_process.stdout, stderr=subprocess.PIPE) + s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_path], stdin=gzip_process.stdout, stderr=subprocess.PIPE) # mysqldumpの標準出力をgzipに接続したため、標準出力をクローズする mysqldump_process.stdout.close() # gzipの標準出力をaws s3 cpに接続したため、標準出力をクローズする @@ -114,6 +114,7 @@ def exec(): return constants.BATCH_EXIT_CODE_SUCCESS logger.info('日次バッチ処理前DBダンプ取得：終了（正常終了）') + logger.info(f'出力ファイルパス: {s3_file_path}') return constants.BATCH_EXIT_CODE_SUCCESS except Exception as e: