From 5c6192463bad355de65eebef18af4023c6689cdb Mon Sep 17 00:00:00 2001 From: "x.azuma.m@nds-tyo.co.jp" Date: Wed, 5 Jul 2023 19:12:44 +0900 Subject: [PATCH 01/30] =?UTF-8?q?=E8=A8=AD=E8=A8=88=E6=9B=B8=E9=80=9A?= =?UTF-8?q?=E3=82=8A=E3=81=AB=E7=92=B0=E5=A2=83=E5=A4=89=E6=95=B0=E5=90=8D?= =?UTF-8?q?=E3=82=92=E3=83=AA=E3=83=95=E3=82=A1=E3=82=AF=E3=82=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/.env.example | 4 ++-- ecs/jskult-batch-daily/src/aws/s3.py | 4 ++-- ecs/jskult-batch-daily/src/system_var/environment.py | 4 ++-- .../tests/batch/vjsk/vjsk_file_check/conftest.py | 4 ++-- ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/conftest.py | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/ecs/jskult-batch-daily/.env.example b/ecs/jskult-batch-daily/.env.example index d109a997..3eec2a82 100644 --- a/ecs/jskult-batch-daily/.env.example +++ b/ecs/jskult-batch-daily/.env.example @@ -13,8 +13,8 @@ JSKULT_CONFIG_BUCKET=********************** JSKULT_CONFIG_CALENDAR_FOLDER=jskult/calendar JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME=jskult_holiday_list.txt JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME=jskult_wholesaler_stock_input_day_list.txt -JSKULT_DATA_BUCKET=********************** -JSKULT_DATA_FOLDER_RECV=********************** +VJSK_DATA_BUCKET=********************** +VJSK_DATA_RECEIVE_FOLDER=********************** # 連携データ抽出期間 SALES_LAUNDERING_EXTRACT_DATE_PERIOD=0 # 洗替対象テーブル名 diff --git a/ecs/jskult-batch-daily/src/aws/s3.py b/ecs/jskult-batch-daily/src/aws/s3.py index 1e685276..8684053d 100644 --- a/ecs/jskult-batch-daily/src/aws/s3.py +++ b/ecs/jskult-batch-daily/src/aws/s3.py @@ -117,8 +117,8 @@ class VjskBackupBucket(JskUltBackupBucket): class VjskReceiveBucket(S3Bucket): - _bucket_name = environment.JSKULT_DATA_BUCKET - _recv_folder = environment.JSKULT_DATA_FOLDER_RECV + _bucket_name = environment.VJSK_DATA_BUCKET + _recv_folder = environment.VJSK_DATA_RECEIVE_FOLDER _s3_file_list = None diff --git a/ecs/jskult-batch-daily/src/system_var/environment.py b/ecs/jskult-batch-daily/src/system_var/environment.py index 7fddde4f..feb8a737 100644 --- a/ecs/jskult-batch-daily/src/system_var/environment.py +++ b/ecs/jskult-batch-daily/src/system_var/environment.py @@ -17,8 +17,8 @@ JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] JSKULT_CONFIG_CALENDAR_FOLDER = os.environ['JSKULT_CONFIG_CALENDAR_FOLDER'] JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME = os.environ['JSKULT_CONFIG_CALENDAR_HOLIDAY_LIST_FILE_NAME'] JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME = os.environ['JSKULT_CONFIG_CALENDAR_WHOLESALER_STOCK_FILE_NAME'] -JSKULT_DATA_BUCKET = os.environ['JSKULT_DATA_BUCKET'] -JSKULT_DATA_FOLDER_RECV = os.environ['JSKULT_DATA_FOLDER_RECV'] +VJSK_DATA_BUCKET = os.environ['VJSK_DATA_BUCKET'] +VJSK_DATA_RECEIVE_FOLDER = os.environ['VJSK_DATA_RECEIVE_FOLDER'] # 初期値がある環境変数 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/conftest.py b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/conftest.py index c315147a..dccdd0df 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/conftest.py +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/conftest.py @@ -15,12 +15,12 @@ def s3_client(): @pytest.fixture def bucket_name(): - return os.environ["JSKULT_DATA_BUCKET"] + return os.environ["VJSK_DATA_BUCKET"] @pytest.fixture def receive_folder(): - return os.environ["JSKULT_DATA_FOLDER_RECV"] + return os.environ["VJSK_DATA_RECEIVE_FOLDER"] # TODO 共通fixtureにして15個固定でput/delete、各個別fixtureで15個から引き算でdeleteする diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/conftest.py b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/conftest.py index ea29eb63..dc77a65f 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/conftest.py +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/conftest.py @@ -15,12 +15,12 @@ def s3_client(): @pytest.fixture def bucket_name(): - return os.environ["JSKULT_DATA_BUCKET"] + return os.environ["VJSK_DATA_BUCKET"] @pytest.fixture def receive_folder(): - return os.environ["JSKULT_DATA_FOLDER_RECV"] + return os.environ["VJSK_DATA_RECEIVE_FOLDER"] @pytest.fixture From 8caa5ab14d35af49cc900968251bbff6920e400e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Mon, 10 Jul 2023 11:54:06 +0900 Subject: [PATCH 02/30] =?UTF-8?q?=E4=BB=AE=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/.dockerignore | 12 + ecs/jskult-dbdump/.env.example | 16 ++ ecs/jskult-dbdump/.gitignore | 10 + ecs/jskult-dbdump/.vscode/launch.json | 17 ++ .../.vscode/recommended_settings.json | 31 +++ ecs/jskult-dbdump/Dockerfile | 28 +++ ecs/jskult-dbdump/Pipfile | 19 ++ ecs/jskult-dbdump/Pipfile.lock | 206 ++++++++++++++++++ ecs/jskult-dbdump/README.md | 48 ++++ .../backup_rds_src05_20230710091352.gz | 0 ecs/jskult-dbdump/entrypoint.py | 10 + ecs/jskult-dbdump/my.cnf | 4 + ecs/jskult-dbdump/src/__init__.py | 0 .../src/batch/batch_functions.py | 118 ++++++++++ .../src/batch/common/__init__.py | 0 .../src/batch/common/batch_context.py | 29 +++ ecs/jskult-dbdump/src/db/__init__.py | 0 ecs/jskult-dbdump/src/db/database.py | 178 +++++++++++++++ ecs/jskult-dbdump/src/error/__init__.py | 0 ecs/jskult-dbdump/src/error/exceptions.py | 10 + ecs/jskult-dbdump/src/jobctrl_dbdump.py | 109 +++++++++ ecs/jskult-dbdump/src/logging/get_logger.py | 37 ++++ ecs/jskult-dbdump/src/system_var/__init__.py | 0 ecs/jskult-dbdump/src/system_var/constants.py | 15 ++ .../src/system_var/environment.py | 21 ++ 25 files changed, 918 insertions(+) create mode 100644 ecs/jskult-dbdump/.dockerignore create mode 100644 ecs/jskult-dbdump/.env.example create mode 100644 ecs/jskult-dbdump/.gitignore create mode 100644 ecs/jskult-dbdump/.vscode/launch.json create mode 100644 ecs/jskult-dbdump/.vscode/recommended_settings.json create mode 100644 ecs/jskult-dbdump/Dockerfile create mode 100644 ecs/jskult-dbdump/Pipfile create mode 100644 ecs/jskult-dbdump/Pipfile.lock create mode 100644 ecs/jskult-dbdump/README.md create mode 100644 ecs/jskult-dbdump/backup_rds_src05_20230710091352.gz create mode 100644 ecs/jskult-dbdump/entrypoint.py create mode 100644 ecs/jskult-dbdump/my.cnf create mode 100644 ecs/jskult-dbdump/src/__init__.py create mode 100644 ecs/jskult-dbdump/src/batch/batch_functions.py create mode 100644 ecs/jskult-dbdump/src/batch/common/__init__.py create mode 100644 ecs/jskult-dbdump/src/batch/common/batch_context.py create mode 100644 ecs/jskult-dbdump/src/db/__init__.py create mode 100644 ecs/jskult-dbdump/src/db/database.py create mode 100644 ecs/jskult-dbdump/src/error/__init__.py create mode 100644 ecs/jskult-dbdump/src/error/exceptions.py create mode 100644 ecs/jskult-dbdump/src/jobctrl_dbdump.py create mode 100644 ecs/jskult-dbdump/src/logging/get_logger.py create mode 100644 ecs/jskult-dbdump/src/system_var/__init__.py create mode 100644 ecs/jskult-dbdump/src/system_var/constants.py create mode 100644 ecs/jskult-dbdump/src/system_var/environment.py diff --git a/ecs/jskult-dbdump/.dockerignore b/ecs/jskult-dbdump/.dockerignore new file mode 100644 index 00000000..8b9da402 --- /dev/null +++ b/ecs/jskult-dbdump/.dockerignore @@ -0,0 +1,12 @@ +tests/* +.coverage +.env +.env.example +.report/* +.vscode/* +.pytest_cache/* +*/__pychache__/* +Dockerfile +pytest.ini +README.md +*.sql diff --git a/ecs/jskult-dbdump/.env.example b/ecs/jskult-dbdump/.env.example new file mode 100644 index 00000000..b331b25e --- /dev/null +++ b/ecs/jskult-dbdump/.env.example @@ -0,0 +1,16 @@ +DB_HOST=************ +DB_PORT=3306 +DB_USERNAME=************ +DB_PASSWORD=************ +DB_SCHEMA=src05 + +JSKULT_BACKUP_BUCKET=mbj-newdwh2021-staging-backup-jskult +JSKULT_CONFIG_BUCKET=mbj-newdwh2021-staging-config + +DUMP_BACKUP_FOLDER=dump + +LOG_LEVEL=INFO +DB_CONNECTION_MAX_RETRY_ATTEMPT=************ +DB_CONNECTION_RETRY_INTERVAL_INIT=************ +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS=************ +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS=************ diff --git a/ecs/jskult-dbdump/.gitignore b/ecs/jskult-dbdump/.gitignore new file mode 100644 index 00000000..bd0b37f8 --- /dev/null +++ b/ecs/jskult-dbdump/.gitignore @@ -0,0 +1,10 @@ +.vscode/settings.json +.env + +# python +__pycache__ + +# python test +.pytest_cache +.coverage +.report/ \ No newline at end of file diff --git a/ecs/jskult-dbdump/.vscode/launch.json b/ecs/jskult-dbdump/.vscode/launch.json new file mode 100644 index 00000000..a0178c26 --- /dev/null +++ b/ecs/jskult-dbdump/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // IntelliSense を使用して利用可能な属性を学べます。 + // 既存の属性の説明をホバーして表示します。 + // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "(DEBUG)jskult batch dbdump", + "type": "python", + "request": "launch", + "program": "entrypoint.py", + "console": "integratedTerminal", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/ecs/jskult-dbdump/.vscode/recommended_settings.json b/ecs/jskult-dbdump/.vscode/recommended_settings.json new file mode 100644 index 00000000..b5e79d73 --- /dev/null +++ b/ecs/jskult-dbdump/.vscode/recommended_settings.json @@ -0,0 +1,31 @@ +{ + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + // 自身の環境に合わせて変えてください + "python.defaultInterpreterPath": "", + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": [ + "--max-line-length=200", + "--ignore=F541" + ], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Path": "autopep8", + "python.formatting.autopep8Args": [ + "--max-line-length", "200", + "--ignore=F541" + ], + "python.testing.pytestArgs": [ + "tests/batch/ultmarc" + ], + + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} diff --git a/ecs/jskult-dbdump/Dockerfile b/ecs/jskult-dbdump/Dockerfile new file mode 100644 index 00000000..f87a7215 --- /dev/null +++ b/ecs/jskult-dbdump/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.9 + +ENV TZ="Asia/Tokyo" + +WORKDIR /usr/src/app +COPY Pipfile Pipfile.lock ./ +RUN \ + apt update -y && \ + # パッケージのセキュリティアップデートのみを適用するコマンド + apt install -y unattended-upgrades && \ + unattended-upgrades && \ + pip install --upgrade pip wheel setuptools && \ + pip install pipenv --no-cache-dir && \ + pipenv install --system --deploy && \ + pip uninstall -y pipenv virtualenv-clone virtualenv + +# mysql-clientと必要なパッケージインストール +RUN apt install -y less vim curl unzip sudo default-mysql-client + +# aws cli v2 のインストール +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +RUN unzip awscliv2.zip +RUN sudo ./aws/install + +COPY src ./src +COPY entrypoint.py entrypoint.py + +CMD ["python", "entrypoint.py"] diff --git a/ecs/jskult-dbdump/Pipfile b/ecs/jskult-dbdump/Pipfile new file mode 100644 index 00000000..65b30a51 --- /dev/null +++ b/ecs/jskult-dbdump/Pipfile @@ -0,0 +1,19 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +sqlalchemy = "*" +tenacity = "*" +pymysql = "*" + +[dev-packages] +autopep8 = "*" +flake8 = "*" + +[requires] +python_version = "3.9" + +[pipenv] +allow_prereleases = true diff --git a/ecs/jskult-dbdump/Pipfile.lock b/ecs/jskult-dbdump/Pipfile.lock new file mode 100644 index 00000000..58713cc6 --- /dev/null +++ b/ecs/jskult-dbdump/Pipfile.lock @@ -0,0 +1,206 @@ +{ + "_meta": { + "hash": { + "sha256": "e2e2efd7ebd6ad719931983f277105dd94c4ebb6363f7c00e75dd8ca05523713" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "greenlet": { + "hashes": [ + "sha256:0a9dfcadc1d79696e90ccb1275c30ad4ec5fd3d1ab3ae6671286fac78ef33435", + "sha256:0f313771cb8ee0a04dfdf586b7d4076180d80c94be09049daeea018089b5b957", + "sha256:17503397bf6cbb5e364217143b6150c540020c51a3f6b08f9a20cd67c25e2ca8", + "sha256:180ec55cb127bc745669eddc9793ffab6e0cf7311e67e1592f183d6ca00d88c1", + "sha256:1b3f3568478bc21b85968e8038c4f98f4bf0039a692791bc324b5e0d1522f4b1", + "sha256:1bd4ea36f0aeb14ca335e0c9594a5aaefa1ac4e2db7d86ba38f0be96166b3102", + "sha256:21ebcb570e0d8501457d6a2695a44c5af3b6c2143dc6644ec73574beba067c90", + "sha256:24071eee113d75fedebaeb86264d94f04b5a24e311c5ba3e8003c07d00112a7e", + "sha256:270432cfdd6a50016b8259b3bbf398a3f7c06a06f2c68c7b93e49f53bc193bcf", + "sha256:271ed380389d2f7e4c1545b6e0837986e62504ab561edbaff05da9c9f3f98f96", + "sha256:2840187a94e258445e62ff1545e34f0b1a14aef4d0078e5c88246688d2b6515e", + "sha256:2cda110faee67613fed221f90467003f477088ef1cc84c8fc88537785a5b4de9", + "sha256:2e160a65cc6023a237be870f2072513747d512a1d018efa083acce0b673cccc0", + "sha256:2fcf7af83516db35af3d0ed5d182dea8585eddd891977adff1b74212f4bfd2fd", + "sha256:36cebce1f30964d5672fd956860e7e7b69772da69658d5743cb676b442eeff36", + "sha256:42bfe67824a9b53e73f568f982f0d1d4c7ac0f587d2e702a23f8a7b505d7b7c2", + "sha256:450a7e52a515402fd110ba807f1a7d464424bfa703be4effbcb97e1dfbfcc621", + "sha256:463d63ca5d8c236788284a9a44b9715372a64d5318a6b5eee36815df1ea0ba3d", + "sha256:4d0c0ffd732466ff324ced144fad55ed5deca36f6036c1d8f04cec69b084c9d6", + "sha256:4ff2a765f4861fc018827eab4df1992f7508d06c62de5d2fe8a6ac2233d4f1d0", + "sha256:53abf19b7dc62795c67b8d0a3d8ef866db166b21017632fff2624cf8fbf3481c", + "sha256:5552d7be37d878e9b6359bbffa0512d857bb9703616a4c0656b49c10739d5971", + "sha256:585810056a8adacd3152945ebfcd25deb58335d41f16ae4e0f3d768918957f9a", + "sha256:5942b1d6ba447cff1ec23a21ec525dde2288f00464950bc647f4e0f03bd537d1", + "sha256:5c355c99be5bb23e85d899b059a4f22fdf8a0741c57e7029425ee63eb436f689", + "sha256:5f61df4fe07864561f49b45c8bd4d2c42e3f03d2872ed05c844902a58b875028", + "sha256:665942d3a954c3e4c976581715f57fb3b86f4cf6bae3ac30b133f8ff777ac6c7", + "sha256:68368e908f14887fb202a81960bfbe3a02d97e6d3fa62b821556463084ffb131", + "sha256:6aac94ff957b5dea0216af71ab59c602e1b947b394e4f5e878a5a65643090038", + "sha256:889934aa8d72b6bfc46babd1dc4b817a56c97ec0f4a10ae7551fb60ab1f96fae", + "sha256:a00550757fca1b9cbc479f8eb1cf3514dbc0103b3f76eae46341c26ddcca67a9", + "sha256:a4a2d6ed0515c05afd5cc435361ced0baabd9ba4536ddfe8ad9a95bcb702c8ce", + "sha256:a8dd92fd76a61af2abc8ccad0c6c6069b3c4ebd4727ecc9a7c33aae37651c8c7", + "sha256:ab81f9ff3e3c2ca65e824454214c10985a846cd9bee5f4d04e15cd875d9fe13b", + "sha256:ac10196b8cde7a082e4e371ff171407270d3337c8d57ed43030094eb01d9c95c", + "sha256:b767930af686551dc96a5eb70af3736709d547ffa275c11a5e820bfb3ae61d8d", + "sha256:b9a1f4d256b81f59ba87bb7a29b9b38b1c018e052dba60a543cb0ddb5062d159", + "sha256:ba94c08321b5d345100fc64eb1ab235f42faf9aabba805cface55ebe677f1c2c", + "sha256:bab71f73001cd15723c4e2ca398f2f48e0a3f584c619eefddb1525e8986e06eb", + "sha256:bce5cf2b0f0b29680396c5c98ab39a011bd70f2dfa8b8a6811a69ee6d920cf9f", + "sha256:c02e514c72e745e49a3ae7e672a1018ba9b68460c21e0361054e956e5d595bc6", + "sha256:c3fb459ced6c5e3b2a895f23f1400f93e9b24d85c30fbe2d637d4f7706a1116b", + "sha256:cd31ab223e43ac64fd23f8f5dad249addadac2a459f040546200acbf7e84e353", + "sha256:ce70aa089ec589b5d5fab388af9f8c9f9dfe8fe4ad844820a92eb240d8628ddf", + "sha256:d47b2e1ad1429da9aa459ef189fbcd8a74ec28a16bc4c3f5f3cf3f88e36535eb", + "sha256:d61bad421c1f496f9fb6114dbd7c30a1dac0e9ff90e9be06f4472cbd8f7a1704", + "sha256:d7ba2e5cb119eddbc10874b41047ad99525e39e397f7aef500e6da0d6f46ab91", + "sha256:dde0ab052c7a1deee8d13d72c37f2afecee30ebdf6eb139790157eaddf04dd61", + "sha256:df34b52aa50a38d7a79f3abc9fda7e400791447aa0400ed895f275f6d8b0bb1f", + "sha256:e0fc20e6e6b298861035a5fc5dcf9fbaa0546318e8bda81112591861a7dcc28f", + "sha256:e20d5e8dc76b73db9280464d6e81bea05e51a99f4d4dd29c5f78dc79f294a5d3", + "sha256:e31d1a33dc9006b278f72cb0aacfe397606c2693aa2fdc0c2f2dcddbad9e0b53", + "sha256:e3a99f890f2cc5535e1b3a90049c6ca9ff9da9ec251cc130c8d269997f9d32ee", + "sha256:e7b192c3df761d0fdd17c2d42d41c28460f124f5922e8bd524018f1d35610682", + "sha256:ed0f4fad4c3656e34d20323a789b6a2d210a6bb82647d9c86dded372f55c58a1", + "sha256:f34ec09702be907727fd479046193725441aaaf7ed4636ca042734f469bb7451", + "sha256:f3530c0ec1fc98c43d5b7061781a8c55bd0db44f789f8152e19d9526cbed6021", + "sha256:f5672082576d0e9f52fa0fa732ff57254d65faeb4a471bc339fe54b58b3e79d2", + "sha256:ffb9f8969789771e95d3c982a36be81f0adfaa7302a1d56e29f168ca15e284b8" + ], + "markers": "platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))", + "version": "==3.0.0a1" + }, + "pymysql": { + "hashes": [ + "sha256:4f13a7df8bf36a51e81dd9f3605fede45a4878fe02f9236349fd82a3f0612f96", + "sha256:8969ec6d763c856f7073c4c64662882675702efcb114b4bcbb955aea3a069fa7" + ], + "index": "pypi", + "version": "==1.1.0" + }, + "sqlalchemy": { + "hashes": [ + "sha256:00aa050faf24ce5f2af643e2b86822fa1d7149649995f11bc1e769bbfbf9010b", + "sha256:09397a18733fa2a4c7680b746094f980060666ee549deafdb5e102a99ce4619b", + "sha256:0f7fdcce52cd882b559a57b484efc92e108efeeee89fab6b623aba1ac68aad2e", + "sha256:10514adc41fc8f5922728fbac13d401a1aefcf037f009e64ca3b92464e33bf0e", + "sha256:10e001a84f820fea2640e4500e12322b03afc31d8f4f6b813b44813b2a7c7e0d", + "sha256:194f2d5a7cb3739875c4d25b3fe288ab0b3dc33f7c857ba2845830c8c51170a0", + "sha256:1aac42a21a7fa6c9665392c840b295962992ddf40aecf0a88073bc5c76728117", + "sha256:1fb792051db66e09c200e7bc3bda3b1eb18a5b8eb153d2cedb2b14b56a68b8cb", + "sha256:2756485f49e7df5c2208bdc64263d19d23eba70666f14ad12d6d8278a2fff65f", + "sha256:2b791577c546b6bbd7b43953565fcb0a2fec63643ad605353dd48afbc3c48317", + "sha256:420bc6d06d4ae7fb6921524334689eebcbea7bf2005efef070a8562cc9527a37", + "sha256:45b07470571bda5ee7f5ec471271bbde97267cc8403fce05e280c36ea73f4754", + "sha256:4ebc542d2289c0b016d6945fd07a7e2e23f4abc41e731ac8ad18a9e0c2fd0ec2", + "sha256:556dc18e39b6edb76239acfd1c010e37395a54c7fde8c57481c15819a3ffb13e", + "sha256:589aba9a35869695b319ed76c6f673d896cd01a7ff78054be1596df7ad9b096f", + "sha256:5c95e3e7cc6285bf7ff263eabb0d3bfe3def9a1ff98124083d45e5ece72f4579", + "sha256:5dd574a37be388512c72fe0d7318cb8e31743a9b2699847a025e0c08c5bf579d", + "sha256:67fbb40db3985c0cfb942fe8853ad94a5e9702d2987dec03abadc2f3b6a24afb", + "sha256:6852cd34d96835e4c9091c1e6087325efb5b607b75fd9f7075616197d1c4688a", + "sha256:69ae0e9509c43474e33152abe1385b8954922544616426bf793481e1a37e094f", + "sha256:6c5bae4c288bda92a7550fe8de9e068c0a7cd56b1c5d888aae5b40f0e13b40bd", + "sha256:774bd401e7993452ba0596e741c0c4d6d22f882dd2a798993859181dbffadc62", + "sha256:79228a7b90d95957354f37b9d46f2cc8926262ae17b0d3ed8f36c892f2a37e06", + "sha256:7b8cba5a25e95041e3413d91f9e50616bcfaec95afa038ce7dc02efefe576745", + "sha256:7db97eabd440327c35b751d5ebf78a107f505586485159bcc87660da8bb1fdca", + "sha256:7ddd6d35c598af872f9a0a5bce7f7c4a1841684a72dab3302e3df7f17d1b5249", + "sha256:82edf3a6090554a83942cec79151d6b5eb96e63d143e80e4cf6671e5d772f6be", + "sha256:8b7b3ebfa9416c8eafaffa65216e229480c495e305a06ba176dcac32710744e6", + "sha256:8da677135eff43502b7afab5a1e641edfb2dc734ba7fc146e9b1b86817a728e2", + "sha256:908c850b98cac1e203ababd4ba76868d19ae0d7172cdc75d3f1b7829b16837d2", + "sha256:9da4ee8f711e077633730955c8f3cd2485c9abf5ea0f80aac23221a3224b9a8c", + "sha256:a6f1d8256d06f58e6ece150fbe05c63c7f9510df99ee8ac37423f5476a2cebb4", + "sha256:afb322ca05e2603deedbcd2e9910f11a3fd2f42bdeafe63018e5641945c7491c", + "sha256:b52c6741073de5a744d27329f9803938dcad5c9fee7e61690c705f72973f4175", + "sha256:ba633b51835036ff0f402c21f3ff567c565a22ff0a5732b060a68f4660e2a38f", + "sha256:bfa1a0f83bdf8061db8d17c2029454722043f1e4dd1b3d3d3120d1b54e75825a", + "sha256:bffd6cd47c2e68970039c0d3e355c9ed761d3ca727b204e63cd294cad0e3df90", + "sha256:d7a2c1e711ce59ac9d0bba780318bcd102d2958bb423209f24c6354d8c4da930", + "sha256:da46beef0ce882546d92b7b2e8deb9e04dbb8fec72945a8eb28b347ca46bc15a", + "sha256:ebdd2418ab4e2e26d572d9a1c03877f8514a9b7436729525aa571862507b3fea", + "sha256:fc44e50f9d5e96af1a561faa36863f9191f27364a4df3eb70bca66e9370480b6" + ], + "index": "pypi", + "version": "==2.0.18" + }, + "tenacity": { + "hashes": [ + "sha256:2f277afb21b851637e8f52e6a613ff08734c347dc19ade928e519d7d2d8569b0", + "sha256:43af037822bd0029025877f3b2d97cc4d7bb0c2991000a3d59d71517c5c969e0" + ], + "index": "pypi", + "version": "==8.2.2" + }, + "typing-extensions": { + "hashes": [ + "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", + "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" + ], + "markers": "python_version >= '3.7'", + "version": "==4.7.1" + } + }, + "develop": { + "autopep8": { + "hashes": [ + "sha256:86e9303b5e5c8160872b2f5ef611161b2893e9bfe8ccc7e2f76385947d57a2f1", + "sha256:f9849cdd62108cb739dbcdbfb7fdcc9a30d1b63c4cc3e1c1f893b5360941b61c" + ], + "index": "pypi", + "version": "==2.0.2" + }, + "flake8": { + "hashes": [ + "sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7", + "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181" + ], + "index": "pypi", + "version": "==6.0.0" + }, + "mccabe": { + "hashes": [ + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053", + "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610" + ], + "markers": "python_version >= '3.6'", + "version": "==2.10.0" + }, + "pyflakes": { + "hashes": [ + "sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf", + "sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd" + ], + "markers": "python_version >= '3.6'", + "version": "==3.0.1" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version < '3.11'", + "version": "==2.0.1" + } + } +} diff --git a/ecs/jskult-dbdump/README.md b/ecs/jskult-dbdump/README.md new file mode 100644 index 00000000..f6e7c2da --- /dev/null +++ b/ecs/jskult-dbdump/README.md @@ -0,0 +1,48 @@ +# 実消化&アルトマーク 月次バッチ + +## 概要 + +実消化&アルトマークの月次バッチ処理。 + +## 環境情報 + +- Python 3.9 +- MySQL 8.23 +- VSCode + +## 環境構築 + +- Python の構築 + + - Merck_NewDWH 開発 2021 の Wiki、[Python 環境構築](https://nds-tyo.backlog.com/alias/wiki/1874930)を参照 + - 「Pipenv の導入」までを行っておくこと + - 構築完了後、プロジェクト配下で以下のコマンドを実行し、Python の仮想環境を作成する + - `pipenv install --dev --python ` + - この手順で出力される仮想環境のパスは、後述する VSCode の設定手順で使用するため、控えておく + +- MySQL の環境構築 + - Windows の場合、以下のリンクからダウンロードする + - + - Docker を利用する場合、「newsdwh-tools」リポジトリの MySQL 設定を使用すると便利 + - 「crm-table-to-ddl」フォルダ内で以下のコマンドを実行すると + - `docker-compose up -d` + - Docker の構築手順は、[Docker のセットアップ手順](https://nds-tyo.backlog.com/alias/wiki/1754332)を参照のこと + - データを投入する + - 立ち上げたデータベースに「src05」スキーマを作成する + - [ローカル開発用データ](https://ndstokyo.sharepoint.com/:f:/r/sites/merck-new-dwh-team/Shared%20Documents/03.NewDWH%E6%A7%8B%E7%AF%89%E3%83%95%E3%82%A7%E3%83%BC%E3%82%BA3/02.%E9%96%8B%E7%99%BA/90.%E9%96%8B%E7%99%BA%E5%85%B1%E6%9C%89/%E3%83%AD%E3%83%BC%E3%82%AB%E3%83%AB%E9%96%8B%E7%99%BA%E7%94%A8%E3%83%87%E3%83%BC%E3%82%BF?csf=1&web=1&e=VVcRUs)をダウンロードし、mysql コマンドを使用して復元する + - `mysql -h <ホスト名> -P <ポート> -u <ユーザー名> -p src05 < src05_dump.sql` +- 環境変数の設定 + - 「.env.example」ファイルをコピーし、「.env」ファイルを作成する + - 環境変数を設定する。設定内容は PRJ メンバーより共有を受けてください +- VSCode の設定 + - 「.vscode/recommended_settings.json」ファイルをコピーし、「settings.json」ファイルを作成する + - 「python.defaultInterpreterPath」を、Python の構築手順で作成した仮想環境のパスに変更する + +## 実行 + +- VSCode 上で「F5」キーを押下すると、バッチ処理が起動する。 +- 「entrypoint.py」が、バッチ処理のエントリーポイント。 +- 実際の処理は、「src/jobctrl_monthly.py」で行っている。 + + +## フォルダ構成（工事中） diff --git a/ecs/jskult-dbdump/backup_rds_src05_20230710091352.gz b/ecs/jskult-dbdump/backup_rds_src05_20230710091352.gz new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/entrypoint.py b/ecs/jskult-dbdump/entrypoint.py new file mode 100644 index 00000000..9a13c457 --- /dev/null +++ b/ecs/jskult-dbdump/entrypoint.py @@ -0,0 +1,10 @@ +"""実消化&アルトマーク 月次バッチのエントリーポイント""" +from src import jobctrl_dbdump + +if __name__ == '__main__': + try: + exit(jobctrl_dbdump.exec()) + except Exception: + # エラーが起きても、正常系のコードで返す。 + # エラーが起きた事実はbatch_process内でログを出す。 + exit(0) diff --git a/ecs/jskult-dbdump/my.cnf b/ecs/jskult-dbdump/my.cnf new file mode 100644 index 00000000..8bdb2c20 --- /dev/null +++ b/ecs/jskult-dbdump/my.cnf @@ -0,0 +1,4 @@ +[client] +user=root +password=admin +host=localhost \ No newline at end of file diff --git a/ecs/jskult-dbdump/src/__init__.py b/ecs/jskult-dbdump/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/batch/batch_functions.py b/ecs/jskult-dbdump/src/batch/batch_functions.py new file mode 100644 index 00000000..da329291 --- /dev/null +++ b/ecs/jskult-dbdump/src/batch/batch_functions.py @@ -0,0 +1,118 @@ +"""バッチ処理の共通関数""" +import logging +import textwrap + +from src.db.database import Database +from src.error.exceptions import BatchOperationException, DBException +from src.system_var import constants + + +def get_batch_statuses() -> tuple[str, str]: + """日付テーブルから、以下を取得して返す。 + - バッチ処理中フラグ + - dump取得状況区分 + + Raises: + BatchOperationException: 日付テーブルが取得できないとき、何らかのエラーが発生したとき + + Returns: + tuple[str, str]: [0]バッチ処理中フラグ，[1]dump取得状況区分 + """ + db = Database.get_instance() + sql = 'SELECT bch_actf, dump_sts_kbn FROM src05.hdke_tbl' + try: + db.connect() + hdke_tbl_result = db.execute_select(sql) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + if len(hdke_tbl_result) == 0: + raise BatchOperationException('日付テーブルが取得できませんでした') + + # 必ず1件取れる + hdke_tbl_record = hdke_tbl_result[0] + batch_processing_flag = hdke_tbl_record['bch_actf'] + dump_status_kbn = hdke_tbl_record['dump_sts_kbn'] + + return batch_processing_flag, dump_status_kbn + + +def update_dump_status_kbn_in_processing() -> None: + """dump取得状況区分を処理中に更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = 'UPDATE src05.hdke_tbl SET dump_sts_kbn = :in_processing' + try: + db.connect() + db.execute(sql, {'in_processing': constants.BATCH_ACTF_BATCH_IN_PROCESSING}) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + return + + +def update_dump_status_kbn_in_error() -> None: + """dump取得状況区分をエラーに更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = """\ + UPDATE src05.hdke_tbl + SET + dump_sts_kbn = :dump_unprocessed + """ + try: + db.connect() + db.execute(sql, { + 'dump_unprocessed': constants.DUMP_STATUS_KBN_ERROR + }) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + return + + +def update_dump_status_kbn_in_complete() -> None: + """dump取得状況区分を正常終了に更新する + + Raises: + BatchOperationException: DB操作の何らかのエラー + """ + db = Database.get_instance() + sql = """\ + UPDATE src05.hdke_tbl + SET + dump_sts_kbn = :dump_unprocessed + """ + try: + db.connect() + db.execute(sql, { + 'dump_unprocessed': constants.DUMP_STATUS_KBN_COMPLETE + }) + except DBException as e: + raise BatchOperationException(e) + finally: + db.disconnect() + + return + + +def logging_sql(logger: logging.Logger, sql: str) -> None: + """SQL文をデバッグログで出力する + + Args: + logger (logging.Logger): ロガー + sql (str): SQL文 + """ + logger.debug(f'\n{"-" * 15}\n{textwrap.dedent(sql)[1:-1]}\n{"-" * 15}') diff --git a/ecs/jskult-dbdump/src/batch/common/__init__.py b/ecs/jskult-dbdump/src/batch/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/batch/common/batch_context.py b/ecs/jskult-dbdump/src/batch/common/batch_context.py new file mode 100644 index 00000000..8c8c12fb --- /dev/null +++ b/ecs/jskult-dbdump/src/batch/common/batch_context.py @@ -0,0 +1,29 @@ +class BatchContext: + __instance = None + __syor_date: str # 処理日(yyyy/mm/dd形式) + __is_arisj_output_day: bool # 月次バッチ起動日フラグ + + def __init__(self) -> None: + self.__is_arisj_output_day = False + + @classmethod + def get_instance(cls): + if cls.__instance is None: + cls.__instance = cls() + return cls.__instance + + @property + def syor_date(self): + return self.__syor_date + + @syor_date.setter + def syor_date(self, syor_date_str: str): + self.__syor_date = syor_date_str + + @property + def is_arisj_output_day(self): + return self.__is_arisj_output_day + + @is_arisj_output_day.setter + def is_arisj_output_day(self, flag: bool): + self.__is_arisj_output_day = flag diff --git a/ecs/jskult-dbdump/src/db/__init__.py b/ecs/jskult-dbdump/src/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/db/database.py b/ecs/jskult-dbdump/src/db/database.py new file mode 100644 index 00000000..f67a21b9 --- /dev/null +++ b/ecs/jskult-dbdump/src/db/database.py @@ -0,0 +1,178 @@ +from sqlalchemy import (Connection, CursorResult, Engine, QueuePool, + create_engine, text) +from sqlalchemy.engine.url import URL +from tenacity import retry, stop_after_attempt, wait_exponential + +from src.error.exceptions import DBException +from src.logging.get_logger import get_logger +from src.system_var import environment + +logger = get_logger(__name__) + + +class Database: + """データベース操作クラス""" + __connection: Connection = None + __engine: Engine = None + __host: str = None + __port: str = None + __username: str = None + __password: str = None + __schema: str = None + __connection_string: str = None + + def __init__(self, username: str, password: str, host: str, port: int, schema: str) -> None: + """このクラスの新たなインスタンスを初期化します + + Args: + username (str): DBユーザー名 + password (str): DBパスワード + host (str): DBホスト名 + port (int): DBポート + schema (str): DBスキーマ名 + """ + self.__username = username + self.__password = password + self.__host = host + self.__port = int(port) + self.__schema = schema + + self.__connection_string = URL.create( + drivername='mysql+pymysql', + username=self.__username, + password=self.__password, + host=self.__host, + port=self.__port, + database=self.__schema, + query={"charset": "utf8mb4"} + ) + + self.__engine = create_engine( + self.__connection_string, + pool_timeout=5, + poolclass=QueuePool + ) + + @classmethod + def get_instance(cls): + """インスタンスを取得します + + Returns: + Database: DB操作クラスインスタンス + """ + return cls( + username=environment.DB_USERNAME, + password=environment.DB_PASSWORD, + host=environment.DB_HOST, + port=environment.DB_PORT, + schema=environment.DB_SCHEMA + ) + + @retry( + wait=wait_exponential( + multiplier=environment.DB_CONNECTION_RETRY_INTERVAL_INIT, + min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, + max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS + ), + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) + def connect(self): + """ + DBに接続します。接続に失敗した場合、リトライします。 + Raises: + DBException: 接続失敗 + """ + try: + self.__connection = self.__engine.connect() + except Exception as e: + raise DBException(e) + + def execute_select(self, select_query: str, parameters=None) -> list[dict]: + """SELECTクエリを実行します。 + + Args: + select_query (str): SELECT文 + parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None. + + Raises: + DBException: DBエラー + + Returns: + list[dict]: カラム名: 値の辞書リスト + """ + if self.__connection is None: + raise DBException('DBに接続していません') + + result = None + try: + # トランザクションが開始している場合は、トランザクションを引き継ぐ + if self.__connection.in_transaction(): + result = self.__connection.execute(text(select_query), parameters) + else: + # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。 + result = self.__execute_with_transaction(select_query, parameters) + except Exception as e: + raise DBException(f'SQL Error: {e}') + + result_rows = result.mappings().all() + return result_rows + + def execute(self, query: str, parameters=None) -> CursorResult: + """SQLクエリを実行します。 + + Args: + query (str): SQL文 + parameters (dict, optional): クエリのプレースホルダーに埋め込む変数の辞書. Defaults to None. + + Raises: + DBException: DBエラー + + Returns: + CursorResult: 取得結果 + """ + if self.__connection is None: + raise DBException('DBに接続していません') + + result = None + try: + # トランザクションが開始している場合は、トランザクションを引き継ぐ + if self.__connection.in_transaction(): + result = self.__connection.execute(text(query), parameters) + else: + # トランザクションが明示的に開始していない場合は、クエリ単位でトランザクションをbegin-commitする。 + result = self.__execute_with_transaction(query, parameters) + except Exception as e: + raise DBException(f'SQL Error: {e}') + + return result + + def begin(self): + """トランザクションを開始します。""" + if not self.__connection.in_transaction(): + self.__connection.begin() + + def commit(self): + """トランザクションをコミットします""" + if self.__connection.in_transaction(): + self.__connection.commit() + + def rollback(self): + """トランザクションをロールバックします""" + if self.__connection.in_transaction(): + self.__connection.rollback() + + def disconnect(self): + """DB接続を切断します。""" + if self.__connection is not None: + self.__connection.close() + self.__connection = None + + def __execute_with_transaction(self, query: str, parameters: dict): + # トランザクションを開始してクエリを実行する + with self.__connection.begin(): + try: + result = self.__connection.execute(text(query), parameters=parameters) + except Exception as e: + self.__connection.rollback() + raise e + # ここでコミットされる + return result diff --git a/ecs/jskult-dbdump/src/error/__init__.py b/ecs/jskult-dbdump/src/error/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/error/exceptions.py b/ecs/jskult-dbdump/src/error/exceptions.py new file mode 100644 index 00000000..055c24f6 --- /dev/null +++ b/ecs/jskult-dbdump/src/error/exceptions.py @@ -0,0 +1,10 @@ +class MeDaCaException(Exception): + pass + + +class DBException(MeDaCaException): + pass + + +class BatchOperationException(MeDaCaException): + pass diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py new file mode 100644 index 00000000..3ae7bdb6 --- /dev/null +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -0,0 +1,109 @@ +"""日次バッチ処理前DBダンプ取得""" + +import datetime +import textwrap +import subprocess +import os +from src.system_var import environment +from src.error.exceptions import BatchOperationException +from src.logging.get_logger import get_logger +from src.system_var import constants +from src.batch.batch_functions import ( + get_batch_statuses, update_dump_status_kbn_in_processing, + update_dump_status_kbn_in_complete, update_dump_status_kbn_in_error) +import gzip + +logger = get_logger('日次バッチ処理前DBダンプ取得') + + +def exec(): + try: + logger.info('日次バッチ処理前DBダンプ取得：開始') + try: + # 日次バッチ処置中フラグ、dump処理状態区分を取得 + batch_processing_flag, dump_status_kbn = get_batch_statuses() + except BatchOperationException as e: + logger.exception(f'日次ジョブ取得エラー（異常終了）\n{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # 日次バッチ処理中の場合、処理は行わない + if batch_processing_flag == constants.BATCH_ACTF_BATCH_IN_PROCESSING: + logger.error('日次ジョブ処理中エラー（異常終了）') + return constants.BATCH_EXIT_CODE_SUCCESS + + # dump処理状態区分が処理中またはエラーの場合、処理は行わない + if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED or dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: + logger.error('dump処理状態区エラー（異常終了）') + return constants.BATCH_EXIT_CODE_SUCCESS + + # dump処理状態区分を処理中に更新 + try: + update_dump_status_kbn_in_processing() + except BatchOperationException as e: + logger.exception(f'dump処理状態区分更新(未処理→処理中) エラー（異常終了）{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # MySQL接続情報を作成する + my_cnf_file_content = f""" + [client] + user={environment.DB_USERNAME} + password={environment.DB_PASSWORD} + host={environment.DB_HOST} + """ + # my.cnfファイルのパス + my_cnf_path = os.path.join('.', 'my.cnf') + + # my.cnfファイルを生成する + with open(my_cnf_path, 'w') as f: + f.write(textwrap.dedent(my_cnf_file_content)[1:-1]) + + dt_now = datetime.datetime.now() + converted_value = dt_now.strftime('%Y%m%d%H%M%S') + file_name = f'backup_rds_src05_{converted_value}.gz' + s3_file_name = f's3://{environment.JSKULT_BACKUP_BUCKET}/{environment.DUMP_BACKUP_FOLDER}/{dt_now.year}/{dt_now.strftime("%m")}/{dt_now.strftime("%d")}/{file_name}' + + mysqldump_cmd = [ + 'mysqldump', + '--user={mysql_user}'.format(mysql_user=environment.DB_USERNAME), + '--password={db_pw}'.format(db_pw=environment.DB_PASSWORD), + '--host={db_host}'.format(db_host=environment.DB_HOST), + '--port={db_port}'.format(db_port=environment.DB_PORT), + '{db_name}'.format(db_name=environment.DB_SCHEMA) + ] + + # mysqldumpコマンドを実行してデータを標準出力に出力 + mysqldump_output = subprocess.check_output(mysqldump_cmd) + + # gzipで圧縮 + compressed_data = gzip.compress(mysqldump_output) + + # AWS CLIを使ってS3にアップロード + aws_cli_cmd = [ + 'aws', + 's3', + 'cp', + '-', + s3_file_name + ] + subprocess.run(aws_cli_cmd, input=compressed_data) + + # dump処理状態区分を正常終了に更新 + try: + update_dump_status_kbn_in_complete() + except BatchOperationException as e: + logger.exception(f'dump処理状態区分更新(処理中→正常終了) エラー（異常終了）\n{e}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # 正常終了を保守ユーザーに通知 + logger.info('[NOTICE]日次バッチ処理前DBダンプ取得：終了（正常終了）') + return constants.BATCH_EXIT_CODE_SUCCESS + + except Exception as e: + # dump処理状態区分をエラーに更新 + try: + update_dump_status_kbn_in_error() + except BatchOperationException as e: + logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）\n{e}') + raise constants.BATCH_EXIT_CODE_SUCCESS + logger.exception(f'日次バッチ処理前DBダンプ取得中に想定外のエラーが発生しました \n{e}') + raise e diff --git a/ecs/jskult-dbdump/src/logging/get_logger.py b/ecs/jskult-dbdump/src/logging/get_logger.py new file mode 100644 index 00000000..f36f1199 --- /dev/null +++ b/ecs/jskult-dbdump/src/logging/get_logger.py @@ -0,0 +1,37 @@ +import logging + +from src.system_var.environment import LOG_LEVEL + +# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する +for name in ["boto3", "botocore", "s3transfer", "urllib3"]: + logging.getLogger(name).setLevel(logging.WARNING) + + +def get_logger(log_name: str) -> logging.Logger: + """一意のログ出力モジュールを取得します。 + + Args: + log_name (str): ロガー名 + + Returns: + _type_: _description_ + """ + logger = logging.getLogger(log_name) + level = logging.getLevelName(LOG_LEVEL) + if not isinstance(level, int): + level = logging.INFO + logger.setLevel(level) + + if not logger.hasHandlers(): + handler = logging.StreamHandler() + logger.addHandler(handler) + + formatter = logging.Formatter( + '%(name)s\t[%(levelname)s]\t%(asctime)s\t%(message)s', + '%Y-%m-%d %H:%M:%S' + ) + + for handler in logger.handlers: + handler.setFormatter(formatter) + + return logger diff --git a/ecs/jskult-dbdump/src/system_var/__init__.py b/ecs/jskult-dbdump/src/system_var/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-dbdump/src/system_var/constants.py b/ecs/jskult-dbdump/src/system_var/constants.py new file mode 100644 index 00000000..993e5a7d --- /dev/null +++ b/ecs/jskult-dbdump/src/system_var/constants.py @@ -0,0 +1,15 @@ +# バッチ正常終了コード +BATCH_EXIT_CODE_SUCCESS = 0 + +# バッチ処理中フラグ：未処理 +BATCH_ACTF_BATCH_UNPROCESSED = '0' +# バッチ処理中フラグ：処理中 +BATCH_ACTF_BATCH_IN_PROCESSING = '1' +# dump取得状態区分：未処理 +DUMP_STATUS_KBN_UNPROCESSED = '0' +# dump取得状態区分：処理中 +DUMP_STATUS_KBN_PROCESSED = '1' +# dump取得状態区分：正常終了 +DUMP_STATUS_KBN_COMPLETE = '2' +# dump取得状態区分：エラー +DUMP_STATUS_KBN_ERROR = '9' diff --git a/ecs/jskult-dbdump/src/system_var/environment.py b/ecs/jskult-dbdump/src/system_var/environment.py new file mode 100644 index 00000000..123935a2 --- /dev/null +++ b/ecs/jskult-dbdump/src/system_var/environment.py @@ -0,0 +1,21 @@ +import os + +# Database +DB_HOST = os.environ['DB_HOST'] +DB_PORT = int(os.environ['DB_PORT']) +DB_USERNAME = os.environ['DB_USERNAME'] +DB_PASSWORD = os.environ['DB_PASSWORD'] +DB_SCHEMA = os.environ['DB_SCHEMA'] + +# AWS +JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] +JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] + +# 初期値がある環境変数 +LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') +DB_CONNECTION_MAX_RETRY_ATTEMPT = int(os.environ.get('DB_CONNECTION_MAX_RETRY_ATTEMPT', 4)) +DB_CONNECTION_RETRY_INTERVAL_INIT = int(os.environ.get('DB_CONNECTION_RETRY_INTERVAL', 5)) +DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MIN_SECONDS', 5)) +DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MAX_SECONDS', 50)) + +DUMP_BACKUP_FOLDER = os.environ['DUMP_BACKUP_FOLDER'] From cf02c6158804df78fd0c5da095c8e71a0088ba41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Mon, 10 Jul 2023 13:52:14 +0900 Subject: [PATCH 03/30] =?UTF-8?q?=E4=BB=AE=E5=AE=8C=E6=88=90=EF=BC=92?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/.env.example | 1 - .../backup_rds_src05_20230710091352.gz | 0 ecs/jskult-dbdump/src/jobctrl_dbdump.py | 18 +++++++++++++++++- .../src/system_var/environment.py | 1 - 4 files changed, 17 insertions(+), 3 deletions(-) delete mode 100644 ecs/jskult-dbdump/backup_rds_src05_20230710091352.gz diff --git a/ecs/jskult-dbdump/.env.example b/ecs/jskult-dbdump/.env.example index b331b25e..6db2d461 100644 --- a/ecs/jskult-dbdump/.env.example +++ b/ecs/jskult-dbdump/.env.example @@ -5,7 +5,6 @@ DB_PASSWORD=************ DB_SCHEMA=src05 JSKULT_BACKUP_BUCKET=mbj-newdwh2021-staging-backup-jskult -JSKULT_CONFIG_BUCKET=mbj-newdwh2021-staging-config DUMP_BACKUP_FOLDER=dump diff --git a/ecs/jskult-dbdump/backup_rds_src05_20230710091352.gz b/ecs/jskult-dbdump/backup_rds_src05_20230710091352.gz deleted file mode 100644 index e69de29b..00000000 diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 3ae7bdb6..42be4e51 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -57,6 +57,23 @@ def exec(): with open(my_cnf_path, 'w') as f: f.write(textwrap.dedent(my_cnf_file_content)[1:-1]) + #*************************************************************** + # # MySQLコマンドを実行する + # command = ['mysqldump', f'--defaults-file={my_cnf_path}', '-P', f"'{environment.DB_PORT}'", environment.DB_SCHEMA] + # mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # # gzipコマンドを実行して圧縮する + # gzip_process = subprocess.Popen(['gzip'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # # aws s3 cpコマンドを実行してアップロードする + # s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_name], stdin=gzip_process.stdout ,stderr=subprocess.PIPE) + # # 出力を取得する + # mysqldump_process.stdout.close() + # _, error = s3_cp_process.communicate() + + # if s3_cp_process.returncode != 0: + # print("Error: ", error.decode("utf-8")) + # raise Exception('Error') + #*************************************************************** + dt_now = datetime.datetime.now() converted_value = dt_now.strftime('%Y%m%d%H%M%S') file_name = f'backup_rds_src05_{converted_value}.gz' @@ -70,7 +87,6 @@ def exec(): '--port={db_port}'.format(db_port=environment.DB_PORT), '{db_name}'.format(db_name=environment.DB_SCHEMA) ] - # mysqldumpコマンドを実行してデータを標準出力に出力 mysqldump_output = subprocess.check_output(mysqldump_cmd) diff --git a/ecs/jskult-dbdump/src/system_var/environment.py b/ecs/jskult-dbdump/src/system_var/environment.py index 123935a2..46f08c5d 100644 --- a/ecs/jskult-dbdump/src/system_var/environment.py +++ b/ecs/jskult-dbdump/src/system_var/environment.py @@ -9,7 +9,6 @@ DB_SCHEMA = os.environ['DB_SCHEMA'] # AWS JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] -JSKULT_CONFIG_BUCKET = os.environ['JSKULT_CONFIG_BUCKET'] # 初期値がある環境変数 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') From 59548a1fa4392ba99b4e721697552b79e64c63a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Mon, 10 Jul 2023 19:44:30 +0900 Subject: [PATCH 04/30] =?UTF-8?q?=E3=83=91=E3=83=A9=E3=83=A1=E3=83=BC?= =?UTF-8?q?=E3=82=BF=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 42be4e51..477a1388 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -57,23 +57,6 @@ def exec(): with open(my_cnf_path, 'w') as f: f.write(textwrap.dedent(my_cnf_file_content)[1:-1]) - #*************************************************************** - # # MySQLコマンドを実行する - # command = ['mysqldump', f'--defaults-file={my_cnf_path}', '-P', f"'{environment.DB_PORT}'", environment.DB_SCHEMA] - # mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # # gzipコマンドを実行して圧縮する - # gzip_process = subprocess.Popen(['gzip'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # # aws s3 cpコマンドを実行してアップロードする - # s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_name], stdin=gzip_process.stdout ,stderr=subprocess.PIPE) - # # 出力を取得する - # mysqldump_process.stdout.close() - # _, error = s3_cp_process.communicate() - - # if s3_cp_process.returncode != 0: - # print("Error: ", error.decode("utf-8")) - # raise Exception('Error') - #*************************************************************** - dt_now = datetime.datetime.now() converted_value = dt_now.strftime('%Y%m%d%H%M%S') file_name = f'backup_rds_src05_{converted_value}.gz' @@ -85,6 +68,10 @@ def exec(): '--password={db_pw}'.format(db_pw=environment.DB_PASSWORD), '--host={db_host}'.format(db_host=environment.DB_HOST), '--port={db_port}'.format(db_port=environment.DB_PORT), + '--no-tablespaces', + '--skip-column-statistics', + '--single-transaction', + '--set-gtid-purged=OFF', '{db_name}'.format(db_name=environment.DB_SCHEMA) ] # mysqldumpコマンドを実行してデータを標準出力に出力 From 47d164a9a254ba6146b86b149f99d931bc4d8df0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Tue, 11 Jul 2023 11:08:09 +0900 Subject: [PATCH 05/30] =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 477a1388..1f821456 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -33,7 +33,7 @@ def exec(): # dump処理状態区分が処理中またはエラーの場合、処理は行わない if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED or dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: - logger.error('dump処理状態区エラー（異常終了）') + logger.error('dump処理状態区分エラー（異常終了）') return constants.BATCH_EXIT_CODE_SUCCESS # dump処理状態区分を処理中に更新 From e87da068bb9986f1bac482cde2cd0ecd6013fc87 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 13:03:54 +0900 Subject: [PATCH 06/30] =?UTF-8?q?feat:=20DB=E5=86=8D=E8=B5=B7=E5=8B=95?= =?UTF-8?q?=E3=80=81=E6=99=82=E9=96=93=E7=B5=8C=E9=81=8E=E5=BE=8C=E3=81=AE?= =?UTF-8?q?=E3=82=B3=E3=83=8D=E3=82=AF=E3=82=B7=E3=83=A7=E3=83=B3=E5=86=8D?= =?UTF-8?q?=E7=A2=BA=E7=AB=8B=E3=81=AE=E4=BB=95=E8=BE=BC=E3=81=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-webapp/src/db/database.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/ecs/jskult-webapp/src/db/database.py b/ecs/jskult-webapp/src/db/database.py index 85759f96..f700c016 100644 --- a/ecs/jskult-webapp/src/db/database.py +++ b/ecs/jskult-webapp/src/db/database.py @@ -1,10 +1,28 @@ from sqlalchemy import (Connection, CursorResult, Engine, QueuePool, - create_engine, text) + create_engine, event, exc, text) from sqlalchemy.engine.url import URL +from sqlalchemy.pool import Pool from src.error.exceptions import DBException +from src.logging.get_logger import get_logger from src.system_var import environment +logger = get_logger('DB接続') + + +@event.listens_for(Pool, 'checkout') +def ping_connection(dbapi_connection, connection_record, connection_proxy): + """コネクションが切れた場合、再接続""" + cursor = dbapi_connection.cursor() + try: + cursor.execute("SELECT 1") + except Exception as e: + logger.info(f'DB接続に失敗したため、リトライします: {e}') + # raise DisconnectionError - pool will try + # connecting again up to three times before raising. + raise exc.DisconnectionError() + cursor.close() + class Database: """データベース操作クラス""" From c7c5eade7b311aea55d45b1f25f950ed2cafae0f Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 14:42:28 +0900 Subject: [PATCH 07/30] =?UTF-8?q?fix:=20my.cnf=E3=81=AFgit=E8=BF=BD?= =?UTF-8?q?=E8=B7=A1=E5=AF=BE=E8=B1=A1=E5=A4=96=E3=81=AB=E3=81=99=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/.gitignore | 1 + ecs/jskult-dbdump/my.cnf | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 ecs/jskult-dbdump/my.cnf diff --git a/ecs/jskult-dbdump/.gitignore b/ecs/jskult-dbdump/.gitignore index bd0b37f8..cf44449e 100644 --- a/ecs/jskult-dbdump/.gitignore +++ b/ecs/jskult-dbdump/.gitignore @@ -1,5 +1,6 @@ .vscode/settings.json .env +my.cnf # python __pycache__ diff --git a/ecs/jskult-dbdump/my.cnf b/ecs/jskult-dbdump/my.cnf deleted file mode 100644 index 8bdb2c20..00000000 --- a/ecs/jskult-dbdump/my.cnf +++ /dev/null @@ -1,4 +0,0 @@ -[client] -user=root -password=admin -host=localhost \ No newline at end of file From 268282486378eb456d639ac8d6510b0e70014aca Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 14:46:46 +0900 Subject: [PATCH 08/30] =?UTF-8?q?feat:=20Docker=E3=82=A4=E3=83=A1=E3=83=BC?= =?UTF-8?q?=E3=82=B8=E3=81=A7mysql-client=208.x=E3=82=92=E4=BD=BF=E7=94=A8?= =?UTF-8?q?=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/Dockerfile | 38 ++++++++++++++-------- ecs/jskult-dbdump/mysql_dpkg_selection.txt | 3 ++ 2 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 ecs/jskult-dbdump/mysql_dpkg_selection.txt diff --git a/ecs/jskult-dbdump/Dockerfile b/ecs/jskult-dbdump/Dockerfile index f87a7215..1d1a865e 100644 --- a/ecs/jskult-dbdump/Dockerfile +++ b/ecs/jskult-dbdump/Dockerfile @@ -1,28 +1,40 @@ -FROM python:3.9 +FROM python:3.9-bullseye ENV TZ="Asia/Tokyo" WORKDIR /usr/src/app COPY Pipfile Pipfile.lock ./ +# apt mysqlパッケージのdpkg次のコマンド注入用 +COPY mysql_dpkg_selection.txt ./ +# 必要なパッケージインストール +RUN apt update && apt install -y less vim curl wget gzip unzip sudo lsb-release + +# # mysqlをインストール +RUN \ + wget https://dev.mysql.com/get/mysql-apt-config_0.8.25-1_all.deb && \ + dpkg -i mysql-apt-config_0.8.25-1_all.deb < mysql_dpkg_selection.txt && \ + apt update && \ + apt install -y mysql-client + +# aws cli v2 のインストール +RUN \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + sudo ./aws/install + +# python関連のライブラリインストール RUN \ - apt update -y && \ - # パッケージのセキュリティアップデートのみを適用するコマンド - apt install -y unattended-upgrades && \ - unattended-upgrades && \ pip install --upgrade pip wheel setuptools && \ pip install pipenv --no-cache-dir && \ pipenv install --system --deploy && \ pip uninstall -y pipenv virtualenv-clone virtualenv -# mysql-clientと必要なパッケージインストール -RUN apt install -y less vim curl unzip sudo default-mysql-client - -# aws cli v2 のインストール -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -RUN unzip awscliv2.zip -RUN sudo ./aws/install +# パッケージのセキュリティアップデートのみを適用するコマンドを実行 +RUN \ + apt install -y unattended-upgrades && \ + unattended-upgrades COPY src ./src -COPY entrypoint.py entrypoint.py +COPY entrypoint.py entrypoint.py CMD ["python", "entrypoint.py"] diff --git a/ecs/jskult-dbdump/mysql_dpkg_selection.txt b/ecs/jskult-dbdump/mysql_dpkg_selection.txt new file mode 100644 index 00000000..d3cb46f9 --- /dev/null +++ b/ecs/jskult-dbdump/mysql_dpkg_selection.txt @@ -0,0 +1,3 @@ +1 +1 +4 \ No newline at end of file From dbfe62e30b7588024c8cf23527cabde4854230d6 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 16:31:14 +0900 Subject: [PATCH 09/30] =?UTF-8?q?feat:=20=E3=83=AD=E3=83=BC=E3=82=AB?= =?UTF-8?q?=E3=83=AB=E3=81=AE=E3=82=B9=E3=83=88=E3=83=AC=E3=83=BC=E3=82=B8?= =?UTF-8?q?=E3=82=92=E7=B5=8C=E7=94=B1=E3=81=9B=E3=81=9A=E3=80=81=E3=83=91?= =?UTF-8?q?=E3=82=A4=E3=83=97=E3=83=A9=E3=82=A4=E3=83=B3=E3=81=AE=E3=81=BF?= =?UTF-8?q?=E3=81=A7=E5=AE=9F=E8=A1=8C=E3=80=82=E3=82=A8=E3=83=A9=E3=83=BC?= =?UTF-8?q?=E3=83=8F=E3=83=B3=E3=83=89=E3=83=AA=E3=83=B3=E3=82=B0=E3=81=AF?= =?UTF-8?q?=E5=AE=9F=E8=A3=85=E3=81=A7=E3=81=8D=E3=81=A6=E3=81=84=E3=81=AA?= =?UTF-8?q?=E3=81=84=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 80 ++++++++++++------------- 1 file changed, 39 insertions(+), 41 deletions(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 1f821456..dcc02211 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -1,17 +1,17 @@ """日次バッチ処理前DBダンプ取得""" import datetime -import textwrap -import subprocess import os -from src.system_var import environment +import subprocess +import textwrap + +from src.batch.batch_functions import (get_batch_statuses, + update_dump_status_kbn_in_complete, + update_dump_status_kbn_in_error, + update_dump_status_kbn_in_processing) from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger -from src.system_var import constants -from src.batch.batch_functions import ( - get_batch_statuses, update_dump_status_kbn_in_processing, - update_dump_status_kbn_in_complete, update_dump_status_kbn_in_error) -import gzip +from src.system_var import constants, environment logger = get_logger('日次バッチ処理前DBダンプ取得') @@ -23,7 +23,7 @@ def exec(): # 日次バッチ処置中フラグ、dump処理状態区分を取得 batch_processing_flag, dump_status_kbn = get_batch_statuses() except BatchOperationException as e: - logger.exception(f'日次ジョブ取得エラー（異常終了）\n{e}') + logger.exception(f'日次ジョブ取得エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS # 日次バッチ処理中の場合、処理は行わない @@ -40,7 +40,7 @@ def exec(): try: update_dump_status_kbn_in_processing() except BatchOperationException as e: - logger.exception(f'dump処理状態区分更新(未処理→処理中) エラー（異常終了）{e}') + logger.exception(f'dump処理状態区分更新(未処理→処理中) エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS # MySQL接続情報を作成する @@ -51,7 +51,7 @@ def exec(): host={environment.DB_HOST} """ # my.cnfファイルのパス - my_cnf_path = os.path.join('.', 'my.cnf') + my_cnf_path = os.path.join('my.cnf') # my.cnfファイルを生成する with open(my_cnf_path, 'w') as f: @@ -62,39 +62,37 @@ def exec(): file_name = f'backup_rds_src05_{converted_value}.gz' s3_file_name = f's3://{environment.JSKULT_BACKUP_BUCKET}/{environment.DUMP_BACKUP_FOLDER}/{dt_now.year}/{dt_now.strftime("%m")}/{dt_now.strftime("%d")}/{file_name}' - mysqldump_cmd = [ - 'mysqldump', - '--user={mysql_user}'.format(mysql_user=environment.DB_USERNAME), - '--password={db_pw}'.format(db_pw=environment.DB_PASSWORD), - '--host={db_host}'.format(db_host=environment.DB_HOST), - '--port={db_port}'.format(db_port=environment.DB_PORT), - '--no-tablespaces', - '--skip-column-statistics', - '--single-transaction', - '--set-gtid-purged=OFF', - '{db_name}'.format(db_name=environment.DB_SCHEMA) + # 'mysqldump --login-path=dwhadmin --no-tablespaces --skip-column-statistics --single-transaction --set-gtid-purged=OFF dwh > /data/mountdwh/backup/%s/before/' + # mysqldumpコマンドを実行する + command = [ + 'mysqldump', + f'--defaults-file={my_cnf_path}', + '-P', + f"{environment.DB_PORT}", + '--no-tablespaces', + '--skip-column-statistics', + '--single-transaction', + '--set-gtid-purged=OFF', + environment.DB_SCHEMA + # 'src05' ] - # mysqldumpコマンドを実行してデータを標準出力に出力 - mysqldump_output = subprocess.check_output(mysqldump_cmd) - - # gzipで圧縮 - compressed_data = gzip.compress(mysqldump_output) - - # AWS CLIを使ってS3にアップロード - aws_cli_cmd = [ - 'aws', - 's3', - 'cp', - '-', - s3_file_name - ] - subprocess.run(aws_cli_cmd, input=compressed_data) - + mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # gzipコマンドを実行してdump結果を圧縮する + gzip_process = subprocess.Popen(['gzip'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # aws s3 cpコマンドを実行してアップロードする + s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_name], stdin=gzip_process.stdout, stderr=subprocess.PIPE) + # 出力を取得する + mysqldump_process.stdout.close() + gzip_process.stdout.close() + _, error = s3_cp_process.communicate() + if s3_cp_process.returncode != 0: + print("Error: ", error.decode("utf-8")) + raise Exception('Error') # dump処理状態区分を正常終了に更新 try: update_dump_status_kbn_in_complete() except BatchOperationException as e: - logger.exception(f'dump処理状態区分更新(処理中→正常終了) エラー（異常終了）\n{e}') + logger.exception(f'dump処理状態区分更新(処理中→正常終了) エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS # 正常終了を保守ユーザーに通知 @@ -106,7 +104,7 @@ def exec(): try: update_dump_status_kbn_in_error() except BatchOperationException as e: - logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）\n{e}') + logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）:{e}') raise constants.BATCH_EXIT_CODE_SUCCESS - logger.exception(f'日次バッチ処理前DBダンプ取得中に想定外のエラーが発生しました \n{e}') + logger.exception(f'日次バッチ処理前DBダンプ取得中に想定外のエラーが発生しました :{e}') raise e From 0b19ba8e04bdd7cc790be65595b77eabac3c52e4 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 17:03:13 +0900 Subject: [PATCH 10/30] =?UTF-8?q?feat:=20=E5=90=84=E3=82=B3=E3=83=9E?= =?UTF-8?q?=E3=83=B3=E3=83=89=E3=81=94=E3=81=A8=E3=81=AB=E3=82=A8=E3=83=A9?= =?UTF-8?q?=E3=83=BC=E3=83=8F=E3=83=B3=E3=83=89=E3=83=AA=E3=83=B3=E3=82=B0?= =?UTF-8?q?=E3=81=99=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 27 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index dcc02211..dc688f69 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -62,8 +62,7 @@ def exec(): file_name = f'backup_rds_src05_{converted_value}.gz' s3_file_name = f's3://{environment.JSKULT_BACKUP_BUCKET}/{environment.DUMP_BACKUP_FOLDER}/{dt_now.year}/{dt_now.strftime("%m")}/{dt_now.strftime("%d")}/{file_name}' - # 'mysqldump --login-path=dwhadmin --no-tablespaces --skip-column-statistics --single-transaction --set-gtid-purged=OFF dwh > /data/mountdwh/backup/%s/before/' - # mysqldumpコマンドを実行する + # mysqldumpコマンドを実行し、dumpを取得する command = [ 'mysqldump', f'--defaults-file={my_cnf_path}', @@ -81,22 +80,32 @@ def exec(): gzip_process = subprocess.Popen(['gzip'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # aws s3 cpコマンドを実行してアップロードする s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_name], stdin=gzip_process.stdout, stderr=subprocess.PIPE) - # 出力を取得する + # mysqldumpの標準出力をgzipに接続したため、標準出力をクローズする mysqldump_process.stdout.close() + # gzipの標準出力をaws s3 cpに接続したため、標準出力をクローズする gzip_process.stdout.close() + + # パイプラインを実行し、エラーハンドリング + _, error = mysqldump_process.communicate() + if mysqldump_process.returncode != 0: + raise BatchOperationException(error.decode('utf-8')) + + _, error = gzip_process.communicate() + if gzip_process.returncode != 0: + raise BatchOperationException(error.decode('utf-8')) + _, error = s3_cp_process.communicate() if s3_cp_process.returncode != 0: - print("Error: ", error.decode("utf-8")) - raise Exception('Error') - # dump処理状態区分を正常終了に更新 + raise BatchOperationException(error.decode('utf-8')) + + # # dump処理状態区分を正常終了に更新 try: update_dump_status_kbn_in_complete() except BatchOperationException as e: logger.exception(f'dump処理状態区分更新(処理中→正常終了) エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS - # 正常終了を保守ユーザーに通知 - logger.info('[NOTICE]日次バッチ処理前DBダンプ取得：終了（正常終了）') + logger.info('日次バッチ処理前DBダンプ取得：終了（正常終了）') return constants.BATCH_EXIT_CODE_SUCCESS except Exception as e: @@ -105,6 +114,6 @@ def exec(): update_dump_status_kbn_in_error() except BatchOperationException as e: logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）:{e}') - raise constants.BATCH_EXIT_CODE_SUCCESS + raise e logger.exception(f'日次バッチ処理前DBダンプ取得中に想定外のエラーが発生しました :{e}') raise e From a108f20a21860a6f6ce1c16569ee2795dab57a7d Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 17:06:39 +0900 Subject: [PATCH 11/30] =?UTF-8?q?feat:=20DB=E6=8E=A5=E7=B6=9A=E3=83=AA?= =?UTF-8?q?=E3=83=88=E3=83=A9=E3=82=A4=E6=99=82=E3=81=ABRetry=E3=82=A8?= =?UTF-8?q?=E3=83=A9=E3=83=BC=E3=81=8C=E8=BF=94=E3=81=95=E3=82=8C=E3=80=81?= =?UTF-8?q?=E6=8D=95=E7=B8=9B=E3=81=A7=E3=81=8D=E3=81=A6=E3=81=84=E3=81=AA?= =?UTF-8?q?=E3=81=8B=E3=81=A3=E3=81=9F=E3=81=9F=E3=82=81=E3=80=81=E4=BF=AE?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/db/database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ecs/jskult-dbdump/src/db/database.py b/ecs/jskult-dbdump/src/db/database.py index f67a21b9..67f5c5a5 100644 --- a/ecs/jskult-dbdump/src/db/database.py +++ b/ecs/jskult-dbdump/src/db/database.py @@ -74,7 +74,9 @@ class Database: min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS ), - stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), + retry_error_cls=DBException + ) def connect(self): """ DBに接続します。接続に失敗した場合、リトライします。 From c87bee4e9ad40213fefc6f376f86c8ddcd2cbb8a Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 17:44:30 +0900 Subject: [PATCH 12/30] =?UTF-8?q?feat:=20=E3=82=A8=E3=83=A9=E3=83=BC?= =?UTF-8?q?=E3=83=8F=E3=83=B3=E3=83=89=E3=83=AA=E3=83=B3=E3=82=B0=E3=81=AE?= =?UTF-8?q?=E3=82=84=E3=82=8A=E6=96=B9=E3=82=92=E7=B5=B1=E4=B8=80=E3=80=82?= =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/Dockerfile | 4 ++-- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/ecs/jskult-dbdump/Dockerfile b/ecs/jskult-dbdump/Dockerfile index 1d1a865e..9c6838cb 100644 --- a/ecs/jskult-dbdump/Dockerfile +++ b/ecs/jskult-dbdump/Dockerfile @@ -4,12 +4,12 @@ ENV TZ="Asia/Tokyo" WORKDIR /usr/src/app COPY Pipfile Pipfile.lock ./ -# apt mysqlパッケージのdpkg次のコマンド注入用 +# mysql-apt-config をdpkgでインストールする際に標準出力に渡す文字列ファイルをコピー COPY mysql_dpkg_selection.txt ./ # 必要なパッケージインストール RUN apt update && apt install -y less vim curl wget gzip unzip sudo lsb-release -# # mysqlをインストール +# mysqlをインストール RUN \ wget https://dev.mysql.com/get/mysql-apt-config_0.8.25-1_all.deb && \ dpkg -i mysql-apt-config_0.8.25-1_all.deb < mysql_dpkg_selection.txt && \ diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index dc688f69..964c34e8 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -88,17 +88,20 @@ def exec(): # パイプラインを実行し、エラーハンドリング _, error = mysqldump_process.communicate() if mysqldump_process.returncode != 0: - raise BatchOperationException(error.decode('utf-8')) + logger.error(f'`mysqldump`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS _, error = gzip_process.communicate() if gzip_process.returncode != 0: - raise BatchOperationException(error.decode('utf-8')) + logger.error(f'`gzip`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS _, error = s3_cp_process.communicate() if s3_cp_process.returncode != 0: - raise BatchOperationException(error.decode('utf-8')) + logger.error(f'`aws s3 cp`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS - # # dump処理状態区分を正常終了に更新 + # dump処理状態区分を正常終了に更新 try: update_dump_status_kbn_in_complete() except BatchOperationException as e: From 1ae009115d6258593741b9eb9102e2cec1007087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Tue, 11 Jul 2023 18:02:52 +0900 Subject: [PATCH 13/30] =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E3=83=AD?= =?UTF-8?q?=E3=82=B0=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 964c34e8..02b8da84 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -23,17 +23,17 @@ def exec(): # 日次バッチ処置中フラグ、dump処理状態区分を取得 batch_processing_flag, dump_status_kbn = get_batch_statuses() except BatchOperationException as e: - logger.exception(f'日次ジョブ取得エラー（異常終了）:{e}') + logger.exception(f'日付テーブル取得エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS # 日次バッチ処理中の場合、処理は行わない if batch_processing_flag == constants.BATCH_ACTF_BATCH_IN_PROCESSING: - logger.error('日次ジョブ処理中エラー（異常終了）') + logger.error('日次バッチ処理中の為、処理を終了') return constants.BATCH_EXIT_CODE_SUCCESS # dump処理状態区分が処理中またはエラーの場合、処理は行わない if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED or dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: - logger.error('dump処理状態区分エラー（異常終了）') + logger.error(f'dump処理状態区分が実行不可な状態の為、処理を終了 dump処理状態区分={dump_status_kbn}') return constants.BATCH_EXIT_CODE_SUCCESS # dump処理状態区分を処理中に更新 @@ -73,7 +73,6 @@ def exec(): '--single-transaction', '--set-gtid-purged=OFF', environment.DB_SCHEMA - # 'src05' ] mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # gzipコマンドを実行してdump結果を圧縮する @@ -117,6 +116,6 @@ def exec(): update_dump_status_kbn_in_error() except BatchOperationException as e: logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）:{e}') - raise e + return constants.BATCH_EXIT_CODE_SUCCESS logger.exception(f'日次バッチ処理前DBダンプ取得中に想定外のエラーが発生しました :{e}') - raise e + return constants.BATCH_EXIT_CODE_SUCCESS From eb1ccebbacc511b746de1902f0b44894962df39a Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 18:13:59 +0900 Subject: [PATCH 14/30] =?UTF-8?q?fix:=20DB=E6=8E=A5=E7=B6=9A=E3=81=AE?= =?UTF-8?q?=E3=83=AA=E3=83=88=E3=83=A9=E3=82=A4=E3=82=A8=E3=83=A9=E3=83=BC?= =?UTF-8?q?=E6=99=82=E3=81=AE=E3=83=8F=E3=83=B3=E3=83=89=E3=83=AA=E3=83=B3?= =?UTF-8?q?=E3=82=B0=E3=81=8C=E6=AD=A3=E3=81=97=E3=81=8F=E3=81=AA=E3=81=8B?= =?UTF-8?q?=E3=81=A3=E3=81=9F=E3=81=9F=E3=82=81=E3=80=81=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/src/db/database.py | 4 +++- ecs/jskult-batch-monthly/src/db/database.py | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ecs/jskult-batch-daily/src/db/database.py b/ecs/jskult-batch-daily/src/db/database.py index 03c9c068..280a9d8b 100644 --- a/ecs/jskult-batch-daily/src/db/database.py +++ b/ecs/jskult-batch-daily/src/db/database.py @@ -83,7 +83,9 @@ class Database: min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS ), - stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), + retry_error_cls=DBException + ) def connect(self): """ DBに接続します。接続に失敗した場合、リトライします。\n diff --git a/ecs/jskult-batch-monthly/src/db/database.py b/ecs/jskult-batch-monthly/src/db/database.py index f67a21b9..d2171fb5 100644 --- a/ecs/jskult-batch-monthly/src/db/database.py +++ b/ecs/jskult-batch-monthly/src/db/database.py @@ -1,11 +1,10 @@ from sqlalchemy import (Connection, CursorResult, Engine, QueuePool, create_engine, text) from sqlalchemy.engine.url import URL -from tenacity import retry, stop_after_attempt, wait_exponential - from src.error.exceptions import DBException from src.logging.get_logger import get_logger from src.system_var import environment +from tenacity import retry, stop_after_attempt, wait_exponential logger = get_logger(__name__) @@ -74,7 +73,9 @@ class Database: min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS ), - stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), + retry_error_cls=DBException + ) def connect(self): """ DBに接続します。接続に失敗した場合、リトライします。 From e27c4f16329987cd0e8dd1e76a39ede1e9e87762 Mon Sep 17 00:00:00 2001 From: "x.azuma.m@nds-tyo.co.jp" Date: Tue, 11 Jul 2023 20:35:59 +0900 Subject: [PATCH 15/30] =?UTF-8?q?=E3=82=BF=E3=83=96=E6=95=B0=E7=95=B0?= =?UTF-8?q?=E5=B8=B8=E6=A4=9C=E7=9F=A5=E3=83=AD=E3=82=B8=E3=83=83=E3=82=AF?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0=E3=80=81=E3=81=8A=E3=82=88=E3=81=B3=E3=83=86?= =?UTF-8?q?=E3=82=B9=E3=83=88=E3=82=B3=E3=83=BC=E3=83=89=E5=AE=9F=E8=A3=85?= =?UTF-8?q?=E3=80=80=E2=80=BB=E3=81=BE=E3=81=A0=E3=82=A8=E3=83=A9=E3=83=BC?= =?UTF-8?q?=E3=81=82=E3=82=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/vjsk/vjsk_data_load_manager.py | 66 +- .../batch/vjsk/vjsk_load/test_vjsk_load.py | 830 ++++++++++++++++-- .../NoData/bio_slip_data_nodatarecord.tsv | 1 + .../testdata/NoData/fcl_mst_nodatarecord.tsv | 1 + .../testdata/NoData/hld_mst_nodatarecord.tsv | 1 + .../NoData/lot_num_mst_nodatarecord.tsv | 1 + .../NoData/mdb_conv_mst_nodatarecord.tsv | 1 + .../NoData/mkr_org_horizon_nodatarecord.tsv | 1 + .../NoData/org_cnv_mst_nodatarecord.tsv | 1 + .../NoData/phm_prd_mst_nodatarecord.tsv | 1 + .../NoData/phm_price_mst_nodatarecord.tsv | 1 + .../NoData/slip_data_nodatarecord.tsv | 1 + .../NoData/stock_slip_data_nodatarecord.tsv | 1 + .../NoData/tran_kbn_mst_nodatarecord.tsv | 1 + .../NoData/vop_hco_merge_nodatarecord.tsv | 1 + .../NoData/whs_customer_mst_nodatarecord.tsv | 1 + .../testdata/NoData/whs_mst_nodatarecord.tsv | 1 + .../bio_slip_data_formaterror.tsv | 5 + .../fcl_mst_formaterror.tsv | 5 + .../hld_mst_formaterror.tsv | 5 + .../lot_num_mst_formaterror.tsv | 5 + .../mdb_conv_mst_formaterror.tsv | 5 + .../mkr_org_horizon_formaterror.tsv | 5 + .../org_cnv_mst_formaterror.tsv | 5 + .../phm_prd_mst_formaterror.tsv | 5 + .../phm_price_mst_formaterror.tsv | 0 .../slip_data_formaterror.tsv | 5 + .../stock_slip_data_formaterror.tsv | 5 + .../tran_kbn_mst_formaterror.tsv | 5 + .../vop_hco_merge_formaterror.tsv | 5 + .../whs_customer_mst_formaterror.tsv | 5 + .../whs_mst_formaterror.tsv | 5 + .../testdata/phm_price_mst_dataerror.tsv | 3 - 33 files changed, 915 insertions(+), 69 deletions(-) create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/bio_slip_data_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/fcl_mst_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/hld_mst_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/lot_num_mst_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mdb_conv_mst_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mkr_org_horizon_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/org_cnv_mst_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_prd_mst_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_price_mst_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/slip_data_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/stock_slip_data_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/tran_kbn_mst_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/vop_hco_merge_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_customer_mst_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_mst_nodatarecord.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/bio_slip_data_formaterror.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/fcl_mst_formaterror.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/hld_mst_formaterror.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/lot_num_mst_formaterror.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mdb_conv_mst_formaterror.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mkr_org_horizon_formaterror.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/org_cnv_mst_formaterror.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/phm_prd_mst_formaterror.tsv rename ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/{ => TestFormatErrorFile}/phm_price_mst_formaterror.tsv (100%) create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/slip_data_formaterror.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/stock_slip_data_formaterror.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/tran_kbn_mst_formaterror.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/vop_hco_merge_formaterror.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_customer_mst_formaterror.tsv create mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_mst_formaterror.tsv delete mode 100644 ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_dataerror.tsv diff --git a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py index cef4e1ec..0cc8a248 100644 --- a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py +++ b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py @@ -1,4 +1,6 @@ # from src.batch.vjsk.vjsk_recv_file_manager import VjskDatFile +import os + from src.batch.vjsk.vjsk_recv_file_mapper import VjskReceiveFileMapper from src.db.database import Database from src.error.exceptions import BatchOperationException @@ -63,21 +65,52 @@ class VjskDataLoadManager: def _get_tsv_last_row_tab_count(src_file_name: str) -> int: # memo: tsvファイルが数百MBに及ぶことを想定して、末尾から1行分を参照する # memo: 前提1 行区切りは LF('\n') + # memo: 前提2 正常時のファイル終端にある文字は、末尾行の LF('\n') + # memo: 前提3 ファイルエンコードはBOM付UTF-8(先頭3byteが b'\xEF' + b'\xBB' + b'\xBF' ) buf_count = 0 + # ファイルサイズ取得 + file_size = os.path.getsize(src_file_name) + + # ファイルサイズが0byteなら処理終了 + if file_size == 0: + return buf_count + # バイナリモードでファイルオープン with open(src_file_name, 'rb') as file: - # ファイルの末尾から2バイト手前に移動 - file.seek(-2, 2) - # 改行文字を見つけるまで逆方向に読み進める - while file.read(1) != b'\n': - # 1バイト戻って再度読み込み - file.seek(-2, 1) - # 末尾行を抽出 - last_line = file.readline().decode().rstrip('\n') - # 末尾行に含まれるタブ文字の数を抽出 + # ファイルポインタを末尾に移動 + file.seek(0, os.SEEK_END) + + # ファイルポインタが先頭+1になるまで逆方向にシークする + while file.tell() > 1: + # 2byte戻って + file.seek(-2, os.SEEK_CUR) + # 1byte読む(同時に+1シークする) + char = file.read(1) + # 行区切りを検出したらループ終了 + # memo: UTF-8 バイトシーケンスとして、b'\n' が全角文字の一部にはならない + if char == b'\n': + break + # ファイル先頭のBOMを検出したらループ終了 + if char == b'\xbf': + break + last_line = file.readline().decode('utf-8-sig').rstrip('\n') buf_count = last_line.count('\t') + # # ファイルの末尾から2バイト手前に、シーク位置を移動 + # # memo: 1byte戻るだけだと、直後のread(1)で末尾行のLFを見てしまうので、-2移動 + # file.seek(-2, os.SEEK_END) + # # 改行文字を見つけるまで逆方向に読み進める + # # memo: read(1)することでシーク位置は1byte前に進む + # while file.read(1) != b'\n': + # # シーク位置を1バイト戻って再度読み込み + # # memo: read(1)でシーク位置が1byte進んでいるので、その戻しと合わせて-2移動 + # file.seek(-2, os.SEEK_CUR) + # # 末尾行を抽出 + # last_line = file.readline().decode().rstrip('\n') + # # 末尾行に含まれるタブ文字の数を抽出 + # buf_count = last_line.count('\t') + return buf_count @classmethod @@ -88,13 +121,14 @@ class VjskDataLoadManager: local_file_name = target["src_file_path"] # tsvファイル末尾行のTABの数が総定数と一致しない場合は例外をスロー - # TODO: ↓↓↓developへのマージを優先させたいので、未テストのロジックはコメントアウトする - # tsv_tabs = self._get_tsv_last_row_tab_count(local_file_name) - # expect_tabs = mapper.get_file_column_separators(target["condkey"]) - # if tsv_tabs != expect_tabs: - # msg = f"受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした local_file_name: {local_file_name}" - # raise BatchOperationException(msg) - # TODO: ↑↑↑developへのマージを優先させたいので、未テストのロジックはコメントアウトする + tsv_tabs = self._get_tsv_last_row_tab_count(local_file_name) + expect_tabs = mapper.get_file_column_separators(target["condkey"]) + if tsv_tabs != expect_tabs: + msg = "受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした" + msg += f" local_file_name: {local_file_name}" + msg += f" 末尾行のtab数: {tsv_tabs}" + msg += f" tab想定数: {expect_tabs}" + raise BatchOperationException(msg) # データベース登録 self._import_to_db(local_file_name, target["condkey"]) diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py index 2dbe2ef5..70504795 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py @@ -1163,56 +1163,6 @@ class TestImportFileToDb: key = f"{receive_folder}/{test_file}" s3_client.delete_object(Bucket=bucket_name, Key=key) - def test_load_data_error(self, mapper): - """ - 観点 - 異常系 : 日付型矛盾のデータ ※製品価格マスタファイルで確認 - 期待値 - 例外が発生する - """ - - # setup - self.batch_context.is_vjsk_stock_import_day = True - - # 処理実行 - target_dict = { - "condkey": mapper.CONDKEY_PHM_PRICE_MST, - "src_file_path": path.join(self.test_file_path_load_individual, "phm_price_mst_dataerror.tsv") - } - - with pytest.raises(BatchOperationException) as e: - VjskDataLoadManager.load(target_dict) - - # 検証 - assert str(e.value).startswith("SQL Error:") > 0 - - # teardown - - def test_load_format_error(self, mapper): - """ - 観点 - 異常系 : tsvファイルが途中で欠落している - 期待値 - 例外が発生する - """ - - # setup - self.batch_context.is_vjsk_stock_import_day = True - - # 処理実行 - target_dict = { - "condkey": mapper.CONDKEY_PHM_PRICE_MST, - "src_file_path": path.join(self.test_file_path_load_individual, "phm_price_mst_formaterror.tsv") - } - - with pytest.raises(BatchOperationException) as e: - VjskDataLoadManager.load(target_dict) - - # 検証 - assert str(e.value).startswith("LOAD文実行時にWARNINGが発生しました。") > 0 - - # teardown - def test_s3backup_ok(self, s3_client, bucket_name, receive_folder, mapper): """ 観点 @@ -1293,3 +1243,783 @@ class TestImportFileToDb: assert str(e.value) == "An error occurred (404) when calling the HeadObject operation: Not Found" # teardown + + def test_load_format_error_01(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_STOCK_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "stock_slip_data_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_02(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "slip_data_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_03(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_ORG_CNV_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "org_cnv_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_04(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_VOP_HCO_MERGE, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "vop_hco_merge_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_05(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_WHS_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "whs_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_06(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_HLD_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "hld_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_07(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_FCL_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "fcl_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_08(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_MKR_ORG_HORIZON, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "mkr_org_horizon_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_09(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_TRAN_KBN_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "tran_kbn_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_10(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_PHM_PRD_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "phm_prd_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_11(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_PHM_PRICE_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "phm_price_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_12(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_WHS_CUSTOMER_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "whs_customer_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_13(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_MDB_CONV_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "mdb_conv_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_14(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_BIO_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "bio_slip_data_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_format_error_15(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_LOT_NUM_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "TestFormatErrorFile", "lot_num_mst_formaterror.tsv") + } + + with pytest.raises(BatchOperationException) as e: + VjskDataLoadManager.load(target_dict) + + # 検証 + assert str(e.value).startswith("受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした") > 0 + + # teardown + print(e) + + def test_load_no_data_ok_01(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_STOCK_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "stock_slip_data_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_02(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "slip_data_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_03(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_ORG_CNV_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "org_cnv_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_04(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_VOP_HCO_MERGE, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "vop_hco_merge_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_05(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_WHS_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "whs_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_06(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_HLD_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "hld_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_07(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_FCL_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "fcl_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_08(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_MKR_ORG_HORIZON, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "mkr_org_horizon_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_09(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_TRAN_KBN_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "tran_kbn_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_10(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_PHM_PRD_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "phm_prd_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_11(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_PHM_PRICE_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "phm_price_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_12(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_WHS_CUSTOMER_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "whs_customer_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_13(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_MDB_CONV_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "mdb_conv_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_14(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_BIO_SLIP_DATA, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "bio_slip_data_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown + + def test_load_no_data_ok_15(self, mapper): + """ + 観点 + 異常系 : tsvファイルが途中で欠落している + 期待値 + 例外が発生する + """ + + # setup + self.batch_context.is_vjsk_stock_import_day = True + + # 処理実行 + target_dict = { + "condkey": mapper.CONDKEY_LOT_NUM_MST, + "src_file_path": path.join(self.test_file_path_load_individual, + "NoData", "lot_num_mst_nodatarecord.tsv") + } + + VjskDataLoadManager.load(target_dict) + + # 検証 + assert True + + # teardown diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/bio_slip_data_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/bio_slip_data_nodatarecord.tsv new file mode 100644 index 00000000..e2dccedb --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/bio_slip_data_nodatarecord.tsv @@ -0,0 +1 @@ +"rec_data" "rec_whs_cd" "rec_whs_sub_cd" "rec_whs_org_cd" "rec_cust_cd" "rec_comm_cd" "rec_tran_kbn" "rec_hsdnYmd_wrk" "rec_hsdnYmd_srk" "rec_urag_no" "rec_comm_nm" "rec_nnskFcl_nm" "rec_nnsk_fcl_addr" "rec_lot_num1" "rec_amt1" "rec_lot_num2" "rec_amt2" "rec_lot_num3" "rec_amt3" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "hsdn_ymd" "exec_dt" "v_tran_cd" "tran_kbn_nm" "whs_org_cd" "v_whsOrg_cd" "whs_org_nm" "whs_org_kn" "v_whs_cd" "whs_nm" "nnsk_cd" "fcl_cd" "fcl_nm" "fcl_kn" "fcl_addr_v" "comm_cd" "comm_nm" "htdnYmd_err_kbn" "prd_exis_kbn" "fcl_exis_kbn" "amt1" "amt2" "amt3" "slip_org_kbn" "bef_slip_mgt_no" "whs_rep_comm_nm" "whs_rep_nnskFcl_nm" "whs_rep_nnsk_fcl_addr" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "err_flg11" "err_flg12" "err_flg13" "err_flg14" "err_flg15" "err_flg16" "err_flg17" "err_flg18" "err_flg19" "err_flg20" "kjyo_ym" "tksNbk_kbn" "fcl_exec_kbn" "rec_sts_kbn" "ins_dt" "ins_usr" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/fcl_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/fcl_mst_nodatarecord.tsv new file mode 100644 index 00000000..ccd44bbd --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/fcl_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"fcl_cd" "sub_no" "start_dt" "end_dt" "closed_dt" "nm" "kn_nm" "sht_nm" "sht_kn_nm" "mkr_cd" "jsk_proc_kbn" "fmt_addr" "fmt_kn_addr" "post_cd" "prft_cd" "prft_nm" "city_nm" "addr_line_1" "tel_no" "admin_kbn" "fcl_type" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/hld_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/hld_mst_nodatarecord.tsv new file mode 100644 index 00000000..27e3a2b4 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/hld_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"v_hld_cd" "sub_no" "nm" "kn_nm" "sht_nm" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/lot_num_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/lot_num_mst_nodatarecord.tsv new file mode 100644 index 00000000..73c23a1f --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/lot_num_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"ser_no" "lot_num" "expr_dt" "frst_mov_dt" "ins_dt" "ins_usr" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mdb_conv_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mdb_conv_mst_nodatarecord.tsv new file mode 100644 index 00000000..d8868820 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mdb_conv_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"hco_vid__v" "sub_no" "mdb_cd" "reliability" "start_dt" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mkr_org_horizon_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mkr_org_horizon_nodatarecord.tsv new file mode 100644 index 00000000..afd4d124 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/mkr_org_horizon_nodatarecord.tsv @@ -0,0 +1 @@ +"vid_kind_1" "v_cd_1" "nm_1" "dsp_odr_1" "vid_kind_2" "v_cd_2" "nm_2" "dsp_odr_2" "vid_kind_3" "v_cd_3" "nm_3" "dsp_odr_3" "vid_kind_4" "v_cd_4" "nm_4" "dsp_odr_4" "vid_kind_5" "v_cd_5" "nm_5" "dsp_odr_5" "vid_kind_6" "v_cd_6" "nm_6" "dsp_odr_6" "vid_kind_7" "v_cd_7" "nm_7" "dsp_odr_7" "vid_kind_8" "v_cd_8" "nm_8" "dsp_odr_8" "vid_kind_9" "v_cd_9" "nm_9" "dsp_odr_9" "vid_kind_10" "v_cd_10" "nm_10" "dsp_odr_10" "v_whs_cd" "start_dt" "end_dt" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/org_cnv_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/org_cnv_mst_nodatarecord.tsv new file mode 100644 index 00000000..a4c52b43 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/org_cnv_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"whs_cd" "whs_sub_cd" "org_cd" "sub_no" "v_org_cd" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_prd_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_prd_mst_nodatarecord.tsv new file mode 100644 index 00000000..a9ce545e --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_prd_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"prd_cd" "sub_no" "prd_nm" "prd_e_nm" "mkr_cd" "mkr_inf_1" "mkr_inf_2" "phm_itm_cd" "itm_nm" "itm_sht_nm" "form_cd" "form_nm" "vol_cd" "vol_nm" "cont_cd" "cont_nm" "pkg_cd" "pkg_nm" "cnv_num" "jsk_start_dt" "prd_sale_kbn" "jsk_proc_kbn" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_price_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_price_mst_nodatarecord.tsv new file mode 100644 index 00000000..362c753c --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/phm_price_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"phm_prd_cd" "phm_price_kind" "sub_no" "price" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/slip_data_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/slip_data_nodatarecord.tsv new file mode 100644 index 00000000..8957ebc7 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/slip_data_nodatarecord.tsv @@ -0,0 +1 @@ +"recvdata" "rec_whs_cd" "rec_whs_sub_cd" "rec_whs_org_cd" "rec_cust_cd" "rec_comm_cd" "rec_tran_kbn" "rev_hsdnYmd_wrk" "rev_hsdnYmd_srk" "rec_urag_no" "rec_amt" "rec_unit_price" "rec_price" "rec_comm_nm" "rec_nnskFcl_nm" "free_item" "rec_nnsk_fcl_addr" "rec_nnsk_fcl_post" "rec_nnsk_fcl_tel" "rec_bef_hsdn_ymd" "rec_bef_slip_no" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "hsdn_ymd" "exec_dt" "v_tran_cd" "tran_kbn_nm" "whs_org_cd" "v_whsOrg_cd" "whs_org_nm" "whs_org_kn" "v_whs_cd" "whs_nm" "nnsk_cd" "fcl_cd" "fcl_kn" "fcl_nm" "fcl_addr_v" "comm_cd" "comm_nm" "nn_amt" "nn_unitPrice" "nn_price" "unit_price" "unit_amt" "drag_price" "drag_amt" "whsPos_err_kbn" "htdnYmd_err_kbn" "prd_exis_kbn" "fcl_exis_kbn" "bef_hsdn_ymd" "bef_slip_no" "slip_org_kbn" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "err_flg11" "err_flg12" "err_flg13" "err_flg14" "err_flg15" "err_flg16" "err_flg17" "err_flg18" "err_flg19" "err_flg20" "kjyo_ym" "tksNbk_kbn" "fcl_exec_kbn" "rec_sts_kbn" "ins_dt" "ins_usr" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/stock_slip_data_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/stock_slip_data_nodatarecord.tsv new file mode 100644 index 00000000..4dcd83cb --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/stock_slip_data_nodatarecord.tsv @@ -0,0 +1 @@ +"rec_data" "rec_whs_cd" "rec_whs_sub_cd" "rec_sto_place" "rec_stock_ymd" "rec_comm_cd" "rec_amt" "rev_stok_no_sign" "rev_jan_cd" "rec_free_item" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "exec_dt" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "rec_sts_kbn" "ins_dt" "ins_usr" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/tran_kbn_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/tran_kbn_mst_nodatarecord.tsv new file mode 100644 index 00000000..67a2ed3d --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/tran_kbn_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"v_tran_cd" "sub_no" "nm" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/vop_hco_merge_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/vop_hco_merge_nodatarecord.tsv new file mode 100644 index 00000000..e707d5cb --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/vop_hco_merge_nodatarecord.tsv @@ -0,0 +1 @@ +"hco_vid__v" "hco_vid__v_merge" "apply_dt" "merge_reason" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_customer_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_customer_mst_nodatarecord.tsv new file mode 100644 index 00000000..dc76587c --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_customer_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"whs_cd" "whs_sub_cd" "customer_cd" "sub_no" "start_dt" "end_dt" "org_cd" "src_org_cd" "nm" "kn_nm" "addr" "kn_addr" "zip_cd" "tel_no" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_mst_nodatarecord.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_mst_nodatarecord.tsv new file mode 100644 index 00000000..6f760e14 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/NoData/whs_mst_nodatarecord.tsv @@ -0,0 +1 @@ +"v_whs_cd" "sub_no" "nm" "kn_nm" "sht_nm" "zip_cd" "addr" "kn_addr" "tel_no" "v_hld_cd" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/bio_slip_data_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/bio_slip_data_formaterror.tsv new file mode 100644 index 00000000..0b359d23 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/bio_slip_data_formaterror.tsv @@ -0,0 +1,5 @@ +"rec_data" "rec_whs_cd" "rec_whs_sub_cd" "rec_whs_org_cd" "rec_cust_cd" "rec_comm_cd" "rec_tran_kbn" "rec_hsdnYmd_wrk" "rec_hsdnYmd_srk" "rec_urag_no" "rec_comm_nm" "rec_nnskFcl_nm" "rec_nnsk_fcl_addr" "rec_lot_num1" "rec_amt1" "rec_lot_num2" "rec_amt2" "rec_lot_num3" "rec_amt3" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "hsdn_ymd" "exec_dt" "v_tran_cd" "tran_kbn_nm" "whs_org_cd" "v_whsOrg_cd" "whs_org_nm" "whs_org_kn" "v_whs_cd" "whs_nm" "nnsk_cd" "fcl_cd" "fcl_nm" "fcl_kn" "fcl_addr_v" "comm_cd" "comm_nm" "htdnYmd_err_kbn" "prd_exis_kbn" "fcl_exis_kbn" "amt1" "amt2" "amt3" "slip_org_kbn" "bef_slip_mgt_no" "whs_rep_comm_nm" "whs_rep_nnskFcl_nm" "whs_rep_nnsk_fcl_addr" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "err_flg11" "err_flg12" "err_flg13" "err_flg14" "err_flg15" "err_flg16" "err_flg17" "err_flg18" "err_flg19" "err_flg20" "kjyo_ym" "tksNbk_kbn" "fcl_exec_kbn" "rec_sts_kbn" "ins_dt" "ins_usr" +"D452960211JD1111311102503851400002304016427519111 496350122ﾊﾞﾍﾞﾝﾁｵﾃﾝﾃｷ200MG 1V ｿｳｺﾞｳﾒﾃﾞｲｶﾙﾆﾎﾝｺｳｶﾝﾋﾞﾂｸﾊﾞｼ ﾀｶｻｷ 753 BAVB007 000003 000000 000000 " "296" "02" "11JD11113111025" "0385140000" "496350122" "111" "230401" "20230401" "6427519" "ﾊﾞﾍﾞﾝﾁｵﾃﾝﾃｷ200MG 1V " "ｿｳｺﾞｳﾒﾃﾞｲｶﾙﾆﾎﾝｺｳｶﾝﾋﾞ" "ﾂｸﾊﾞｼ ﾀｶｻｷ 753 " "BAVB007 " "000003" " " "000000" " " "000000" "20230403" "J" "VJSK-BIO_J_MERCK_2023040300.txt" "J2023040300000126" "129" "20230401" "202305082041" "110" "売上" "11JD" "300001370" "川崎南支店" "" "200000007" "アルフレッサ株式会社" "0385140000" "670235967013012526" "医療法人社団こうかん会　日本鋼管病院" "ｲﾘｮｳﾎｳｼﾞﾝｼｬﾀﾞﾝｺｳｶﾝｶｲ ﾆﾎﾝｺｳｶﾝﾋﾞｮｳｲﾝ" "210-0852 神奈川県川崎市川崎区鋼管通１−２−１" "496350122" "ﾊﾞﾍﾞﾝﾁｵ 注射剤 200mg 1VIAL" "" "1" "" "3" "0" "0" "J" "" "" "" "" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "202304" "" "" "0" "23-04-03 20:42:11" "system" +"D452960211G11111377452402930640002304016433215111 496300127ｺﾞﾅ-ﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ ｾｺﾑ)ｵｷﾞｸﾎﾞﾋﾞﾖｳｲﾝ ﾄｳｷﾖｳﾄ ｼﾌﾞﾔｸ ｼﾞﾝｸﾞｳﾏｴ 1-5-1 GF4C001 000002 000000 000000 " "296" "02" "11G111113774524" "0293064000" "496300127" "111" "230401" "20230401" "6433215" "ｺﾞﾅ-ﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ " "ｾｺﾑ)ｵｷﾞｸﾎﾞﾋﾞﾖｳｲﾝ " "ﾄｳｷﾖｳﾄ ｼﾌﾞﾔｸ ｼﾞﾝｸﾞｳﾏｴ 1-5-1 " "GF4C001 " "000002" " " "000000" " " "000000" "20230403" "J" "VJSK-BIO_J_MERCK_2023040300.txt" "J2023040300000127" "130" "20230401" "202305082041" "110" "売上" "11G1" "300001351" "杉並・中野支店" "" "200000007" "アルフレッサ株式会社" "0293064000" "670234652241314835" "医療法人財団荻窪病院　荻窪病院" "ｲﾘｮｳﾎｳｼﾞﾝｻﾞｲﾀﾞﾝｵｷﾞｸﾎﾞﾋﾞｮｳｲﾝ ｵｷﾞｸﾎﾞﾋﾞｮｳｲﾝ" "167-0035 東京都杉並区今川３−１−２４" "496300127" "ｺﾞﾅｰﾙｴﾌ 皮下注ﾍﾟﾝ 450IU 1PEN" "" "1" "" "2" "0" "0" "J" "" "" "" "" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "202304" "" "" "0" "23-04-03 20:42:11" "system" +"D452960211V11121120604204799500002304016461276111 496300134ｺﾞﾅ-ﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ ﾆﾁｲｻ-ﾋﾞｽｷﾕｳｼﾕｳﾌｸｵｶｻﾝﾌｸｵｶｼ ｻﾜﾗｸ ﾓﾓﾁﾊﾏ 1-7-5 7F GF9C002 000010 000000 000000 " "296" "02" "11V111211206042" "0479950000" "496300134" "111" "230401" "20230401" "6461276" "ｺﾞﾅ-ﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ " "ﾆﾁｲｻ-ﾋﾞｽｷﾕｳｼﾕｳﾌｸｵｶｻﾝ" "ﾌｸｵｶｼ ｻﾜﾗｸ ﾓﾓﾁﾊﾏ 1-7-5 7F " "GF9C002 " "000010" " " "000000" " " "000000" "20230403" "J" "VJSK-BIO_J_MERCK_2023040300.txt" "J2023040300000128" "131" "20230401" "202305082041" "110" "売上" "11V1" "300001491" "福岡第一支店" "" "200000007" "アルフレッサ株式会社" "0479950000" "670235883412145206" "医療法人社団高邦会　福岡山王病院" "ｲﾘｮｳﾎｳｼﾞﾝｼｬﾀﾞﾝｺｳﾎｳｶｲ ﾌｸｵｶｻﾝﾉｳﾋﾞｮｳｲﾝ" "814-0001 福岡県福岡市早良区百道浜３−６−４５" "496300134" "ｺﾞﾅｰﾙｴﾌ 皮下注ﾍﾟﾝ 900IU 1PEN" "" "1" "" "10" "0" "0" "J" "" "" "" "" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "202304" "" "" "0" "23-04-03 20:42:11" "system" +"D452960211JB1121309300202875030002304016523689111 496300127ｺﾞﾅ-ﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ ｲﾂｶﾝﾄﾞｳﾔﾂｷﾖｸ0561 ｶﾜｻｷｼ ｱｻｵｸ ﾌﾙｻﾜ 172-1 GF4C001 000001 000000 000000 " "296" "02" "11JB11213093002" "02875 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/fcl_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/fcl_mst_formaterror.tsv new file mode 100644 index 00000000..0125ec7d --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/fcl_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"fcl_cd" "sub_no" "start_dt" "end_dt" "closed_dt" "nm" "kn_nm" "sht_nm" "sht_kn_nm" "mkr_cd" "jsk_proc_kbn" "fmt_addr" "fmt_kn_addr" "post_cd" "prft_cd" "prft_nm" "city_nm" "addr_line_1" "tel_no" "admin_kbn" "fcl_type" "rec_sts_kbn" "ins_dt" "upd_dt" +"670229430760653825" "0" "20000101" "99991231" "" "駅前町歯科診療所" "ｴｷﾏｴﾁｮｳｼｶｼﾝﾘｮｳｼﾞｮ" "駅前町歯科診療所" "ｴｷﾏｴﾁｮｳｼｶｼﾝﾘｮｳｼﾞｮ" "" "0" "700-0023 岡山県岡山市北区駅前町１−６−２０" "ｵｶﾔﾏｹﾝ ｵｶﾔﾏｼｷﾀｸ ｴｷﾏｴﾁｮｳ1-6-20" "700-0023" "33" "岡山県" "岡山市北区" "駅前町１−６−２０" "0862236468" "33101" "30" "1" "" "" +"670229435466662922" "0" "20000101" "99991231" "" "医療法人社団仁卓会　ほりかわ歯科クリニック" "ｲﾘｮｳﾎｳｼﾞﾝｼｬﾀﾞﾝｼﾞﾝﾀｸｶｲ ﾎﾘｶﾜｼｶｸﾘﾆｯｸ" "ほりかわ歯科クリニック　（医社）" "ﾎﾘｶﾜｼｶｸﾘﾆｯｸ (ｲｼｬ)" "" "0" "675-0101 兵庫県加古川市平岡町新在家１５７３−１−４Ｆ" "ﾋｮｳｺﾞｹﾝ ｶｺｶﾞﾜｼ ﾋﾗｵｶﾁｮｳｼﾝｻﾞｲｹ1573-1-4F" "675-0101" "28" "兵庫県" "加古川市" "平岡町新在家１５７３−１−４Ｆ" "0794244617" "28210" "30" "1" "" "" +"670229435785430019" "0" "20000101" "99991231" "" "株式会社コミュニティメディカル　なつめ薬局　千歳船橋店" "ｶﾌﾞｼｷｶﾞｲｼｬｺﾐｭﾆﾃｨﾒﾃﾞｨｶﾙ ﾅﾂﾒﾔｯｷｮｸ ﾁﾄｾﾌﾅﾊﾞｼﾃﾝ" "なつめ薬局　千歳船橋店　（株）" "ﾅﾂﾒﾔｯｷｮｸ ﾁﾄｾﾌﾅﾊﾞｼﾃﾝ (ｶ)" "" "0" "156-0054 東京都世田谷区桜丘２−２４−２" "ﾄｳｷｮｳﾄ ｾﾀｶﾞﾔｸ ｻｸﾗｶﾞｵｶ2-24-2" "156-0054" "13" "東京都" "世田谷区" "桜丘２−２４−２" "0364136189" "13112" "20" "1" "" "" +"670229447437206529" "0" "20000101" "20230407" "2023-04-07" "ヒカリ薬局" "ﾋｶﾘﾔｯｷｮｸ" "ヒカリ薬局" "ﾋｶﾘﾔｯｷｮｸ" "" "0" "670-0955 兵庫県 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/hld_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/hld_mst_formaterror.tsv new file mode 100644 index 00000000..9060a7da --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/hld_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"v_hld_cd" "sub_no" "nm" "kn_nm" "sht_nm" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" +"10001" "1" "卸ホールディングス名01-1" "オロシホールディングスメイ01-1" "卸名01-1" "20230101" "99991231" "1" "0" "23-05-09 12:00:01" "23-05-09 13:00:01" +"10001" "2" "卸ホールディングス名01-2" "オロシホールディングスメイ01-2" "卸名01-2" "20230102" "99991231" "1" "0" "23-05-09 12:00:02" "23-05-09 13:00:02" +"10001" "3" "卸ホールディングス名01-3" "オロシホールディングスメイ01-3" "卸名01-3" "20230103" "99991231" "1" "0" "23-05-09 12:00:03" "23-05-09 13:00:03" +"10001" "4" "卸ホールディングス名01-4" "オロシホールディングスメイ01-4" "卸 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/lot_num_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/lot_num_mst_formaterror.tsv new file mode 100644 index 00000000..6978f030 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/lot_num_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"ser_no" "lot_num" "expr_dt" "frst_mov_dt" "ins_dt" "ins_usr" +"F0110601" "BAVA001" "20230331" "20210510" "23-05-08 20:40:41" "batch" +"F0110601" "BAVA002" "20230331" "20210615" "23-05-08 20:40:41" "batch" +"F0110601" "BAVA003" "20231031" "20210719" "23-05-08 20:40:41" "batch" +"F0110601" "BAVA004" "2 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mdb_conv_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mdb_conv_mst_formaterror.tsv new file mode 100644 index 00000000..5b4d688d --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mdb_conv_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"hco_vid__v" "sub_no" "mdb_cd" "reliability" "start_dt" "rec_sts_kbn" "ins_dt" "upd_dt" +"670229780011959315" "1" "003410424" "0" "20020601" "0" "22-03-09 13:56:19" "22-03-09 13:56:19" +"670230081112654862" "0" "004101420" "0" "20000101" "9" "17-10-17 17:06:52" "22-03-09 14:17:34" +"670230081112654862" "1" "004104997" "2" "20000101" "0" "22-03-09 14:17:33" "22-03-09 14:17:33" +"670230100414841865" "0" "003622111" "3" \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mkr_org_horizon_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mkr_org_horizon_formaterror.tsv new file mode 100644 index 00000000..6f108bcf --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/mkr_org_horizon_formaterror.tsv @@ -0,0 +1,5 @@ +"vid_kind_1" "v_cd_1" "nm_1" "dsp_odr_1" "vid_kind_2" "v_cd_2" "nm_2" "dsp_odr_2" "vid_kind_3" "v_cd_3" "nm_3" "dsp_odr_3" "vid_kind_4" "v_cd_4" "nm_4" "dsp_odr_4" "vid_kind_5" "v_cd_5" "nm_5" "dsp_odr_5" "vid_kind_6" "v_cd_6" "nm_6" "dsp_odr_6" "vid_kind_7" "v_cd_7" "nm_7" "dsp_odr_7" "vid_kind_8" "v_cd_8" "nm_8" "dsp_odr_8" "vid_kind_9" "v_cd_9" "nm_9" "dsp_odr_9" "vid_kind_10" "v_cd_10" "nm_10" "dsp_odr_10" "v_whs_cd" "start_dt" "end_dt" "rec_sts_kbn" "ins_dt" "upd_dt" +"3" "300003202" "その他営業本部卸" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "3" "300003217" "試薬岐阜（回収）" "0" "200000007" "20190401" "99991231" "0" "19-04-11 11:30:59" "23-04-12 17:52:38" +"3" "300003138" "北関東甲信越営業本部" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "3" "300003195" "首都圏移管組織" "0" "200000007" "20190401" "99991231" "0" "19-04-11 11:30:59" "23-04-12 17:52:38" +"3" "300003202" "その他営業本部卸" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "3" "300003226" "大垣（回収）" "0" "200000007" "20190401" "99991231" "0" "19-04-11 11:30:59" "23-04-12 17:52:38" +"a" "300003144" "メディカル営業本部1" "1" "b" "300003202" "東海スタッフ医療2" "2" "c" "300003203" "東海スタ \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/org_cnv_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/org_cnv_mst_formaterror.tsv new file mode 100644 index 00000000..6151cd4e --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/org_cnv_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"whs_cd" "whs_sub_cd" "org_cd" "sub_no" "v_org_cd" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" +"042" "01" "361007" "0" "300006657" "20230401" "99991231" "0" "0" "23-04-12 11:24:06" "23-04-12 11:24:06" +"042" "01" "381207" "0" "300006658" "20230401" "99991231" "0" "0" "23-04-12 11:24:27" "23-04-12 11:24:27" +"080" "00" "02780" "0" "300006526" "20220401" "99991231" "0" "9" "22-04-11 15:57:35" "23-04-12 10:46:48" +"080" "00" "02780" "1" "300006526" "20220401" \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/phm_prd_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/phm_prd_mst_formaterror.tsv new file mode 100644 index 00000000..8324689b --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/phm_prd_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"prd_cd" "sub_no" "prd_nm" "prd_e_nm" "mkr_cd" "mkr_inf_1" "mkr_inf_2" "phm_itm_cd" "itm_nm" "itm_sht_nm" "form_cd" "form_nm" "vol_cd" "vol_nm" "cont_cd" "cont_nm" "pkg_cd" "pkg_nm" "cnv_num" "jsk_start_dt" "prd_sale_kbn" "jsk_proc_kbn" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" +"496201110" "0" "ｾﾛﾌｪﾝ 錠剤 50mg 30TAB" "" "F21206A0" "セロフェン 錠 50MG" "SEROPHENE TAB. 50 MG. (30)" "001" "セロフェン" "SP" "F003" "錠剤" "0000" "" "V009" "50mg" "P007" "30TAB" "30" "" "0" "0" "20080101" "20190930" "140" "0" "17-11-08 16:52:41" "19-09-19 11:42:45" +"496201127" "0" "ｾﾛﾌｪﾝ 錠剤 50mg 30TAB" "" "F21206A0" "セロフェン 錠 50MG" "SEROPHENE TAB. 50 MG. (30)" "001" "セロフェン" "SP" "F003" "錠剤" "0000" "" "V009" "50mg" "P007" "30TAB" "30" "" "0" "0" "20070401" "20190930" "150" "0" "17-11-08 16:52:41" "19-09-19 11:42:45" +"496300110" "2" "ｺﾞﾅｰﾙｴﾌ 皮下注ﾍﾟﾝ 300IU 1PEN" "" "F1990608" "ゴナールエフ皮下注ペン 300" "GONAL-F PEN 300IU (1) - JPN" "005" "セロスティム" "ST" "F005" "皮下注ペン" "0000" "" "V017" "300IU" "P011" "1PEN" "1" "" "0" "0" "20190501" "20190930" "100" "9" "19-04-23 16:35:36" "19-04-23 16:40:38" +"496300127" "2" "ｺﾞﾅｰﾙｴﾌ 皮下注ﾍﾟﾝ 450IU 1PEN" "" "F19D0608" "ゴナールエフ皮下注ペン450" "Gonalef Pen 450 (1)" "008" "BDマイクロファインプラス" "MF" "F005" "皮下注ペン" "0000" "" "V018" "450IU" "P011" "1PEN" "1" "" "0" "0" "20190501" "20190930" "120" "9" "19-04-23 1 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/phm_price_mst_formaterror.tsv similarity index 100% rename from ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_formaterror.tsv rename to ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/phm_price_mst_formaterror.tsv diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/slip_data_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/slip_data_formaterror.tsv new file mode 100644 index 00000000..7b4b20b0 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/slip_data_formaterror.tsv @@ -0,0 +1,5 @@ +"recvdata" "rec_whs_cd" "rec_whs_sub_cd" "rec_whs_org_cd" "rec_cust_cd" "rec_comm_cd" "rec_tran_kbn" "rev_hsdnYmd_wrk" "rev_hsdnYmd_srk" "rec_urag_no" "rec_amt" "rec_unit_price" "rec_price" "rec_comm_nm" "rec_nnskFcl_nm" "free_item" "rec_nnsk_fcl_addr" "rec_nnsk_fcl_post" "rec_nnsk_fcl_tel" "rec_bef_hsdn_ymd" "rec_bef_slip_no" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "hsdn_ymd" "exec_dt" "v_tran_cd" "tran_kbn_nm" "whs_org_cd" "v_whsOrg_cd" "whs_org_nm" "whs_org_kn" "v_whs_cd" "whs_nm" "nnsk_cd" "fcl_cd" "fcl_kn" "fcl_nm" "fcl_addr_v" "comm_cd" "comm_nm" "nn_amt" "nn_unitPrice" "nn_price" "unit_price" "unit_amt" "drag_price" "drag_amt" "whsPos_err_kbn" "htdnYmd_err_kbn" "prd_exis_kbn" "fcl_exis_kbn" "bef_hsdn_ymd" "bef_slip_no" "slip_org_kbn" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "err_flg11" "err_flg12" "err_flg13" "err_flg14" "err_flg15" "err_flg16" "err_flg17" "err_flg18" "err_flg19" "err_flg20" "kjyo_ym" "tksNbk_kbn" "fcl_exec_kbn" "rec_sts_kbn" "ins_dt" "ins_usr" +"D4420202011611A4 0183733 23030133625911102303 4963500230000020003110000000622000000000000000000000000 ｱ-ﾋﾞﾀﾂｸｽﾁﾕｳｼﾔｴｷ100MG 100MG 20MLX1Vﾊｺﾀﾞﾃﾁﾕｳｵｳﾋﾞﾖｳｲﾝ 00000408585ﾊｺﾀﾞﾃｼﾎﾝﾁﾖｳ33ﾊﾞﾝ2ｺﾞｳ " "202" "02" "011611A4 " "0183733 " "496350023" "110" "230301" "20230301" "3362591" "000002" "00031100" "0000062200" "ｱ-ﾋﾞﾀﾂｸｽﾁﾕｳｼﾔｴｷ100MG 100MG 20MLX1V" "ﾊｺﾀﾞﾃﾁﾕｳｵｳﾋﾞﾖｳｲﾝ " "0408585ﾊｺﾀﾞﾃｼﾎﾝﾁﾖｳ33ﾊﾞﾝ2ｺﾞｳ " "ﾊｺﾀﾞﾃｼﾎﾝﾁﾖｳ33ﾊﾞﾝ2ｺﾞｳ " "0408585" "" "000000" " " "20230222" "J" "VJSK_J_MERCK_2023022" "J2023022200000022" "29" "20230301" "202303142041" "110" "売上" "01161" "300000383" "函館支店" "" "200000016" "株式会社スズケン" "0183733 " "670234934576694289" "ｼｬｶｲﾌｸｼﾎｳｼﾞﾝﾊｺﾀﾞﾃｺｳｾｲｲﾝ ﾊｺﾀﾞﾃﾁｭｳｵｳﾋﾞｮｳｲﾝ" "社会福祉法人函館厚生院　函館中央病院" "040-0011 北海道函館市本町３３−２" "496350023" "ｱｰﾋﾞﾀｯｸｽ 注射剤 100mg 1VIAL" "2" "31100" "62200" "31438" "62876" "35309" "70618" "" "" "1" "" "" "" "J" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "202303" "" "" "0" "23-03-14 20:41:26" "SYSTEM" +"D4420202011611A4 0183733 23030133625921102303 4963500230000080003110000002488000000000000000000000000 ｱ-ﾋﾞﾀﾂｸｽﾁﾕｳｼﾔｴｷ100MG 100MG 20MLX1Vﾊｺﾀﾞﾃﾁﾕｳｵｳﾋﾞﾖｳｲﾝ 00000408585ﾊｺﾀﾞﾃｼﾎﾝﾁﾖｳ33ﾊﾞﾝ2ｺﾞｳ " "202" "02" "011611A4 " "0183733 " "496350023" "110" "230301" "20230301" "3362592" "000008" "00031100" "0000248800" "ｱ-ﾋﾞﾀﾂｸｽﾁﾕｳｼﾔｴｷ100MG 100MG 20MLX1V" "ﾊｺﾀﾞﾃﾁﾕｳｵｳﾋﾞﾖｳｲﾝ " "0408585ﾊｺﾀﾞﾃｼﾎﾝﾁﾖｳ33ﾊﾞﾝ2ｺﾞｳ " "ﾊｺﾀﾞﾃｼﾎﾝﾁﾖｳ33ﾊﾞﾝ2ｺﾞｳ " "0408585" "" "000000" " " "20230222" "J" "VJSK_J_MERCK_2023022" "J2023022200000023" "30" "20230301" "202303142041" "110" "売上" "01161" "300000383" "函館支店" "" "200000016" "株式会社スズケン" "0183733 " "670234934576694289" "ｼｬｶｲﾌｸｼﾎｳｼﾞﾝﾊｺﾀﾞﾃｺｳｾｲｲﾝ ﾊｺﾀﾞﾃﾁｭｳｵｳﾋﾞｮｳｲﾝ" "社会福祉法人函館厚生院　函館中央病院" "040-0011 北海道函館市本町３３−２" "496350023" "ｱｰﾋﾞﾀｯｸｽ 注射剤 100mg 1VIAL" "8" "31100" "248800" "31438" "251504" "35309" "282472" "" "" "1" "" "" "" "J" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "202303" "" "" "0" "23-03-14 20:41:26" "SYSTEM" +"D4416101311101A8 5140013 23030173719811122303 4963500230000120002738100003285720000000000000000000000 ｱ-ﾋﾞﾀﾂｸｽﾁﾕｳｼﾔｴｷ100MG 100MG 20MLX1Vﾄﾂﾄﾘﾆﾂｾｷﾋﾞﾖｳｲﾝ 00006808517ﾄﾂﾄﾘｹﾝﾄﾂﾄﾘｼｼﾖｳﾄｸﾁﾖｳ117 " "161" "01" "311101A8 " "5140013 " "496350023" "112" "230301" "20230301" "7371981" "000012" "00027381" "0000328572" "ｱ-ﾋﾞﾀﾂｸｽﾁﾕｳｼﾔｴｷ100MG 100MG 20MLX1V" "ﾄﾂﾄﾘﾆﾂｾｷﾋﾞﾖｳｲﾝ " "6808517ﾄﾂﾄﾘｹﾝﾄﾂﾄﾘｼｼﾖｳﾄｸﾁﾖｳ117 " "ﾄﾂﾄﾘｹﾝﾄﾂﾄﾘｼｼﾖｳﾄｸﾁﾖｳ117 " "6808517" "" "000000" " " "20230224" "J" "VJSK_J_MERCK_2023022" "J2023022400000011" "16" "20230301" "202303142041" "110" "売上" "31110" "300000391" "鳥取支店" "" "200000015" "株式会社サンキ" "5140013 " "670237031040828444" "ﾆﾎﾝｾｷｼﾞｭｳｼﾞｼｬ ﾄｯﾄﾘｾｷｼﾞｭｳｼﾞﾋﾞｮｳｲﾝ" "日本赤十字社　鳥取赤十字病院" "680-0017 鳥取県鳥取市尚徳町１１７" "496350023" "ｱｰﾋﾞﾀｯｸｽ 注射剤 100mg 1VIAL" "12" "27381" "328572" "31438" "377256" "35309" "423708" "" "" "1" "" "" "" "J" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "202303" "" "" "0" "23-03-14 20:41:26" "SYSTEM" +"D4416101311 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/stock_slip_data_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/stock_slip_data_formaterror.tsv new file mode 100644 index 00000000..aa3cb22e --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/stock_slip_data_formaterror.tsv @@ -0,0 +1,5 @@ +"rec_data" "rec_whs_cd" "rec_whs_sub_cd" "rec_sto_place" "rec_stock_ymd" "rec_comm_cd" "rec_amt" "rev_stok_no_sign" "rev_jan_cd" "rec_free_item" "rec_ymd" "sale_data_cat" "slip_file_nm" "slip_mgt_no" "row_num" "exec_dt" "err_flg1" "err_flg2" "err_flg3" "err_flg4" "err_flg5" "err_flg6" "err_flg7" "err_flg8" "err_flg9" "err_flg10" "rec_sts_kbn" "ins_dt" "ins_usr" +"D463630101 23022849630021900003500000 ｾﾄﾛﾀｲﾄﾞﾁﾕｳｼﾔﾖｳ0.25MG 1V" "363" "01" "01 " "230228" "496300219" "000035" "0" "0000" " ｾﾄﾛﾀｲﾄﾞﾁﾕｳｼﾔﾖｳ0.25MG 1V" "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000059" "59" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" +"D4625301026 2302284963001270000040000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ40 " "253" "01" "026 " "230228" "496300127" "000004" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ450 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000060" "60" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" +"D4625301026 2302284963001340000220000001ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ40 " "253" "01" "026 " "230228" "496300134" "000022" "0" "0000" "01ｺﾞﾅｰﾙｴﾌﾋｶﾁﾕｳﾍﾟﾝ900 1ﾄｳ40 " "20230314" "J" "VJSK-STOCK_J_MERCK_2023031400.txt" "J2023031400000061" "61" "202303142041" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "0" "23-03-14 20:41:57" "SYSTEM" +"D4625301026 2302284963004170000500000001ｵﾋﾞﾄﾞﾚﾙﾋｶﾁﾕｳｼﾘﾝｼﾞ250MCG 140 " "2 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/tran_kbn_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/tran_kbn_mst_formaterror.tsv new file mode 100644 index 00000000..34692f31 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/tran_kbn_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"v_tran_cd" "sub_no" "nm" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" +"10001" "1" "Veeva取引区分名01-1" "20230101" "99991231" "1" "0" "2023-05-09 12:00:01" "2023-05-09 13:00:01" +"10001" "2" "Veeva取引区分名01-2" "20230102" "99991231" "1" "0" "2023-05-09 12:00:02" "2023-05-09 13:00:02" +"10001" "3" "Veeva取引区分名01-3" "20230103" "99991231" "1" "0" "2023-05-09 12:00:03" "2023-05-09 13:00:03" +"10001" "4" "Veeva取引区分名01-4" "20230104" "9 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/vop_hco_merge_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/vop_hco_merge_formaterror.tsv new file mode 100644 index 00000000..f4e8956e --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/vop_hco_merge_formaterror.tsv @@ -0,0 +1,5 @@ +"hco_vid__v" "hco_vid__v_merge" "apply_dt" "merge_reason" +"100000001" "900000001" "20230509" "事由01" +"100000002" "900000002" "20230509" "事由02" +"100000003" "900000003" "20230509" "事由03" +"100000004" "900000004" \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_customer_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_customer_mst_formaterror.tsv new file mode 100644 index 00000000..0ec0c4cb --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_customer_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"whs_cd" "whs_sub_cd" "customer_cd" "sub_no" "start_dt" "end_dt" "org_cd" "src_org_cd" "nm" "kn_nm" "addr" "kn_addr" "zip_cd" "tel_no" "rec_sts_kbn" "ins_dt" "upd_dt" +"006" "01" "1002900000" "0" "20000101" "99991231" "11" "1131A2283316" "辻内科小児科医院　　　　　　　　　　　　　　　" "ﾂｼﾞﾅｲｶｼﾖｳﾆｶ ｲｲﾝ" "長崎県　佐世保市皆瀬町２９　　　　　　　　　　　　　　　　　　　" "ﾅｶﾞｻｷｹﾝ ｻｾﾎﾞｼｶｲｾﾞﾁﾖｳ 29" "8570144" "0956492319" "0" "23-04-14 11:53:14" "23-04-14 11:53:14" +"006" "01" "1005400000" "0" "20000101" "99991231" "12" "1211C3415515" "医療法人　愛恵会　佐世保愛恵病院　　　　　　　" "ｲﾘﾖｳﾎｳｼﾞﾝｱｲｹｲｶｲｻｾﾎﾞｱｲｹｲﾋﾞ" "長崎県　佐世保市瀬戸越４丁目　２−１５　　　　　　　　　　　　　" "ﾅｶﾞｻｷｹﾝ ｻｾﾎﾞｼｾﾄｺﾞｼ 4ﾁﾖｳﾒ 2-15" "8570134" "0956493335" "0" "23-04-14 11:53:14" "23-04-14 11:53:14" +"006" "01" "1007200000" "0" "20000101" "99991231" "11" "1131A2407312" "医療法人　山祇診療所　　　　　　　　　　　　　" "ｲﾘﾖｳﾎｳｼﾞﾝﾔﾏｽﾞﾐｼﾝﾘﾖｳｼﾞﾖ" "長崎県　佐世保市山祇町　１９−３６　　　　　　　　　　　　　　　" "ﾅｶﾞｻｷｹﾝ ｻｾﾎﾞｼﾔﾏｽﾞﾐﾁｮｳ" "8570822" "0956313633" "0" "23-04-14 11:53:14" "23-04-14 11:53:14" +"006" "01" "1007800000" "0" "20000101" "999912 \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_mst_formaterror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_mst_formaterror.tsv new file mode 100644 index 00000000..82239889 --- /dev/null +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/TestFormatErrorFile/whs_mst_formaterror.tsv @@ -0,0 +1,5 @@ +"v_whs_cd" "sub_no" "nm" "kn_nm" "sht_nm" "zip_cd" "addr" "kn_addr" "tel_no" "v_hld_cd" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" +"200000002" "0" "株式会社モロオ" "カナ01" "モロオ" "1110001" "住所01" "ジュウショ01" "00-0000-0001" "0" "20000101" "99991231" "20" "0" "16-04-15 16:25:33" "16-04-15 16:25:33" +"200000005" "0" "岩渕薬品株式会社" "カナ02" "岩渕薬品" "1110002" "住所02" "ジュウショ02" "00-0000-0002" "0" "20000101" "99991231" "50" "0" "16-04-15 16:25:33" "16-04-15 16:25:33" +"200000009" "0" "株式会社マルタケ" "カナ03" "マルタケ" "1110003" "住所03" "ジュウショ03" "00-0000-0003" "0" "20000101" "99991231" "90" "0" "16-04-15 16:25:33" "16-04-15 16:25:33" +"200000010" "0" "株式会社ファイネス" "カナ04" "ファイネス" "1110004" "住所04" "ジュ \ No newline at end of file diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_dataerror.tsv b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_dataerror.tsv deleted file mode 100644 index 3df124c1..00000000 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/testdata/phm_price_mst_dataerror.tsv +++ /dev/null @@ -1,3 +0,0 @@ -"phm_prd_cd" "phm_price_kind" "sub_no" "price" "start_dt" "end_dt" "dsp_odr" "rec_sts_kbn" "ins_dt" "upd_dt" -"123456701" "01" "1" "12345.6" "yyyy0401" "20190930" "10" "0" "18-03-07 09:48:00" "19-09-19 11:23:47" -"123456701" "02" "1" "12587.8" "20191001" "99991231" "10" "0" "19-09-19 11:24:05" "19-09-19 11:24:05" From c29044873096a87ef03d0546c70274ada88d4299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 12 Jul 2023 10:48:07 +0900 Subject: [PATCH 16/30] =?UTF-8?q?=E6=8C=87=E6=91=98=E4=BA=8B=E9=A0=85?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/README.md | 6 ++-- ecs/jskult-dbdump/entrypoint.py | 2 +- .../src/batch/batch_functions.py | 4 +-- .../src/batch/common/batch_context.py | 29 ------------------- ecs/jskult-dbdump/src/db/database.py | 7 +---- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 13 +++++---- .../src/system_var/environment.py | 2 +- 7 files changed, 15 insertions(+), 48 deletions(-) delete mode 100644 ecs/jskult-dbdump/src/batch/common/batch_context.py diff --git a/ecs/jskult-dbdump/README.md b/ecs/jskult-dbdump/README.md index f6e7c2da..7d8f0ccd 100644 --- a/ecs/jskult-dbdump/README.md +++ b/ecs/jskult-dbdump/README.md @@ -1,8 +1,8 @@ -# 実消化&アルトマーク 月次バッチ +# 日次バッチ処理前DBダンプ取得　 ## 概要 -実消化&アルトマークの月次バッチ処理。 +日次バッチ処理前DBダンプ取得処理。 ## 環境情報 @@ -42,7 +42,7 @@ - VSCode 上で「F5」キーを押下すると、バッチ処理が起動する。 - 「entrypoint.py」が、バッチ処理のエントリーポイント。 -- 実際の処理は、「src/jobctrl_monthly.py」で行っている。 +- 実際の処理は、「src/jobctrl_dbdump.py」で行っている。 ## フォルダ構成（工事中） diff --git a/ecs/jskult-dbdump/entrypoint.py b/ecs/jskult-dbdump/entrypoint.py index 9a13c457..543cb134 100644 --- a/ecs/jskult-dbdump/entrypoint.py +++ b/ecs/jskult-dbdump/entrypoint.py @@ -1,4 +1,4 @@ -"""実消化&アルトマーク 月次バッチのエントリーポイント""" +"""実消化&アルトマーク 日次バッチ処理前DBダンプ取得のエントリーポイント""" from src import jobctrl_dbdump if __name__ == '__main__': diff --git a/ecs/jskult-dbdump/src/batch/batch_functions.py b/ecs/jskult-dbdump/src/batch/batch_functions.py index da329291..87231e17 100644 --- a/ecs/jskult-dbdump/src/batch/batch_functions.py +++ b/ecs/jskult-dbdump/src/batch/batch_functions.py @@ -58,7 +58,7 @@ def update_dump_status_kbn_in_processing() -> None: return -def update_dump_status_kbn_in_error() -> None: +def update_dump_status_kbn_error() -> None: """dump取得状況区分をエラーに更新する Raises: @@ -83,7 +83,7 @@ def update_dump_status_kbn_in_error() -> None: return -def update_dump_status_kbn_in_complete() -> None: +def update_dump_status_kbn_complete() -> None: """dump取得状況区分を正常終了に更新する Raises: diff --git a/ecs/jskult-dbdump/src/batch/common/batch_context.py b/ecs/jskult-dbdump/src/batch/common/batch_context.py deleted file mode 100644 index 8c8c12fb..00000000 --- a/ecs/jskult-dbdump/src/batch/common/batch_context.py +++ /dev/null @@ -1,29 +0,0 @@ -class BatchContext: - __instance = None - __syor_date: str # 処理日(yyyy/mm/dd形式) - __is_arisj_output_day: bool # 月次バッチ起動日フラグ - - def __init__(self) -> None: - self.__is_arisj_output_day = False - - @classmethod - def get_instance(cls): - if cls.__instance is None: - cls.__instance = cls() - return cls.__instance - - @property - def syor_date(self): - return self.__syor_date - - @syor_date.setter - def syor_date(self, syor_date_str: str): - self.__syor_date = syor_date_str - - @property - def is_arisj_output_day(self): - return self.__is_arisj_output_day - - @is_arisj_output_day.setter - def is_arisj_output_day(self, flag: bool): - self.__is_arisj_output_day = flag diff --git a/ecs/jskult-dbdump/src/db/database.py b/ecs/jskult-dbdump/src/db/database.py index 67f5c5a5..ee127d51 100644 --- a/ecs/jskult-dbdump/src/db/database.py +++ b/ecs/jskult-dbdump/src/db/database.py @@ -4,11 +4,8 @@ from sqlalchemy.engine.url import URL from tenacity import retry, stop_after_attempt, wait_exponential from src.error.exceptions import DBException -from src.logging.get_logger import get_logger from src.system_var import environment -logger = get_logger(__name__) - class Database: """データベース操作クラス""" @@ -74,9 +71,7 @@ class Database: min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS ), - stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), - retry_error_cls=DBException - ) + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) def connect(self): """ DBに接続します。接続に失敗した場合、リトライします。 diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 02b8da84..7eba902a 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -6,8 +6,8 @@ import subprocess import textwrap from src.batch.batch_functions import (get_batch_statuses, - update_dump_status_kbn_in_complete, - update_dump_status_kbn_in_error, + update_dump_status_kbn_complete, + update_dump_status_kbn_error, update_dump_status_kbn_in_processing) from src.error.exceptions import BatchOperationException from src.logging.get_logger import get_logger @@ -28,12 +28,12 @@ def exec(): # 日次バッチ処理中の場合、処理は行わない if batch_processing_flag == constants.BATCH_ACTF_BATCH_IN_PROCESSING: - logger.error('日次バッチ処理中の為、処理を終了') + logger.error('日次バッチ処理中の為、日次バッチ処理前DBダンプ取得を終了します。') return constants.BATCH_EXIT_CODE_SUCCESS # dump処理状態区分が処理中またはエラーの場合、処理は行わない if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED or dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: - logger.error(f'dump処理状態区分が実行不可な状態の為、処理を終了 dump処理状態区分={dump_status_kbn}') + logger.error(f'ダンプ取得が実行不可能な状態の為、日次バッチ処理前DBダンプ取得を終了します。 dump処理状態区分={dump_status_kbn}') return constants.BATCH_EXIT_CODE_SUCCESS # dump処理状態区分を処理中に更新 @@ -74,6 +74,7 @@ def exec(): '--set-gtid-purged=OFF', environment.DB_SCHEMA ] + mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # gzipコマンドを実行してdump結果を圧縮する gzip_process = subprocess.Popen(['gzip'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -102,7 +103,7 @@ def exec(): # dump処理状態区分を正常終了に更新 try: - update_dump_status_kbn_in_complete() + update_dump_status_kbn_complete() except BatchOperationException as e: logger.exception(f'dump処理状態区分更新(処理中→正常終了) エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS @@ -113,7 +114,7 @@ def exec(): except Exception as e: # dump処理状態区分をエラーに更新 try: - update_dump_status_kbn_in_error() + update_dump_status_kbn_error() except BatchOperationException as e: logger.exception(f'dump処理状態区分更新(処理中→エラー) エラー（異常終了）:{e}') return constants.BATCH_EXIT_CODE_SUCCESS diff --git a/ecs/jskult-dbdump/src/system_var/environment.py b/ecs/jskult-dbdump/src/system_var/environment.py index 46f08c5d..ff510581 100644 --- a/ecs/jskult-dbdump/src/system_var/environment.py +++ b/ecs/jskult-dbdump/src/system_var/environment.py @@ -9,6 +9,7 @@ DB_SCHEMA = os.environ['DB_SCHEMA'] # AWS JSKULT_BACKUP_BUCKET = os.environ['JSKULT_BACKUP_BUCKET'] +DUMP_BACKUP_FOLDER = os.environ['DUMP_BACKUP_FOLDER'] # 初期値がある環境変数 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') @@ -17,4 +18,3 @@ DB_CONNECTION_RETRY_INTERVAL_INIT = int(os.environ.get('DB_CONNECTION_RETRY_INTE DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MIN_SECONDS', 5)) DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS = int(os.environ.get('DB_CONNECTION_RETRY_MAX_SECONDS', 50)) -DUMP_BACKUP_FOLDER = os.environ['DUMP_BACKUP_FOLDER'] From 3c3178fe70becf2c581ff65c56f336d3df53debb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 12 Jul 2023 13:15:37 +0900 Subject: [PATCH 17/30] =?UTF-8?q?=E3=82=A8=E3=83=A9=E3=83=BC=E3=81=AE?= =?UTF-8?q?=E6=96=87=E8=A8=80=E3=82=92=E3=82=8F=E3=81=91=E3=81=BE=E3=81=97?= =?UTF-8?q?=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index 7eba902a..eebbaf3e 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -31,8 +31,13 @@ def exec(): logger.error('日次バッチ処理中の為、日次バッチ処理前DBダンプ取得を終了します。') return constants.BATCH_EXIT_CODE_SUCCESS - # dump処理状態区分が処理中またはエラーの場合、処理は行わない - if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED or dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: + # dump処理状態区分が処理中の場合、処理は行わない + if dump_status_kbn == constants.DUMP_STATUS_KBN_PROCESSED: + logger.error(f'ダンプ取得中の為、日次バッチ処理前DBダンプ取得を終了します。 dump処理状態区分={dump_status_kbn}') + return constants.BATCH_EXIT_CODE_SUCCESS + + # dump処理状態区分がエラーの場合、処理は行わない + if dump_status_kbn == constants.DUMP_STATUS_KBN_ERROR: logger.error(f'ダンプ取得が実行不可能な状態の為、日次バッチ処理前DBダンプ取得を終了します。 dump処理状態区分={dump_status_kbn}') return constants.BATCH_EXIT_CODE_SUCCESS From e0797fae1f293f0f88edef160f08e8c1bcd68513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 12 Jul 2023 14:26:43 +0900 Subject: [PATCH 18/30] =?UTF-8?q?=E9=96=93=E9=81=95=E3=81=88=E3=81=A6?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E3=81=97=E3=81=A6=E3=81=97=E3=81=BE=E3=81=A3?= =?UTF-8?q?=E3=81=9F=E3=81=AE=E3=82=92=E6=88=BB=E3=81=97=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/db/database.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ecs/jskult-dbdump/src/db/database.py b/ecs/jskult-dbdump/src/db/database.py index ee127d51..b675381c 100644 --- a/ecs/jskult-dbdump/src/db/database.py +++ b/ecs/jskult-dbdump/src/db/database.py @@ -71,7 +71,9 @@ class Database: min=environment.DB_CONNECTION_RETRY_INTERVAL_MIN_SECONDS, max=environment.DB_CONNECTION_RETRY_INTERVAL_MAX_SECONDS ), - stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT)) + stop=stop_after_attempt(environment.DB_CONNECTION_MAX_RETRY_ATTEMPT), + retry_error_cls=DBException + ) def connect(self): """ DBに接続します。接続に失敗した場合、リトライします。 From 5ebd71b03d47b05b211c0fdb6004c6d65e5c8ce2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=87=8E=E9=96=93?= Date: Wed, 12 Jul 2023 14:48:10 +0900 Subject: [PATCH 19/30] =?UTF-8?q?gzip=E3=82=B3=E3=83=9E=E3=83=B3=E3=83=89?= =?UTF-8?q?=E3=81=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index eebbaf3e..c93b2a0f 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -82,7 +82,7 @@ def exec(): mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # gzipコマンドを実行してdump結果を圧縮する - gzip_process = subprocess.Popen(['gzip'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + gzip_process = subprocess.Popen(['gzip', '-c'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # aws s3 cpコマンドを実行してアップロードする s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_name], stdin=gzip_process.stdout, stderr=subprocess.PIPE) # mysqldumpの標準出力をgzipに接続したため、標準出力をクローズする From b3759b4ae9144a227881559837f7772df88483aa Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Thu, 13 Jul 2023 10:29:01 +0900 Subject: [PATCH 20/30] =?UTF-8?q?feat:=20dump=E5=8F=96=E5=BE=97=E6=99=82?= =?UTF-8?q?=E3=80=81=E5=87=BA=E5=8A=9B=E5=85=88=E3=81=AE=E3=83=91=E3=82=B9?= =?UTF-8?q?=E3=82=92=E3=83=AD=E3=82=B0=E5=87=BA=E5=8A=9B=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-dbdump/Dockerfile | 2 +- ecs/jskult-dbdump/src/jobctrl_dbdump.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ecs/jskult-dbdump/Dockerfile b/ecs/jskult-dbdump/Dockerfile index 9c6838cb..3c410b72 100644 --- a/ecs/jskult-dbdump/Dockerfile +++ b/ecs/jskult-dbdump/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-bullseye +FROM python:3.9 ENV TZ="Asia/Tokyo" diff --git a/ecs/jskult-dbdump/src/jobctrl_dbdump.py b/ecs/jskult-dbdump/src/jobctrl_dbdump.py index c93b2a0f..3dac256f 100644 --- a/ecs/jskult-dbdump/src/jobctrl_dbdump.py +++ b/ecs/jskult-dbdump/src/jobctrl_dbdump.py @@ -64,8 +64,8 @@ def exec(): dt_now = datetime.datetime.now() converted_value = dt_now.strftime('%Y%m%d%H%M%S') - file_name = f'backup_rds_src05_{converted_value}.gz' - s3_file_name = f's3://{environment.JSKULT_BACKUP_BUCKET}/{environment.DUMP_BACKUP_FOLDER}/{dt_now.year}/{dt_now.strftime("%m")}/{dt_now.strftime("%d")}/{file_name}' + dump_file_name = f'backup_rds_src05_{converted_value}.gz' + s3_file_path = f's3://{environment.JSKULT_BACKUP_BUCKET}/{environment.DUMP_BACKUP_FOLDER}/{dt_now.year}/{dt_now.strftime("%m")}/{dt_now.strftime("%d")}/{dump_file_name}' # mysqldumpコマンドを実行し、dumpを取得する command = [ @@ -79,12 +79,12 @@ def exec(): '--set-gtid-purged=OFF', environment.DB_SCHEMA ] - + mysqldump_process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # gzipコマンドを実行してdump結果を圧縮する gzip_process = subprocess.Popen(['gzip', '-c'], stdin=mysqldump_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) # aws s3 cpコマンドを実行してアップロードする - s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_name], stdin=gzip_process.stdout, stderr=subprocess.PIPE) + s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', '-', s3_file_path], stdin=gzip_process.stdout, stderr=subprocess.PIPE) # mysqldumpの標準出力をgzipに接続したため、標準出力をクローズする mysqldump_process.stdout.close() # gzipの標準出力をaws s3 cpに接続したため、標準出力をクローズする @@ -114,6 +114,7 @@ def exec(): return constants.BATCH_EXIT_CODE_SUCCESS logger.info('日次バッチ処理前DBダンプ取得：終了（正常終了）') + logger.info(f'出力ファイルパス: {s3_file_path}') return constants.BATCH_EXIT_CODE_SUCCESS except Exception as e: From ecbff882bf2819e0d56edf452522596be5fb9168 Mon Sep 17 00:00:00 2001 From: "x.azuma.m@nds-tyo.co.jp" Date: Thu, 13 Jul 2023 15:41:38 +0900 Subject: [PATCH 21/30] =?UTF-8?q?=E3=83=90=E3=82=B0=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit BOM3byte目の0xBFを判定するとき、UTF-8バイトシーケンスに含まれる0xBFを誤検知して、正しく末尾行が抽出できなかった --- .../src/batch/vjsk/vjsk_data_load_manager.py | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py index 0cc8a248..11764edf 100644 --- a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py +++ b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py @@ -91,26 +91,14 @@ class VjskDataLoadManager: # memo: UTF-8 バイトシーケンスとして、b'\n' が全角文字の一部にはならない if char == b'\n': break - # ファイル先頭のBOMを検出したらループ終了 - if char == b'\xbf': + # ファイル先頭のBOM3byte目の BF を検出したらループ終了 + # memo: UTF-8 バイトシーケンスとして、b'\xbf' が全角文字の一部の可能性がある(例:全角片仮名の「タ」) + # memo: charに代入したときのfile.read(1)によって、ファイルポインタは2→3になっている前提のロジック + if char == b'\xbf' and file.tell() == 3: break last_line = file.readline().decode('utf-8-sig').rstrip('\n') buf_count = last_line.count('\t') - # # ファイルの末尾から2バイト手前に、シーク位置を移動 - # # memo: 1byte戻るだけだと、直後のread(1)で末尾行のLFを見てしまうので、-2移動 - # file.seek(-2, os.SEEK_END) - # # 改行文字を見つけるまで逆方向に読み進める - # # memo: read(1)することでシーク位置は1byte前に進む - # while file.read(1) != b'\n': - # # シーク位置を1バイト戻って再度読み込み - # # memo: read(1)でシーク位置が1byte進んでいるので、その戻しと合わせて-2移動 - # file.seek(-2, os.SEEK_CUR) - # # 末尾行を抽出 - # last_line = file.readline().decode().rstrip('\n') - # # 末尾行に含まれるタブ文字の数を抽出 - # buf_count = last_line.count('\t') - return buf_count @classmethod From 241838ef71e29a02f3a7db343ad6ffa7e5133da3 Mon Sep 17 00:00:00 2001 From: "x.azuma.m@nds-tyo.co.jp" Date: Fri, 14 Jul 2023 12:54:59 +0900 Subject: [PATCH 22/30] =?UTF-8?q?=E3=82=B3=E3=83=A1=E3=83=B3=E3=83=88?= =?UTF-8?q?=E6=95=B4=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/vjsk/vjsk_data_load_manager.py | 35 +++++++++++++ .../src/batch/vjsk/vjsk_recv_file_mapper.py | 51 +++++++++++++++++++ .../batch/vjsk/vjsk_file_check/conftest.py | 1 - .../batch/vjsk/vjsk_load/test_vjsk_load.py | 3 ++ 4 files changed, 89 insertions(+), 1 deletion(-) diff --git a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py index 11764edf..af23490b 100644 --- a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py +++ b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py @@ -11,10 +11,21 @@ mapper = VjskReceiveFileMapper() class VjskDataLoadManager: + """ + Ｖ実消化データ取込機能クラス + """ + def __init__(self): pass def _import_to_db(src_file_name: str, condkey: str): + """ + 概要 + 指定されたtsvファイル src_file_name を策席スキーマに登録する + 引数 + src_file_name: ローカルストレージにある取込対象tsvファイルパス + condkey: 受領データの種類を一意に示す値(VjskReceiveFileMapperクラスで管理されているCONDKEY値) + """ logger.debug(f"_import_to_db start (src_file_name : {src_file_name}, condkey : {condkey})") db = Database.get_instance() @@ -24,6 +35,7 @@ class VjskDataLoadManager: upsert_sql = mapper.get_upsert_sql(condkey) try: + # データベース接続 db.connect() db.execute("SET SESSION sql_mode = 'TRADITIONAL';") @@ -52,17 +64,27 @@ class VjskDataLoadManager: # MEMO: https://docs.sqlalchemy.org/en/14/core/connections.html#sqlalchemy.engine.BaseCursorResult.rowcount logger.info(f'{table_name_org}を{table_name_src}にUPSERT') + # データベースコミット db.commit() + except Exception as e: db.rollback() raise BatchOperationException(e) + finally: + # データベース切断 db.disconnect() logger.debug("_import_to_db done") return def _get_tsv_last_row_tab_count(src_file_name: str) -> int: + """ + 概要 + 指定されたtsvファイル src_file_name の末尾行に含まれるタブ文字数を取得する + 引数 + src_file_name: ローカルストレージにある取込対象tsvファイルパス + """ # memo: tsvファイルが数百MBに及ぶことを想定して、末尾から1行分を参照する # memo: 前提1 行区切りは LF('\n') # memo: 前提2 正常時のファイル終端にある文字は、末尾行の LF('\n') @@ -103,12 +125,25 @@ class VjskDataLoadManager: @classmethod def load(self, target: dict): + """ + 概要 + 取込対象受領ファイル target をデータベースに登録する + 引数 + target: { + condkey: 受領データの種類を一意に示す値(VjskReceiveFileMapperクラスで管理されているCONDKEY値) + src_file_path: ローカルストレージにある取込対象tsvファイルパス + } + """ logger.debug(f'load start target:{target}') # S3からローカルストレージにdownloadした登録対象のtsvファイルパスを取得 local_file_name = target["src_file_path"] # tsvファイル末尾行のTABの数が総定数と一致しない場合は例外をスロー + # memo: + # 対向元システムで生成されるファイルは稀に途中欠落が発生することがある。 + # これを、ファイルMySQL8.0のLOADステートメントで発生するWARNING/ERRORでは検知不可能なので、 + # LOADステートメント実行前に、物理的に途中欠落があるかを検知してエラーとすることが目的。 tsv_tabs = self._get_tsv_last_row_tab_count(local_file_name) expect_tabs = mapper.get_file_column_separators(target["condkey"]) if tsv_tabs != expect_tabs: diff --git a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_recv_file_mapper.py b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_recv_file_mapper.py index 19f70067..21264c42 100644 --- a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_recv_file_mapper.py +++ b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_recv_file_mapper.py @@ -2,6 +2,9 @@ import textwrap class VjskReceiveFileMapper: + """ + Ｖ実消化データファイルI/Fマッピング定義クラス + """ CONDKEY_SLIP_DATA = "SLIP_DATA" # 販売実績データ CONDKEY_HLD_MST = "HLD_MST" # Ｖ卸ホールディングスマスタ CONDKEY_WHS_MST = "WHS_MST" # Ｖ卸マスタ @@ -1489,27 +1492,75 @@ class VjskReceiveFileMapper: return ret def get_data_name(self, condkey: str) -> str: + """ + 概要 + 受領ファイルI/Fが想定する、ファイル論理名を取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return self._get_interface_property(condkey, self._KEY_DATA_NAME) def get_file_prefix(self, condkey: str) -> str: + """ + 概要 + 受領ファイルI/Fが想定する、ファイル名接頭辞を取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return self._get_interface_property(condkey, self._KEY_FILE_PREFIX) def get_file_suffix(self, condkey: str) -> str: + """ + 概要 + 受領ファイルI/Fが想定する、ファイル拡張子を取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return self._get_interface_property(condkey, self._KEY_FILE_SUFFIX) def get_file_column_separators(self, condkey: str) -> int: + """ + 概要 + 受領ファイルI/Fが想定する、1行あたりのタブ文字数を取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return int(self._get_interface_property(condkey, self._KEY_FILE_COLUMN_SEPARATORS)) def get_org_table(self, condkey: str) -> str: + """ + 概要 + 受領ファイルI/Fが想定する、LOAD先ロードスキーマテーブル名を取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return self._get_interface_property(condkey, self._KEY_ORG_TABLE) def get_src_table(self, condkey: str) -> str: + """ + 概要 + 受領ファイルI/Fが想定する、登録先蓄積スキーマテーブル名を取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return self._get_interface_property(condkey, self._KEY_SRC_TABLE) def get_upsert_sql(self, condkey: str) -> str: + """ + 概要 + 受領ファイルI/Fが想定する、upsert (ロードスキーマ→蓄積スキーマ) CMLステートメントを取得する + 引数 + condkey: 受領データの種類を一意に示す値(このクラスのメンバ CONDKEY_* の値) + """ return self._get_interface_property(condkey, self._KEY_UPSERT_SQL) def get_condkey_by_s3_file_path(self, s3_file_path: str) -> str: + """ + 概要 + S3受領バケットに受領したファイル名から、I/F想定に該当する condkey 値を取得する + 引数 + s3_file_path: S3受領バケットにある受領したファイルパス + """ ret = None filename = s3_file_path[s3_file_path.rfind("/") + 1:] for condkey in self._VJSK_INTERFACE_MAPPING: diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/conftest.py b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/conftest.py index dccdd0df..bf9b2629 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/conftest.py +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_file_check/conftest.py @@ -23,7 +23,6 @@ def receive_folder(): return os.environ["VJSK_DATA_RECEIVE_FOLDER"] -# TODO 共通fixtureにして15個固定でput/delete、各個別fixtureで15個から引き算でdeleteする @pytest.fixture def init_check_received_files_ok1(s3_client, bucket_name, receive_folder): # setup diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py index 70504795..d2fb07ac 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py @@ -17,6 +17,9 @@ from tests.testing_vjsk_utility import (assert_table_results, class TestImportFileToDb: + """ + Ｖ実消化データ取込－データベース登録処理 テストクラス + """ db: Database batch_context: BatchContext test_file_path_import_all: str From de9bf1a0649d2cf6050a14e93e7b7223ccf90971 Mon Sep 17 00:00:00 2001 From: "x.azuma.m@nds-tyo.co.jp" Date: Fri, 14 Jul 2023 18:44:26 +0900 Subject: [PATCH 23/30] =?UTF-8?q?=E3=83=86=E3=82=B9=E3=83=88=E3=82=B3?= =?UTF-8?q?=E3=83=BC=E3=83=89=E5=AE=9F=E8=A1=8C=E7=94=A8=E3=82=B3=E3=83=9E?= =?UTF-8?q?=E3=83=B3=E3=83=89=E8=BF=BD=E5=8A=A0=E3=80=81=E3=81=8A=E3=82=88?= =?UTF-8?q?=E3=81=B3readme=E3=81=AE=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/Pipfile | 4 +- ecs/jskult-batch-daily/README.md | 100 +++++++++++++++++++++++++++---- 2 files changed, 90 insertions(+), 14 deletions(-) diff --git a/ecs/jskult-batch-daily/Pipfile b/ecs/jskult-batch-daily/Pipfile index 1e6adf91..6dc0871b 100644 --- a/ecs/jskult-batch-daily/Pipfile +++ b/ecs/jskult-batch-daily/Pipfile @@ -5,7 +5,9 @@ name = "pypi" [scripts] "test:ultmarc" = "pytest tests/batch/ultmarc/" -"test:ultmarc:cov" = "pytest --cov=src/batch/ultmarc/ --cov-branch --cov-report=term-missing tests/batch/ultmarc/" +"test:ultmarc:cov" = "pytest --cov=src/batch/ultmarc/ --cov-branch --cov-report=term-missing tests/batch/vjsk/" +"test:vjsk" = "pytest tests/batch/vjsk/" +"test:vjsk:cov" = "pytest --cov=src/batch/vjsk/ --cov-branch --cov-report=term-missing tests/batch/vjsk/" [packages] boto3 = "*" diff --git a/ecs/jskult-batch-daily/README.md b/ecs/jskult-batch-daily/README.md index d0f37149..144cf9b8 100644 --- a/ecs/jskult-batch-daily/README.md +++ b/ecs/jskult-batch-daily/README.md @@ -44,7 +44,7 @@ - 「entrypoint.py」が、バッチ処理のエントリーポイント。 - 実際の処理は、「src/jobctrl_daily.py」で行っている。 -## 単体テスト +## 単体テスト(アルトマーク取込処理) アルトマーク取込処理は、単体テストコードを使用してテスト自動化を行う @@ -68,7 +68,7 @@ | コマンド | 概要 | | ---------------- | -------------------------------------------------------------------------------------------- | | test:ultmarc | tests/batch/ultmarc フォルダ配下のユニットテストを実行する | -| test:ultmarc:cov | tests/batch/ultmarc フォルダ配下のユニットテストを実行し、テストカバレッジを取得する(C1, C2) | +| test:ultmarc:cov | tests/batch/ultmarc フォルダ配下のユニットテストを実行し、テストカバレッジを取得する(C0, C1) | ### テスト共通関数の仕様 @@ -156,6 +156,43 @@ - テスト結果データと期待値データを突き合わせ、期待値どおりとなっているかを確認する - ignore_col_nameに指定したカラムは、呼び出し元のテストコード内で個別に突き合わせする + +## 単体テスト(実消化データ取込処理) + +実消化データは、単体テストコードを使用してテスト自動化を行う + +### テスト準備 + +※単体テスト(アルトマーク取込処理)と同じ + +### テスト用のサブコマンド一覧 + +- `pipenv run`のあとに、サブコマンドとしてユーザー定義スクリプトを実行できる + - `Pipfile`内の「scripts」セクションに宣言されている + +| コマンド | 概要 | +| ---------------- | -------------------------------------------------------------------------------------------- | +| test:vjsk | tests/batch/vjsk フォルダ配下のユニットテストを実行する | +| test:vjsk:cov | tests/batch/vjsk フォルダ配下のユニットテストを実行し、テストカバレッジを取得する(C0, C1) | + +### テスト共通関数の仕様 + +- tests/testing_vjsk_utility.py内の共通関数の仕様について記載する + +#### create_vjsk_assertion_list + +- 概要 + - DB登録期待値リストを作成する +- Args: + - file_path (str): DB登録期待値ファイル(tsvファイル)のパス + - memo: ※DB登録期待値ファイルの前提 + - memo: 受領データファイルと同じ + - memo: BOM付きtsv形式 + - memo: 一行目はカラム名になっているヘッダ行 + - Returns: + - List(dict) DB登録期待値辞書リスト + + ## フォルダ構成 ```text @@ -203,16 +240,53 @@ └── tests -- ユニットテストのルートディレクト ├── batch │ └── ultmarc -- アルトマーク関連のユニットテストを格納する - │ └── utmp_tables - │ └── table_mapper -- 以下、マッパークラス単位でフォルダを切る - │ └── com_alma - │ ├── test_com_alma_mapper.py -- テストコード本体 - │ ├── com_alma_insert.csv -- S3に配置される想定のテストCSVデータ。ケースごとに用意する。 - │ ... - │ ├── db_com_alma_before_update.csv -- テスト時に事前にDBに登録しておくデータ。CSVで用意する。 - │ ... - │ ├── expect_com_alma_insert.csv -- テストの期待値データ。CSVで用意する。 - │ ... + │ │ └── utmp_tables + │ │ └── table_mapper -- 以下、マッパークラス単位でフォルダを切る + │ │ └── com_alma + │ │ ├── test_com_alma_mapper.py -- テストコード本体 + │ │ ├── com_alma_insert.csv -- S3に配置される想定のテストCSVデータ。ケースごとに用意する。 + │ │ ... + │ │ ├── db_com_alma_before_update.csv -- テスト時に事前にDBに登録しておくデータ。CSVで用意する。 + │ │ ... + │ │ ├── expect_com_alma_insert.csv -- テストの期待値データ。CSVで用意する。 + │ │ ... + │ └─vjsk -- 実消化データ取込処理関連のユニットテストを格納する + │ │ + │ ├─vjsk_file_check -- 受領ファイルチェック処理関連のユニットテストを格納する + │ │ ├─conftest.py -- テスト内で共通利用できるフィクスチャの宣言 + │ │ └─test_vjsk_file_check.py -- テストクラス本体 + │ │ + │ └─vjsk_load -- 受領データ登録処理関連のユニットテストを格納する + │ │ conftest.py -- テスト内で共通利用できるフィクスチャの宣言 + │ │ test_vjsk_load.py -- テストクラス本体 + │ │ + │ └─testdata -- テストモジュールが使用するテストデータを格納する + │ │ bio_slip_data_202304280000.tsv -- 正常ケースの単体確認用 + │ │ ... -- *20230428* は新規4件の登録確認用 + │ │ whs_mst_202304290000.tsv -- *20230429* は更新2件+追加新規2件の登録確認用 + │ │ + │ ├─NoData -- 正常ケースの単体確認用 + │ │ bio_slip_data_nodatarecord.tsv -- ヘッダ行のみでデータが0件の動作確認用 + │ │ ... + │ │ whs_mst_nodatarecord.tsv + │ │ + │ ├─TestFormatErrorFile -- 異常ケースの単体確認用 + │ │ bio_slip_data_formaterror.tsv -- 末尾行のタブ数が想定と異なる(ファイル欠落がある)ときの動作確認用 + │ │ ... + │ │ whs_mst_formaterror.tsv + │ │ + │ ├─TestImportFileToDb -- 正常ケースの単体確認用 + │ │ bio_slip_data_202304270000.gz -- 対向元システムから送られてきた状態(gz圧縮)の受領データファイルの動作確認用 + │ │ ... + │ │ whs_mst_202304270000.gz + │ │ + │ └─UnzipError -- 異常ケースの単体確認用 + │ bio_slip_data_202304270000.gz -- gz圧縮ファイルが解凍できないときの動作確認用 + │ ... + │ whs_mst_202304270000.gz + │ ├── conftest.py -- テスト内で共通利用できるフィクスチャを宣言する(執筆時点ではDBのみ) - └── testing_utility.py -- テストの共通関数 + ├── testing_utility.py -- テストの共通関数 + └── testing_vjsk_utility.py -- テストの共通関数(実消化データ取込処理関連) ``` + From 63c3340fe8bb91256b286355eac6c500f5359868 Mon Sep 17 00:00:00 2001 From: "x.azuma.m@nds-tyo.co.jp" Date: Fri, 14 Jul 2023 18:51:49 +0900 Subject: [PATCH 24/30] =?UTF-8?q?=E3=82=B3=E3=83=94=E3=83=9A=E5=A2=97?= =?UTF-8?q?=E6=AE=96=E3=81=97=E3=81=A6=E5=A4=89=E3=81=88=E3=82=8B=E3=82=82?= =?UTF-8?q?=E3=81=AE=E3=82=92=E5=A4=89=E3=81=88=E3=81=99=E3=81=8E=E3=81=A6?= =?UTF-8?q?=E3=81=9F=E3=81=AE=E3=81=A7=E5=85=83=E3=81=AB=E6=88=BB=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-batch-daily/Pipfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/jskult-batch-daily/Pipfile b/ecs/jskult-batch-daily/Pipfile index 6dc0871b..a40e6c17 100644 --- a/ecs/jskult-batch-daily/Pipfile +++ b/ecs/jskult-batch-daily/Pipfile @@ -5,7 +5,7 @@ name = "pypi" [scripts] "test:ultmarc" = "pytest tests/batch/ultmarc/" -"test:ultmarc:cov" = "pytest --cov=src/batch/ultmarc/ --cov-branch --cov-report=term-missing tests/batch/vjsk/" +"test:ultmarc:cov" = "pytest --cov=src/batch/ultmarc/ --cov-branch --cov-report=term-missing tests/batch/ultmarc/" "test:vjsk" = "pytest tests/batch/vjsk/" "test:vjsk:cov" = "pytest --cov=src/batch/vjsk/ --cov-branch --cov-report=term-missing tests/batch/vjsk/" From d2d5b5f9b6ac56921abc4e616c2c3163fa712b12 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Jul 2023 09:22:49 +0900 Subject: [PATCH 25/30] =?UTF-8?q?feat:=20=E3=83=AA=E3=82=B9=E3=83=88?= =?UTF-8?q?=E3=82=A2=E7=94=A8=E3=82=B9=E3=82=AF=E3=83=AA=E3=83=97=E3=83=88?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-restore-backup/.dockerignore | 12 +++ ecs/jskult-restore-backup/.env.example | 9 +++ ecs/jskult-restore-backup/.gitignore | 11 +++ ecs/jskult-restore-backup/.vscode/launch.json | 17 +++++ .../.vscode/recommended_settings.json | 31 ++++++++ ecs/jskult-restore-backup/Dockerfile | 40 ++++++++++ ecs/jskult-restore-backup/Pipfile | 16 ++++ ecs/jskult-restore-backup/Pipfile.lock | 69 +++++++++++++++++ ecs/jskult-restore-backup/README.md | 65 ++++++++++++++++ ecs/jskult-restore-backup/entrypoint.py | 10 +++ .../mysql_dpkg_selection.txt | 3 + ecs/jskult-restore-backup/src/__init__.py | 0 .../src/logging/get_logger.py | 37 ++++++++++ .../src/restore_backup.py | 74 +++++++++++++++++++ .../src/system_var/__init__.py | 0 .../src/system_var/constants.py | 2 + .../src/system_var/environment.py | 14 ++++ 17 files changed, 410 insertions(+) create mode 100644 ecs/jskult-restore-backup/.dockerignore create mode 100644 ecs/jskult-restore-backup/.env.example create mode 100644 ecs/jskult-restore-backup/.gitignore create mode 100644 ecs/jskult-restore-backup/.vscode/launch.json create mode 100644 ecs/jskult-restore-backup/.vscode/recommended_settings.json create mode 100644 ecs/jskult-restore-backup/Dockerfile create mode 100644 ecs/jskult-restore-backup/Pipfile create mode 100644 ecs/jskult-restore-backup/Pipfile.lock create mode 100644 ecs/jskult-restore-backup/README.md create mode 100644 ecs/jskult-restore-backup/entrypoint.py create mode 100644 ecs/jskult-restore-backup/mysql_dpkg_selection.txt create mode 100644 ecs/jskult-restore-backup/src/__init__.py create mode 100644 ecs/jskult-restore-backup/src/logging/get_logger.py create mode 100644 ecs/jskult-restore-backup/src/restore_backup.py create mode 100644 ecs/jskult-restore-backup/src/system_var/__init__.py create mode 100644 ecs/jskult-restore-backup/src/system_var/constants.py create mode 100644 ecs/jskult-restore-backup/src/system_var/environment.py diff --git a/ecs/jskult-restore-backup/.dockerignore b/ecs/jskult-restore-backup/.dockerignore new file mode 100644 index 00000000..8b9da402 --- /dev/null +++ b/ecs/jskult-restore-backup/.dockerignore @@ -0,0 +1,12 @@ +tests/* +.coverage +.env +.env.example +.report/* +.vscode/* +.pytest_cache/* +*/__pychache__/* +Dockerfile +pytest.ini +README.md +*.sql diff --git a/ecs/jskult-restore-backup/.env.example b/ecs/jskult-restore-backup/.env.example new file mode 100644 index 00000000..2bda940a --- /dev/null +++ b/ecs/jskult-restore-backup/.env.example @@ -0,0 +1,9 @@ +DB_HOST=************ +DB_PORT=3306 +DB_USERNAME=************ +DB_PASSWORD=************ +DB_SCHEMA=src05 + +JSKULT_BACKUP_DUMP_S3_FILE_PATH=******************* + +LOG_LEVEL=INFO diff --git a/ecs/jskult-restore-backup/.gitignore b/ecs/jskult-restore-backup/.gitignore new file mode 100644 index 00000000..cf44449e --- /dev/null +++ b/ecs/jskult-restore-backup/.gitignore @@ -0,0 +1,11 @@ +.vscode/settings.json +.env +my.cnf + +# python +__pycache__ + +# python test +.pytest_cache +.coverage +.report/ \ No newline at end of file diff --git a/ecs/jskult-restore-backup/.vscode/launch.json b/ecs/jskult-restore-backup/.vscode/launch.json new file mode 100644 index 00000000..a0178c26 --- /dev/null +++ b/ecs/jskult-restore-backup/.vscode/launch.json @@ -0,0 +1,17 @@ +{ + // IntelliSense を使用して利用可能な属性を学べます。 + // 既存の属性の説明をホバーして表示します。 + // 詳細情報は次を確認してください: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + + { + "name": "(DEBUG)jskult batch dbdump", + "type": "python", + "request": "launch", + "program": "entrypoint.py", + "console": "integratedTerminal", + "justMyCode": true + } + ] +} \ No newline at end of file diff --git a/ecs/jskult-restore-backup/.vscode/recommended_settings.json b/ecs/jskult-restore-backup/.vscode/recommended_settings.json new file mode 100644 index 00000000..b5e79d73 --- /dev/null +++ b/ecs/jskult-restore-backup/.vscode/recommended_settings.json @@ -0,0 +1,31 @@ +{ + "[python]": { + "editor.defaultFormatter": null, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } + }, + // 自身の環境に合わせて変えてください + "python.defaultInterpreterPath": "", + "python.linting.lintOnSave": true, + "python.linting.enabled": true, + "python.linting.pylintEnabled": false, + "python.linting.flake8Enabled": true, + "python.linting.flake8Args": [ + "--max-line-length=200", + "--ignore=F541" + ], + "python.formatting.provider": "autopep8", + "python.formatting.autopep8Path": "autopep8", + "python.formatting.autopep8Args": [ + "--max-line-length", "200", + "--ignore=F541" + ], + "python.testing.pytestArgs": [ + "tests/batch/ultmarc" + ], + + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true +} diff --git a/ecs/jskult-restore-backup/Dockerfile b/ecs/jskult-restore-backup/Dockerfile new file mode 100644 index 00000000..3c410b72 --- /dev/null +++ b/ecs/jskult-restore-backup/Dockerfile @@ -0,0 +1,40 @@ +FROM python:3.9 + +ENV TZ="Asia/Tokyo" + +WORKDIR /usr/src/app +COPY Pipfile Pipfile.lock ./ +# mysql-apt-config をdpkgでインストールする際に標準出力に渡す文字列ファイルをコピー +COPY mysql_dpkg_selection.txt ./ +# 必要なパッケージインストール +RUN apt update && apt install -y less vim curl wget gzip unzip sudo lsb-release + +# mysqlをインストール +RUN \ + wget https://dev.mysql.com/get/mysql-apt-config_0.8.25-1_all.deb && \ + dpkg -i mysql-apt-config_0.8.25-1_all.deb < mysql_dpkg_selection.txt && \ + apt update && \ + apt install -y mysql-client + +# aws cli v2 のインストール +RUN \ + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip && \ + sudo ./aws/install + +# python関連のライブラリインストール +RUN \ + pip install --upgrade pip wheel setuptools && \ + pip install pipenv --no-cache-dir && \ + pipenv install --system --deploy && \ + pip uninstall -y pipenv virtualenv-clone virtualenv + +# パッケージのセキュリティアップデートのみを適用するコマンドを実行 +RUN \ + apt install -y unattended-upgrades && \ + unattended-upgrades + +COPY src ./src +COPY entrypoint.py entrypoint.py + +CMD ["python", "entrypoint.py"] diff --git a/ecs/jskult-restore-backup/Pipfile b/ecs/jskult-restore-backup/Pipfile new file mode 100644 index 00000000..f032c7ea --- /dev/null +++ b/ecs/jskult-restore-backup/Pipfile @@ -0,0 +1,16 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[packages] + +[dev-packages] +autopep8 = "*" +flake8 = "*" + +[requires] +python_version = "3.9" + +[pipenv] +allow_prereleases = true diff --git a/ecs/jskult-restore-backup/Pipfile.lock b/ecs/jskult-restore-backup/Pipfile.lock new file mode 100644 index 00000000..f948e3df --- /dev/null +++ b/ecs/jskult-restore-backup/Pipfile.lock @@ -0,0 +1,69 @@ +{ + "_meta": { + "hash": { + "sha256": "cc5f54bfb2073051a26f113ceac64e12fdd0bf8faa36f1a42210cc9c921c134b" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.9" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": {}, + "develop": { + "autopep8": { + "hashes": [ + "sha256:86e9303b5e5c8160872b2f5ef611161b2893e9bfe8ccc7e2f76385947d57a2f1", + "sha256:f9849cdd62108cb739dbcdbfb7fdcc9a30d1b63c4cc3e1c1f893b5360941b61c" + ], + "index": "pypi", + "version": "==2.0.2" + }, + "flake8": { + "hashes": [ + "sha256:3833794e27ff64ea4e9cf5d410082a8b97ff1a06c16aa3d2027339cd0f1195c7", + "sha256:c61007e76655af75e6785a931f452915b371dc48f56efd765247c8fe68f2b181" + ], + "index": "pypi", + "version": "==6.0.0" + }, + "mccabe": { + "hashes": [ + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.0" + }, + "pycodestyle": { + "hashes": [ + "sha256:347187bdb476329d98f695c213d7295a846d1152ff4fe9bacb8a9590b8ee7053", + "sha256:8a4eaf0d0495c7395bdab3589ac2db602797d76207242c17d470186815706610" + ], + "markers": "python_version >= '3.6'", + "version": "==2.10.0" + }, + "pyflakes": { + "hashes": [ + "sha256:ec55bf7fe21fff7f1ad2f7da62363d749e2a470500eab1b555334b67aa1ef8cf", + "sha256:ec8b276a6b60bd80defed25add7e439881c19e64850afd9b346283d4165fd0fd" + ], + "markers": "python_version >= '3.6'", + "version": "==3.0.1" + }, + "tomli": { + "hashes": [ + "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" + ], + "markers": "python_version < '3.11'", + "version": "==2.0.1" + } + } +} diff --git a/ecs/jskult-restore-backup/README.md b/ecs/jskult-restore-backup/README.md new file mode 100644 index 00000000..f955ff97 --- /dev/null +++ b/ecs/jskult-restore-backup/README.md @@ -0,0 +1,65 @@ +# 実消化＆アルトマークダンプ復元スクリプト + +## 概要 + +実消化＆アルトマークダンプ復元スクリプト + +## 環境情報 + +- Python 3.9 +- MySQL 8.23 +- VSCode + +## 環境構築 + +- Python の構築 + + - Merck_NewDWH 開発 2021 の Wiki、[Python 環境構築](https://nds-tyo.backlog.com/alias/wiki/1874930)を参照 + - 「Pipenv の導入」までを行っておくこと + - 構築完了後、プロジェクト配下で以下のコマンドを実行し、Python の仮想環境を作成する + - `pipenv install --dev --python ` + - この手順で出力される仮想環境のパスは、後述する VSCode の設定手順で使用するため、控えておく + +- MySQL の環境構築 + - Windows の場合、以下のリンクからダウンロードする + - + - Docker を利用する場合、「newsdwh-tools」リポジトリの MySQL 設定を使用すると便利 + - 「crm-table-to-ddl」フォルダ内で以下のコマンドを実行すると + - `docker-compose up -d` + - Docker の構築手順は、[Docker のセットアップ手順](https://nds-tyo.backlog.com/alias/wiki/1754332)を参照のこと + - データを投入する + - 立ち上げたデータベースに「src05」スキーマを作成する + - [ローカル開発用データ](https://ndstokyo.sharepoint.com/:f:/r/sites/merck-new-dwh-team/Shared%20Documents/03.NewDWH%E6%A7%8B%E7%AF%89%E3%83%95%E3%82%A7%E3%83%BC%E3%82%BA3/02.%E9%96%8B%E7%99%BA/90.%E9%96%8B%E7%99%BA%E5%85%B1%E6%9C%89/%E3%83%AD%E3%83%BC%E3%82%AB%E3%83%AB%E9%96%8B%E7%99%BA%E7%94%A8%E3%83%87%E3%83%BC%E3%82%BF?csf=1&web=1&e=VVcRUs)をダウンロードし、mysql コマンドを使用して復元する + - `mysql -h <ホスト名> -P <ポート> -u <ユーザー名> -p src05 < src05_dump.sql` +- 環境変数の設定 + - 「.env.example」ファイルをコピーし、「.env」ファイルを作成する + - 環境変数を設定する。設定内容は PRJ メンバーより共有を受けてください +- VSCode の設定 + - 「.vscode/recommended_settings.json」ファイルをコピーし、「settings.json」ファイルを作成する + - 「python.defaultInterpreterPath」を、Python の構築手順で作成した仮想環境のパスに変更する + +## 実行 + +- VSCode 上で「F5」キーを押下すると、バッチ処理が起動する。 +- 「entrypoint.py」が、バッチ処理のエントリーポイント。 +- 実際の処理は、「src/restore_backup.py」で行っている。 + +## フォルダ構成 + +```txt +. +├── .env.example -- 環境変数のサンプル値 +├── Dockerfile -- Dockerイメージ作成用 +├── Pipfile -- pythonの依存関係管理 +├── Pipfile.lock -- 依存関係バージョン固定 +├── README.md -- 当ファイル +├── entrypoint.py -- エントリーポイントとなるファイル +├── mysql_dpkg_selection.txt -- Dockerイメージでdpkgを使うときに外部から注入する選択値 +└── src -- ソースコードフォルダ + ├── logging + │ └── get_logger.py -- ロガー + ├── restore_backup.py -- dump復元処理本体 + └── system_var + ├── constants.py -- 定数ファイル + └── environment.py -- 環境変数ファイル +``` diff --git a/ecs/jskult-restore-backup/entrypoint.py b/ecs/jskult-restore-backup/entrypoint.py new file mode 100644 index 00000000..2bacdf0c --- /dev/null +++ b/ecs/jskult-restore-backup/entrypoint.py @@ -0,0 +1,10 @@ +"""実消化&アルトマーク 日次バッチ処理前DBダンプ取得のエントリーポイント""" +from src import restore_backup + +if __name__ == '__main__': + try: + exit(restore_backup.exec()) + except Exception: + # エラーが起きても、正常系のコードで返す。 + # エラーが起きた事実はbatch_process内でログを出す。 + exit(0) diff --git a/ecs/jskult-restore-backup/mysql_dpkg_selection.txt b/ecs/jskult-restore-backup/mysql_dpkg_selection.txt new file mode 100644 index 00000000..d3cb46f9 --- /dev/null +++ b/ecs/jskult-restore-backup/mysql_dpkg_selection.txt @@ -0,0 +1,3 @@ +1 +1 +4 \ No newline at end of file diff --git a/ecs/jskult-restore-backup/src/__init__.py b/ecs/jskult-restore-backup/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-restore-backup/src/logging/get_logger.py b/ecs/jskult-restore-backup/src/logging/get_logger.py new file mode 100644 index 00000000..f36f1199 --- /dev/null +++ b/ecs/jskult-restore-backup/src/logging/get_logger.py @@ -0,0 +1,37 @@ +import logging + +from src.system_var.environment import LOG_LEVEL + +# boto3関連モジュールのログレベルを事前に個別指定し、モジュール内のDEBUGログの表示を抑止する +for name in ["boto3", "botocore", "s3transfer", "urllib3"]: + logging.getLogger(name).setLevel(logging.WARNING) + + +def get_logger(log_name: str) -> logging.Logger: + """一意のログ出力モジュールを取得します。 + + Args: + log_name (str): ロガー名 + + Returns: + _type_: _description_ + """ + logger = logging.getLogger(log_name) + level = logging.getLevelName(LOG_LEVEL) + if not isinstance(level, int): + level = logging.INFO + logger.setLevel(level) + + if not logger.hasHandlers(): + handler = logging.StreamHandler() + logger.addHandler(handler) + + formatter = logging.Formatter( + '%(name)s\t[%(levelname)s]\t%(asctime)s\t%(message)s', + '%Y-%m-%d %H:%M:%S' + ) + + for handler in logger.handlers: + handler.setFormatter(formatter) + + return logger diff --git a/ecs/jskult-restore-backup/src/restore_backup.py b/ecs/jskult-restore-backup/src/restore_backup.py new file mode 100644 index 00000000..da481eaf --- /dev/null +++ b/ecs/jskult-restore-backup/src/restore_backup.py @@ -0,0 +1,74 @@ +"""実消化＆アルトマークダンプ復元スクリプト""" + +import os +import re +import subprocess +import textwrap + +from src.logging.get_logger import get_logger +from src.system_var import constants, environment + +logger = get_logger('実消化＆アルトマークダンプ復元スクリプト') + + +def exec(): + try: + logger.info('実消化＆アルトマークダンプ復元スクリプト：開始') + + # MySQL接続情報を作成する + my_cnf_file_content = f""" + [client] + user={environment.DB_USERNAME} + password={environment.DB_PASSWORD} + host={environment.DB_HOST} + """ + # my.cnfファイルのパス + my_cnf_path = os.path.join('my.cnf') + + # my.cnfファイルを生成する + with open(my_cnf_path, 'w') as f: + f.write(textwrap.dedent(my_cnf_file_content)[1:-1]) + + # 復元対象のダンプファイルを特定 + s3_file_path = environment.JSKULT_BACKUP_DUMP_S3_FILE_PATH + + # S3 URIになっているか確認 + s3_file_path_pattern = r"s3://([\w-]+)/dump/(\d{4})/(\d{2})/(\d{2})/backup_rds_(\d+).gz" + if re.match(s3_file_path_pattern, s3_file_path) is None: + logger.warning('環境変数[JSKULT_BACKUP_DUMP_S3_FILE_PATH]に指定された、S3 URIの形式が不正です。') + return constants.BATCH_EXIT_CODE_SUCCESS + + # aws s3 cpコマンドを実行してdumpファイルを標準出力に取得する + s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', s3_file_path, '-'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # gunzipコマンドを実行してdumpファイルを解凍する + gunzip_process = subprocess.Popen(['gzip', '-c'], stdin=s3_cp_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # mysqldumpコマンドを実行し、dumpを取得する + mysql_process = subprocess.Popen( + ['mysql', f'--defaults-file={my_cnf_path}', '-P', f"{environment.DB_PORT}", environment.DB_SCHEMA], + stdin=gunzip_process.stdout, stderr=subprocess.PIPE + ) + # aws s3 cpの標準出力をgunzipに接続したため、標準出力をクローズする + s3_cp_process.stdout.close() + # gunzipの標準出力をaws mysqlに接続したため、標準出力をクローズする + gunzip_process.stdout.close() + + # パイプラインを実行し、エラーハンドリング + _, error = s3_cp_process.communicate() + if s3_cp_process.returncode != 0: + logger.error(f'`aws s3 cp`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS + _, error = gunzip_process.communicate() + if gunzip_process.returncode != 0: + logger.error(f'`gunzip`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS + _, error = mysql_process.communicate() + if mysql_process.returncode != 0: + logger.error(f'`mysql`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + return constants.BATCH_EXIT_CODE_SUCCESS + + logger.info('[NOTICE]実消化＆アルトマークダンプ復元スクリプト：終了（正常終了）') + return constants.BATCH_EXIT_CODE_SUCCESS + + except Exception as e: + logger.exception(f'実消化＆アルトマークダンプ復元スクリプト中に想定外のエラーが発生しました :{e}') + return constants.BATCH_EXIT_CODE_SUCCESS diff --git a/ecs/jskult-restore-backup/src/system_var/__init__.py b/ecs/jskult-restore-backup/src/system_var/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/ecs/jskult-restore-backup/src/system_var/constants.py b/ecs/jskult-restore-backup/src/system_var/constants.py new file mode 100644 index 00000000..8a555af3 --- /dev/null +++ b/ecs/jskult-restore-backup/src/system_var/constants.py @@ -0,0 +1,2 @@ +# バッチ正常終了コード +BATCH_EXIT_CODE_SUCCESS = 0 diff --git a/ecs/jskult-restore-backup/src/system_var/environment.py b/ecs/jskult-restore-backup/src/system_var/environment.py new file mode 100644 index 00000000..4ab083e0 --- /dev/null +++ b/ecs/jskult-restore-backup/src/system_var/environment.py @@ -0,0 +1,14 @@ +import os + +# Database +DB_HOST = os.environ['DB_HOST'] +DB_PORT = int(os.environ['DB_PORT']) +DB_USERNAME = os.environ['DB_USERNAME'] +DB_PASSWORD = os.environ['DB_PASSWORD'] +DB_SCHEMA = os.environ['DB_SCHEMA'] + +# AWS +JSKULT_BACKUP_DUMP_S3_FILE_PATH = os.environ['JSKULT_BACKUP_DUMP_FILE_PATH'] + +# 初期値がある環境変数 +LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') From c16d00c3b2c1148f8182a74a2cbb52ec50a13c51 Mon Sep 17 00:00:00 2001 From: "x.azuma.m@nds-tyo.co.jp" Date: Tue, 18 Jul 2023 13:52:58 +0900 Subject: [PATCH 26/30] =?UTF-8?q?PR=E6=8C=87=E6=91=98=E5=8F=8D=E6=98=A0?= =?UTF-8?q?=E3=80=80=E8=AA=AD=E3=81=BF=E3=82=84=E3=81=99=E3=81=8F=E3=81=99?= =?UTF-8?q?=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/batch/vjsk/vjsk_data_load_manager.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py index af23490b..6006c030 100644 --- a/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py +++ b/ecs/jskult-batch-daily/src/batch/vjsk/vjsk_data_load_manager.py @@ -147,11 +147,13 @@ class VjskDataLoadManager: tsv_tabs = self._get_tsv_last_row_tab_count(local_file_name) expect_tabs = mapper.get_file_column_separators(target["condkey"]) if tsv_tabs != expect_tabs: - msg = "受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした" - msg += f" local_file_name: {local_file_name}" - msg += f" 末尾行のtab数: {tsv_tabs}" - msg += f" tab想定数: {expect_tabs}" - raise BatchOperationException(msg) + msg = [ + "受領tsvファイルの末尾行のTABの数が総定数と一致しませんでした", + f"local_file_name: {local_file_name}", + f"末尾行のtab数: {tsv_tabs}", + f"tab想定数: {expect_tabs}" + ] + raise BatchOperationException(' '.join(msg)) # データベース登録 self._import_to_db(local_file_name, target["condkey"]) From 3b02642144c8ccaa45021d758255400970ccd4e5 Mon Sep 17 00:00:00 2001 From: "x.azuma.m@nds-tyo.co.jp" Date: Tue, 18 Jul 2023 13:56:45 +0900 Subject: [PATCH 27/30] =?UTF-8?q?PR=E6=8C=87=E6=91=98=E5=8F=8D=E6=98=A0?= =?UTF-8?q?=E3=80=80=E3=83=86=E3=82=B9=E3=83=88=E3=83=A1=E3=82=BD=E3=83=83?= =?UTF-8?q?=E3=83=89=E8=AA=AC=E6=98=8E=E3=81=AE=E8=AA=A4=E8=A8=98=E8=A8=82?= =?UTF-8?q?=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/vjsk/vjsk_load/test_vjsk_load.py | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py index d2fb07ac..6b45f96e 100644 --- a/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py +++ b/ecs/jskult-batch-daily/tests/batch/vjsk/vjsk_load/test_vjsk_load.py @@ -1655,9 +1655,9 @@ class TestImportFileToDb: def test_load_no_data_ok_01(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1680,9 +1680,9 @@ class TestImportFileToDb: def test_load_no_data_ok_02(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1705,9 +1705,9 @@ class TestImportFileToDb: def test_load_no_data_ok_03(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1730,9 +1730,9 @@ class TestImportFileToDb: def test_load_no_data_ok_04(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1755,9 +1755,9 @@ class TestImportFileToDb: def test_load_no_data_ok_05(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1780,9 +1780,9 @@ class TestImportFileToDb: def test_load_no_data_ok_06(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1805,9 +1805,9 @@ class TestImportFileToDb: def test_load_no_data_ok_07(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1830,9 +1830,9 @@ class TestImportFileToDb: def test_load_no_data_ok_08(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1855,9 +1855,9 @@ class TestImportFileToDb: def test_load_no_data_ok_09(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1880,9 +1880,9 @@ class TestImportFileToDb: def test_load_no_data_ok_10(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1905,9 +1905,9 @@ class TestImportFileToDb: def test_load_no_data_ok_11(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1930,9 +1930,9 @@ class TestImportFileToDb: def test_load_no_data_ok_12(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1955,9 +1955,9 @@ class TestImportFileToDb: def test_load_no_data_ok_13(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -1980,9 +1980,9 @@ class TestImportFileToDb: def test_load_no_data_ok_14(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup @@ -2005,9 +2005,9 @@ class TestImportFileToDb: def test_load_no_data_ok_15(self, mapper): """ 観点 - 異常系 : tsvファイルが途中で欠落している + 正常系 : tsvファイルレコード0件の取込 期待値 - 例外が発生する + 例外が発生しないこと """ # setup From 7d7e0640570cf3938c3df099f90f5ae3b09332ec Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Jul 2023 15:23:57 +0900 Subject: [PATCH 28/30] =?UTF-8?q?fix:=20=E3=82=B9=E3=83=86=E3=83=BC?= =?UTF-8?q?=E3=82=B8=E3=83=B3=E3=82=B0=E7=92=B0=E5=A2=83=E3=81=A7=E5=AE=9F?= =?UTF-8?q?=E8=A1=8C=E3=81=A7=E3=81=8D=E3=82=8B=E3=82=88=E3=81=86=E3=81=AB?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-restore-backup/.gitignore | 3 +- .../src/restore_backup.py | 39 ++++++++----------- .../src/system_var/environment.py | 2 +- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/ecs/jskult-restore-backup/.gitignore b/ecs/jskult-restore-backup/.gitignore index cf44449e..d9a271c3 100644 --- a/ecs/jskult-restore-backup/.gitignore +++ b/ecs/jskult-restore-backup/.gitignore @@ -8,4 +8,5 @@ __pycache__ # python test .pytest_cache .coverage -.report/ \ No newline at end of file +.report/ +*.gz \ No newline at end of file diff --git a/ecs/jskult-restore-backup/src/restore_backup.py b/ecs/jskult-restore-backup/src/restore_backup.py index da481eaf..439fe4cf 100644 --- a/ecs/jskult-restore-backup/src/restore_backup.py +++ b/ecs/jskult-restore-backup/src/restore_backup.py @@ -33,39 +33,34 @@ def exec(): s3_file_path = environment.JSKULT_BACKUP_DUMP_S3_FILE_PATH # S3 URIになっているか確認 - s3_file_path_pattern = r"s3://([\w-]+)/dump/(\d{4})/(\d{2})/(\d{2})/backup_rds_(\d+).gz" + s3_file_path_pattern = r"s3://(.+)/dump/(\d{4})/(\d{2})/(\d{2})/backup_rds_src05_(\d+).gz" if re.match(s3_file_path_pattern, s3_file_path) is None: logger.warning('環境変数[JSKULT_BACKUP_DUMP_S3_FILE_PATH]に指定された、S3 URIの形式が不正です。') return constants.BATCH_EXIT_CODE_SUCCESS - # aws s3 cpコマンドを実行してdumpファイルを標準出力に取得する - s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', s3_file_path, '-'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # gunzipコマンドを実行してdumpファイルを解凍する - gunzip_process = subprocess.Popen(['gzip', '-c'], stdin=s3_cp_process.stdout, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - # mysqldumpコマンドを実行し、dumpを取得する - mysql_process = subprocess.Popen( - ['mysql', f'--defaults-file={my_cnf_path}', '-P', f"{environment.DB_PORT}", environment.DB_SCHEMA], - stdin=gunzip_process.stdout, stderr=subprocess.PIPE - ) - # aws s3 cpの標準出力をgunzipに接続したため、標準出力をクローズする - s3_cp_process.stdout.close() - # gunzipの標準出力をaws mysqlに接続したため、標準出力をクローズする - gunzip_process.stdout.close() - - # パイプラインを実行し、エラーハンドリング + # aws s3 cpコマンドを実行してdumpファイルをローカルにダウンロードする + s3_cp_process = subprocess.Popen(['aws', 's3', 'cp', s3_file_path, './dump.gz'], stderr=subprocess.PIPE) _, error = s3_cp_process.communicate() if s3_cp_process.returncode != 0: logger.error(f'`aws s3 cp`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') return constants.BATCH_EXIT_CODE_SUCCESS - _, error = gunzip_process.communicate() - if gunzip_process.returncode != 0: - logger.error(f'`gunzip`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') - return constants.BATCH_EXIT_CODE_SUCCESS + # S3コマンドの標準エラーはクローズしておく + s3_cp_process.stderr.close() + + # gzipコマンドを実行してdumpファイルを解凍する + gzip_process = subprocess.Popen(['gunzip', '-c', './dump.gz'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # mysqlコマンドを実行し、dumpを復元する + mysql_process = subprocess.Popen( + ['mysql', f'--defaults-file={my_cnf_path}', '-P', f"{environment.DB_PORT}", environment.DB_SCHEMA], + stdin=gzip_process.stdout, stderr=subprocess.PIPE + ) + # gzipの標準出力をmysqlに接続したため、標準出力をクローズする + gzip_process.stdout.close() + _, error = mysql_process.communicate() if mysql_process.returncode != 0: - logger.error(f'`mysql`実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') + logger.error(f'コマンド実行時にエラーが発生しました。{"" if error is None else error.decode("utf-8")}') return constants.BATCH_EXIT_CODE_SUCCESS - logger.info('[NOTICE]実消化＆アルトマークダンプ復元スクリプト：終了（正常終了）') return constants.BATCH_EXIT_CODE_SUCCESS diff --git a/ecs/jskult-restore-backup/src/system_var/environment.py b/ecs/jskult-restore-backup/src/system_var/environment.py index 4ab083e0..ee0e9be3 100644 --- a/ecs/jskult-restore-backup/src/system_var/environment.py +++ b/ecs/jskult-restore-backup/src/system_var/environment.py @@ -8,7 +8,7 @@ DB_PASSWORD = os.environ['DB_PASSWORD'] DB_SCHEMA = os.environ['DB_SCHEMA'] # AWS -JSKULT_BACKUP_DUMP_S3_FILE_PATH = os.environ['JSKULT_BACKUP_DUMP_FILE_PATH'] +JSKULT_BACKUP_DUMP_S3_FILE_PATH = os.environ['JSKULT_BACKUP_DUMP_S3_FILE_PATH'] # 初期値がある環境変数 LOG_LEVEL = os.environ.get('LOG_LEVEL', 'INFO') From 5c310433ac7dfbf9ca4d26c1107d911e4b137029 Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Tue, 18 Jul 2023 16:27:57 +0900 Subject: [PATCH 29/30] =?UTF-8?q?feat:=20=E8=A8=AD=E5=AE=9A=E3=83=95?= =?UTF-8?q?=E3=82=A1=E3=82=A4=E3=83=AB=E5=91=A8=E3=82=8A=E4=BF=AE=E6=AD=A3?= =?UTF-8?q?=E3=80=82*.gz=E3=82=92=E7=9C=81=E3=81=84=E3=81=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-restore-backup/.dockerignore | 1 + ecs/jskult-restore-backup/.gitignore | 6 +++++- ecs/jskult-restore-backup/README.md | 4 +++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ecs/jskult-restore-backup/.dockerignore b/ecs/jskult-restore-backup/.dockerignore index 8b9da402..d826198a 100644 --- a/ecs/jskult-restore-backup/.dockerignore +++ b/ecs/jskult-restore-backup/.dockerignore @@ -10,3 +10,4 @@ Dockerfile pytest.ini README.md *.sql +*.gz diff --git a/ecs/jskult-restore-backup/.gitignore b/ecs/jskult-restore-backup/.gitignore index d9a271c3..339d80cc 100644 --- a/ecs/jskult-restore-backup/.gitignore +++ b/ecs/jskult-restore-backup/.gitignore @@ -1,6 +1,5 @@ .vscode/settings.json .env -my.cnf # python __pycache__ @@ -9,4 +8,9 @@ __pycache__ .pytest_cache .coverage .report/ + +# mysql config file +my.cnf + +# compress file *.gz \ No newline at end of file diff --git a/ecs/jskult-restore-backup/README.md b/ecs/jskult-restore-backup/README.md index f955ff97..e9dfa8f7 100644 --- a/ecs/jskult-restore-backup/README.md +++ b/ecs/jskult-restore-backup/README.md @@ -48,7 +48,9 @@ ```txt . -├── .env.example -- 環境変数のサンプル値 +├── .env.example -- ローカル実行用の環境変数のサンプル値。 +├── .dockerignore -- docker build時のコンテキストに含めるファイルの抑制リスト +├── .gitignore -- Git差分管理除外リスト ├── Dockerfile -- Dockerイメージ作成用 ├── Pipfile -- pythonの依存関係管理 ├── Pipfile.lock -- 依存関係バージョン固定 From 0679ad944c946dca0d6d0ccf6164d1f5927bf3de Mon Sep 17 00:00:00 2001 From: "shimoda.m@nds-tyo.co.jp" Date: Wed, 19 Jul 2023 15:13:44 +0900 Subject: [PATCH 30/30] =?UTF-8?q?style:=20=E3=82=B3=E3=83=A1=E3=83=B3?= =?UTF-8?q?=E3=83=88=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecs/jskult-restore-backup/entrypoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecs/jskult-restore-backup/entrypoint.py b/ecs/jskult-restore-backup/entrypoint.py index 2bacdf0c..70f48060 100644 --- a/ecs/jskult-restore-backup/entrypoint.py +++ b/ecs/jskult-restore-backup/entrypoint.py @@ -1,4 +1,4 @@ -"""実消化&アルトマーク 日次バッチ処理前DBダンプ取得のエントリーポイント""" +"""実消化&アルトマーク DBダンプ復元のエントリーポイント""" from src import restore_backup if __name__ == '__main__':