diff --git a/data_migration_tools/server/package-lock.json b/data_migration_tools/server/package-lock.json index c1c2143..1a8aff6 100644 --- a/data_migration_tools/server/package-lock.json +++ b/data_migration_tools/server/package-lock.json @@ -26,6 +26,7 @@ "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cookie-parser": "^1.4.6", + "csv": "^6.3.6", "multer": "^1.4.5-lts.1", "mysql2": "^2.3.3", "reflect-metadata": "^0.1.13", @@ -4049,6 +4050,35 @@ "node": ">= 8" } }, + "node_modules/csv": { + "version": "6.3.6", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.6.tgz", + "integrity": "sha512-jsEsX2HhGp7xiwrJu5srQavKsh+HUJcCi78Ar3m4jlmFKRoTkkMy7ZZPP+LnQChmaztW+uj44oyfMb59daAs/Q==", + "dependencies": { + "csv-generate": "^4.3.1", + "csv-parse": "^5.5.3", + "csv-stringify": "^6.4.5", + "stream-transform": "^3.3.0" + }, + "engines": { + "node": ">= 0.1.90" + } + }, + "node_modules/csv-generate": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.3.1.tgz", + "integrity": "sha512-7YeeJq+44/I/O5N2sr2qBMcHZXhpfe38eh7DOFxyMtYO+Pir7kIfgFkW5MPksqKqqR6+/wX7UGoZm1Ot11151w==" + }, + "node_modules/csv-parse": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz", + "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A==" + }, + "node_modules/csv-stringify": { + "version": "6.4.5", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.5.tgz", + "integrity": "sha512-SPu1Vnh8U5EnzpNOi1NDBL5jU5Rx7DVHr15DNg9LXDTAbQlAVAmEbVt16wZvEW9Fu9Qt4Ji8kmeCJ2B1+4rFTQ==" + }, "node_modules/date-fns": { "version": "2.30.0", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", @@ -8654,6 +8684,11 @@ "npm": ">=6" } }, + "node_modules/stream-transform": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.0.tgz", + "integrity": "sha512-pG1NeDdmErNYKtvTpFayrEueAmL0xVU5wd22V7InGnatl4Ocq3HY7dcXIKj629kXvYQvglCC7CeDIGAlx1RNGA==" + }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -13027,6 +13062,32 @@ "which": "^2.0.1" } }, + "csv": { + "version": "6.3.6", + "resolved": "https://registry.npmjs.org/csv/-/csv-6.3.6.tgz", + "integrity": "sha512-jsEsX2HhGp7xiwrJu5srQavKsh+HUJcCi78Ar3m4jlmFKRoTkkMy7ZZPP+LnQChmaztW+uj44oyfMb59daAs/Q==", + "requires": { + "csv-generate": "^4.3.1", + "csv-parse": "^5.5.3", + "csv-stringify": "^6.4.5", + "stream-transform": "^3.3.0" + } + }, + "csv-generate": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/csv-generate/-/csv-generate-4.3.1.tgz", + "integrity": "sha512-7YeeJq+44/I/O5N2sr2qBMcHZXhpfe38eh7DOFxyMtYO+Pir7kIfgFkW5MPksqKqqR6+/wX7UGoZm1Ot11151w==" + }, + "csv-parse": { + "version": "5.5.3", + "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-5.5.3.tgz", + "integrity": "sha512-v0KW6C0qlZzoGjk6u5tLmVfyZxNgPGXZsWTXshpAgKVGmGXzaVWGdlCFxNx5iuzcXT/oJN1HHM9DZKwtAtYa+A==" + }, + "csv-stringify": { + "version": "6.4.5", + "resolved": "https://registry.npmjs.org/csv-stringify/-/csv-stringify-6.4.5.tgz", + "integrity": "sha512-SPu1Vnh8U5EnzpNOi1NDBL5jU5Rx7DVHr15DNg9LXDTAbQlAVAmEbVt16wZvEW9Fu9Qt4Ji8kmeCJ2B1+4rFTQ==" + }, "date-fns": { "version": "2.30.0", "resolved": "https://registry.npmjs.org/date-fns/-/date-fns-2.30.0.tgz", @@ -16557,6 +16618,11 @@ "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz", "integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==" }, + "stream-transform": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/stream-transform/-/stream-transform-3.3.0.tgz", + "integrity": "sha512-pG1NeDdmErNYKtvTpFayrEueAmL0xVU5wd22V7InGnatl4Ocq3HY7dcXIKj629kXvYQvglCC7CeDIGAlx1RNGA==" + }, "streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", diff --git a/data_migration_tools/server/package.json b/data_migration_tools/server/package.json index a2299cb..a0502b7 100644 --- a/data_migration_tools/server/package.json +++ b/data_migration_tools/server/package.json @@ -45,7 +45,8 @@ "reflect-metadata": "^0.1.13", "rxjs": "^7.8.0", "swagger-cli": "^4.0.4", - "typeorm": "^0.3.20" + "typeorm": "^0.3.20", + "csv": "^6.3.6" }, "devDependencies": { "@types/express": "^4.17.17", diff --git a/data_migration_tools/server/src/app.module.ts b/data_migration_tools/server/src/app.module.ts index 6935bba..bcf32fe 100644 --- a/data_migration_tools/server/src/app.module.ts +++ b/data_migration_tools/server/src/app.module.ts @@ -27,6 +27,10 @@ import { DeleteService } from "./features/delete/delete.service"; import { TransferModule } from "./features/transfer/transfer.module"; import { TransferController } from "./features/transfer/transfer.controller"; import { TransferService } from "./features/transfer/transfer.service"; +import { VerificationController } from "./features/verification/verification.controller"; +import { VerificationService } from "./features/verification/verification.service"; +import { VerificationModule } from "./features/verification/verification.module"; + @Module({ imports: [ ServeStaticModule.forRoot({ @@ -41,6 +45,7 @@ import { TransferService } from "./features/transfer/transfer.service"; UsersModule, TransferModule, RegisterModule, + VerificationModule, AccountsRepositoryModule, UsersRepositoryModule, SortCriteriaRepositoryModule, @@ -70,6 +75,7 @@ import { TransferService } from "./features/transfer/transfer.service"; UsersController, DeleteController, TransferController, + VerificationController, ], providers: [ RegisterService, @@ -77,6 +83,7 @@ import { TransferService } from "./features/transfer/transfer.service"; UsersService, DeleteService, TransferService, + VerificationService, ], }) export class AppModule { diff --git a/data_migration_tools/server/src/common/types/types.ts b/data_migration_tools/server/src/common/types/types.ts index 69ef466..529a4ad 100644 --- a/data_migration_tools/server/src/common/types/types.ts +++ b/data_migration_tools/server/src/common/types/types.ts @@ -34,6 +34,11 @@ export class csvInputFile { wt19: string; wt20: string; } + +export class csvInputFileWithRow extends csvInputFile { + row: number; +} + export class AccountsFileType { accountId: number; type: string; @@ -93,6 +98,22 @@ export class CardLicensesFile { updated_by?: string; } +export class AccountsMappingFile { + accountIdText: string; + accountIdNumber: number; +} + +export class VerificationResultDetails { + input: string; + inputRow: number; + diffTargetTable: string; + columnName: string; + fileData: string; + databaseData: string; + reason: string; +} + + export function isAccountsFileArray(obj: any): obj is AccountsFile[] { return Array.isArray(obj) && obj.every((item) => isAccountsFile(item)); } @@ -202,3 +223,65 @@ export function isCardLicensesFile(obj: any): obj is CardLicensesFile { (obj.updated_by === null || typeof obj.updated_by === "string") ); } + +export function isAccountsMappingFileArray( + obj: any +): obj is AccountsMappingFile[] { + return Array.isArray(obj) && obj.every((item) => isAccountsMappingFile(item)); +} +export function isAccountsMappingFile(obj: any): obj is AccountsMappingFile { + return ( + typeof obj === "object" && + obj !== null && + "accountIdText" in obj && + "accountIdNumber" in obj && + typeof obj.accountIdText === "string" && + typeof obj.accountIdNumber === "number" + ); +} + +export function isCsvInputFileForValidateArray(obj: any): obj is csvInputFile[] { + return ( + Array.isArray(obj) && obj.every((item) => isCsvInputFileForValidate(item)) + ); +} + +export function isCsvInputFileForValidate(obj: any): obj is csvInputFile { + return ( + typeof obj === "object" && + "type" in obj && + "account_id" in obj && + "parent_id" in obj && + "email" in obj && + "company_name" in obj && + "first_name" in obj && + "last_name" in obj && + "country" in obj && + "state" in obj && + "start_date" in obj && + "expired_date" in obj && + "user_email" in obj && + "author_id" in obj && + "recording_mode" in obj && + "wt1" in obj && + "wt2" in obj && + "wt3" in obj && + "wt4" in obj && + "wt5" in obj && + "wt6" in obj && + "wt7" in obj && + "wt8" in obj && + "wt9" in obj && + "wt10" in obj && + "wt11" in obj && + "wt12" in obj && + "wt13" in obj && + "wt14" in obj && + "wt15" in obj && + "wt16" in obj && + "wt17" in obj && + "wt18" in obj && + "wt19" in obj && + "wt20" in obj + ); +} diff --git a/data_migration_tools/server/src/features/transfer/transfer.controller.ts b/data_migration_tools/server/src/features/transfer/transfer.controller.ts index 7916f27..ee38969 100644 --- a/data_migration_tools/server/src/features/transfer/transfer.controller.ts +++ b/data_migration_tools/server/src/features/transfer/transfer.controller.ts @@ -13,7 +13,7 @@ import { Request } from "express"; import { transferRequest, transferResponse } from "./types/types"; import { TransferService } from "./transfer.service"; import { makeContext } from "../../common/log"; -import { csvInputFile } from "../../common/types/types"; +import { csvInputFile, AccountsMappingFile } from "../../common/types/types"; import { makeErrorResponse } from "src/common/errors/makeErrorResponse"; import { AUTO_INCREMENT_START } from "../../constants"; @ApiTags("transfer") @@ -149,6 +149,21 @@ export class TransferController { accountIdListArray.forEach((accountId, index) => { accountIdMap.set(accountId, index + AUTO_INCREMENT_START); }); + + // アカウントID numberとstring対応表の出力 + const accountsMappingFiles: AccountsMappingFile[] = []; + accountIdMap.forEach((value, key) => { + const accountsMappingFile = new AccountsMappingFile(); + accountsMappingFile.accountIdNumber = value; + accountsMappingFile.accountIdText = key + accountsMappingFiles.push(accountsMappingFile) + }); + + fs.writeFileSync( + `${inputFilePath}account_map.json`, + JSON.stringify(accountsMappingFiles) + ); + // CSVファイルの変換 const transferResponseCsv = await this.transferService.transferInputData( context, diff --git a/data_migration_tools/server/src/features/verification/types/types.ts b/data_migration_tools/server/src/features/verification/types/types.ts new file mode 100644 index 0000000..2eda873 --- /dev/null +++ b/data_migration_tools/server/src/features/verification/types/types.ts @@ -0,0 +1,9 @@ +import { ApiProperty } from '@nestjs/swagger'; + +export class VerificationRequest { + @ApiProperty() + inputFilePath: string; +} + +export class VerificationResponse {} + diff --git a/data_migration_tools/server/src/features/verification/verification.controller.ts b/data_migration_tools/server/src/features/verification/verification.controller.ts new file mode 100644 index 0000000..31cde89 --- /dev/null +++ b/data_migration_tools/server/src/features/verification/verification.controller.ts @@ -0,0 +1,148 @@ +import { + Body, + Controller, + HttpStatus, + Post, + Req, + Logger, + HttpException, +} from "@nestjs/common"; +import { makeErrorResponse } from "../../common/error/makeErrorResponse"; +import fs from "fs"; +import { ApiOperation, ApiResponse, ApiTags } from "@nestjs/swagger"; +import { Request } from "express"; +import { VerificationRequest, VerificationResponse } from "./types/types"; +import { VerificationService } from "./verification.service"; +import { makeContext } from "../../common/log"; +import { + csvInputFileWithRow, + isAccountsMappingFileArray, + isCardLicensesFileArray, + isCsvInputFileForValidateArray, +} from "../../common/types/types"; +import * as csv from "csv"; + +@ApiTags("verification") +@Controller("verification") +export class VerificationController { + private readonly logger = new Logger(VerificationController.name); + constructor(private readonly verificationService: VerificationService) {} + + @Post() + @ApiResponse({ + status: HttpStatus.OK, + type: VerificationResponse, + description: "成功時のレスポンス", + }) + @ApiResponse({ + status: HttpStatus.INTERNAL_SERVER_ERROR, + description: "想定外のサーバーエラー", + }) + @ApiOperation({ operationId: "dataVerification" }) + async dataVerification( + @Body() body: VerificationRequest, + @Req() req: Request + ): Promise { + const context = makeContext("iko", "varification"); + + const inputFilePath = body.inputFilePath; + + this.logger.log( + `[IN] [${context.getTrackingId()}] ${ + this.dataVerification.name + } | params: { inputFilePath: ${inputFilePath}};` + ); + + try { + // 読み込みファイルのフルパス + const accountTransitionFileFullPath = + inputFilePath + "Account_transition.csv"; + const accountMapFileFullPath = inputFilePath + "account_map.json"; + const cardLicensesFileFullPath = inputFilePath + "cardLicenses.json"; + + // ファイル存在チェックと読み込み + if (!fs.existsSync(accountTransitionFileFullPath)) { + this.logger.error( + `file not exists from ${accountTransitionFileFullPath}` + ); + throw new Error( + `file not exists from ${accountTransitionFileFullPath}` + ); + } + + if (!fs.existsSync(accountMapFileFullPath)) { + this.logger.error(`file not exists from ${accountMapFileFullPath}`); + throw new Error(`file not exists from ${accountMapFileFullPath}`); + } + + if (!fs.existsSync(cardLicensesFileFullPath)) { + this.logger.error(`file not exists from ${cardLicensesFileFullPath}`); + throw new Error(`file not exists from ${cardLicensesFileFullPath}`); + } + + // カードライセンスの登録用ファイル読み込み + const cardLicensesObject = JSON.parse( + fs.readFileSync(cardLicensesFileFullPath, "utf8") + ); + + // 型ガード（cardLicenses） + if (!isCardLicensesFileArray(cardLicensesObject)) { + throw new Error("input file is not cardLicensesInputFiles"); + } + + // アカウントIDマッピング用ファイル読み込み + const accountsMapObject = JSON.parse( + fs.readFileSync(accountMapFileFullPath, "utf8") + ); + + // 型ガード（accountsMapingFile） + if (!isAccountsMappingFileArray(accountsMapObject)) { + throw new Error("input file is not accountsMapingFile"); + } + + // 移行用csvファイルの読み込み（csv parse） + fs.createReadStream(accountTransitionFileFullPath).pipe( + csv.parse({ columns: true, delimiter: "," }, (err, csvInputFiles) => { + // 型ガード（csvInputFile） + if (!isCsvInputFileForValidateArray(csvInputFiles)) { + throw new Error("input file is not csvInputFile"); + } + + const csvInputFileswithRows: csvInputFileWithRow[] = []; + let rowCount = 2; // csvの何行目かを表す変数。ヘッダ行があるので2から開始 + for (const csvInputFile of csvInputFiles) { + const csvInputFileswithRow: csvInputFileWithRow = { + ...csvInputFile, + row: rowCount + }; + csvInputFileswithRows.push(csvInputFileswithRow); + rowCount = rowCount + 1; + } + this.verificationService.varificationData( + context, + inputFilePath, + csvInputFileswithRows, + accountsMapObject, + cardLicensesObject + ); + }) + ); + + return {}; + } catch (e) { + this.logger.error(`[${context.getTrackingId()}] error=${e}`); + throw new HttpException( + makeErrorResponse("E009999"), + HttpStatus.INTERNAL_SERVER_ERROR + ); + } finally { + this.logger.log( + `[OUT] [${context.getTrackingId()}] ${this.dataVerification.name}` + ); + } + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/data_migration_tools/server/src/features/verification/verification.module.ts b/data_migration_tools/server/src/features/verification/verification.module.ts new file mode 100644 index 0000000..51b9892 --- /dev/null +++ b/data_migration_tools/server/src/features/verification/verification.module.ts @@ -0,0 +1,17 @@ +import { Module } from "@nestjs/common"; +import { VerificationController } from "./verification.controller"; +import { VerificationService } from "./verification.service"; +import { LicensesRepositoryModule } from "../../repositories/licenses/licenses.repository.module"; +import { AccountsRepositoryModule } from "../../repositories/accounts/accounts.repository.module"; +import { UsersRepositoryModule } from "../../repositories//users/users.repository.module"; + +@Module({ + imports: [ + LicensesRepositoryModule, + AccountsRepositoryModule, + UsersRepositoryModule, + ], + controllers: [VerificationController], + providers: [VerificationService], +}) +export class VerificationModule {} diff --git a/data_migration_tools/server/src/features/verification/verification.service.ts b/data_migration_tools/server/src/features/verification/verification.service.ts new file mode 100644 index 0000000..1de9596 --- /dev/null +++ b/data_migration_tools/server/src/features/verification/verification.service.ts @@ -0,0 +1,695 @@ +import { HttpException, HttpStatus, Injectable, Logger } from "@nestjs/common"; +import { Context } from "../../common/log"; +import { + AccountsMappingFile, + CardLicensesFile, + csvInputFileWithRow, + VerificationResultDetails, +} from "../../common/types/types"; +import { + AUTO_INCREMENT_START, + MIGRATION_TYPE, + COUNTRY_LIST, +} from "../../constants/index"; + +import { makeErrorResponse } from "../../common/error/makeErrorResponse"; +import { LicensesRepositoryService } from "../../repositories/licenses/licenses.repository.service"; +import { + License, + CardLicense, +} from "../../repositories/licenses/entity/license.entity"; +import { AccountsRepositoryService } from "../../repositories/accounts/accounts.repository.service"; +import { UsersRepositoryService } from "../../repositories//users/users.repository.service"; +import { Account } from "src/repositories/accounts/entity/account.entity"; +import fs from "fs"; + +@Injectable() +export class VerificationService { + constructor( + private readonly AccountsRepository: AccountsRepositoryService, + private readonly UsersRepository: UsersRepositoryService, + private readonly licensesRepository: LicensesRepositoryService + ) {} + private readonly logger = new Logger(VerificationService.name); + + /** + * Verification Data + * @param inputFilePath: string + */ + async varificationData( + context: Context, + inputFilePath: string, + csvInputFiles: csvInputFileWithRow[], + accountsMappingInputFiles: AccountsMappingFile[], + cardlicensesInputFiles: CardLicensesFile[] + ): Promise { + // パラメータ内容が長大なのでログには出さない + this.logger.log( + `[IN] [${context.getTrackingId()}] ${this.varificationData.name}` + ); + + // this.logger.log(csvInputFiles); + try { + // 件数情報の取得 + this.logger.log(`入力ファイルから件数情報を取得する`); + + const accountCountFromFile = csvInputFiles.filter( + (item) => item.type !== "USER" && item.type !== "Country" + ).length; + const cardLicensesCountFromFile = cardlicensesInputFiles.length; + + const licensesCountFromFile = + csvInputFiles.filter( + (item) => + item.type === "USER" && item.expired_date !== "9999/12/31 23:59:59" + ).length + cardLicensesCountFromFile; + + // 管理ユーザ数のカウント + const administratorCountFromFile = accountCountFromFile; + // 一般ユーザ数のカウント + const normaluserCountFromFile = csvInputFiles.filter( + (item) => item.type === "USER" && item.user_email.length !== 0 + ).length; + + // ユーザ重複数のカウント + let mailAdresses: string[] = []; + csvInputFiles.forEach((item) => { + // メールアドレスの要素を配列に追加（入力データとして管理者とユーザの両方に入ることはない） + if (item.email.length !== 0) { + mailAdresses.push(item.email); + } + if (item.user_email.length !== 0) { + mailAdresses.push(item.user_email); + } + }); + + // 重複する要素を抽出 + const duplicates: { [key: string]: number } = {}; + mailAdresses.forEach((str) => { + duplicates[str] = (duplicates[str] || 0) + 1; + }); + + // 重複する要素と件数を表示 + let duplicateCount = 0; + Object.keys(duplicates).forEach((key) => { + const count = duplicates[key]; + if (count > 1) { + // 重複件数をカウント + duplicateCount = duplicateCount + (count - 1); + //console.log(`${key}が${count}件`); + } + }); + const userCountFromFile = + administratorCountFromFile + normaluserCountFromFile - duplicateCount; + + this.logger.log(`accountCountFromFile=${accountCountFromFile}`); + this.logger.log(`cardLicensesCountFromFile=${cardLicensesCountFromFile}`); + this.logger.log(`licensesCountFromFile=${licensesCountFromFile}`); + this.logger.log(`userCountFromFile=${userCountFromFile}`); + + // DBから情報を取得する + this.logger.log(`DBの情報を取得する`); + + const accounts = await this.AccountsRepository.getAllAccounts(context); + const users = await this.UsersRepository.getAllUsers(context); + const licenses = await this.licensesRepository.getAllLicenses(context); + const cardLicenses = await this.licensesRepository.getAllCardLicense( + context + ); + + // DB件数のカウント + this.logger.log(`DBの情報から件数を取得する`); + const accountsCountFromDB = accounts.length; + const usersCountFromDB = users.length; + const licensesCountFromDB = licenses.length; + const cardLicensesCountFromDB = cardLicenses.length; + + this.logger.log(`accountsCountFromDB=${accountsCountFromDB}`); + this.logger.log(`usersCountFromDB=${usersCountFromDB}`); + this.logger.log(`licensesCountFromDB=${licensesCountFromDB}`); + this.logger.log(`cardLicensesCountFromDB=${cardLicensesCountFromDB}`); + + // エラー情報の定義 + const VerificationResultDetails: VerificationResultDetails[] = []; + + // カードライセンス関連の情報突き合わせ + this.logger.log(`カードライセンス関連の情報突き合わせ`); + const isCardDetailNoError = compareCardLicenses( + VerificationResultDetails, + cardlicensesInputFiles, + cardLicenses, + licenses + ); + + // ライセンス関連の情報突き合わせ + this.logger.log(`ライセンス関連の情報突き合わせ`); + const isLicensesDetailNoError = compareLicenses( + VerificationResultDetails, + csvInputFiles.filter( + (item) => + item.type === "USER" && item.expired_date !== "9999/12/31 23:59:59" + ), + licenses.filter((item) => item.expiry_date !== null), + accountsMappingInputFiles + ); + + // アカウント情報の突き合わせ + this.logger.log(`アカウント関連の情報突き合わせ`); + const isAccountsDetailNoError = compareAccounts( + VerificationResultDetails, + csvInputFiles.filter( + (item) => item.type !== "USER" && item.type !== "Country" + ), + csvInputFiles.filter((item) => item.type === "Country"), + accounts, + accountsMappingInputFiles + ); + + // 結果の判定と出力 + this.logger.log(`結果の判定と出力`); + const isAccountCountNoDifference = + accountCountFromFile === accountsCountFromDB; + const isUsersCountNoDifference = userCountFromFile === usersCountFromDB; + const isLicensesCountNoDifference = + licensesCountFromFile === licensesCountFromDB; + const isCardLicensesCountNoDifference = + cardLicensesCountFromFile === cardLicensesCountFromDB; + const isNoDetailError = VerificationResultDetails.length === 0; + + const isSummaryNoError = + isAccountCountNoDifference && + isUsersCountNoDifference && + isLicensesCountNoDifference && + isCardLicensesCountNoDifference && + isNoDetailError; + + const summaryString = ` +サマリファイル： + +比較結果：${isSummaryNoError ? "OK" : "NG"} + +件数： +アカウント：${ + isAccountCountNoDifference ? "OK" : "NG" + }（csv件数:${accountCountFromFile}/DB件数:${accountsCountFromDB}） +ライセンス：${ + isLicensesCountNoDifference ? "OK" : "NG" + }（csv件数:${licensesCountFromFile}/DB件数:${licensesCountFromDB}） +カードライセンス：${ + isCardLicensesCountNoDifference ? "OK" : "NG" + }（csv件数:${cardLicensesCountFromFile}/DB件数:${cardLicensesCountFromDB}） +ユーザー：${ + isUsersCountNoDifference ? "OK" : "NG" + }（csv件数:${userCountFromFile}/DB件数:${usersCountFromDB}） + +項目比較： +アカウント：${isAccountsDetailNoError ? "OK" : "NG"} +カードライセンス：${isCardDetailNoError ? "OK" : "NG"} +ライセンス：${isLicensesDetailNoError ? "OK" : "NG"} +`; + + // サマリファイルの書き込み + fs.writeFileSync(`${inputFilePath}resultsummary.txt`, summaryString); + + // 詳細ファイルの書き込み + // 配列をJSON文字列に変換 + const jsonContent = JSON.stringify(VerificationResultDetails, null, 2); + + // JSONをファイルに書き込み + fs.writeFileSync(`${inputFilePath}resultdetail.json`, jsonContent); + } catch (e) { + this.logger.error(`[${context.getTrackingId()}] error=${e}`); + throw new HttpException( + makeErrorResponse("E009999"), + HttpStatus.INTERNAL_SERVER_ERROR + ); + } finally { + this.logger.log( + `[OUT] [${context.getTrackingId()}] ${this.varificationData.name}` + ); + } + } +} + +// dateを任意のフォーマットに変換する +const getFormattedDate = (date: Date | null, format: string) => { + if (!date) { + return null; + } + const symbol = { + M: date.getMonth() + 1, + d: date.getDate(), + h: date.getHours(), + m: date.getMinutes(), + s: date.getSeconds(), + }; + + const formatted = format.replace(/(M+|d+|h+|m+|s+)/g, (v) => + ( + (v.length > 1 ? "0" : "") + symbol[v.slice(-1) as keyof typeof symbol] + ).slice(-2) + ); + + return formatted.replace(/(y+)/g, (v) => + date.getFullYear().toString().slice(-v.length) + ); +}; + +// 親の階層がcountryの場合、countryの親を返却する +function transrateCountryHierarchy( + countriesFromFile: csvInputFileWithRow[], + targetParentAccountIdString: string +): string { + for (const countryFromFile of countriesFromFile) { + if (countryFromFile.account_id === targetParentAccountIdString) { + return countryFromFile.parent_id; + } + } + return targetParentAccountIdString; +} + +// アカウントID（number）を対応するアカウントID（string）に変換する +function findAccountIdText( + accountsMappings: AccountsMappingFile[], + targetAccountIdNumber: number +): string { + if (targetAccountIdNumber == null) { + return ""; + } + for (const accountsMapping of accountsMappings) { + if (accountsMapping.accountIdNumber === targetAccountIdNumber) { + return accountsMapping.accountIdText; + } + } + return `NO_MATCHED_ACCOUNTID_${targetAccountIdNumber}`; // マッチするものが見つからない場合 +} + +// 階層（number）を対応する階層（string）に変換する +function getMigrationTypeByNumber(numberValue: number): string { + switch (numberValue) { + case 1: + return MIGRATION_TYPE.ADMINISTRATOR; + case 2: + return MIGRATION_TYPE.BC; + case 3: + return MIGRATION_TYPE.DISTRIBUTOR; + case 4: + return MIGRATION_TYPE.DEALER; + case 5: + return MIGRATION_TYPE.CUSTOMER; + default: + return `NO_MATCHED_TIER_${numberValue}`; + } +} + +// 国（省略版）を対応する国（非省略版）に変換する +function getCountryLabelByValue(value: string): string { + const country = COUNTRY_LIST.find((country) => country.value === value); + return country ? country.label : `NO_MATCHED_COUNTRY_${value}`; +} + +// カードライセンス情報の突き合わせを行い、エラー時はエラー情報配列に情報追加する +function compareCardLicenses( + VerificationResultDetails: VerificationResultDetails[], + cardlicensesInputFiles: CardLicensesFile[], + cardLicenses: CardLicense[], + licenses: License[] +): boolean { + let isNoError = true; + + let row = 1; // カードライセンスファイルの行数 + for (const cardlicensesInputFile of cardlicensesInputFiles) { + const filterdCardLicenses = cardLicenses.filter( + (cardLicenses) => + cardLicenses.card_license_key === cardlicensesInputFile.card_license_key + ); + + if (filterdCardLicenses.length === 0) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "cardLicenses", + columnName: "card_license_key", + fileData: cardlicensesInputFile.card_license_key, + databaseData: "-", + reason: "レコード無し", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + + /* issue_idは自動採番のため比較しない + if (cardlicensesInputFile.issue_id !== filterdCardLicenses[0].issue_id) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + diffTargetTable: "cardLicenses", + columnName: "issue_id", + fileData: cardlicensesInputFile.issue_id.toString(), + databaseData: filterdCardLicenses[0].issue_id.toString(), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + */ + + const formattedActivated = getFormattedDate( + filterdCardLicenses[0].activated_at, + `yyyy/MM/dd hh:mm:ss` + ); + if (cardlicensesInputFile.activated_at !== formattedActivated) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "cardLicenses", + columnName: "activated_at", + fileData: cardlicensesInputFile.activated_at, + databaseData: formattedActivated, + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + + const filterdLicenses = licenses.filter( + (licenses) => licenses.id === filterdCardLicenses[0].license_id + ); + if (filterdLicenses.length === 0) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "id", + fileData: filterdCardLicenses[0].license_id.toString(), + databaseData: "-", + reason: "紐つくライセンスのレコード無し", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + + if (filterdLicenses[0].expiry_date !== null) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "expiry_date", + fileData: null, + databaseData: getFormattedDate( + filterdLicenses[0].expiry_date, + `yyyy/MM/dd hh:mm:ss` + ), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + + if (filterdLicenses[0].account_id !== AUTO_INCREMENT_START) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "account_id", + fileData: AUTO_INCREMENT_START.toString(), + databaseData: filterdLicenses[0].account_id.toString(), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + + if (filterdLicenses[0].type !== "CARD") { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "type", + fileData: "CARD", + databaseData: filterdLicenses[0].type, + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + + if (filterdLicenses[0].status !== "Unallocated") { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "status", + fileData: "Unallocated", + databaseData: filterdLicenses[0].status, + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + if (filterdLicenses[0].allocated_user_id !== null) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "allocated_user_id", + fileData: null, + databaseData: filterdLicenses[0].allocated_user_id.toString(), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + if (filterdLicenses[0].order_id !== null) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "order_id", + fileData: null, + databaseData: filterdLicenses[0].order_id.toString(), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + if (filterdLicenses[0].deleted_at !== null) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "deleted_at", + fileData: null, + databaseData: getFormattedDate( + filterdLicenses[0].deleted_at, + `yyyy/MM/dd hh:mm:ss` + ), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + if (filterdLicenses[0].delete_order_id !== null) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "cardLicenses", + inputRow: row, + diffTargetTable: "licenses", + columnName: "delete_order_id", + fileData: null, + databaseData: filterdLicenses[0].delete_order_id.toString(), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + row = row + 1; + } + return isNoError; +} + +// ライセンス情報の突き合わせを行い、エラー時はエラー情報配列に情報追加する +function compareLicenses( + VerificationResultDetails: VerificationResultDetails[], + licensesFromFile: csvInputFileWithRow[], + licensesFromDatabase: License[], + accountsMappingInputFiles: AccountsMappingFile[] +): boolean { + let isNoError = true; + for (let i = 0; i < licensesFromFile.length; i++) { + if ( + !licensesFromDatabase[i] || + licensesFromFile[i].account_id !== + findAccountIdText( + accountsMappingInputFiles, + licensesFromDatabase[i].account_id + ) + ) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: licensesFromFile[i].row, + diffTargetTable: "licenses", + columnName: "account_id", + fileData: licensesFromFile[i].account_id, + databaseData: licensesFromDatabase[i] + ? findAccountIdText( + accountsMappingInputFiles, + licensesFromDatabase[i].account_id + ) + `(${licensesFromDatabase[i].account_id})` + : "undifined", + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + if ( + !licensesFromDatabase[i] || + licensesFromFile[i].expired_date !== + getFormattedDate( + licensesFromDatabase[i].expiry_date, + `yyyy/MM/dd hh:mm:ss` + ) + ) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: licensesFromFile[i].row, + diffTargetTable: "licenses", + columnName: "expired_date", + fileData: licensesFromFile[i].expired_date, + databaseData: licensesFromDatabase[i] + ? getFormattedDate( + licensesFromDatabase[i].expiry_date, + `yyyy/MM/dd hh:mm:ss` + ) + : "undifined", + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + } + } + return isNoError; +} + +// アカウント情報の突き合わせを行い、エラー時はエラー情報配列に情報追加する +function compareAccounts( + VerificationResultDetails: VerificationResultDetails[], + accountsFromFile: csvInputFileWithRow[], + countriesFromFile: csvInputFileWithRow[], + accountsFromDatabase: Account[], + accountsMappingInputFiles: AccountsMappingFile[] +): boolean { + let isNoError = true; + for (const accountFromFile of accountsFromFile) { + // DBレコードの存在チェック + const filterdAccounts = accountsFromDatabase.filter( + (accountsFromDatabase) => + findAccountIdText( + accountsMappingInputFiles, + accountsFromDatabase.id + ) === accountFromFile.account_id + ); + + if (filterdAccounts.length === 0) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: accountFromFile.row, + diffTargetTable: "accounts", + columnName: "account_id", + fileData: accountFromFile.account_id, + databaseData: "-", + reason: "レコード無し", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + + // 項目チェック（parent_account_id） + const transratedParentId = transrateCountryHierarchy( + countriesFromFile, + accountFromFile.parent_id + ); + if ( + transratedParentId !== + findAccountIdText( + accountsMappingInputFiles, + filterdAccounts[0].parent_account_id + ) + ) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: accountFromFile.row, + diffTargetTable: "accounts", + columnName: "parent_account_id", + fileData: + transratedParentId === accountFromFile.parent_id + ? accountFromFile.parent_id + : `${transratedParentId}(${accountFromFile.parent_id})`, + databaseData: + findAccountIdText( + accountsMappingInputFiles, + filterdAccounts[0].parent_account_id + ) + `(${filterdAccounts[0].parent_account_id})`, + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + + // 項目チェック（tier） + if ( + accountFromFile.type !== getMigrationTypeByNumber(filterdAccounts[0].tier) + ) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: accountFromFile.row, + diffTargetTable: "accounts", + columnName: "tier", + fileData: accountFromFile.type, + databaseData: getMigrationTypeByNumber(filterdAccounts[0].tier), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + + // 項目チェック（country） + if ( + accountFromFile.country !== + getCountryLabelByValue(filterdAccounts[0].country) + ) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: accountFromFile.row, + diffTargetTable: "accounts", + columnName: "country", + fileData: accountFromFile.country, + databaseData: getCountryLabelByValue(filterdAccounts[0].country), + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + + // 項目チェック（company_name） + if (accountFromFile.company_name !== filterdAccounts[0].company_name) { + const VerificationResultDetailsOne: VerificationResultDetails = { + input: "Account_transition", + inputRow: accountFromFile.row, + diffTargetTable: "accounts", + columnName: "company_name", + fileData: accountFromFile.company_name, + databaseData: filterdAccounts[0].company_name, + reason: "内容不一致", + }; + VerificationResultDetails.push(VerificationResultDetailsOne); + isNoError = false; + continue; + } + } + return isNoError; +} diff --git a/data_migration_tools/server/src/repositories/accounts/accounts.repository.service.ts b/data_migration_tools/server/src/repositories/accounts/accounts.repository.service.ts index 96d5030..d73563a 100644 --- a/data_migration_tools/server/src/repositories/accounts/accounts.repository.service.ts +++ b/data_migration_tools/server/src/repositories/accounts/accounts.repository.service.ts @@ -163,4 +163,21 @@ export class AccountsRepositoryService { ); }); } + + /** + * アカウント情報を全件取得する + * @returns Account[] + */ + async getAllAccounts( + context: Context, + ): Promise { + return await this.dataSource.transaction(async (entityManager) => { + const accountsRepo = entityManager.getRepository(Account); + + const accouts = accountsRepo.find({ + comment: `${context.getTrackingId()}_${new Date().toUTCString()}`, + }); + return accouts; + }); + } } diff --git a/data_migration_tools/server/src/repositories/licenses/licenses.repository.service.ts b/data_migration_tools/server/src/repositories/licenses/licenses.repository.service.ts index df1c190..7a9f46d 100644 --- a/data_migration_tools/server/src/repositories/licenses/licenses.repository.service.ts +++ b/data_migration_tools/server/src/repositories/licenses/licenses.repository.service.ts @@ -166,4 +166,34 @@ export class LicensesRepositoryService { return {}; }); } + + /** + * ライセンス情報を全件取得する + * @returns License[] + */ + async getAllLicenses(context: Context): Promise { + return await this.dataSource.transaction(async (entityManager) => { + const licenseRepo = entityManager.getRepository(License); + + const licenses = licenseRepo.find({ + comment: `${context.getTrackingId()}_${new Date().toUTCString()}`, + }); + return licenses; + }); + } + + /** + * カードライセンス情報を全件取得する + * @returns CardLicense[] + */ + async getAllCardLicense(context: Context): Promise { + return await this.dataSource.transaction(async (entityManager) => { + const cardLicenseRepo = entityManager.getRepository(CardLicense); + + const cardLicenses = cardLicenseRepo.find({ + comment: `${context.getTrackingId()}_${new Date().toUTCString()}`, + }); + return cardLicenses; + }); + } } diff --git a/data_migration_tools/server/src/repositories/users/users.repository.service.ts b/data_migration_tools/server/src/repositories/users/users.repository.service.ts index ebb4dc3..d060de7 100644 --- a/data_migration_tools/server/src/repositories/users/users.repository.service.ts +++ b/data_migration_tools/server/src/repositories/users/users.repository.service.ts @@ -138,4 +138,20 @@ export class UsersRepositoryService { await deleteEntity(usersRepo, { id: userId }, this.isCommentOut, context); }); } + + /** + * ユーザー情報を全件取得する + * @returns User[] + */ + async getAllUsers(context: Context): Promise { + return await this.dataSource.transaction(async (entityManager) => { + const userRepo = entityManager.getRepository(User); + + const users = userRepo.find({ + comment: `${context.getTrackingId()}_${new Date().toUTCString()}`, + }); + return users; + }); + } } +