diff --git a/azure-pipelines-staging.yml b/azure-pipelines-staging.yml index 65c19c4..16550b1 100644 --- a/azure-pipelines-staging.yml +++ b/azure-pipelines-staging.yml @@ -196,22 +196,12 @@ jobs: displayName: Bash Script (Test) inputs: targetType: inline + workingDirectory: dictation_function/.devcontainer script: | - cd dictation_function - npm run test - env: - TENANT_NAME: xxxxxxxxxxxx - SIGNIN_FLOW_NAME: xxxxxxxxxxxx - ADB2C_TENANT_ID: $(adb2c-tenant-id) - ADB2C_CLIENT_ID: $(adb2c-client-id) - ADB2C_CLIENT_SECRET: $(adb2c-client-secret) - ADB2C_ORIGIN: xxxxxx - SENDGRID_API_KEY: $(sendgrid-api-key) - MAIL_FROM: xxxxxx - APP_DOMAIN: xxxxxxxxx - REDIS_HOST: xxxxxxxxxxxx - REDIS_PORT: 0 - REDIS_PASSWORD: xxxxxxxxxxxx + docker-compose -f pipeline-docker-compose.yml build + docker-compose -f pipeline-docker-compose.yml up -d + docker-compose exec -T dictation_function sudo npm ci + docker-compose exec -T dictation_function sudo npm run test - task: Docker@0 displayName: build inputs: diff --git a/dictation_function/.devcontainer/Dockerfile b/dictation_function/.devcontainer/Dockerfile index c33b122..97e310d 100644 --- a/dictation_function/.devcontainer/Dockerfile +++ b/dictation_function/.devcontainer/Dockerfile @@ -6,7 +6,7 @@ FROM node:18.17.1-buster RUN /bin/cp /usr/share/zoneinfo/Asia/Tokyo /etc/localtime && \ echo "Asia/Tokyo" > /etc/timezone - + # Options for setup script ARG INSTALL_ZSH="true" ARG UPGRADE_PACKAGES="false" @@ -24,6 +24,21 @@ RUN bash /tmp/library-scripts/common-debian.sh "${INSTALL_ZSH}" "${USERNAME}" "$ && apt-get install default-jre -y \ && apt-get clean -y && rm -rf /var/lib/apt/lists/* /tmp/library-scripts +# COPY --from=golang:1.18-buster /usr/local/go/ /usr/local/go/ +ENV GO111MODULE=auto +COPY library-scripts/go-debian.sh /tmp/library-scripts/ +RUN bash /tmp/library-scripts/go-debian.sh "1.18" "/usr/local/go" "${GOPATH}" "${USERNAME}" "false" \ + && apt-get clean -y && rm -rf /tmp/library-scripts +ENV PATH="/usr/local/go/bin:${PATH}" +RUN mkdir -p /tmp/gotools \ + && cd /tmp/gotools \ + && export GOPATH=/tmp/gotools \ + && export GOCACHE=/tmp/gotools/cache \ + # sql-migrate + && go install github.com/rubenv/sql-migrate/sql-migrate@v1.1.2 \ + && mv /tmp/gotools/bin/* ${TARGET_GOPATH}/bin/ \ + && rm -rf /tmp/gotools + # Update NPM RUN npm install -g npm diff --git a/dictation_function/.devcontainer/docker-compose.yml b/dictation_function/.devcontainer/docker-compose.yml index ebcd52d..78d2750 100644 --- a/dictation_function/.devcontainer/docker-compose.yml +++ b/dictation_function/.devcontainer/docker-compose.yml @@ -16,6 +16,15 @@ services: - CHOKIDAR_USEPOLLING=true networks: - external + test_mysql_db: + image: mysql:8.0-bullseye + environment: + MYSQL_ROOT_PASSWORD: root_password + MYSQL_DATABASE: odms + MYSQL_USER: user + MYSQL_PASSWORD: password + networks: + - external networks: external: name: omds_network diff --git a/dictation_function/.devcontainer/pipeline-docker-compose.yml b/dictation_function/.devcontainer/pipeline-docker-compose.yml new file mode 100644 index 0000000..16a94b6 --- /dev/null +++ b/dictation_function/.devcontainer/pipeline-docker-compose.yml @@ -0,0 +1,31 @@ +version: "3" + +services: + dictation_function: + build: . + working_dir: /app/dictation_function + volumes: + - ../../:/app + - node_modules:/app/dictation_function/node_modules + environment: + - CHOKIDAR_USEPOLLING=true + depends_on: + - test_mysql_db + networks: + - dictation_function_network + test_mysql_db: + image: mysql:8.0-bullseye + environment: + MYSQL_ROOT_PASSWORD: root_password + MYSQL_DATABASE: odms + MYSQL_USER: user + MYSQL_PASSWORD: password + networks: + - dictation_function_network +networks: + dictation_function_network: + name: test_dictation_function_network + +# Data Volume として永続化する +volumes: + node_modules: diff --git a/dictation_function/.env.test b/dictation_function/.env.test new file mode 100644 index 0000000..5026a47 --- /dev/null +++ b/dictation_function/.env.test @@ -0,0 +1,35 @@ +STAGE=local +TENANT_NAME=tenantoname +SIGNIN_FLOW_NAME=b2c_1_signin_dev +ADB2C_TENANT_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +ADB2C_CLIENT_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +ADB2C_CLIENT_ID=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +ADB2C_CLIENT_SECRET=xxxxxxxxxxxxxxxxxxxxxxxxxxxxx +ADB2C_ORIGIN=https://xxxxxxx.b2clogin.com/xxxxxxxx.onmicrosoft.com/b2c_1_signin_dev/ +KEY_VAULT_NAME=xxxxxxxxxxxxxxx +SENDGRID_API_KEY=SG.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +MAIL_FROM=noreply@se0223.com +APP_DOMAIN=http://localhost:8081/ +REDIS_HOST=redis-cache +REDIS_PORT=6379 +REDIS_PASSWORD=omdsredispass +ADB2C_CACHE_TTL=86400 +STORAGE_ACCOUNT_NAME_US=saxxxxxxxxx +STORAGE_ACCOUNT_NAME_AU=saxxxxxxxxx +STORAGE_ACCOUNT_NAME_EU=saxxxxxxxxx +STORAGE_ACCOUNT_NAME_IMPORT=saxxxxxxxxx +STORAGE_ACCOUNT_NAME_ANALYSIS=saxxxxxxxxx +STORAGE_ACCOUNT_KEY_US=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx== +STORAGE_ACCOUNT_KEY_AU=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx== +STORAGE_ACCOUNT_KEY_EU=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx== +STORAGE_ACCOUNT_KEY_IMPORT=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx== +STORAGE_ACCOUNT_KEY_ANALYSIS=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx== +STORAGE_ACCOUNT_ENDPOINT_US=https://saxxxxxxxxx.blob.core.windows.net/ +STORAGE_ACCOUNT_ENDPOINT_AU=https://saxxxxxxxxx.blob.core.windows.net/ +STORAGE_ACCOUNT_ENDPOINT_EU=https://saxxxxxxxxx.blob.core.windows.net/ +STORAGE_ACCOUNT_ENDPOINT_IMPORT=https://saxxxxxxxxx.blob.core.windows.net/ +STORAGE_ACCOUNT_ENDPOINT_ANALYSIS=https://saxxxxxxxxx.blob.core.windows.net/ +BASE_PATH=http://localhost:8081 +ACCESS_TOKEN_LIFETIME_WEB=7200000 +JWT_PRIVATE_KEY="-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEA5IZZNgDew9eGmuFTezwdHYLSaJvUPPIKYoiOeVLD1paWNI51\n7Vkaoh0ngprcKOdv6T1N07V4igK7mOim2zY3yCTR6wcWR3PfFJrl9vh5SOo79koZ\noJb27YiM4jtxfx2dezzp0T2GoNR5rRolPUbWFJXnDe0DVXYXpJLb4LAlF2XAyYX0\nSYKUVUsJnzm5k4xbXtnwPwVbpm0EdswBE6qSfiL9zWk9dvHoKzSnfSDzDFoFcEoV\nchawzYXf/MM1YR4wo5XyzECc6Q5Ah4z522//mBNNaDHv83Yuw3mGShT73iJ0JQdk\nTturshv2Ecma38r6ftrIwNYXw4VVatJM8+GOOQIDAQABAoIBADrwp7u097+dK/tw\nWD61n3DIGAqg/lmFt8X4IH8MKLSE/FKr16CS1bqwOEuIM3ZdUtDeXd9Xs7IsyEPE\n5ZwuXK7DSF0M4+Mj8Ip49Q0Aww9aUoLQU9HGfgN/r4599GTrt31clZXA/6Mlighq\ncOZgCcEfdItz8OMu5SQuOIW4CKkCuaWnPOP26UqZocaXNZfpZH0iFLATMMH/TT8x\nay9ToHTQYE17ijdQ/EOLSwoeDV1CU1CIE3P4YfLJjvpKptly5dTevriHEzBi70Jx\n/KEPUn9Jj2gZafrUxRVhmMbm1zkeYxL3gsqRuTzRjEeeILuZhSJyCkQZyUNARxsg\nQY4DZfECgYEA+YLKUtmYTx60FS6DJ4s31TAsXY8kwhq/lB9E3GBZKDd0DPayXEeK\n4UWRQDTT6MI6fedW69FOZJ5sFLp8HQpcssb4Weq9PCpDhNTx8MCbdH3Um5QR3vfW\naKq/1XM8MDUnx5XcNYd87Aw3azvJAvOPr69as8IPnj6sKaRR9uQjbYUCgYEA6nfV\n5j0qmn0EJXZJblk4mvvjLLoWSs17j9YlrZJlJxXMDFRYtgnelv73xMxOMvcGoxn5\nifs7dpaM2x5EmA6jVU5sYaB/beZGEPWqPYGyjIwXPvUGAAv8Gbnvpp+xlSco/Dum\nIq0w+43ry5/xWh6CjfrvKV0J2bDOiJwPEdu/8iUCgYEAnBBSvL+dpN9vhFAzeOh7\nY71eAqcmNsLEUcG9MJqTKbSFwhYMOewF0iHRWHeylEPokhfBJn8kqYrtz4lVWFTC\n5o/Nh3BsLNXCpbMMIapXkeWiti1HgE9ErPMgSkJpwz18RDpYIqM8X+jEQS6D7HSr\nyxfDg+w+GJza0rEVE3hfMIECgYBw+KZ2VfhmEWBjEHhXE+QjQMR3s320MwebCUqE\nNCpKx8TWF/naVC0MwfLtvqbbBY0MHyLN6d//xpA9r3rLbRojqzKrY2KiuDYAS+3n\nzssRzxoQOozWju+8EYu30/ADdqfXyIHG6X3VZs87AGiQzGyJLmP3oR1y5y7MQa09\nJI16hQKBgHK5uwJhGa281Oo5/FwQ3uYLymbNwSGrsOJXiEu2XwJEXwVi2ELOKh4/\n03pBk3Kva3fIwEK+vCzDNnxShIQqBE76/2I1K1whOfoUehhYvKHGaXl2j70Zz9Ks\nrkGW1cx7p+yDqATDrwHBHTHFh5bUTTn8dN40n0e0W/llurpbBkJM\n-----END RSA PRIVATE KEY-----\n" +JWT_PUBLIC_KEY="-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA5IZZNgDew9eGmuFTezwd\nHYLSaJvUPPIKYoiOeVLD1paWNI517Vkaoh0ngprcKOdv6T1N07V4igK7mOim2zY3\nyCTR6wcWR3PfFJrl9vh5SOo79koZoJb27YiM4jtxfx2dezzp0T2GoNR5rRolPUbW\nFJXnDe0DVXYXpJLb4LAlF2XAyYX0SYKUVUsJnzm5k4xbXtnwPwVbpm0EdswBE6qS\nfiL9zWk9dvHoKzSnfSDzDFoFcEoVchawzYXf/MM1YR4wo5XyzECc6Q5Ah4z522//\nmBNNaDHv83Yuw3mGShT73iJ0JQdkTturshv2Ecma38r6ftrIwNYXw4VVatJM8+GO\nOQIDAQAB\n-----END PUBLIC KEY-----\n" \ No newline at end of file diff --git a/dictation_function/.gitignore b/dictation_function/.gitignore index b351586..27683c5 100644 --- a/dictation_function/.gitignore +++ b/dictation_function/.gitignore @@ -11,6 +11,8 @@ Publish *.Cache project.lock.json +.test/ + /packages /TestResults diff --git a/dictation_function/package.json b/dictation_function/package.json index fce3022..0be5119 100644 --- a/dictation_function/package.json +++ b/dictation_function/package.json @@ -9,7 +9,7 @@ "clean": "rimraf dist", "prestart": "npm run clean && npm run build", "start": "func start", - "test": "jest", + "test": "sql-migrate up -config=/app/dictation_server/db/dbconfig.yml -env=test && jest -w 1", "codegen": "sh codegen.sh" }, "dependencies": { diff --git a/dictation_function/src/blobstorage/audioBlobStorage.service.ts b/dictation_function/src/blobstorage/audioBlobStorage.service.ts new file mode 100644 index 0000000..4acf23a --- /dev/null +++ b/dictation_function/src/blobstorage/audioBlobStorage.service.ts @@ -0,0 +1,146 @@ +import { + BlobServiceClient, + ContainerClient, + StorageSharedKeyCredential, +} from "@azure/storage-blob"; +import { + BLOB_STORAGE_REGION_AU, + BLOB_STORAGE_REGION_EU, + BLOB_STORAGE_REGION_US, +} from "../constants"; +import { InvocationContext } from "@azure/functions"; + +export class AudioBlobStorageService { + private readonly sharedKeyCredentialUS: StorageSharedKeyCredential; + private readonly sharedKeyCredentialEU: StorageSharedKeyCredential; + private readonly sharedKeyCredentialAU: StorageSharedKeyCredential; + + private readonly blobServiceClientUS: BlobServiceClient; + private readonly blobServiceClientEU: BlobServiceClient; + private readonly blobServiceClientAU: BlobServiceClient; + + constructor() { + if ( + !process.env.STORAGE_ACCOUNT_ENDPOINT_US || + !process.env.STORAGE_ACCOUNT_ENDPOINT_AU || + !process.env.STORAGE_ACCOUNT_ENDPOINT_EU || + !process.env.STORAGE_ACCOUNT_NAME_US || + !process.env.STORAGE_ACCOUNT_KEY_US || + !process.env.STORAGE_ACCOUNT_NAME_AU || + !process.env.STORAGE_ACCOUNT_KEY_AU || + !process.env.STORAGE_ACCOUNT_NAME_EU || + !process.env.STORAGE_ACCOUNT_KEY_EU + ) { + throw new Error("Storage account information is missing"); + } + + // リージョンごとのSharedKeyCredentialを生成 + this.sharedKeyCredentialUS = new StorageSharedKeyCredential( + process.env.STORAGE_ACCOUNT_NAME_US, + process.env.STORAGE_ACCOUNT_KEY_US + ); + this.sharedKeyCredentialAU = new StorageSharedKeyCredential( + process.env.STORAGE_ACCOUNT_NAME_AU, + process.env.STORAGE_ACCOUNT_KEY_AU + ); + this.sharedKeyCredentialEU = new StorageSharedKeyCredential( + process.env.STORAGE_ACCOUNT_NAME_EU, + process.env.STORAGE_ACCOUNT_KEY_EU + ); + + // Audioファイルの保存先のリージョンごとにBlobServiceClientを生成 + this.blobServiceClientUS = new BlobServiceClient( + process.env.STORAGE_ACCOUNT_ENDPOINT_US, + this.sharedKeyCredentialUS + ); + this.blobServiceClientAU = new BlobServiceClient( + process.env.STORAGE_ACCOUNT_ENDPOINT_AU, + this.sharedKeyCredentialAU + ); + this.blobServiceClientEU = new BlobServiceClient( + process.env.STORAGE_ACCOUNT_ENDPOINT_EU, + this.sharedKeyCredentialEU + ); + } + + /** + * 指定されたファイルを削除します。 + * @param context + * @param accountId + * @param country + * @param fileName + * @returns file + */ + async deleteFile( + context: InvocationContext, + accountId: number, + country: string, + fileName: string + ): Promise { + context.log( + `[IN] ${this.deleteFile.name} | params: { ` + + `accountId: ${accountId} ` + + `country: ${country} ` + + `fileName: ${fileName} };` + ); + + try { + // 国に応じたリージョンでコンテナ名を指定してClientを取得 + const containerClient = this.getContainerClient( + context, + accountId, + country + ); + // コンテナ内のBlobパス名を指定してClientを取得 + const blobClient = containerClient.getBlobClient(fileName); + + const { succeeded, errorCode, date } = await blobClient.deleteIfExists(); + context.log( + `succeeded: ${succeeded}, errorCode: ${errorCode}, date: ${date}` + ); + + // 失敗時、Blobが存在しない場合以外はエラーとして例外をスローする + // Blob不在の場合のエラーコードは「BlobNotFound」以下を参照 + // https://learn.microsoft.com/ja-jp/rest/api/storageservices/blob-service-error-codes + if (!succeeded && errorCode !== "BlobNotFound") { + throw new Error( + `delete blob failed. succeeded: ${succeeded}, errorCode: ${errorCode}, date: ${date}` + ); + } + } catch (e) { + context.error(`error=${e}`); + throw e; + } finally { + context.log(`[OUT] ${this.deleteFile.name}`); + } + } + + /** + * 指定してアカウントIDと国に応じたリージョンのコンテナクライアントを取得します。 + * @param context + * @param accountId + * @param country + * @returns + */ + private getContainerClient( + context: InvocationContext, + accountId: number, + country: string + ): ContainerClient { + context.log( + `[IN] ${this.getContainerClient.name} | params: { ` + + `accountId: ${accountId}; country: ${country} };` + ); + + const containerName = `account-${accountId}`; + if (BLOB_STORAGE_REGION_US.includes(country)) { + return this.blobServiceClientUS.getContainerClient(containerName); + } else if (BLOB_STORAGE_REGION_AU.includes(country)) { + return this.blobServiceClientAU.getContainerClient(containerName); + } else if (BLOB_STORAGE_REGION_EU.includes(country)) { + return this.blobServiceClientEU.getContainerClient(containerName); + } else { + throw new Error("invalid country"); + } + } +} diff --git a/dictation_function/src/constants/index.ts b/dictation_function/src/constants/index.ts index 6f94278..ca6be1e 100644 --- a/dictation_function/src/constants/index.ts +++ b/dictation_function/src/constants/index.ts @@ -338,6 +338,12 @@ export const SYSTEM_IMPORT_USERS = "import-users"; export const ROW_START_INDEX = 2; +/** + * ファイル保持日数の初期値 + * @const {number} + */ +export const FILE_RETENTION_DAYS_DEFAULT = 30; + /** * ライセンス数推移出力機能のCSVヘッダ * @const {string[]} diff --git a/dictation_function/src/database/initializeDataSource.ts b/dictation_function/src/database/initializeDataSource.ts index e573669..f323d54 100644 --- a/dictation_function/src/database/initializeDataSource.ts +++ b/dictation_function/src/database/initializeDataSource.ts @@ -6,8 +6,11 @@ import { LicenseAllocationHistoryArchive, LicenseArchive, } from "../entity/license.entity"; -import { InvocationContext, } from "@azure/functions"; -import { DataSource} from "typeorm"; +import { InvocationContext } from "@azure/functions"; +import { DataSource } from "typeorm"; +import { Task } from "../entity/task.entity"; +import { AudioFile } from "../entity/audio_file.entity"; +import { AudioOptionItem } from "../entity/audio_option_item.entity"; export async function initializeDataSource( context: InvocationContext @@ -25,6 +28,9 @@ export async function initializeDataSource( UserArchive, Account, AccountArchive, + Task, + AudioFile, + AudioOptionItem, License, LicenseArchive, LicenseAllocationHistory, diff --git a/dictation_function/src/entity/account.entity.ts b/dictation_function/src/entity/account.entity.ts index eed6190..043be02 100644 --- a/dictation_function/src/entity/account.entity.ts +++ b/dictation_function/src/entity/account.entity.ts @@ -11,6 +11,7 @@ import { JoinColumn, OneToMany, } from "typeorm"; +import { FILE_RETENTION_DAYS_DEFAULT } from "../constants"; @Entity({ name: "accounts" }) export class Account { @@ -47,6 +48,12 @@ export class Account { @Column({ nullable: true, type: "bigint", transformer: bigintTransformer }) active_worktype_id: number | null; + @Column({ default: false }) + auto_file_delete: boolean; + + @Column({ default: FILE_RETENTION_DAYS_DEFAULT }) + file_retention_days: number; + @Column({ nullable: true, type: "datetime" }) deleted_at: Date | null; diff --git a/dictation_function/src/entity/audio_file.entity.ts b/dictation_function/src/entity/audio_file.entity.ts new file mode 100644 index 0000000..4de4b82 --- /dev/null +++ b/dictation_function/src/entity/audio_file.entity.ts @@ -0,0 +1,40 @@ +import { Entity, Column, PrimaryGeneratedColumn, OneToOne } from "typeorm"; + +@Entity({ name: "audio_files" }) +export class AudioFile { + @PrimaryGeneratedColumn() + id: number; + + @Column() + account_id: number; + @Column() + owner_user_id: number; + @Column() + url: string; + @Column() + file_name: string; + @Column() + author_id: string; + @Column() + work_type_id: string; + @Column() + started_at: Date; + @Column({ type: "time" }) + duration: string; + @Column() + finished_at: Date; + @Column() + uploaded_at: Date; + @Column() + file_size: number; + @Column() + priority: string; + @Column() + audio_format: string; + @Column({ nullable: true, type: "varchar" }) + comment: string | null; + @Column({ nullable: true, type: "datetime" }) + deleted_at: Date | null; + @Column() + is_encrypted: boolean; +} diff --git a/dictation_function/src/entity/audio_option_item.entity.ts b/dictation_function/src/entity/audio_option_item.entity.ts new file mode 100644 index 0000000..a3fe3cc --- /dev/null +++ b/dictation_function/src/entity/audio_option_item.entity.ts @@ -0,0 +1,13 @@ +import { Entity, Column, PrimaryGeneratedColumn } from "typeorm"; + +@Entity({ name: "audio_option_items" }) +export class AudioOptionItem { + @PrimaryGeneratedColumn() + id: number; + @Column() + audio_file_id: number; + @Column() + label: string; + @Column() + value: string; +} diff --git a/dictation_function/src/entity/task.entity.ts b/dictation_function/src/entity/task.entity.ts new file mode 100644 index 0000000..ceb96fc --- /dev/null +++ b/dictation_function/src/entity/task.entity.ts @@ -0,0 +1,59 @@ +import { AudioOptionItem } from "./audio_option_item.entity"; +import { AudioFile } from "./audio_file.entity"; +import { + Entity, + Column, + PrimaryGeneratedColumn, + OneToOne, + JoinColumn, + OneToMany, + ManyToOne, + CreateDateColumn, + UpdateDateColumn, +} from "typeorm"; +import { bigintTransformer } from "../common/entity"; +import { Account } from "./account.entity"; + +@Entity({ name: "tasks" }) +export class Task { + @PrimaryGeneratedColumn() + id: number; + @Column() + job_number: string; + @Column() + account_id: number; + @Column({ nullable: true, type: "boolean" }) + is_job_number_enabled: boolean | null; + @Column() + audio_file_id: number; + @Column() + status: string; + @Column({ nullable: true, type: "bigint", transformer: bigintTransformer }) + typist_user_id: number | null; + @Column() + priority: string; + @Column({ nullable: true, type: "bigint", transformer: bigintTransformer }) + template_file_id: number | null; + @Column({ nullable: true, type: "datetime" }) + started_at: Date | null; + @Column({ nullable: true, type: "datetime" }) + finished_at: Date | null; + + @Column({ nullable: true, type: "datetime" }) + created_by: string | null; + + @CreateDateColumn({ + default: () => "datetime('now', 'localtime')", + type: "datetime", + }) // defaultはSQLite用設定値.本番用は別途migrationで設定 + created_at: Date; + + @Column({ nullable: true, type: "datetime" }) + updated_by: string | null; + + @UpdateDateColumn({ + default: () => "datetime('now', 'localtime')", + type: "datetime", + }) // defaultはSQLite用設定値.本番用は別途migrationで設定 + updated_at: Date; +} diff --git a/dictation_function/src/functions/deleteAudioFiles.ts b/dictation_function/src/functions/deleteAudioFiles.ts new file mode 100644 index 0000000..b710969 --- /dev/null +++ b/dictation_function/src/functions/deleteAudioFiles.ts @@ -0,0 +1,187 @@ +import { app, InvocationContext, Timer } from "@azure/functions"; +import { DataSource, In } from "typeorm"; +import { Task } from "../entity/task.entity"; +import { AudioFile } from "../entity/audio_file.entity"; +import { AudioOptionItem } from "../entity/audio_option_item.entity"; +import { MANUAL_RECOVERY_REQUIRED } from "../constants"; +import * as dotenv from "dotenv"; +import { initializeDataSource } from "../database/initializeDataSource"; +import { AudioBlobStorageService } from "../blobstorage/audioBlobStorage.service"; + +app.timer("deleteAudioFiles", { + schedule: "0 3 * * *", // 毎日UTC 3:00に実行 + handler: deleteAudioFiles, +}); + +// 削除対象となる音声ファイルとタスクを特定する為の情報 +type TargetTaskInfo = { + /** + * タスクID + * column: tasks.id + */ + id: number; + + /** + * ファイルID + * column: audio_files.id + */ + audio_file_id: number; + + /** + * ファイル名 + * column: audio_files.file_name + */ + file_name: string; + + /** + * アカウントID + * column: accounts.id + */ + account_id: number; + + /** + * アカウントの所属する地域 + * column: accounts.country + */ + country: string; +}; + +export async function deleteAudioFilesProcessing( + context: InvocationContext, + dataSource: DataSource, + blobStorageService: AudioBlobStorageService, + now: Date +): Promise { + context.log(`[IN] deleteAudioFilesProcessing. now=${now}`); + try { + // 削除対象のタスクとファイルを取得 + const targets = await getProcessTargets(dataSource, now); + context.log(`delete targets: ${JSON.stringify(targets)}`); + + // DBからレコードを削除 + await deleteRecords(dataSource, targets); + + // タスクに紐づくファイルを削除 + for (const target of targets) { + try { + const { account_id, country, file_name } = target; + await blobStorageService.deleteFile( + context, + account_id, + country, + file_name + ); + context.log(`file delete success. target=${JSON.stringify(target)}`); + } catch (e) { + context.log( + `${MANUAL_RECOVERY_REQUIRED} file delete failed. target=${JSON.stringify( + target + )}` + ); + } + } + } catch (error) { + // DB関連で例外が発生した場合、アラートを出す為のログを出力する + context.error( + `${MANUAL_RECOVERY_REQUIRED} Failed to execute auto file deletion function. error=${error}` + ); + throw error; + } finally { + context.log(`[OUT] deleteAudioFilesProcessing`); + } +} + +async function deleteAudioFiles( + myTimer: Timer, + context: InvocationContext +): Promise { + context.log("[IN]deleteAudioFiles"); + + dotenv.config({ path: ".env" }); + dotenv.config({ path: ".env.local", override: true }); + const dataSource = await initializeDataSource(context); + try { + await deleteAudioFilesProcessing( + context, + dataSource, + new AudioBlobStorageService(), + new Date() + ); + } catch (e) { + context.log("deleteAudioFilesProcessing failed."); + context.error(e); + throw e; + } finally { + await dataSource.destroy(); + context.log("[OUT]deleteAudioFiles"); + } +} + +/** + * タスクに紐づくファイルを削除する + * @param dataSource + * @param targets + * @returns Promise + */ +// テスト容易性を高めて開発効率を上げるため、本来はexport不要だがexportを行う 2024/03/13 +export async function deleteRecords( + dataSource: DataSource, + targets: TargetTaskInfo[] +): Promise { + const taskIds = targets.map((target) => target.id); + const audioFileIds = targets.map((target) => target.audio_file_id); + + // taskIdsに紐づくタスクを削除 + await dataSource.transaction(async (manager) => { + const taskRepository = manager.getRepository(Task); + const audioFileRepository = manager.getRepository(AudioFile); + const audioOptionItem = manager.getRepository(AudioOptionItem); + + // tasksテーブルから削除 + await taskRepository.delete({ + id: In(taskIds), + }); + // audio_filesテーブルから削除 + await audioFileRepository.delete({ + id: In(audioFileIds), + }); + // audio_option_itemsテーブルから削除 + await audioOptionItem.delete({ + audio_file_id: In(audioFileIds), + }); + }); +} + +/* + * ファイル削除対象のタスクを取得する + * @param dataSource + * @param now + * @returns Promise<{ account_id: number; audio_file_id: number; file_name: string; }[]> + */ +// テスト容易性を高めて開発効率を上げるため、本来はexport不要だがexportを行う 2024/03/13 +export async function getProcessTargets( + dataSource: DataSource, + now: Date +): Promise { + return await dataSource + .createQueryBuilder(Task, "tasks") + // SELECTでエイリアスを指定して、結果オブジェクトのプロパティとTargetTaskInfoのプロパティを一致させる + .select("tasks.id", "id") + .addSelect("tasks.audio_file_id", "audio_file_id") + .addSelect("audio_files.file_name", "file_name") + .addSelect("accounts.id", "account_id") + .addSelect("accounts.country", "country") + .where("tasks.finished_at IS NOT NULL") + .innerJoin("accounts", "accounts", "accounts.id = tasks.account_id") + .innerJoin( + "audio_files", + "audio_files", + "audio_files.id = tasks.audio_file_id" + ) + .andWhere("accounts.auto_file_delete = true") + .andWhere( + "DATE_ADD(tasks.finished_at, INTERVAL accounts.file_retention_days DAY) < :now", + { now } + ) + .getRawMany(); +} diff --git a/dictation_function/src/test/analysisLicenses.spec.ts b/dictation_function/src/test/analysisLicenses.spec.ts index e2cfb0b..75e694c 100644 --- a/dictation_function/src/test/analysisLicenses.spec.ts +++ b/dictation_function/src/test/analysisLicenses.spec.ts @@ -33,7 +33,7 @@ import { BlobstorageService } from "../blobstorage/blobstorage.service"; import { User, UserArchive } from "../entity/user.entity"; describe("analysisLicenses", () => { dotenv.config({ path: ".env" }); - dotenv.config({ path: ".env.local", override: true }); + dotenv.config({ path: ".env.test", override: true }); let source: DataSource | null = null; beforeEach(async () => { diff --git a/dictation_function/src/test/common/init.ts b/dictation_function/src/test/common/init.ts new file mode 100644 index 0000000..bfe1512 --- /dev/null +++ b/dictation_function/src/test/common/init.ts @@ -0,0 +1,21 @@ +import { DataSource } from 'typeorm'; + +export const truncateAllTable = async (source: DataSource) => { + const entities = source.entityMetadatas; + const queryRunner = source.createQueryRunner(); + + try { + await queryRunner.startTransaction(); + await queryRunner.query('SET FOREIGN_KEY_CHECKS=0'); + for (const entity of entities) { + await queryRunner.query(`TRUNCATE TABLE \`${entity.tableName}\``); + } + await queryRunner.query('SET FOREIGN_KEY_CHECKS=1'); + await queryRunner.commitTransaction(); + } catch (err) { + await queryRunner.rollbackTransaction(); + throw err; + } finally { + await queryRunner.release(); + } +}; diff --git a/dictation_function/src/test/common/logger.ts b/dictation_function/src/test/common/logger.ts new file mode 100644 index 0000000..19f5124 --- /dev/null +++ b/dictation_function/src/test/common/logger.ts @@ -0,0 +1,129 @@ +import { Logger, QueryRunner } from "typeorm"; +import * as fs from "fs"; +import * as path from "path"; + +interface IOutput { + initialize(): void; + write(message: string): void; +} + +class ConsoleOutput implements IOutput { + initialize(): void { + // do nothing + } + + write(message: string): void { + console.log(message); + } +} + +class FileOutput implements IOutput { + private logPath = path.join("/app/dictation_function/.test", "logs"); + private fileName = new Date().getTime(); + + initialize(): void { + if (!fs.existsSync(this.logPath)) { + fs.mkdirSync(this.logPath, { recursive: true }); + } + } + + write(message: string): void { + const logFile = path.join(this.logPath, `${this.fileName}.log`); + fs.appendFileSync(logFile, `${message}\n`); + } +} + +class NoneOutput implements IOutput { + initialize(): void { + // do nothing + } + + write(message: string): void { + // do nothing + } +} + +export class TestLogger implements Logger { + out: IOutput; + + constructor(output: "none" | "file" | "console") { + switch (output) { + case "none": + this.out = new NoneOutput(); + break; + case "file": + this.out = new FileOutput(); + break; + case "console": + this.out = new ConsoleOutput(); + break; + default: + this.out = new NoneOutput(); + break; + } + this.out.initialize(); + } + + private write(message: string): void { + this.out.write(message); + } + + logQuery(query: string, parameters?: any[], queryRunner?: QueryRunner) { + const raw = `Query: ${query} -- Parameters: ${JSON.stringify(parameters)}`; + // ex: 2024-03-08T06:38:43.125Z を TIME という文字列に置換 + const dateRemoved = raw.replace( + /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/g, + "TIME" + ); + // ex: /* コメント内容 */ を /* コメント */ という文字列に置換 + const commentRemoved = dateRemoved.replace( + /\/\*.*\*\//g, + "/* RequestID */" + ); + + // UUIDを固定文字列に置換する ex: 88a9c78e-115a-439c-9e23-731d649f0c27 を XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX という文字列に置換 + const uuidRemoved = commentRemoved.replace( + /[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}/g, + "XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX" + ); + this.write(uuidRemoved); + } + + logQueryError( + error: string, + query: string, + parameters?: any[], + queryRunner?: QueryRunner + ) { + this.write( + `ERROR: ${error} -- Query: ${query} -- Parameters: ${JSON.stringify( + parameters + )}` + ); + } + + logQuerySlow( + time: number, + query: string, + parameters?: any[], + queryRunner?: QueryRunner + ) { + this.write( + `SLOW QUERY: ${time}ms -- Query: ${query} -- Parameters: ${JSON.stringify( + parameters + )}` + ); + } + + logSchemaBuild(message: string, queryRunner?: QueryRunner) { + this.write(`Schema Build: ${message}`); + } + + logMigration(message: string, queryRunner?: QueryRunner) { + this.write(`Migration: ${message}`); + } + + log(level: "log" | "info" | "warn", message: any, queryRunner?: QueryRunner) { + this.write(`${level.toUpperCase()}: ${message}`); + } +} diff --git a/dictation_function/src/test/common/utility.ts b/dictation_function/src/test/common/utility.ts index c167750..d9c6a07 100644 --- a/dictation_function/src/test/common/utility.ts +++ b/dictation_function/src/test/common/utility.ts @@ -1,14 +1,19 @@ import { v4 as uuidv4 } from "uuid"; -import { DataSource } from "typeorm"; +import { DataSource, In } from "typeorm"; import { User, UserArchive } from "../../entity/user.entity"; import { Account, AccountArchive } from "../../entity/account.entity"; -import { ADMIN_ROLES, USER_ROLES } from "../../constants"; +import { ADMIN_ROLES, TASK_STATUS, USER_ROLES } from "../../constants"; import { License, LicenseAllocationHistory, LicenseArchive, LicenseAllocationHistoryArchive, } from "../../entity/license.entity"; +import { bigintTransformer } from "../../common/entity"; +import { Task } from "../../entity/task.entity"; +import { AudioFile } from "../../entity/audio_file.entity"; +import { AudioOptionItem } from "../../entity/audio_option_item.entity"; +import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity"; type InitialTestDBState = { tier1Accounts: { account: Account; users: User[] }[]; @@ -41,6 +46,8 @@ type OverrideUserArchive = Omit< "id" | "account" | "license" | "userGroupMembers" >; +type OverrideTask = Omit; + type AccountDefault = { [K in keyof OverrideAccount]?: OverrideAccount[K] }; type UserDefault = { [K in keyof OverrideUser]?: OverrideUser[K] }; type AccountArchiveDefault = { @@ -49,6 +56,7 @@ type AccountArchiveDefault = { type UserArchiveDefault = { [K in keyof OverrideUserArchive]?: OverrideUserArchive[K]; }; +type TaskDefault = { [K in keyof OverrideTask]?: OverrideTask[K] }; /** * テスト ユーティリティ: 指定したプロパティを上書きしたユーザーを作成する @@ -117,6 +125,8 @@ export const makeTestAccount = async ( locked: d?.locked ?? false, company_name: d?.company_name ?? "test inc.", verified: d?.verified ?? true, + auto_file_delete: d?.auto_file_delete ?? false, + file_retention_days: d?.file_retention_days ?? 30, deleted_at: d?.deleted_at ?? "", created_by: d?.created_by ?? "test_runner", created_at: d?.created_at ?? new Date(), @@ -124,7 +134,7 @@ export const makeTestAccount = async ( updated_at: d?.updated_at ?? new Date(), }); const result = identifiers.pop() as Account; - accountId = result.id; + accountId = bigintTransformer.from(result.id); } { const d = defaultAdminUserValue; @@ -148,7 +158,7 @@ export const makeTestAccount = async ( }); const result = identifiers.pop() as User; - userId = result.id; + userId = bigintTransformer.from(result.id); } // Accountの管理者を設定する @@ -471,3 +481,176 @@ export const createLicenseAllocationHistoryArchive = async ( }); identifiers.pop() as LicenseAllocationHistoryArchive; }; + +export const makeTestTask = async ( + datasource: DataSource, + accountId: number, + ownerUserId: number, + identifierName: string, + defaultTaskValue?: TaskDefault +): Promise<{ task: Task; file: AudioFile; options: AudioOptionItem[] }> => { + const d = defaultTaskValue; + + // AudioFileを作成 + const { identifiers: identifiers1 } = await datasource + .getRepository(AudioFile) + .insert({ + account_id: accountId, + owner_user_id: ownerUserId, + url: `https://example.com/${identifierName}`, + file_name: `test${identifierName}.wav`, + author_id: "test_author", + work_type_id: "test_work_type", + started_at: new Date(), + duration: "00:00:00", + finished_at: new Date(), + uploaded_at: new Date(), + file_size: 1024, + priority: "01", + audio_format: "wav", + comment: `test_comment_${identifierName}`, + deleted_at: new Date(), + is_encrypted: false, + }); + + const result = identifiers1.pop() as AudioFile; + const audioFileId = bigintTransformer.from(result.id); + + // AudioFileを取得 + const file = await datasource.getRepository(AudioFile).findOne({ + where: { + id: audioFileId, + }, + }); + if (!file) { + throw new Error("Unexpected null"); + } + + // Taskを作成 + const { identifiers: identifiers2 } = await datasource + .getRepository(Task) + .insert({ + job_number: d?.job_number ?? "0001", + account_id: accountId, + is_job_number_enabled: d?.is_job_number_enabled ?? true, + audio_file_id: file.id, + status: d?.status ?? "Uploaded", + typist_user_id: d?.typist_user_id, + priority: d?.priority ?? "01", + template_file_id: d?.template_file_id, + started_at: d?.started_at ?? new Date(), + finished_at: d?.finished_at ?? new Date(), + created_by: d?.created_by ?? "test_runner", + created_at: d?.created_at ?? new Date(), + updated_by: d?.updated_by ?? "updater", + updated_at: d?.updated_at ?? new Date(), + }); + const result2 = identifiers2.pop() as Task; + const taskId = bigintTransformer.from(result2.id); + + const task = await datasource.getRepository(Task).findOne({ + where: { + id: taskId, + }, + }); + if (!task) { + throw new Error("Unexpected null"); + } + + // AudioOptionItemを作成 + const item01 = await datasource.getRepository(AudioOptionItem).insert({ + audio_file_id: audioFileId, + label: `test_option_label_${identifierName}_01`, + value: `test_option_value_${identifierName}_01`, + }); + const item02 = await datasource.getRepository(AudioOptionItem).insert({ + audio_file_id: audioFileId, + label: `test_option_label_${identifierName}_02`, + value: `test_option_value_${identifierName}_02`, + }); + const optionItemResult01 = item01.identifiers.pop() as AudioOptionItem; + const optionItemResult02 = item02.identifiers.pop() as AudioOptionItem; + const optionItemID01 = bigintTransformer.from(optionItemResult01.id); + const optionItemID02 = bigintTransformer.from(optionItemResult02.id); + + const optionItems = await datasource.getRepository(AudioOptionItem).find({ + where: { + id: In([optionItemID01, optionItemID02]), + }, + }); + + return { task, file, options: optionItems }; +}; + +export const makeManyTestTasks = async ( + datasource: DataSource, + inputFiles: QueryDeepPartialEntity[], + task_finished_at: Date +): Promise => { + const fileRepository = datasource.getRepository(AudioFile); + const result = await fileRepository.insert(inputFiles); + const audioFileIds = result.identifiers.map((id) => id.id); + const files = await fileRepository.find({ + where: { + id: In(audioFileIds), + }, + }); + + const tasks = files.map((file, index): QueryDeepPartialEntity => { + return { + job_number: `0001_${index}`, + account_id: file.account_id, + is_job_number_enabled: true, + audio_file_id: file.id, + status: TASK_STATUS.FINISHED, + typist_user_id: null, + priority: "01", + template_file_id: null, + started_at: new Date(), + finished_at: task_finished_at, + created_by: "test_runner", + created_at: new Date(), + updated_by: "updater", + updated_at: new Date(), + }; + }); + + const taskRepository = datasource.getRepository(Task); + const x = await taskRepository.insert(tasks); + + const partialOptions = files.flatMap( + (file, index): QueryDeepPartialEntity[] => { + return [ + { + audio_file_id: file.id, + label: `test_option_label_${index}_01`, + value: `test_option_value_${index}_01`, + }, + { + audio_file_id: file.id, + label: `test_option_label_${index}_02`, + value: `test_option_value_${index}_02`, + }, + ]; + } + ); + + const optionRepository = datasource.getRepository(AudioOptionItem); + await optionRepository.insert(partialOptions); +}; + +export const getTasks = async (datasource: DataSource): Promise => { + return await datasource.getRepository(Task).find(); +}; + +export const getAudioFiles = async ( + datasource: DataSource +): Promise => { + return await datasource.getRepository(AudioFile).find(); +}; + +export const getAudioOptionItems = async ( + datasource: DataSource +): Promise => { + return await datasource.getRepository(AudioOptionItem).find(); +}; diff --git a/dictation_function/src/test/deleteAudioFiles.spec.ts b/dictation_function/src/test/deleteAudioFiles.spec.ts new file mode 100644 index 0000000..f165e4b --- /dev/null +++ b/dictation_function/src/test/deleteAudioFiles.spec.ts @@ -0,0 +1,1496 @@ +import * as dotenv from "dotenv"; +import { InvocationContext } from "@azure/functions"; +import { DataSource } from "typeorm"; +import { truncateAllTable } from "./common/init"; +import { + deleteAudioFilesProcessing, + deleteRecords, + getProcessTargets, +} from "../functions/deleteAudioFiles"; +import { + getAudioFiles, + getAudioOptionItems, + getTasks, + makeManyTestTasks, + makeTestAccount, + makeTestTask, +} from "./common/utility"; +import { TASK_STATUS } from "../constants"; +import { TestLogger } from "./common/logger"; +import { AudioFile } from "../entity/audio_file.entity"; +import { QueryDeepPartialEntity } from "typeorm/query-builder/QueryPartialEntity"; +import { AudioBlobStorageService } from "../blobstorage/audioBlobStorage.service"; +import { User, UserArchive } from "../entity/user.entity"; +import { Account, AccountArchive } from "../entity/account.entity"; +import { Task } from "../entity/task.entity"; +import { + License, + LicenseAllocationHistory, + LicenseAllocationHistoryArchive, + LicenseArchive, +} from "../entity/license.entity"; +import { AudioOptionItem } from "../entity/audio_option_item.entity"; + +describe("getProcessTargets | 削除対象を特定するQueryが正常に動作するか確認する", () => { + let source: DataSource | null = null; + beforeAll(async () => { + dotenv.config({ path: ".env" }); + dotenv.config({ path: ".env.test", override: true }); + if (source == null) { + source = await (async () => { + const s = new DataSource({ + type: "mysql", + host: "test_mysql_db", + port: 3306, + username: "user", + password: "password", + database: "odms", + entities: [__dirname + "/../../**/*.entity{.ts,.js}"], + synchronize: false, // trueにすると自動的にmigrationが行われるため注意 + logger: new TestLogger("none"), + logging: true, + }); + return await s.initialize(); + })(); + } + }); + + beforeEach(async () => { + if (source) { + await truncateAllTable(source); + } + }); + + afterAll(async () => { + await source?.destroy(); + source = null; + }); + + it("ファイル削除対象のタスクが存在しない場合、空の配列が取得できる", async () => { + if (!source) fail(); + + // 2024/02/29 00:00:00を"今"とする + const now = new Date("2024-02-29T00:00:00Z"); + + const { account, admin } = await makeTestAccount(source, { + file_retention_days: 2, + auto_file_delete: true, + }); + + { + // ファイル削除対象のタスクが存在しない場合、空の配列が取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([]); + } + + // ちょうど2日前(削除対象外)のタスクを作成 + await makeTestTask(source, account.id, admin.id, "case01", { + status: TASK_STATUS.FINISHED, + job_number: "job01", + finished_at: new Date("2024-02-27T00:00:00Z"), + }); + // ちょうど1日前(削除対象外)のタスクを作成 + await makeTestTask(source, account.id, admin.id, "case02", { + status: TASK_STATUS.FINISHED, + job_number: "job02", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + + { + // ファイル削除対象のタスクが存在しない場合、空の配列が取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([]); + } + }); + + it("ファイル削除対象のタスク情報のみを取得できる(対象となる期限切れのタスク情報のみが取れる)", async () => { + if (!source) fail(); + + // 2024/02/29 00:00:00を"今"とする + const now = new Date("2024-02-29T00:00:00Z"); + + const { account, admin } = await makeTestAccount(source, { + file_retention_days: 2, + auto_file_delete: true, + }); + + // ちょうど2日前(削除対象外)のタスクを作成 + await makeTestTask(source, account.id, admin.id, "case01", { + status: TASK_STATUS.FINISHED, + job_number: "job01", + finished_at: new Date("2024-02-27T00:00:00Z"), + }); + // ちょうど1日前(削除対象外)のタスクを作成 + await makeTestTask(source, account.id, admin.id, "case02", { + status: TASK_STATUS.FINISHED, + job_number: "job02", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + // 2日と1秒前(削除対象)のタスクを作成 + const { task, file } = await makeTestTask( + source, + account.id, + admin.id, + "case03", + { + status: TASK_STATUS.FINISHED, + job_number: "job03", + finished_at: new Date("2024-02-26T23:59:59Z"), + } + ); + + { + // ファイル削除対象のタスク情報1件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task.id, + audio_file_id: file.id, + file_name: file.file_name, + account_id: account.id, + country: account.country, + }, + ]); + } + + // ちょうど3日前（削除対象）のタスクを作成 + const { task: task2, file: file2 } = await makeTestTask( + source, + account.id, + admin.id, + "case04", + { + status: TASK_STATUS.FINISHED, + job_number: "job04", + finished_at: new Date("2024-02-26T00:00:00Z"), + } + ); + + { + // ファイル削除対象のタスク情報2件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task.id, + audio_file_id: file.id, + file_name: file.file_name, + account_id: account.id, + country: account.country, + }, + { + id: task2.id, + audio_file_id: file2.id, + file_name: file2.file_name, + account_id: account.id, + country: account.country, + }, + ]); + } + + // ちょうど1日前(削除対象外)のタスクを作成 + await makeTestTask(source, account.id, admin.id, "case05", { + status: TASK_STATUS.FINISHED, + job_number: "job05", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + + { + // ファイル削除対象のタスク情報2件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task.id, + audio_file_id: file.id, + file_name: file.file_name, + account_id: account.id, + country: account.country, + }, + { + id: task2.id, + audio_file_id: file2.id, + file_name: file2.file_name, + account_id: account.id, + country: account.country, + }, + ]); + } + + // 100日と5時間前（削除対象）のタスクを作成 + const { task: task3, file: file3 } = await makeTestTask( + source, + account.id, + admin.id, + "case06", + { + status: TASK_STATUS.FINISHED, + job_number: "job06", + finished_at: new Date("2023-11-20T19:00:00Z"), + } + ); + + { + // ファイル削除対象のタスク情報3件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task.id, + audio_file_id: file.id, + file_name: file.file_name, + account_id: account.id, + country: account.country, + }, + { + id: task2.id, + audio_file_id: file2.id, + file_name: file2.file_name, + account_id: account.id, + country: account.country, + }, + { + id: task3.id, + audio_file_id: file3.id, + file_name: file3.file_name, + account_id: account.id, + country: account.country, + }, + ]); + } + + // 1日後（削除対象外。本来はミリ秒単位の未来方向の時刻違いを想定）のタスクを作成 + await makeTestTask(source, account.id, admin.id, "case07", { + status: TASK_STATUS.FINISHED, + job_number: "job07", + finished_at: new Date("2024-03-01T00:00:00Z"), + }); + + { + // ファイル削除対象のタスク情報3件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task.id, + audio_file_id: file.id, + file_name: file.file_name, + account_id: account.id, + country: account.country, + }, + { + id: task2.id, + audio_file_id: file2.id, + file_name: file2.file_name, + account_id: account.id, + country: account.country, + }, + { + id: task3.id, + audio_file_id: file3.id, + file_name: file3.file_name, + account_id: account.id, + country: account.country, + }, + ]); + } + }); + + it("ファイル削除対象のタスク情報のみを取得できる(auto_file_delete=falseのアカウントの情報のタスク情報は取れない)", async () => { + if (!source) fail(); + + // 2024/02/29 00:00:00を"今"とする + const now = new Date("2024-02-29T00:00:00Z"); + + // auto_file_deleteがtrueののアカウントを作成 + const { account: account01, admin: admin01 } = await makeTestAccount( + source, + { + file_retention_days: 2, + auto_file_delete: true, + } + ); + + // auto_file_deleteがfalseののアカウントを作成 + const { account: account02, admin: admin02 } = await makeTestAccount( + source, + { + file_retention_days: 2, + auto_file_delete: false, + } + ); + + // ちょうど2日前(削除対象外)のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case01", { + status: TASK_STATUS.FINISHED, + job_number: "job01", + finished_at: new Date("2024-02-27T00:00:00Z"), + }); + await makeTestTask(source, account02.id, admin02.id, "case01", { + status: TASK_STATUS.FINISHED, + job_number: "job01", + finished_at: new Date("2024-02-27T00:00:00Z"), + }); + + // ちょうど1日前(削除対象外)のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case02", { + status: TASK_STATUS.FINISHED, + job_number: "job02", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + await makeTestTask(source, account02.id, admin02.id, "case02", { + status: TASK_STATUS.FINISHED, + job_number: "job02", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + + // 2日と1秒前(削除対象)のタスクを作成 + const { task, file } = await makeTestTask( + source, + account01.id, + admin01.id, + "case03", + { + status: TASK_STATUS.FINISHED, + job_number: "job03", + finished_at: new Date("2024-02-26T23:59:59Z"), + } + ); + await makeTestTask(source, account02.id, admin02.id, "case03", { + status: TASK_STATUS.FINISHED, + job_number: "job03", + finished_at: new Date("2024-02-26T23:59:59Z"), + }); + + { + // ファイル削除対象のタスク情報1件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task.id, + audio_file_id: file.id, + file_name: file.file_name, + account_id: account01.id, + country: account01.country, + }, + ]); + } + + { + // ファイル削除対象のタスク情報1件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task.id, + audio_file_id: file.id, + file_name: file.file_name, + account_id: account01.id, + country: account01.country, + }, + ]); + } + + // ちょうど3日前（削除対象）のタスクを作成 + const { task: task2, file: file2 } = await makeTestTask( + source, + account01.id, + admin01.id, + "case04", + { + status: TASK_STATUS.FINISHED, + job_number: "job04", + finished_at: new Date("2024-02-26T00:00:00Z"), + } + ); + await makeTestTask(source, account02.id, admin02.id, "case04", { + status: TASK_STATUS.FINISHED, + job_number: "job04", + finished_at: new Date("2024-02-26T00:00:00Z"), + }); + + { + // ファイル削除対象のタスク情報2件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task.id, + audio_file_id: file.id, + file_name: file.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task2.id, + audio_file_id: file2.id, + file_name: file2.file_name, + account_id: account01.id, + country: account01.country, + }, + ]); + } + + // ちょうど1日前(削除対象外)のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case05", { + status: TASK_STATUS.FINISHED, + job_number: "job05", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + await makeTestTask(source, account02.id, admin02.id, "case05", { + status: TASK_STATUS.FINISHED, + job_number: "job05", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + + { + // ファイル削除対象のタスク情報2件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task.id, + audio_file_id: file.id, + file_name: file.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task2.id, + audio_file_id: file2.id, + file_name: file2.file_name, + account_id: account01.id, + country: account01.country, + }, + ]); + } + + // 100日と5時間前（削除対象）のタスクを作成 + const { task: task3, file: file3 } = await makeTestTask( + source, + account01.id, + admin01.id, + "case06", + { + status: TASK_STATUS.FINISHED, + job_number: "job06", + finished_at: new Date("2023-11-20T19:00:00Z"), + } + ); + await makeTestTask(source, account02.id, admin02.id, "case06", { + status: TASK_STATUS.FINISHED, + job_number: "job06", + finished_at: new Date("2023-11-20T19:00:00Z"), + }); + + { + // ファイル削除対象のタスク情報3件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task.id, + audio_file_id: file.id, + file_name: file.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task2.id, + audio_file_id: file2.id, + file_name: file2.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task3.id, + audio_file_id: file3.id, + file_name: file3.file_name, + account_id: account01.id, + country: account01.country, + }, + ]); + } + + // 1日後（削除対象外。本来はミリ秒単位の未来方向の時刻違いを想定）のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case07", { + status: TASK_STATUS.FINISHED, + job_number: "job07", + finished_at: new Date("2024-03-01T00:00:00Z"), + }); + await makeTestTask(source, account02.id, admin02.id, "case07", { + status: TASK_STATUS.FINISHED, + job_number: "job07", + finished_at: new Date("2024-03-01T00:00:00Z"), + }); + + { + // ファイル削除対象のタスク情報3件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task.id, + audio_file_id: file.id, + file_name: file.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task2.id, + audio_file_id: file2.id, + file_name: file2.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task3.id, + audio_file_id: file3.id, + file_name: file3.file_name, + account_id: account01.id, + country: account01.country, + }, + ]); + } + }); + + it("ファイル削除対象のタスク情報のみを取得できる(auto_file_delete=trueのアカウントの情報のタスク情報は全て取れる)", async () => { + if (!source) fail(); + + // 2024/02/29 00:00:00を"今"とする + const now = new Date("2024-02-29T00:00:00Z"); + + // auto_file_deleteがtrueののアカウントを作成 + const { account: account01, admin: admin01 } = await makeTestAccount( + source, + { + file_retention_days: 2, + auto_file_delete: true, + } + ); + const { account: account02, admin: admin02 } = await makeTestAccount( + source, + { + file_retention_days: 2, + auto_file_delete: true, + } + ); + // auto_file_deleteがfalseののアカウントを作成 + const { account: account03, admin: admin03 } = await makeTestAccount( + source, + { + file_retention_days: 2, + auto_file_delete: false, + } + ); + + // ちょうど2日前(削除対象外)のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case01", { + status: TASK_STATUS.FINISHED, + job_number: "job01", + finished_at: new Date("2024-02-27T00:00:00Z"), + }); + await makeTestTask(source, account02.id, admin02.id, "case01", { + status: TASK_STATUS.FINISHED, + job_number: "job01", + finished_at: new Date("2024-02-27T00:00:00Z"), + }); + await makeTestTask(source, account03.id, admin03.id, "case01", { + status: TASK_STATUS.FINISHED, + job_number: "job01", + finished_at: new Date("2024-02-27T00:00:00Z"), + }); + + // ちょうど1日前(削除対象外)のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case02", { + status: TASK_STATUS.FINISHED, + job_number: "job02", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + await makeTestTask(source, account02.id, admin02.id, "case02", { + status: TASK_STATUS.FINISHED, + job_number: "job02", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + await makeTestTask(source, account03.id, admin03.id, "case02", { + status: TASK_STATUS.FINISHED, + job_number: "job02", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + + // 2日と1秒前(削除対象)のタスクを作成 + const { task: task01, file: file01 } = await makeTestTask( + source, + account01.id, + admin01.id, + "case03", + { + status: TASK_STATUS.FINISHED, + job_number: "job03", + finished_at: new Date("2024-02-26T23:59:59Z"), + } + ); + const { task: task02, file: file02 } = await makeTestTask( + source, + account02.id, + admin02.id, + "case03", + { + status: TASK_STATUS.FINISHED, + job_number: "job03", + finished_at: new Date("2024-02-26T23:59:59Z"), + } + ); + await makeTestTask(source, account03.id, admin03.id, "case03", { + status: TASK_STATUS.FINISHED, + job_number: "job03", + finished_at: new Date("2024-02-26T23:59:59Z"), + }); + + { + // ファイル削除対象のタスク情報2件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task01.id, + audio_file_id: file01.id, + file_name: file01.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task02.id, + audio_file_id: file02.id, + file_name: file02.file_name, + account_id: account02.id, + country: account02.country, + }, + ]); + } + + // ちょうど3日前（削除対象）のタスクを作成 + const { task: task03, file: file03 } = await makeTestTask( + source, + account01.id, + admin01.id, + "case04", + { + status: TASK_STATUS.FINISHED, + job_number: "job04", + finished_at: new Date("2024-02-26T00:00:00Z"), + } + ); + const { task: task04, file: file04 } = await makeTestTask( + source, + account02.id, + admin02.id, + "case04", + { + status: TASK_STATUS.FINISHED, + job_number: "job04", + finished_at: new Date("2024-02-26T00:00:00Z"), + } + ); + await makeTestTask(source, account03.id, admin03.id, "case04", { + status: TASK_STATUS.FINISHED, + job_number: "job04", + finished_at: new Date("2024-02-26T00:00:00Z"), + }); + + { + // ファイル削除対象のタスク情報4件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task01.id, + audio_file_id: file01.id, + file_name: file01.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task02.id, + audio_file_id: file02.id, + file_name: file02.file_name, + account_id: account02.id, + country: account02.country, + }, + { + id: task03.id, + audio_file_id: file03.id, + file_name: file03.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task04.id, + audio_file_id: file04.id, + file_name: file04.file_name, + account_id: account02.id, + country: account02.country, + }, + ]); + } + + // ちょうど1日前(削除対象外)のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case05", { + status: TASK_STATUS.FINISHED, + job_number: "job05", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + await makeTestTask(source, account02.id, admin02.id, "case05", { + status: TASK_STATUS.FINISHED, + job_number: "job05", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + await makeTestTask(source, account03.id, admin03.id, "case05", { + status: TASK_STATUS.FINISHED, + job_number: "job05", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + + { + // ファイル削除対象のタスク情報4件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task01.id, + audio_file_id: file01.id, + file_name: file01.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task02.id, + audio_file_id: file02.id, + file_name: file02.file_name, + account_id: account02.id, + country: account02.country, + }, + { + id: task03.id, + audio_file_id: file03.id, + file_name: file03.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task04.id, + audio_file_id: file04.id, + file_name: file04.file_name, + account_id: account02.id, + country: account02.country, + }, + ]); + } + + // 100日と5時間前（削除対象）のタスクを作成 + const { task: task05, file: file05 } = await makeTestTask( + source, + account01.id, + admin01.id, + "case06", + { + status: TASK_STATUS.FINISHED, + job_number: "job06", + finished_at: new Date("2023-11-20T19:00:00Z"), + } + ); + const { task: task06, file: file06 } = await makeTestTask( + source, + account02.id, + admin02.id, + "case06", + { + status: TASK_STATUS.FINISHED, + job_number: "job06", + finished_at: new Date("2023-11-20T19:00:00Z"), + } + ); + await makeTestTask(source, account03.id, admin03.id, "case06", { + status: TASK_STATUS.FINISHED, + job_number: "job06", + finished_at: new Date("2023-11-20T19:00:00Z"), + }); + + { + // ファイル削除対象のタスク情報6件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task01.id, + audio_file_id: file01.id, + file_name: file01.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task02.id, + audio_file_id: file02.id, + file_name: file02.file_name, + account_id: account02.id, + country: account02.country, + }, + { + id: task03.id, + audio_file_id: file03.id, + file_name: file03.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task04.id, + audio_file_id: file04.id, + file_name: file04.file_name, + account_id: account02.id, + country: account02.country, + }, + { + id: task05.id, + audio_file_id: file05.id, + file_name: file05.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task06.id, + audio_file_id: file06.id, + file_name: file06.file_name, + account_id: account02.id, + country: account02.country, + }, + ]); + } + + // 1日後（削除対象外。本来はミリ秒単位の未来方向の時刻違いを想定）のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case07", { + status: TASK_STATUS.FINISHED, + job_number: "job07", + finished_at: new Date("2024-03-01T00:00:00Z"), + }); + await makeTestTask(source, account02.id, admin02.id, "case07", { + status: TASK_STATUS.FINISHED, + job_number: "job07", + finished_at: new Date("2024-03-01T00:00:00Z"), + }); + + { + // ファイル削除対象のタスク情報6件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task01.id, + audio_file_id: file01.id, + file_name: file01.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task02.id, + audio_file_id: file02.id, + file_name: file02.file_name, + account_id: account02.id, + country: account02.country, + }, + { + id: task03.id, + audio_file_id: file03.id, + file_name: file03.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task04.id, + audio_file_id: file04.id, + file_name: file04.file_name, + account_id: account02.id, + country: account02.country, + }, + { + id: task05.id, + audio_file_id: file05.id, + file_name: file05.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task06.id, + audio_file_id: file06.id, + file_name: file06.file_name, + account_id: account02.id, + country: account02.country, + }, + ]); + } + }); +}); + +describe("deleteRecords | 削除対象タスク等を削除できる", () => { + let source: DataSource | null = null; + beforeAll(async () => { + dotenv.config({ path: ".env" }); + dotenv.config({ path: ".env.local", override: true }); + if (source == null) { + source = await (async () => { + const s = new DataSource({ + type: "mysql", + host: "test_mysql_db", + port: 3306, + username: "user", + password: "password", + database: "odms", + entities: [ + User, + UserArchive, + Account, + AccountArchive, + Task, + AudioFile, + AudioOptionItem, + License, + LicenseArchive, + LicenseAllocationHistory, + LicenseAllocationHistoryArchive, + ], + synchronize: false, // trueにすると自動的にmigrationが行われるため注意 + logger: new TestLogger("none"), + logging: true, + }); + return await s.initialize(); + })(); + } + }); + + beforeEach(async () => { + if (source) { + await truncateAllTable(source); + } + }); + + afterAll(async () => { + await source?.destroy(); + source = null; + }); + + it("競合により存在しないタスクを削除しようとしても例外は発生せず、成功扱いとなる", async () => { + if (!source) fail(); + + await deleteRecords(source, [ + { + id: 1, + audio_file_id: 1, + file_name: "test", + account_id: 1, + country: "US", + }, + ]); + }); + + it("対象としたタスクやAudioFile等が削除される", async () => { + if (!source) fail(); + + // 2024/02/29 00:00:00を"今"とする + const now = new Date("2024-02-29T00:00:00Z"); + + // auto_file_deleteがtrueののアカウントを作成 + const { account: account01, admin: admin01 } = await makeTestAccount( + source, + { + file_retention_days: 2, + auto_file_delete: true, + } + ); + const { account: account02, admin: admin02 } = await makeTestAccount( + source, + { + file_retention_days: 2, + auto_file_delete: true, + } + ); + // auto_file_deleteがfalseののアカウントを作成 + const { account: account03, admin: admin03 } = await makeTestAccount( + source, + { + file_retention_days: 2, + auto_file_delete: false, + } + ); + + // ちょうど2日前(削除対象外)のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case01", { + status: TASK_STATUS.FINISHED, + job_number: "job01", + finished_at: new Date("2024-02-27T00:00:00Z"), + }); + await makeTestTask(source, account02.id, admin02.id, "case01", { + status: TASK_STATUS.FINISHED, + job_number: "job01", + finished_at: new Date("2024-02-27T00:00:00Z"), + }); + await makeTestTask(source, account03.id, admin03.id, "case01", { + status: TASK_STATUS.FINISHED, + job_number: "job01", + finished_at: new Date("2024-02-27T00:00:00Z"), + }); + + // ちょうど1日前(削除対象外)のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case02", { + status: TASK_STATUS.FINISHED, + job_number: "job02", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + await makeTestTask(source, account02.id, admin02.id, "case02", { + status: TASK_STATUS.FINISHED, + job_number: "job02", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + await makeTestTask(source, account03.id, admin03.id, "case02", { + status: TASK_STATUS.FINISHED, + job_number: "job02", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + + // 2日と1秒前(削除対象)のタスクを作成 + const { task: task01, file: file01 } = await makeTestTask( + source, + account01.id, + admin01.id, + "case03", + { + status: TASK_STATUS.FINISHED, + job_number: "job03", + finished_at: new Date("2024-02-26T23:59:59Z"), + } + ); + const { task: task02, file: file02 } = await makeTestTask( + source, + account02.id, + admin02.id, + "case03", + { + status: TASK_STATUS.FINISHED, + job_number: "job03", + finished_at: new Date("2024-02-26T23:59:59Z"), + } + ); + await makeTestTask(source, account03.id, admin03.id, "case03", { + status: TASK_STATUS.FINISHED, + job_number: "job03", + finished_at: new Date("2024-02-26T23:59:59Z"), + }); + + // ちょうど3日前（削除対象）のタスクを作成 + const { task: task03, file: file03 } = await makeTestTask( + source, + account01.id, + admin01.id, + "case04", + { + status: TASK_STATUS.FINISHED, + job_number: "job04", + finished_at: new Date("2024-02-26T00:00:00Z"), + } + ); + const { task: task04, file: file04 } = await makeTestTask( + source, + account02.id, + admin02.id, + "case04", + { + status: TASK_STATUS.FINISHED, + job_number: "job04", + finished_at: new Date("2024-02-26T00:00:00Z"), + } + ); + await makeTestTask(source, account03.id, admin03.id, "case04", { + status: TASK_STATUS.FINISHED, + job_number: "job04", + finished_at: new Date("2024-02-26T00:00:00Z"), + }); + + // ちょうど1日前(削除対象外)のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case05", { + status: TASK_STATUS.FINISHED, + job_number: "job05", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + await makeTestTask(source, account02.id, admin02.id, "case05", { + status: TASK_STATUS.FINISHED, + job_number: "job05", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + await makeTestTask(source, account03.id, admin03.id, "case05", { + status: TASK_STATUS.FINISHED, + job_number: "job05", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + + // 100日と5時間前（削除対象）のタスクを作成 + const { task: task05, file: file05 } = await makeTestTask( + source, + account01.id, + admin01.id, + "case06", + { + status: TASK_STATUS.FINISHED, + job_number: "job06", + finished_at: new Date("2023-11-20T19:00:00Z"), + } + ); + const { task: task06, file: file06 } = await makeTestTask( + source, + account02.id, + admin02.id, + "case06", + { + status: TASK_STATUS.FINISHED, + job_number: "job06", + finished_at: new Date("2023-11-20T19:00:00Z"), + } + ); + await makeTestTask(source, account03.id, admin03.id, "case06", { + status: TASK_STATUS.FINISHED, + job_number: "job06", + finished_at: new Date("2023-11-20T19:00:00Z"), + }); + + // 1日後（削除対象外。本来はミリ秒単位の未来方向の時刻違いを想定）のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case07", { + status: TASK_STATUS.FINISHED, + job_number: "job07", + finished_at: new Date("2024-03-01T00:00:00Z"), + }); + await makeTestTask(source, account02.id, admin02.id, "case07", { + status: TASK_STATUS.FINISHED, + job_number: "job07", + finished_at: new Date("2024-03-01T00:00:00Z"), + }); + + { + // ファイル削除対象のタスク情報6件のみを取得できる + const result = await getProcessTargets(source, now); + result.sort((a, b) => a.id - b.id); + expect(result).toEqual([ + { + id: task01.id, + audio_file_id: file01.id, + file_name: file01.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task02.id, + audio_file_id: file02.id, + file_name: file02.file_name, + account_id: account02.id, + country: account02.country, + }, + { + id: task03.id, + audio_file_id: file03.id, + file_name: file03.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task04.id, + audio_file_id: file04.id, + file_name: file04.file_name, + account_id: account02.id, + country: account02.country, + }, + { + id: task05.id, + audio_file_id: file05.id, + file_name: file05.file_name, + account_id: account01.id, + country: account01.country, + }, + { + id: task06.id, + audio_file_id: file06.id, + file_name: file06.file_name, + account_id: account02.id, + country: account02.country, + }, + ]); + + { + // DB全体のレコードを確認 + const tasks = await getTasks(source); + expect(tasks.length).toEqual(20); + const files = await getAudioFiles(source); + expect(files.length).toEqual(20); + const optionItems = await getAudioOptionItems(source); + expect(optionItems.length).toEqual(40); + } + + // 削除対象のタスク情報を削除 + await deleteRecords(source, result); + + // 削除後のタスク情報を取得 + const result2 = await getProcessTargets(source, now); + // 削除対象のタスク情報が削除されているので取得が0件になる + expect(result2).toEqual([]); + + { + // DB全体のレコードを確認 + // 削除対象外のタスク情報のみが残っている + const tasks = await getTasks(source); + expect(tasks.length).toEqual(14); + const files = await getAudioFiles(source); + expect(files.length).toEqual(14); + const optionItems = await getAudioOptionItems(source); + expect(optionItems.length).toEqual(28); + } + } + }); + + it("対象としたタスクやAudioFile等が大量に存在しても削除される", async () => { + if (!source) fail(); + // 2024/02/29 00:00:00を"今"とする + const now = new Date("2024-02-29T00:00:00Z"); + + // "大量"の数を定義 + const count = 10000; + + // auto_file_deleteがtrueののアカウントを作成 + const { account, admin } = await makeTestAccount(source, { + file_retention_days: 30, + auto_file_delete: true, + }); + + // ファイルを10000件作成 + const createdFiles = [...Array(count).keys()].map( + (index): QueryDeepPartialEntity => { + return { + account_id: account.id, + owner_user_id: admin.id, + url: `https://example.com/${index}`, + file_name: `test${index}.wav`, + author_id: "test_author", + work_type_id: "test_work_type", + started_at: new Date(), + duration: "00:00:00", + finished_at: new Date(), + uploaded_at: new Date(), + file_size: 1024, + priority: "01", + audio_format: "wav", + comment: `test_comment_${index}`, + deleted_at: new Date(), + is_encrypted: false, + }; + } + ); + + // ファイルを元に、10年前に完了した扱いのタスクを作成 + const finished_at = new Date("2014-02-26T23:59:59Z"); + await makeManyTestTasks(source, createdFiles, finished_at); + + { + // DB全体のレコードを確認 + const tasks = await getTasks(source); + expect(tasks.length).toEqual(count); + + const files = await getAudioFiles(source); + expect(files.length).toEqual(count); + + const optionItems = await getAudioOptionItems(source); + expect(optionItems.length).toEqual(2 * count); + } + + const result = await getProcessTargets(source, now); + expect(result.length).toEqual(count); + + await deleteRecords(source, result); + + { + // 削除後のタスク情報を取得 + const tasks = await getTasks(source); + expect(tasks.length).toEqual(0); + + const files = await getAudioFiles(source); + expect(files.length).toEqual(0); + + const optionItems = await getAudioOptionItems(source); + expect(optionItems.length).toEqual(0); + } + }, 100000); +}); + +describe("deleteAudioFilesProcessing", () => { + let source: DataSource | null = null; + beforeAll(async () => { + dotenv.config({ path: ".env" }); + dotenv.config({ path: ".env.test", override: true }); + if (source == null) { + source = await (async () => { + const s = new DataSource({ + type: "mysql", + host: "test_mysql_db", + port: 3306, + username: "user", + password: "password", + database: "odms", + entities: [__dirname + "/../../**/*.entity{.ts,.js}"], + synchronize: false, // trueにすると自動的にmigrationが行われるため注意 + logger: new TestLogger("none"), + logging: true, + }); + return await s.initialize(); + })(); + } + }); + + beforeEach(async () => { + if (source) { + await truncateAllTable(source); + } + }); + + afterAll(async () => { + await source?.destroy(); + source = null; + }); + + it("BlobとDBの削除が正常に行われる", async () => { + if (!source) fail(); + + // 2024/02/29 00:00:00を"今"とする + const now = new Date("2024-02-29T00:00:00Z"); + + const { account: account01, admin: admin01 } = await makeTestAccount( + source, + { + file_retention_days: 2, + auto_file_delete: true, + } + ); + + const { account: account02, admin: admin02 } = await makeTestAccount( + source, + { + file_retention_days: 2, + auto_file_delete: true, + country: "JP", + } + ); + + // ちょうど2日前(削除対象外)のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case01", { + status: TASK_STATUS.FINISHED, + job_number: "job01", + finished_at: new Date("2024-02-27T00:00:00Z"), + }); + // ちょうど1日前(削除対象外)のタスクを作成 + await makeTestTask(source, account01.id, admin01.id, "case02", { + status: TASK_STATUS.FINISHED, + job_number: "job02", + finished_at: new Date("2024-02-28T00:00:00Z"), + }); + // 2日と1秒前(削除対象)のタスクを作成 + const { file: file1 } = await makeTestTask( + source, + account01.id, + admin01.id, + "case03", + { + status: TASK_STATUS.FINISHED, + job_number: "job03", + finished_at: new Date("2024-02-26T23:59:59Z"), + } + ); + // 2日と1秒前(削除対象)のタスクを作成 + const { file: file2 } = await makeTestTask( + source, + account01.id, + admin01.id, + "case04", + { + status: TASK_STATUS.FINISHED, + job_number: "job04", + finished_at: new Date("2024-02-26T23:59:59Z"), + } + ); + + // 2日と1秒前(削除対象)のタスクを作成 + const { file: file3 } = await makeTestTask( + source, + account02.id, + admin02.id, + "case05", + { + status: TASK_STATUS.FINISHED, + job_number: "job05", + finished_at: new Date("2024-02-26T23:59:59Z"), + } + ); + + const args: { accountId: number; fileName: string; country: string }[] = []; + const blobstorage = new AudioBlobStorageService(); + Object.defineProperty(blobstorage, blobstorage.deleteFile.name, { + value: async ( + context: InvocationContext, + accountId: number, + country: string, + fileName: string + ): Promise => { + args.push({ accountId, country, fileName }); + }, + writable: true, + }); + + { + // DB全体のレコードを確認 + const tasks = await getTasks(source); + expect(tasks.length).toEqual(5); + const files = await getAudioFiles(source); + expect(files.length).toEqual(5); + const optionItems = await getAudioOptionItems(source); + expect(optionItems.length).toEqual(10); + } + + const context = new InvocationContext(); + await deleteAudioFilesProcessing(context, source, blobstorage, now); + + // 想定通りの呼び出しが行われているか + { + const { accountId, country, fileName } = args[0]; + expect(fileName).toEqual(file1.file_name); + expect(accountId).toEqual(account01.id); + expect(country).toEqual(account01.country); + } + { + const { accountId, country, fileName } = args[1]; + expect(fileName).toEqual(file2.file_name); + expect(accountId).toEqual(account01.id); + expect(country).toEqual(account01.country); + } + { + const { accountId, country, fileName } = args[2]; + expect(fileName).toEqual(file3.file_name); + expect(accountId).toEqual(account02.id); + expect(country).toEqual(account02.country); + } + + { + // DB全体のレコードを確認 + const tasks = await getTasks(source); + expect(tasks.length).toEqual(2); + const files = await getAudioFiles(source); + expect(files.length).toEqual(2); + const optionItems = await getAudioOptionItems(source); + expect(optionItems.length).toEqual(4); + } + }); +}); diff --git a/dictation_function/src/test/importUsers.spec.ts b/dictation_function/src/test/importUsers.spec.ts index 9c07fb7..d35bb73 100644 --- a/dictation_function/src/test/importUsers.spec.ts +++ b/dictation_function/src/test/importUsers.spec.ts @@ -11,7 +11,7 @@ import { AxiosRequestConfig, AxiosResponse } from "axios"; describe("importUsersProcessing", () => { dotenv.config({ path: ".env" }); - dotenv.config({ path: ".env.local", override: true }); + dotenv.config({ path: ".env.test", override: true }); it("stage.jsonがない状態でユーザー追加できること", async () => { const context = new InvocationContext(); diff --git a/dictation_function/src/test/licenseAlert.spec.ts b/dictation_function/src/test/licenseAlert.spec.ts index c3891a0..58548db 100644 --- a/dictation_function/src/test/licenseAlert.spec.ts +++ b/dictation_function/src/test/licenseAlert.spec.ts @@ -18,7 +18,7 @@ import { promisify } from "util"; describe("licenseAlert", () => { dotenv.config({ path: ".env" }); - dotenv.config({ path: ".env.local", override: true }); + dotenv.config({ path: ".env.test", override: true }); let source: DataSource | null = null; const redisClient = createClient(); @@ -80,7 +80,7 @@ describe("licenseAlert", () => { // redisからキャッシュが削除されていることを確認 const getAsync = promisify(redisClient.keys).bind(redisClient); const keys = await getAsync(`*`); - expect(keys).toHaveLength(0); + expect(keys).toHaveLength(0); }); it("ライセンス在庫不足メール、ライセンス失効警告メールが送信されること", async () => { diff --git a/dictation_function/src/test/licenseAutoAllocation.spec.ts b/dictation_function/src/test/licenseAutoAllocation.spec.ts index d0bf9ef..5f1261b 100644 --- a/dictation_function/src/test/licenseAutoAllocation.spec.ts +++ b/dictation_function/src/test/licenseAutoAllocation.spec.ts @@ -18,7 +18,7 @@ import { InvocationContext } from "@azure/functions"; describe("licenseAutoAllocation", () => { dotenv.config({ path: ".env" }); - dotenv.config({ path: ".env.local", override: true }); + dotenv.config({ path: ".env.test", override: true }); let source: DataSource | null = null; beforeEach(async () => { source = new DataSource({