diff --git a/src/database/migrations/08-OptimizeDBIndices.ts b/src/database/migrations/08-OptimizeDBIndices.ts new file mode 100644 index 0000000..cc89c97 --- /dev/null +++ b/src/database/migrations/08-OptimizeDBIndices.ts @@ -0,0 +1,72 @@ +import { MigrationInterface, QueryRunner, TableIndex } from "typeorm"; + +export class OptimizeDBIndices0000000000008 implements MigrationInterface { + async up(queryRunner: QueryRunner): Promise { + await queryRunner.createIndices("artist", [ + new TableIndex({ + // This index helps with the "update artist" job + name: "IDX_ARTIST_UPDATED_AT", + columnNames: ["updatedAt"], + }), + ]); + + await queryRunner.createIndices("listen", [ + new TableIndex({ + // This index helps with the "getCrawlableUserInfo" query + name: "IDX_LISTEN_USER_ID_PLAYED_AT", + columnNames: ["userId", "playedAt"], + }), + ]); + + // handled by Primary Key on (albumId, artistId) + await queryRunner.dropIndex("album_artist", "IDX_ALBUM_ARTISTS_ALBUM_ID"); + + // handled by Primary Key on (artistId, genreId) + await queryRunner.dropIndex("artist_genres", "IDX_ARTIST_GENRES_ARTIST_ID"); + + // handled by IDX_LISTEN_UNIQUE on (trackId, userId, playedAt) + await queryRunner.dropIndex("listen", "IDX_LISTEN_TRACK_ID"); + // handled by IDX_LISTEN_USER_ID_PLAYED_AT on (userId, playedAt) + await queryRunner.dropIndex("listen", "IDX_LISTEN_USER_ID"); + + // handled by Primary Key on (trackId, artistId) + await queryRunner.dropIndex("track_artists", "IDX_TRACK_ARTISTS_TRACK_ID"); + } + + async down(queryRunner: QueryRunner): Promise { + await queryRunner.createIndices("album_artist", [ + new TableIndex({ + name: "IDX_ALBUM_ARTISTS_ALBUM_ID", + columnNames: ["albumId"], + }), + ]); + + await queryRunner.createIndices("artist_genres", [ + new TableIndex({ + name: "IDX_ARTIST_GENRES_ARTIST_ID", + columnNames: ["artistId"], + }), + ]); + + await queryRunner.createIndices("listen", [ + new TableIndex({ + name: "IDX_LISTEN_TRACK_ID", + columnNames: ["trackId"], + }), + new TableIndex({ + name: "IDX_LISTEN_USER_ID", + columnNames: ["userId"], + }), + ]); + + await queryRunner.createIndices("track_artists", [ + new TableIndex({ + name: "IDX_TRACK_ARTISTS_TRACK_ID", + columnNames: ["trackId"], + }), + ]); + + await queryRunner.dropIndex("artist", "IDX_ARTIST_UPDATED_AT"); + await queryRunner.dropIndex("listen", "IDX_LISTEN_USER_ID_PLAYED_AT"); + } +} diff --git a/src/listens/listens.service.ts b/src/listens/listens.service.ts index a3906c3..fea7e90 100644 --- a/src/listens/listens.service.ts +++ b/src/listens/listens.service.ts @@ -77,16 +77,6 @@ export class ListensService { }); } - async getMostRecentListenPerUser(): Promise { - return this.listenRepository - .createQueryBuilder("listen") - .leftJoinAndSelect("listen.user", "user") - .distinctOn(["user.id"]) - .orderBy({ "user.id": "ASC", "listen.playedAt": "DESC" }) - .limit(1) - .getMany(); - } - getScopedQueryBuilder(): ListenScopes { return this.listenRepository.scoped; } diff --git a/src/sources/scheduler.service.ts b/src/sources/scheduler.service.ts index d1cea24..041395b 100644 --- a/src/sources/scheduler.service.ts +++ b/src/sources/scheduler.service.ts @@ -53,7 +53,7 @@ export class SchedulerService implements OnApplicationBootstrap { const INACTIVE_CUTOFF_MSEC = 60 * 60 * 1000; await Promise.all( - userInfo.map(({ user, lastListen }) => { + userInfo.map(({ userID, lastListen }) => { let pollRate = POLL_RATE_INACTIVE_SEC; const timeSinceLastListen = new Date().getTime() - lastListen.getTime(); @@ -62,10 +62,10 @@ export class SchedulerService implements OnApplicationBootstrap { } this.importSpotifyJobService.sendThrottled( - { userID: user.id }, + { userID }, {}, pollRate, - user.id, + userID, ); }), ); diff --git a/src/sources/spotify/spotify.service.ts b/src/sources/spotify/spotify.service.ts index a73e317..dbb309b 100644 --- a/src/sources/spotify/spotify.service.ts +++ b/src/sources/spotify/spotify.service.ts @@ -43,27 +43,31 @@ export class SpotifyService { ) {} @Span() - async getCrawlableUserInfo(): Promise<{ user: User; lastListen: Date }[]> { - // All of this is kinda inefficient, we do two db queries and join in code, - // i can't be bothered to do this properly in the db for now. - // Should be refactored if listory gets hundreds of users (lol). + async getCrawlableUserInfo(): Promise< + { userID: string; lastListen: Date }[] + > { + const users = await this.usersService + .getQueryBuilder() + .select(`user.id as "userID"`) + .addSelect(`listen."playedAt" as "lastListen"`) + .leftJoin( + (qb) => + qb + .distinctOn(["listen.userId"]) + .select(`listen."userId"`) + .addSelect(`listen."playedAt"`) + .from("listen", "listen") + .orderBy("listen.userId", "DESC") + .addOrderBy("listen.playedAt", "DESC"), + "listen", + `listen."userId" = user.id`, + ) + .getRawMany<{ userID: string; lastListen?: Date }>(); - const [users, listens] = await Promise.all([ - this.usersService.findAll(), - this.listensService.getMostRecentListenPerUser(), - ]); - - return users.map((user) => { - const lastListen = listens.find((listen) => listen.user.id === user.id); - - return { - user, - // Return 1970 if no listen exists - lastListen: lastListen ? lastListen.playedAt : new Date(0), - }; - }); - - return; + return users.map(({ userID, lastListen }) => ({ + userID, + lastListen: lastListen || new Date(0), + })); } @ImportSpotifyJob.Handle() diff --git a/src/users/users.service.ts b/src/users/users.service.ts index 364ff7f..7af4263 100644 --- a/src/users/users.service.ts +++ b/src/users/users.service.ts @@ -1,3 +1,4 @@ +import { SelectQueryBuilder } from "typeorm"; import { JobService } from "@apricote/nest-pg-boss"; import { Injectable, NotFoundException } from "@nestjs/common"; import { IImportSpotifyJob, ImportSpotifyJob } from "../sources/jobs"; @@ -65,4 +66,8 @@ export class UsersService { user.spotify = spotify; await this.userRepository.save(user); } + + getQueryBuilder(): SelectQueryBuilder { + return this.userRepository.createQueryBuilder("user"); + } }