diff --git a/package-lock.json b/package-lock.json index e2c26b4..72b0dc0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.22.0", "license": "MIT", "dependencies": { + "@apricote/nest-pg-boss": "1.0.0", "@hapi/joi": "17.1.1", "@narando/nest-axios-interceptor": "2.2.0", "@nestjs/axios": "0.1.0", @@ -18,7 +19,6 @@ "@nestjs/jwt": "10.0.2", "@nestjs/passport": "9.0.3", "@nestjs/platform-express": "9.3.9", - "@nestjs/schedule": "2.2.0", "@nestjs/serve-static": "3.0.1", "@nestjs/swagger": "6.2.1", "@nestjs/terminus": "9.2.1", @@ -356,6 +356,18 @@ "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==", "dev": true }, + "node_modules/@apricote/nest-pg-boss": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@apricote/nest-pg-boss/-/nest-pg-boss-1.0.0.tgz", + "integrity": "sha512-jYe9Bj/qBUxEWfnzKly6TTyg+x3A5uuQ1ocP0SXRyXHgtdXKUIf4AF4OZdgcX2qR9yrHNdi3lcoj9P2Qk+vvbw==", + "peerDependencies": { + "@nestjs/common": "^9.3.3", + "@nestjs/core": "^9.3.3", + "pg-boss": "^8.4.1", + "reflect-metadata": "^0.1.13", + "rxjs": "^7.2.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.18.6", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", @@ -2208,20 +2220,6 @@ "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz", "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg==" }, - "node_modules/@nestjs/schedule": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-2.2.0.tgz", - "integrity": "sha512-wrDnUONTxBkD6lTWh9ecYk/kvJTbA3PylotjBoRsECmcS1SNvgInFXuL38UnHiFnXM3CHSFnzRLB259Bc1mOdQ==", - "dependencies": { - "cron": "2.2.0", - "uuid": "9.0.0" - }, - "peerDependencies": { - "@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0", - "@nestjs/core": "^7.0.0 || ^8.0.0 || ^9.0.0", - "reflect-metadata": "^0.1.12" - } - }, "node_modules/@nestjs/schematics": { "version": "9.0.4", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-9.0.4.tgz", @@ -4511,6 +4509,19 @@ "node": ">= 6.0.0" } }, + "node_modules/aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "peer": true, + "dependencies": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/ajv": { "version": "8.11.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", @@ -5358,6 +5369,15 @@ "validator": "^13.7.0" } }, + "node_modules/clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "peer": true, + "engines": { + "node": ">=6" + } + }, "node_modules/cli-boxes": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz", @@ -5695,12 +5715,16 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "devOptional": true }, - "node_modules/cron": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cron/-/cron-2.2.0.tgz", - "integrity": "sha512-GPiI3OgMv83XRtEUc2gUdaLvJhO3XbLN288layOBkDTupg0RK5IECNGpkykIMHg+muVR2bxt29b0xvCAcBrjYQ==", + "node_modules/cron-parser": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.8.1.tgz", + "integrity": "sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==", + "peer": true, "dependencies": { "luxon": "^3.2.1" + }, + "engines": { + "node": ">=12.0.0" } }, "node_modules/cross-spawn": { @@ -5840,6 +5864,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/delay": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz", + "integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==", + "peer": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -7784,6 +7820,15 @@ "node": ">=0.8.19" } }, + "node_modules/indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "peer": true, + "engines": { + "node": ">=8" + } + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -9417,6 +9462,12 @@ "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" }, + "node_modules/lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", + "peer": true + }, "node_modules/lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -9503,6 +9554,7 @@ "version": "3.3.0", "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz", "integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==", + "peer": true, "engines": { "node": ">=12" } @@ -10184,6 +10236,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "peer": true, + "dependencies": { + "aggregate-error": "^3.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -10459,6 +10526,24 @@ } } }, + "node_modules/pg-boss": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/pg-boss/-/pg-boss-8.4.1.tgz", + "integrity": "sha512-8Zqy4h4Ib/l21lEC21uVmCdZnP5qjqwVolDAP2SBqEDaAdkOaWVm3VjyJ+a/GKDtjqpTvyn/jY/7IeBeAM++Jg==", + "peer": true, + "dependencies": { + "cron-parser": "^4.0.0", + "delay": "^5.0.0", + "lodash.debounce": "^4.0.8", + "p-map": "^4.0.0", + "pg": "^8.5.1", + "serialize-error": "^8.1.0", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=14" + } + }, "node_modules/pg-connection-string": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", @@ -11515,6 +11600,21 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, + "node_modules/serialize-error": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", + "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", + "peer": true, + "dependencies": { + "type-fest": "^0.20.2" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/serialize-javascript": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", @@ -13528,6 +13628,12 @@ } } }, + "@apricote/nest-pg-boss": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@apricote/nest-pg-boss/-/nest-pg-boss-1.0.0.tgz", + "integrity": "sha512-jYe9Bj/qBUxEWfnzKly6TTyg+x3A5uuQ1ocP0SXRyXHgtdXKUIf4AF4OZdgcX2qR9yrHNdi3lcoj9P2Qk+vvbw==", + "requires": {} + }, "@babel/code-frame": { "version": "7.18.6", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz", @@ -14889,15 +14995,6 @@ } } }, - "@nestjs/schedule": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-2.2.0.tgz", - "integrity": "sha512-wrDnUONTxBkD6lTWh9ecYk/kvJTbA3PylotjBoRsECmcS1SNvgInFXuL38UnHiFnXM3CHSFnzRLB259Bc1mOdQ==", - "requires": { - "cron": "2.2.0", - "uuid": "9.0.0" - } - }, "@nestjs/schematics": { "version": "9.0.4", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-9.0.4.tgz", @@ -16600,6 +16697,16 @@ "debug": "4" } }, + "aggregate-error": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz", + "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==", + "peer": true, + "requires": { + "clean-stack": "^2.0.0", + "indent-string": "^4.0.0" + } + }, "ajv": { "version": "8.11.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz", @@ -17220,6 +17327,12 @@ "validator": "^13.7.0" } }, + "clean-stack": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz", + "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==", + "peer": true + }, "cli-boxes": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz", @@ -17477,10 +17590,11 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "devOptional": true }, - "cron": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/cron/-/cron-2.2.0.tgz", - "integrity": "sha512-GPiI3OgMv83XRtEUc2gUdaLvJhO3XbLN288layOBkDTupg0RK5IECNGpkykIMHg+muVR2bxt29b0xvCAcBrjYQ==", + "cron-parser": { + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.8.1.tgz", + "integrity": "sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==", + "peer": true, "requires": { "luxon": "^3.2.1" } @@ -17591,6 +17705,12 @@ "object-keys": "^1.1.1" } }, + "delay": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz", + "integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==", + "peer": true + }, "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -19050,6 +19170,12 @@ "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==", "dev": true }, + "indent-string": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz", + "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==", + "peer": true + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -20279,6 +20405,12 @@ "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==" }, + "lodash.debounce": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz", + "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==", + "peer": true + }, "lodash.memoize": { "version": "4.1.2", "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz", @@ -20348,7 +20480,8 @@ "luxon": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz", - "integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==" + "integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==", + "peer": true }, "macos-release": { "version": "2.5.0", @@ -20841,6 +20974,15 @@ "p-limit": "^3.0.2" } }, + "p-map": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", + "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==", + "peer": true, + "requires": { + "aggregate-error": "^3.0.0" + } + }, "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -21040,6 +21182,21 @@ "pgpass": "1.x" } }, + "pg-boss": { + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/pg-boss/-/pg-boss-8.4.1.tgz", + "integrity": "sha512-8Zqy4h4Ib/l21lEC21uVmCdZnP5qjqwVolDAP2SBqEDaAdkOaWVm3VjyJ+a/GKDtjqpTvyn/jY/7IeBeAM++Jg==", + "peer": true, + "requires": { + "cron-parser": "^4.0.0", + "delay": "^5.0.0", + "lodash.debounce": "^4.0.8", + "p-map": "^4.0.0", + "pg": "^8.5.1", + "serialize-error": "^8.1.0", + "uuid": "^9.0.0" + } + }, "pg-connection-string": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz", @@ -21817,6 +21974,15 @@ } } }, + "serialize-error": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz", + "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==", + "peer": true, + "requires": { + "type-fest": "^0.20.2" + } + }, "serialize-javascript": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz", diff --git a/package.json b/package.json index 0d99c61..7c86c09 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "test:e2e": "jest --config ./apps/listory/test/jest-e2e.json" }, "dependencies": { + "@apricote/nest-pg-boss": "1.0.0", "@hapi/joi": "17.1.1", "@narando/nest-axios-interceptor": "2.2.0", "@nestjs/axios": "0.1.0", @@ -35,7 +36,6 @@ "@nestjs/jwt": "10.0.2", "@nestjs/passport": "9.0.3", "@nestjs/platform-express": "9.3.9", - "@nestjs/schedule": "2.2.0", "@nestjs/serve-static": "3.0.1", "@nestjs/swagger": "6.2.1", "@nestjs/terminus": "9.2.1", diff --git a/src/app.module.ts b/src/app.module.ts index a4facf4..a38baef 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -1,5 +1,4 @@ import { Module } from "@nestjs/common"; -import { ScheduleModule } from "@nestjs/schedule"; import { ServeStaticModule } from "@nestjs/serve-static"; import { RavenModule } from "nest-raven"; import { join } from "path"; @@ -14,13 +13,14 @@ import { ReportsModule } from "./reports/reports.module"; import { SourcesModule } from "./sources/sources.module"; import { UsersModule } from "./users/users.module"; import { OpenTelemetryModule } from "./open-telemetry/open-telemetry.module"; +import { JobQueueModule } from "./job-queue/job-queue.module"; @Module({ imports: [ LoggerModule, ConfigModule, DatabaseModule, - ScheduleModule.forRoot(), + JobQueueModule, ServeStaticModule.forRoot({ rootPath: join(__dirname, "..", "static"), exclude: ["/api*"], diff --git a/src/job-queue/job-queue.module.ts b/src/job-queue/job-queue.module.ts new file mode 100644 index 0000000..5d4aece --- /dev/null +++ b/src/job-queue/job-queue.module.ts @@ -0,0 +1,23 @@ +import { Module } from "@nestjs/common"; +import { PGBossModule } from "@apricote/nest-pg-boss"; +import { ConfigService } from "@nestjs/config"; + +@Module({ + imports: [ + PGBossModule.forRootAsync({ + application_name: "listory", + useFactory: (config: ConfigService) => ({ + // Connection details + host: config.get("DB_HOST"), + user: config.get("DB_USERNAME"), + password: config.get("DB_PASSWORD"), + database: config.get("DB_DATABASE"), + schema: "public", + max: config.get("DB_POOL_MAX"), + }), + inject: [ConfigService], + }), + ], + exports: [PGBossModule], +}) +export class JobQueueModule {} diff --git a/src/sources/jobs.ts b/src/sources/jobs.ts new file mode 100644 index 0000000..bd40d47 --- /dev/null +++ b/src/sources/jobs.ts @@ -0,0 +1,14 @@ +import { createJob } from "@apricote/nest-pg-boss"; + +export type ICrawlerSupervisorJob = {}; +export const CrawlerSupervisorJob = createJob( + "spotify-crawler-supervisor" +); + +export type IUpdateSpotifyLibraryJob = {}; +export const UpdateSpotifyLibraryJob = createJob( + "update-spotify-library" +); + +export type IImportSpotifyJob = { userID: string }; +export const ImportSpotifyJob = createJob("import-spotify"); diff --git a/src/sources/scheduler.service.ts b/src/sources/scheduler.service.ts index 1295c2f..51ed6a8 100644 --- a/src/sources/scheduler.service.ts +++ b/src/sources/scheduler.service.ts @@ -1,8 +1,15 @@ import { Injectable, Logger, OnApplicationBootstrap } from "@nestjs/common"; import { ConfigService } from "@nestjs/config"; -import { SchedulerRegistry } from "@nestjs/schedule"; -import { captureException } from "@sentry/node"; import { SpotifyService } from "./spotify/spotify.service"; +import { + CrawlerSupervisorJob, + ICrawlerSupervisorJob, + IImportSpotifyJob, + ImportSpotifyJob, + IUpdateSpotifyLibraryJob, + UpdateSpotifyLibraryJob, +} from "./jobs"; +import { JobService } from "@apricote/nest-pg-boss"; @Injectable() export class SchedulerService implements OnApplicationBootstrap { @@ -10,38 +17,42 @@ export class SchedulerService implements OnApplicationBootstrap { constructor( private readonly config: ConfigService, - private readonly registry: SchedulerRegistry, - private readonly spotifyService: SpotifyService + private readonly spotifyService: SpotifyService, + @CrawlerSupervisorJob.Inject() + private readonly superviseImportJobsJobService: JobService, + @ImportSpotifyJob.Inject() + private readonly importSpotifyJobService: JobService, + @UpdateSpotifyLibraryJob.Inject() + private readonly updateSpotifyLibraryJobService: JobService ) {} - onApplicationBootstrap() { - this.setupSpotifyCrawler(); - this.setupSpotifyMusicLibraryUpdater(); + async onApplicationBootstrap() { + await this.setupSpotifyCrawlerSupervisor(); + await this.setupSpotifyMusicLibraryUpdater(); } - private setupSpotifyCrawler() { - const callback = () => - this.spotifyService.runCrawlerForAllUsers().catch((err) => { - captureException(err); - this.logger.error(`Spotify crawler loop crashed! ${err.stack}`); - }); - const timeoutMs = - this.config.get("SPOTIFY_FETCH_INTERVAL_SEC") * 1000; - - const interval = setInterval(callback, timeoutMs); - - this.registry.addInterval("crawler_spotify", interval); + private async setupSpotifyCrawlerSupervisor(): Promise { + await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {}); } - private setupSpotifyMusicLibraryUpdater() { - const callback = () => { - this.spotifyService.runUpdaterForAllEntities(); - }; - const timeoutMs = - this.config.get("SPOTIFY_UPDATE_INTERVAL_SEC") * 1000; + @CrawlerSupervisorJob.Handle() + async superviseImportJobs(): Promise { + this.logger.log("Starting crawler jobs"); + const users = await this.spotifyService.getCrawlableUserInfo(); - const interval = setInterval(callback, timeoutMs); + await Promise.all( + users.map((user) => + this.importSpotifyJobService.sendOnce({ userID: user.id }, {}, user.id) + ) + ); + } - this.registry.addInterval("updater_spotify", interval); + private async setupSpotifyMusicLibraryUpdater() { + await this.updateSpotifyLibraryJobService.schedule("*/1 * * * *", {}, {}); + } + + @UpdateSpotifyLibraryJob.Handle() + async updateSpotifyLibrary() { + this.spotifyService.runUpdaterForAllEntities(); } } diff --git a/src/sources/sources.module.ts b/src/sources/sources.module.ts index 8f267d1..6512a7c 100644 --- a/src/sources/sources.module.ts +++ b/src/sources/sources.module.ts @@ -1,9 +1,22 @@ import { Module } from "@nestjs/common"; +import { PGBossModule } from "@apricote/nest-pg-boss"; +import { + CrawlerSupervisorJob, + ImportSpotifyJob, + UpdateSpotifyLibraryJob, +} from "./jobs"; import { SchedulerService } from "./scheduler.service"; import { SpotifyModule } from "./spotify/spotify.module"; @Module({ - imports: [SpotifyModule], + imports: [ + SpotifyModule, + PGBossModule.forJobs([ + CrawlerSupervisorJob, + ImportSpotifyJob, + UpdateSpotifyLibraryJob, + ]), + ], providers: [SchedulerService], }) export class SourcesModule {} diff --git a/src/sources/spotify/spotify.module.ts b/src/sources/spotify/spotify.module.ts index 00dbf7a..1df6bbf 100644 --- a/src/sources/spotify/spotify.module.ts +++ b/src/sources/spotify/spotify.module.ts @@ -1,13 +1,16 @@ +import { PGBossModule } from "@apricote/nest-pg-boss"; import { Module } from "@nestjs/common"; import { ListensModule } from "../../listens/listens.module"; import { MusicLibraryModule } from "../../music-library/music-library.module"; import { UsersModule } from "../../users/users.module"; +import { ImportSpotifyJob } from "../jobs"; import { SpotifyApiModule } from "./spotify-api/spotify-api.module"; import { SpotifyAuthModule } from "./spotify-auth/spotify-auth.module"; import { SpotifyService } from "./spotify.service"; @Module({ imports: [ + PGBossModule.forJobs([ImportSpotifyJob]), UsersModule, ListensModule, MusicLibraryModule, @@ -17,4 +20,6 @@ import { SpotifyService } from "./spotify.service"; providers: [SpotifyService], exports: [SpotifyService], }) -export class SpotifyModule {} +export class SpotifyModule { + constructor(private readonly spotifyService: SpotifyService) {} +} diff --git a/src/sources/spotify/spotify.service.ts b/src/sources/spotify/spotify.service.ts index cafb3f7..0dbf44c 100644 --- a/src/sources/spotify/spotify.service.ts +++ b/src/sources/spotify/spotify.service.ts @@ -8,6 +8,11 @@ import { MusicLibraryService } from "../../music-library/music-library.service"; import { Track } from "../../music-library/track.entity"; import { User } from "../../users/user.entity"; import { UsersService } from "../../users/users.service"; +import { + IImportSpotifyJob, + ImportSpotifyJob, + UpdateSpotifyLibraryJob, +} from "../jobs"; import { AlbumObject } from "./spotify-api/entities/album-object"; import { ArtistObject } from "./spotify-api/entities/artist-object"; import { PlayHistoryObject } from "./spotify-api/entities/play-history-object"; @@ -31,15 +36,19 @@ export class SpotifyService { ) {} @Span() - async runCrawlerForAllUsers(): Promise { - this.logger.debug("Starting Spotify crawler loop"); - const users = await this.usersService.findAll(); + async getCrawlableUserInfo(): Promise { + return this.usersService.findAll(); + } - for (const user of users) { - // We want to run this sequentially to avoid rate limits - // eslint-disable-next-line no-await-in-loop - await this.crawlListensForUser(user); + @ImportSpotifyJob.Handle() + async importSpotifyJobHandler({ userID }: IImportSpotifyJob): Promise { + const user = await this.usersService.findById(userID); + if (!user) { + this.logger.warn("User for import job not found", { userID }); + return; } + + await this.crawlListensForUser(user); } @Span() @@ -130,6 +139,7 @@ export class SpotifyService { } @Span() + @UpdateSpotifyLibraryJob.Handle() async runUpdaterForAllEntities(): Promise { this.logger.debug("Starting Spotify updater loop");