apricote/Listory
1
0
Fork 0
mirror of https://github.com/apricote/Listory.git synced 2026-01-13 13:11:02 +00:00
Code Issues Projects Releases Packages Wiki Activity

feat(api): use nest-pg-boss for spotify interactions

Browse source
This commit is contained in:
Julian Tölle 2022-06-29 21:29:00 +02:00
parent cd672a408e
commit b9f92bbdfa
9 changed files with 314 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

232
package-lock.json generated
View file

@ -9,6 +9,7 @@
"version": "1.22.0",
"license": "MIT",
"dependencies": {
"@apricote/nest-pg-boss": "1.0.0",
"@hapi/joi": "17.1.1",
"@narando/nest-axios-interceptor": "2.2.0",
"@nestjs/axios": "0.1.0",
@ -18,7 +19,6 @@
"@nestjs/jwt": "10.0.2",
"@nestjs/passport": "9.0.3",
"@nestjs/platform-express": "9.3.9",
"@nestjs/schedule": "2.2.0",
"@nestjs/serve-static": "3.0.1",
"@nestjs/swagger": "6.2.1",
"@nestjs/terminus": "9.2.1",
@ -356,6 +356,18 @@
"integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
"dev": true
},
"node_modules/@apricote/nest-pg-boss": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@apricote/nest-pg-boss/-/nest-pg-boss-1.0.0.tgz",
"integrity": "sha512-jYe9Bj/qBUxEWfnzKly6TTyg+x3A5uuQ1ocP0SXRyXHgtdXKUIf4AF4OZdgcX2qR9yrHNdi3lcoj9P2Qk+vvbw==",
"peerDependencies": {
"@nestjs/common": "^9.3.3",
"@nestjs/core": "^9.3.3",
"pg-boss": "^8.4.1",
"reflect-metadata": "^0.1.13",
"rxjs": "^7.2.0"
}
},
"node_modules/@babel/code-frame": {
"version": "7.18.6",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz",
@ -2208,20 +2220,6 @@
"resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
"integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
},
"node_modules/@nestjs/schedule": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-2.2.0.tgz",
"integrity": "sha512-wrDnUONTxBkD6lTWh9ecYk/kvJTbA3PylotjBoRsECmcS1SNvgInFXuL38UnHiFnXM3CHSFnzRLB259Bc1mOdQ==",
"dependencies": {
"cron": "2.2.0",
"uuid": "9.0.0"
},
"peerDependencies": {
"@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0",
"@nestjs/core": "^7.0.0 || ^8.0.0 || ^9.0.0",
"reflect-metadata": "^0.1.12"
}
},
"node_modules/@nestjs/schematics": {
"version": "9.0.4",
"resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-9.0.4.tgz",
@ -4511,6 +4509,19 @@
"node": ">= 6.0.0"
}
},
"node_modules/aggregate-error": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
"integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
"peer": true,
"dependencies": {
"clean-stack": "^2.0.0",
"indent-string": "^4.0.0"
},
"engines": {
"node": ">=8"
}
},
"node_modules/ajv": {
"version": "8.11.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz",
@ -5358,6 +5369,15 @@
"validator": "^13.7.0"
}
},
"node_modules/clean-stack": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
"integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==",
"peer": true,
"engines": {
"node": ">=6"
}
},
"node_modules/cli-boxes": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz",
@ -5695,12 +5715,16 @@
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
"devOptional": true
},
"node_modules/cron": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/cron/-/cron-2.2.0.tgz",
"integrity": "sha512-GPiI3OgMv83XRtEUc2gUdaLvJhO3XbLN288layOBkDTupg0RK5IECNGpkykIMHg+muVR2bxt29b0xvCAcBrjYQ==",
"node_modules/cron-parser": {
"version": "4.8.1",
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.8.1.tgz",
"integrity": "sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==",
"peer": true,
"dependencies": {
"luxon": "^3.2.1"
},
"engines": {
"node": ">=12.0.0"
}
},
"node_modules/cross-spawn": {
@ -5840,6 +5864,18 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/delay": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
"integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==",
"peer": true,
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@ -7784,6 +7820,15 @@
"node": ">=0.8.19"
}
},
"node_modules/indent-string": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
"integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
"peer": true,
"engines": {
"node": ">=8"
}
},
"node_modules/inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@ -9417,6 +9462,12 @@
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
},
"node_modules/lodash.debounce": {
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
"integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==",
"peer": true
},
"node_modules/lodash.memoize": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
@ -9503,6 +9554,7 @@
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz",
"integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==",
"peer": true,
"engines": {
"node": ">=12"
}
@ -10184,6 +10236,21 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/p-map": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
"integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
"peer": true,
"dependencies": {
"aggregate-error": "^3.0.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/p-try": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
@ -10459,6 +10526,24 @@
}
}
},
"node_modules/pg-boss": {
"version": "8.4.1",
"resolved": "https://registry.npmjs.org/pg-boss/-/pg-boss-8.4.1.tgz",
"integrity": "sha512-8Zqy4h4Ib/l21lEC21uVmCdZnP5qjqwVolDAP2SBqEDaAdkOaWVm3VjyJ+a/GKDtjqpTvyn/jY/7IeBeAM++Jg==",
"peer": true,
"dependencies": {
"cron-parser": "^4.0.0",
"delay": "^5.0.0",
"lodash.debounce": "^4.0.8",
"p-map": "^4.0.0",
"pg": "^8.5.1",
"serialize-error": "^8.1.0",
"uuid": "^9.0.0"
},
"engines": {
"node": ">=14"
}
},
"node_modules/pg-connection-string": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz",
@ -11515,6 +11600,21 @@
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
},
"node_modules/serialize-error": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz",
"integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==",
"peer": true,
"dependencies": {
"type-fest": "^0.20.2"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/serialize-javascript": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz",
@ -13528,6 +13628,12 @@
}
}
},
"@apricote/nest-pg-boss": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/@apricote/nest-pg-boss/-/nest-pg-boss-1.0.0.tgz",
"integrity": "sha512-jYe9Bj/qBUxEWfnzKly6TTyg+x3A5uuQ1ocP0SXRyXHgtdXKUIf4AF4OZdgcX2qR9yrHNdi3lcoj9P2Qk+vvbw==",
"requires": {}
},
"@babel/code-frame": {
"version": "7.18.6",
"resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz",
@ -14889,15 +14995,6 @@
}
}
},
"@nestjs/schedule": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-2.2.0.tgz",
"integrity": "sha512-wrDnUONTxBkD6lTWh9ecYk/kvJTbA3PylotjBoRsECmcS1SNvgInFXuL38UnHiFnXM3CHSFnzRLB259Bc1mOdQ==",
"requires": {
"cron": "2.2.0",
"uuid": "9.0.0"
}
},
"@nestjs/schematics": {
"version": "9.0.4",
"resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-9.0.4.tgz",
@ -16600,6 +16697,16 @@
"debug": "4"
}
},
"aggregate-error": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
"integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
"peer": true,
"requires": {
"clean-stack": "^2.0.0",
"indent-string": "^4.0.0"
}
},
"ajv": {
"version": "8.11.0",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz",
@ -17220,6 +17327,12 @@
"validator": "^13.7.0"
}
},
"clean-stack": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
"integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==",
"peer": true
},
"cli-boxes": {
"version": "2.2.1",
"resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz",
@ -17477,10 +17590,11 @@
"integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
"devOptional": true
},
"cron": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/cron/-/cron-2.2.0.tgz",
"integrity": "sha512-GPiI3OgMv83XRtEUc2gUdaLvJhO3XbLN288layOBkDTupg0RK5IECNGpkykIMHg+muVR2bxt29b0xvCAcBrjYQ==",
"cron-parser": {
"version": "4.8.1",
"resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.8.1.tgz",
"integrity": "sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==",
"peer": true,
"requires": {
"luxon": "^3.2.1"
}
@ -17591,6 +17705,12 @@
"object-keys": "^1.1.1"
}
},
"delay": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
"integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==",
"peer": true
},
"delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@ -19050,6 +19170,12 @@
"integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==",
"dev": true
},
"indent-string": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
"integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
"peer": true
},
"inflight": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@ -20279,6 +20405,12 @@
"resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
"integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
},
"lodash.debounce": {
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
"integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==",
"peer": true
},
"lodash.memoize": {
"version": "4.1.2",
"resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
@ -20348,7 +20480,8 @@
"luxon": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz",
"integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg=="
"integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==",
"peer": true
},
"macos-release": {
"version": "2.5.0",
@ -20841,6 +20974,15 @@
"p-limit": "^3.0.2"
}
},
"p-map": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
"integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
"peer": true,
"requires": {
"aggregate-error": "^3.0.0"
}
},
"p-try": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
@ -21040,6 +21182,21 @@
"pgpass": "1.x"
}
},
"pg-boss": {
"version": "8.4.1",
"resolved": "https://registry.npmjs.org/pg-boss/-/pg-boss-8.4.1.tgz",
"integrity": "sha512-8Zqy4h4Ib/l21lEC21uVmCdZnP5qjqwVolDAP2SBqEDaAdkOaWVm3VjyJ+a/GKDtjqpTvyn/jY/7IeBeAM++Jg==",
"peer": true,
"requires": {
"cron-parser": "^4.0.0",
"delay": "^5.0.0",
"lodash.debounce": "^4.0.8",
"p-map": "^4.0.0",
"pg": "^8.5.1",
"serialize-error": "^8.1.0",
"uuid": "^9.0.0"
}
},
"pg-connection-string": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz",
@ -21817,6 +21974,15 @@
}
}
},
"serialize-error": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz",
"integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==",
"peer": true,
"requires": {
"type-fest": "^0.20.2"
}
},
"serialize-javascript": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz",

2
package.json
View file

@ -26,6 +26,7 @@
"test:e2e": "jest --config ./apps/listory/test/jest-e2e.json"
},
"dependencies": {
"@apricote/nest-pg-boss": "1.0.0",
"@hapi/joi": "17.1.1",
"@narando/nest-axios-interceptor": "2.2.0",
"@nestjs/axios": "0.1.0",
@ -35,7 +36,6 @@
"@nestjs/jwt": "10.0.2",
"@nestjs/passport": "9.0.3",
"@nestjs/platform-express": "9.3.9",
"@nestjs/schedule": "2.2.0",
"@nestjs/serve-static": "3.0.1",
"@nestjs/swagger": "6.2.1",
"@nestjs/terminus": "9.2.1",

4
src/app.module.ts
View file

@ -1,5 +1,4 @@
import { Module } from "@nestjs/common";
import { ScheduleModule } from "@nestjs/schedule";
import { ServeStaticModule } from "@nestjs/serve-static";
import { RavenModule } from "nest-raven";
import { join } from "path";
@ -14,13 +13,14 @@ import { ReportsModule } from "./reports/reports.module";
import { SourcesModule } from "./sources/sources.module";
import { UsersModule } from "./users/users.module";
import { OpenTelemetryModule } from "./open-telemetry/open-telemetry.module";
import { JobQueueModule } from "./job-queue/job-queue.module";
@Module({
imports: [
LoggerModule,
ConfigModule,
DatabaseModule,
ScheduleModule.forRoot(),
JobQueueModule,
ServeStaticModule.forRoot({
rootPath: join(__dirname, "..", "static"),
exclude: ["/api*"],

23
src/job-queue/job-queue.module.ts Normal file
View file

@ -0,0 +1,23 @@
import { Module } from "@nestjs/common";
import { PGBossModule } from "@apricote/nest-pg-boss";
import { ConfigService } from "@nestjs/config";
@Module({
imports: [
PGBossModule.forRootAsync({
application_name: "listory",
useFactory: (config: ConfigService) => ({
// Connection details
host: config.get<string>("DB_HOST"),
user: config.get<string>("DB_USERNAME"),
password: config.get<string>("DB_PASSWORD"),
database: config.get<string>("DB_DATABASE"),
schema: "public",
max: config.get<number>("DB_POOL_MAX"),
}),
inject: [ConfigService],
}),
],
exports: [PGBossModule],
})
export class JobQueueModule {}

14
src/sources/jobs.ts Normal file
View file

@ -0,0 +1,14 @@
import { createJob } from "@apricote/nest-pg-boss";
export type ICrawlerSupervisorJob = {};
export const CrawlerSupervisorJob = createJob<ICrawlerSupervisorJob>(
"spotify-crawler-supervisor"
);
export type IUpdateSpotifyLibraryJob = {};
export const UpdateSpotifyLibraryJob = createJob<IUpdateSpotifyLibraryJob>(
"update-spotify-library"
);
export type IImportSpotifyJob = { userID: string };
export const ImportSpotifyJob = createJob<IImportSpotifyJob>("import-spotify");

65
src/sources/scheduler.service.ts
View file

@ -1,8 +1,15 @@
import { Injectable, Logger, OnApplicationBootstrap } from "@nestjs/common";
import { ConfigService } from "@nestjs/config";
import { SchedulerRegistry } from "@nestjs/schedule";
import { captureException } from "@sentry/node";
import { SpotifyService } from "./spotify/spotify.service";
import {
CrawlerSupervisorJob,
ICrawlerSupervisorJob,
IImportSpotifyJob,
ImportSpotifyJob,
IUpdateSpotifyLibraryJob,
UpdateSpotifyLibraryJob,
} from "./jobs";
import { JobService } from "@apricote/nest-pg-boss";
@Injectable()
export class SchedulerService implements OnApplicationBootstrap {
@ -10,38 +17,42 @@ export class SchedulerService implements OnApplicationBootstrap {
constructor(
private readonly config: ConfigService,
private readonly registry: SchedulerRegistry,
private readonly spotifyService: SpotifyService
private readonly spotifyService: SpotifyService,
@CrawlerSupervisorJob.Inject()
private readonly superviseImportJobsJobService: JobService<ICrawlerSupervisorJob>,
@ImportSpotifyJob.Inject()
private readonly importSpotifyJobService: JobService<IImportSpotifyJob>,
@UpdateSpotifyLibraryJob.Inject()
private readonly updateSpotifyLibraryJobService: JobService<IUpdateSpotifyLibraryJob>
) {}
onApplicationBootstrap() {
this.setupSpotifyCrawler();
this.setupSpotifyMusicLibraryUpdater();
async onApplicationBootstrap() {
await this.setupSpotifyCrawlerSupervisor();
await this.setupSpotifyMusicLibraryUpdater();
}
private setupSpotifyCrawler() {
const callback = () =>
this.spotifyService.runCrawlerForAllUsers().catch((err) => {
captureException(err);
this.logger.error(`Spotify crawler loop crashed! ${err.stack}`);
});
const timeoutMs =
this.config.get<number>("SPOTIFY_FETCH_INTERVAL_SEC") * 1000;
const interval = setInterval(callback, timeoutMs);
this.registry.addInterval("crawler_spotify", interval);
private async setupSpotifyCrawlerSupervisor(): Promise<void> {
await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {});
}
private setupSpotifyMusicLibraryUpdater() {
const callback = () => {
this.spotifyService.runUpdaterForAllEntities();
};
const timeoutMs =
this.config.get<number>("SPOTIFY_UPDATE_INTERVAL_SEC") * 1000;
@CrawlerSupervisorJob.Handle()
async superviseImportJobs(): Promise<void> {
this.logger.log("Starting crawler jobs");
const users = await this.spotifyService.getCrawlableUserInfo();
const interval = setInterval(callback, timeoutMs);
await Promise.all(
users.map((user) =>
this.importSpotifyJobService.sendOnce({ userID: user.id }, {}, user.id)
)
);
}
this.registry.addInterval("updater_spotify", interval);
private async setupSpotifyMusicLibraryUpdater() {
await this.updateSpotifyLibraryJobService.schedule("*/1 * * * *", {}, {});
}
@UpdateSpotifyLibraryJob.Handle()
async updateSpotifyLibrary() {
this.spotifyService.runUpdaterForAllEntities();
}
}

15
src/sources/sources.module.ts
View file

@ -1,9 +1,22 @@
import { Module } from "@nestjs/common";
import { PGBossModule } from "@apricote/nest-pg-boss";
import {
CrawlerSupervisorJob,
ImportSpotifyJob,
UpdateSpotifyLibraryJob,
} from "./jobs";
import { SchedulerService } from "./scheduler.service";
import { SpotifyModule } from "./spotify/spotify.module";
@Module({
imports: [SpotifyModule],
imports: [
SpotifyModule,
PGBossModule.forJobs([
CrawlerSupervisorJob,
ImportSpotifyJob,
UpdateSpotifyLibraryJob,
]),
],
providers: [SchedulerService],
})
export class SourcesModule {}

7
src/sources/spotify/spotify.module.ts
View file

@ -1,13 +1,16 @@
import { PGBossModule } from "@apricote/nest-pg-boss";
import { Module } from "@nestjs/common";
import { ListensModule } from "../../listens/listens.module";
import { MusicLibraryModule } from "../../music-library/music-library.module";
import { UsersModule } from "../../users/users.module";
import { ImportSpotifyJob } from "../jobs";
import { SpotifyApiModule } from "./spotify-api/spotify-api.module";
import { SpotifyAuthModule } from "./spotify-auth/spotify-auth.module";
import { SpotifyService } from "./spotify.service";
@Module({
imports: [
PGBossModule.forJobs([ImportSpotifyJob]),
UsersModule,
ListensModule,
MusicLibraryModule,
@ -17,4 +20,6 @@ import { SpotifyService } from "./spotify.service";
providers: [SpotifyService],
exports: [SpotifyService],
})
export class SpotifyModule {}
export class SpotifyModule {
constructor(private readonly spotifyService: SpotifyService) {}
}

24
src/sources/spotify/spotify.service.ts
View file

@ -8,6 +8,11 @@ import { MusicLibraryService } from "../../music-library/music-library.service";
import { Track } from "../../music-library/track.entity";
import { User } from "../../users/user.entity";
import { UsersService } from "../../users/users.service";
import {
IImportSpotifyJob,
ImportSpotifyJob,
UpdateSpotifyLibraryJob,
} from "../jobs";
import { AlbumObject } from "./spotify-api/entities/album-object";
import { ArtistObject } from "./spotify-api/entities/artist-object";
import { PlayHistoryObject } from "./spotify-api/entities/play-history-object";
@ -31,15 +36,19 @@ export class SpotifyService {
) {}
@Span()
async runCrawlerForAllUsers(): Promise<void> {
this.logger.debug("Starting Spotify crawler loop");
const users = await this.usersService.findAll();
async getCrawlableUserInfo(): Promise<User[]> {
return this.usersService.findAll();
}
for (const user of users) {
// We want to run this sequentially to avoid rate limits
// eslint-disable-next-line no-await-in-loop
await this.crawlListensForUser(user);
@ImportSpotifyJob.Handle()
async importSpotifyJobHandler({ userID }: IImportSpotifyJob): Promise<void> {
const user = await this.usersService.findById(userID);
if (!user) {
this.logger.warn("User for import job not found", { userID });
return;
}
await this.crawlListensForUser(user);
}
@Span()
@ -130,6 +139,7 @@ export class SpotifyService {
}
@Span()
@UpdateSpotifyLibraryJob.Handle()
async runUpdaterForAllEntities(): Promise<void> {
this.logger.debug("Starting Spotify updater loop");