feat(api): use nest-pg-boss for spotify interactions

package-lock.json

@@ -9,6 +9,7 @@
       "version": "1.22.0",
       "license": "MIT",
       "dependencies": {
+        "@apricote/nest-pg-boss": "1.0.0",
         "@hapi/joi": "17.1.1",
         "@narando/nest-axios-interceptor": "2.2.0",
         "@nestjs/axios": "0.1.0",
@@ -18,7 +19,6 @@
         "@nestjs/jwt": "10.0.2",
         "@nestjs/passport": "9.0.3",
         "@nestjs/platform-express": "9.3.9",
-        "@nestjs/schedule": "2.2.0",
         "@nestjs/serve-static": "3.0.1",
         "@nestjs/swagger": "6.2.1",
         "@nestjs/terminus": "9.2.1",
@@ -356,6 +356,18 @@
       "integrity": "sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==",
       "dev": true
     },
+    "node_modules/@apricote/nest-pg-boss": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@apricote/nest-pg-boss/-/nest-pg-boss-1.0.0.tgz",
+      "integrity": "sha512-jYe9Bj/qBUxEWfnzKly6TTyg+x3A5uuQ1ocP0SXRyXHgtdXKUIf4AF4OZdgcX2qR9yrHNdi3lcoj9P2Qk+vvbw==",
+      "peerDependencies": {
+        "@nestjs/common": "^9.3.3",
+        "@nestjs/core": "^9.3.3",
+        "pg-boss": "^8.4.1",
+        "reflect-metadata": "^0.1.13",
+        "rxjs": "^7.2.0"
+      }
+    },
     "node_modules/@babel/code-frame": {
       "version": "7.18.6",
       "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz",
@@ -2208,20 +2220,6 @@
       "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.5.0.tgz",
       "integrity": "sha512-336iVw3rtn2BUK7ORdIAHTyxHGRIHVReokCR3XjbckJMK7ms8FysBfhLR8IXnAgy7T0PTPNBWKiH514FOW/WSg=="
     },
-    "node_modules/@nestjs/schedule": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-2.2.0.tgz",
-      "integrity": "sha512-wrDnUONTxBkD6lTWh9ecYk/kvJTbA3PylotjBoRsECmcS1SNvgInFXuL38UnHiFnXM3CHSFnzRLB259Bc1mOdQ==",
-      "dependencies": {
-        "cron": "2.2.0",
-        "uuid": "9.0.0"
-      },
-      "peerDependencies": {
-        "@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0",
-        "@nestjs/core": "^7.0.0 || ^8.0.0 || ^9.0.0",
-        "reflect-metadata": "^0.1.12"
-      }
-    },
     "node_modules/@nestjs/schematics": {
       "version": "9.0.4",
       "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-9.0.4.tgz",
@@ -4511,6 +4509,19 @@
         "node": ">= 6.0.0"
       }
     },
+    "node_modules/aggregate-error": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
+      "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
+      "peer": true,
+      "dependencies": {
+        "clean-stack": "^2.0.0",
+        "indent-string": "^4.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/ajv": {
       "version": "8.11.0",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz",
@@ -5358,6 +5369,15 @@
         "validator": "^13.7.0"
       }
     },
+    "node_modules/clean-stack": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
+      "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==",
+      "peer": true,
+      "engines": {
+        "node": ">=6"
+      }
+    },
     "node_modules/cli-boxes": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz",
@@ -5695,12 +5715,16 @@
       "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
       "devOptional": true
     },
-    "node_modules/cron": {
+    "node_modules/cron-parser": {
-      "version": "2.2.0",
+      "version": "4.8.1",
-      "resolved": "https://registry.npmjs.org/cron/-/cron-2.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.8.1.tgz",
-      "integrity": "sha512-GPiI3OgMv83XRtEUc2gUdaLvJhO3XbLN288layOBkDTupg0RK5IECNGpkykIMHg+muVR2bxt29b0xvCAcBrjYQ==",
+      "integrity": "sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==",
+      "peer": true,
       "dependencies": {
         "luxon": "^3.2.1"
+      },
+      "engines": {
+        "node": ">=12.0.0"
       }
     },
     "node_modules/cross-spawn": {
@@ -5840,6 +5864,18 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/delay": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
+      "integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==",
+      "peer": true,
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@@ -7784,6 +7820,15 @@
         "node": ">=0.8.19"
       }
     },
+    "node_modules/indent-string": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
+      "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
+      "peer": true,
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/inflight": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@@ -9417,6 +9462,12 @@
       "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
       "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
     },
+    "node_modules/lodash.debounce": {
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
+      "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==",
+      "peer": true
+    },
     "node_modules/lodash.memoize": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
@@ -9503,6 +9554,7 @@
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz",
       "integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==",
+      "peer": true,
       "engines": {
         "node": ">=12"
       }
@@ -10184,6 +10236,21 @@
         "url": "https://github.com/sponsors/sindresorhus"
       }
     },
+    "node_modules/p-map": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
+      "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
+      "peer": true,
+      "dependencies": {
+        "aggregate-error": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/p-try": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
@@ -10459,6 +10526,24 @@
         }
       }
     },
+    "node_modules/pg-boss": {
+      "version": "8.4.1",
+      "resolved": "https://registry.npmjs.org/pg-boss/-/pg-boss-8.4.1.tgz",
+      "integrity": "sha512-8Zqy4h4Ib/l21lEC21uVmCdZnP5qjqwVolDAP2SBqEDaAdkOaWVm3VjyJ+a/GKDtjqpTvyn/jY/7IeBeAM++Jg==",
+      "peer": true,
+      "dependencies": {
+        "cron-parser": "^4.0.0",
+        "delay": "^5.0.0",
+        "lodash.debounce": "^4.0.8",
+        "p-map": "^4.0.0",
+        "pg": "^8.5.1",
+        "serialize-error": "^8.1.0",
+        "uuid": "^9.0.0"
+      },
+      "engines": {
+        "node": ">=14"
+      }
+    },
     "node_modules/pg-connection-string": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz",
@@ -11515,6 +11600,21 @@
       "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
       "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="
     },
+    "node_modules/serialize-error": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz",
+      "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==",
+      "peer": true,
+      "dependencies": {
+        "type-fest": "^0.20.2"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
+    },
     "node_modules/serialize-javascript": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz",
@@ -13528,6 +13628,12 @@
         }
       }
     },
+    "@apricote/nest-pg-boss": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@apricote/nest-pg-boss/-/nest-pg-boss-1.0.0.tgz",
+      "integrity": "sha512-jYe9Bj/qBUxEWfnzKly6TTyg+x3A5uuQ1ocP0SXRyXHgtdXKUIf4AF4OZdgcX2qR9yrHNdi3lcoj9P2Qk+vvbw==",
+      "requires": {}
+    },
     "@babel/code-frame": {
       "version": "7.18.6",
       "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.18.6.tgz",
@@ -14889,15 +14995,6 @@
         }
       }
     },
-    "@nestjs/schedule": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-2.2.0.tgz",
-      "integrity": "sha512-wrDnUONTxBkD6lTWh9ecYk/kvJTbA3PylotjBoRsECmcS1SNvgInFXuL38UnHiFnXM3CHSFnzRLB259Bc1mOdQ==",
-      "requires": {
-        "cron": "2.2.0",
-        "uuid": "9.0.0"
-      }
-    },
     "@nestjs/schematics": {
       "version": "9.0.4",
       "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-9.0.4.tgz",
@@ -16600,6 +16697,16 @@
         "debug": "4"
       }
     },
+    "aggregate-error": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/aggregate-error/-/aggregate-error-3.1.0.tgz",
+      "integrity": "sha512-4I7Td01quW/RpocfNayFdFVk1qSuoh0E7JrbRJ16nH01HhKFQ88INq9Sd+nd72zqRySlr9BmDA8xlEJ6vJMrYA==",
+      "peer": true,
+      "requires": {
+        "clean-stack": "^2.0.0",
+        "indent-string": "^4.0.0"
+      }
+    },
     "ajv": {
       "version": "8.11.0",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.11.0.tgz",
@@ -17220,6 +17327,12 @@
         "validator": "^13.7.0"
       }
     },
+    "clean-stack": {
+      "version": "2.2.0",
+      "resolved": "https://registry.npmjs.org/clean-stack/-/clean-stack-2.2.0.tgz",
+      "integrity": "sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==",
+      "peer": true
+    },
     "cli-boxes": {
       "version": "2.2.1",
       "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-2.2.1.tgz",
@@ -17477,10 +17590,11 @@
       "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==",
       "devOptional": true
     },
-    "cron": {
+    "cron-parser": {
-      "version": "2.2.0",
+      "version": "4.8.1",
-      "resolved": "https://registry.npmjs.org/cron/-/cron-2.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.8.1.tgz",
-      "integrity": "sha512-GPiI3OgMv83XRtEUc2gUdaLvJhO3XbLN288layOBkDTupg0RK5IECNGpkykIMHg+muVR2bxt29b0xvCAcBrjYQ==",
+      "integrity": "sha512-jbokKWGcyU4gl6jAfX97E1gDpY12DJ1cLJZmoDzaAln/shZ+S3KBFBuA2Q6WeUN4gJf/8klnV1EfvhA2lK5IRQ==",
+      "peer": true,
       "requires": {
         "luxon": "^3.2.1"
       }
@@ -17591,6 +17705,12 @@
         "object-keys": "^1.1.1"
       }
     },
+    "delay": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/delay/-/delay-5.0.0.tgz",
+      "integrity": "sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw==",
+      "peer": true
+    },
     "delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@@ -19050,6 +19170,12 @@
       "integrity": "sha512-JmXMZ6wuvDmLiHEml9ykzqO6lwFbof0GG4IkcGaENdCRDDmMVnny7s5HsIgHCbaq0w2MyPhDqkhTUgS2LU2PHA==",
       "dev": true
     },
+    "indent-string": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
+      "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
+      "peer": true
+    },
     "inflight": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@@ -20279,6 +20405,12 @@
       "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz",
       "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA=="
     },
+    "lodash.debounce": {
+      "version": "4.0.8",
+      "resolved": "https://registry.npmjs.org/lodash.debounce/-/lodash.debounce-4.0.8.tgz",
+      "integrity": "sha512-FT1yDzDYEoYWhnSGnpE/4Kj1fLZkDFyqRb7fNt6FdYOSxlUWAtp42Eh6Wb0rGIv/m9Bgo7x4GhQbm5Ys4SG5ow==",
+      "peer": true
+    },
     "lodash.memoize": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/lodash.memoize/-/lodash.memoize-4.1.2.tgz",
@@ -20348,7 +20480,8 @@
     "luxon": {
       "version": "3.3.0",
       "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.3.0.tgz",
-      "integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg=="
+      "integrity": "sha512-An0UCfG/rSiqtAIiBPO0Y9/zAnHUZxAMiCpTd5h2smgsj7GGmcenvrvww2cqNA8/4A5ZrD1gJpHN2mIHZQF+Mg==",
+      "peer": true
     },
     "macos-release": {
       "version": "2.5.0",
@@ -20841,6 +20974,15 @@
         "p-limit": "^3.0.2"
       }
     },
+    "p-map": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
+      "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
+      "peer": true,
+      "requires": {
+        "aggregate-error": "^3.0.0"
+      }
+    },
     "p-try": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
@@ -21040,6 +21182,21 @@
         "pgpass": "1.x"
       }
     },
+    "pg-boss": {
+      "version": "8.4.1",
+      "resolved": "https://registry.npmjs.org/pg-boss/-/pg-boss-8.4.1.tgz",
+      "integrity": "sha512-8Zqy4h4Ib/l21lEC21uVmCdZnP5qjqwVolDAP2SBqEDaAdkOaWVm3VjyJ+a/GKDtjqpTvyn/jY/7IeBeAM++Jg==",
+      "peer": true,
+      "requires": {
+        "cron-parser": "^4.0.0",
+        "delay": "^5.0.0",
+        "lodash.debounce": "^4.0.8",
+        "p-map": "^4.0.0",
+        "pg": "^8.5.1",
+        "serialize-error": "^8.1.0",
+        "uuid": "^9.0.0"
+      }
+    },
     "pg-connection-string": {
       "version": "2.5.0",
       "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.5.0.tgz",
@@ -21817,6 +21974,15 @@
         }
       }
     },
+    "serialize-error": {
+      "version": "8.1.0",
+      "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-8.1.0.tgz",
+      "integrity": "sha512-3NnuWfM6vBYoy5gZFvHiYsVbafvI9vZv/+jlIigFn4oP4zjNPK3LhcY0xSCgeb1a5L8jO71Mit9LlNoi2UfDDQ==",
+      "peer": true,
+      "requires": {
+        "type-fest": "^0.20.2"
+      }
+    },
     "serialize-javascript": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.0.tgz",

package.json

@@ -26,6 +26,7 @@
     "test:e2e": "jest --config ./apps/listory/test/jest-e2e.json"
   },
   "dependencies": {
+    "@apricote/nest-pg-boss": "1.0.0",
     "@hapi/joi": "17.1.1",
     "@narando/nest-axios-interceptor": "2.2.0",
     "@nestjs/axios": "0.1.0",
@@ -35,7 +36,6 @@
     "@nestjs/jwt": "10.0.2",
     "@nestjs/passport": "9.0.3",
     "@nestjs/platform-express": "9.3.9",
-    "@nestjs/schedule": "2.2.0",
     "@nestjs/serve-static": "3.0.1",
     "@nestjs/swagger": "6.2.1",
     "@nestjs/terminus": "9.2.1",

src/app.module.ts

@@ -1,5 +1,4 @@
 import { Module } from "@nestjs/common";
-import { ScheduleModule } from "@nestjs/schedule";
 import { ServeStaticModule } from "@nestjs/serve-static";
 import { RavenModule } from "nest-raven";
 import { join } from "path";
@@ -14,13 +13,14 @@ import { ReportsModule } from "./reports/reports.module";
 import { SourcesModule } from "./sources/sources.module";
 import { UsersModule } from "./users/users.module";
 import { OpenTelemetryModule } from "./open-telemetry/open-telemetry.module";
+import { JobQueueModule } from "./job-queue/job-queue.module";
 
 @Module({
   imports: [
     LoggerModule,
     ConfigModule,
     DatabaseModule,
-    ScheduleModule.forRoot(),
+    JobQueueModule,
     ServeStaticModule.forRoot({
       rootPath: join(__dirname, "..", "static"),
       exclude: ["/api*"],

src/job-queue/job-queue.module.ts

@@ -0,0 +1,23 @@
+import { Module } from "@nestjs/common";
+import { PGBossModule } from "@apricote/nest-pg-boss";
+import { ConfigService } from "@nestjs/config";
+
+@Module({
+  imports: [
+    PGBossModule.forRootAsync({
+      application_name: "listory",
+      useFactory: (config: ConfigService) => ({
+        // Connection details
+        host: config.get<string>("DB_HOST"),
+        user: config.get<string>("DB_USERNAME"),
+        password: config.get<string>("DB_PASSWORD"),
+        database: config.get<string>("DB_DATABASE"),
+        schema: "public",
+        max: config.get<number>("DB_POOL_MAX"),
+      }),
+      inject: [ConfigService],
+    }),
+  ],
+  exports: [PGBossModule],
+})
+export class JobQueueModule {}

src/sources/jobs.ts

@@ -0,0 +1,14 @@
+import { createJob } from "@apricote/nest-pg-boss";
+
+export type ICrawlerSupervisorJob = {};
+export const CrawlerSupervisorJob = createJob<ICrawlerSupervisorJob>(
+  "spotify-crawler-supervisor"
+);
+
+export type IUpdateSpotifyLibraryJob = {};
+export const UpdateSpotifyLibraryJob = createJob<IUpdateSpotifyLibraryJob>(
+  "update-spotify-library"
+);
+
+export type IImportSpotifyJob = { userID: string };
+export const ImportSpotifyJob = createJob<IImportSpotifyJob>("import-spotify");

src/sources/scheduler.service.ts

@@ -1,8 +1,15 @@
 import { Injectable, Logger, OnApplicationBootstrap } from "@nestjs/common";
 import { ConfigService } from "@nestjs/config";
-import { SchedulerRegistry } from "@nestjs/schedule";
-import { captureException } from "@sentry/node";
 import { SpotifyService } from "./spotify/spotify.service";
+import {
+  CrawlerSupervisorJob,
+  ICrawlerSupervisorJob,
+  IImportSpotifyJob,
+  ImportSpotifyJob,
+  IUpdateSpotifyLibraryJob,
+  UpdateSpotifyLibraryJob,
+} from "./jobs";
+import { JobService } from "@apricote/nest-pg-boss";
 
 @Injectable()
 export class SchedulerService implements OnApplicationBootstrap {
@@ -10,38 +17,42 @@ export class SchedulerService implements OnApplicationBootstrap {
 
   constructor(
     private readonly config: ConfigService,
-    private readonly registry: SchedulerRegistry,
+    private readonly spotifyService: SpotifyService,
-    private readonly spotifyService: SpotifyService
+    @CrawlerSupervisorJob.Inject()
+    private readonly superviseImportJobsJobService: JobService<ICrawlerSupervisorJob>,
+    @ImportSpotifyJob.Inject()
+    private readonly importSpotifyJobService: JobService<IImportSpotifyJob>,
+    @UpdateSpotifyLibraryJob.Inject()
+    private readonly updateSpotifyLibraryJobService: JobService<IUpdateSpotifyLibraryJob>
   ) {}
 
-  onApplicationBootstrap() {
+  async onApplicationBootstrap() {
-    this.setupSpotifyCrawler();
+    await this.setupSpotifyCrawlerSupervisor();
-    this.setupSpotifyMusicLibraryUpdater();
+    await this.setupSpotifyMusicLibraryUpdater();
   }
 
-  private setupSpotifyCrawler() {
+  private async setupSpotifyCrawlerSupervisor(): Promise<void> {
-    const callback = () =>
+    await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {});
-      this.spotifyService.runCrawlerForAllUsers().catch((err) => {
-        captureException(err);
-        this.logger.error(`Spotify crawler loop crashed! ${err.stack}`);
-      });
-    const timeoutMs =
-      this.config.get<number>("SPOTIFY_FETCH_INTERVAL_SEC") * 1000;
-
-    const interval = setInterval(callback, timeoutMs);
-
-    this.registry.addInterval("crawler_spotify", interval);
   }
 
-  private setupSpotifyMusicLibraryUpdater() {
+  @CrawlerSupervisorJob.Handle()
-    const callback = () => {
+  async superviseImportJobs(): Promise<void> {
-      this.spotifyService.runUpdaterForAllEntities();
+    this.logger.log("Starting crawler jobs");
-    };
+    const users = await this.spotifyService.getCrawlableUserInfo();
-    const timeoutMs =
-      this.config.get<number>("SPOTIFY_UPDATE_INTERVAL_SEC") * 1000;
 
-    const interval = setInterval(callback, timeoutMs);
+    await Promise.all(
+      users.map((user) =>
+        this.importSpotifyJobService.sendOnce({ userID: user.id }, {}, user.id)
+      )
+    );
+  }
 
-    this.registry.addInterval("updater_spotify", interval);
+  private async setupSpotifyMusicLibraryUpdater() {
+    await this.updateSpotifyLibraryJobService.schedule("*/1 * * * *", {}, {});
+  }
+
+  @UpdateSpotifyLibraryJob.Handle()
+  async updateSpotifyLibrary() {
+    this.spotifyService.runUpdaterForAllEntities();
   }
 }

src/sources/sources.module.ts

@@ -1,9 +1,22 @@
 import { Module } from "@nestjs/common";
+import { PGBossModule } from "@apricote/nest-pg-boss";
+import {
+  CrawlerSupervisorJob,
+  ImportSpotifyJob,
+  UpdateSpotifyLibraryJob,
+} from "./jobs";
 import { SchedulerService } from "./scheduler.service";
 import { SpotifyModule } from "./spotify/spotify.module";
 
 @Module({
-  imports: [SpotifyModule],
+  imports: [
+    SpotifyModule,
+    PGBossModule.forJobs([
+      CrawlerSupervisorJob,
+      ImportSpotifyJob,
+      UpdateSpotifyLibraryJob,
+    ]),
+  ],
   providers: [SchedulerService],
 })
 export class SourcesModule {}

src/sources/spotify/spotify.module.ts

@@ -1,13 +1,16 @@
+import { PGBossModule } from "@apricote/nest-pg-boss";
 import { Module } from "@nestjs/common";
 import { ListensModule } from "../../listens/listens.module";
 import { MusicLibraryModule } from "../../music-library/music-library.module";
 import { UsersModule } from "../../users/users.module";
+import { ImportSpotifyJob } from "../jobs";
 import { SpotifyApiModule } from "./spotify-api/spotify-api.module";
 import { SpotifyAuthModule } from "./spotify-auth/spotify-auth.module";
 import { SpotifyService } from "./spotify.service";
 
 @Module({
   imports: [
+    PGBossModule.forJobs([ImportSpotifyJob]),
     UsersModule,
     ListensModule,
     MusicLibraryModule,
@@ -17,4 +20,6 @@ import { SpotifyService } from "./spotify.service";
   providers: [SpotifyService],
   exports: [SpotifyService],
 })
-export class SpotifyModule {}
+export class SpotifyModule {
+  constructor(private readonly spotifyService: SpotifyService) {}
+}

src/sources/spotify/spotify.service.ts

@@ -8,6 +8,11 @@ import { MusicLibraryService } from "../../music-library/music-library.service";
 import { Track } from "../../music-library/track.entity";
 import { User } from "../../users/user.entity";
 import { UsersService } from "../../users/users.service";
+import {
+  IImportSpotifyJob,
+  ImportSpotifyJob,
+  UpdateSpotifyLibraryJob,
+} from "../jobs";
 import { AlbumObject } from "./spotify-api/entities/album-object";
 import { ArtistObject } from "./spotify-api/entities/artist-object";
 import { PlayHistoryObject } from "./spotify-api/entities/play-history-object";
@@ -31,15 +36,19 @@ export class SpotifyService {
   ) {}
 
   @Span()
-  async runCrawlerForAllUsers(): Promise<void> {
+  async getCrawlableUserInfo(): Promise<User[]> {
-    this.logger.debug("Starting Spotify crawler loop");
+    return this.usersService.findAll();
-    const users = await this.usersService.findAll();
+  }
 
-    for (const user of users) {
+  @ImportSpotifyJob.Handle()
-      // We want to run this sequentially to avoid rate limits
+  async importSpotifyJobHandler({ userID }: IImportSpotifyJob): Promise<void> {
-      // eslint-disable-next-line no-await-in-loop
+    const user = await this.usersService.findById(userID);
-      await this.crawlListensForUser(user);
+    if (!user) {
+      this.logger.warn("User for import job not found", { userID });
+      return;
     }
+
+    await this.crawlListensForUser(user);
   }
 
   @Span()
@@ -130,6 +139,7 @@ export class SpotifyService {
   }
 
   @Span()
+  @UpdateSpotifyLibraryJob.Handle()
   async runUpdaterForAllEntities(): Promise<void> {
     this.logger.debug("Starting Spotify updater loop");