.dockerignore

@@ -19,6 +19,5 @@
 !frontend/tsconfig.json
 !frontend/vite.config.js
 !frontend/index.html
-!frontend/*.d.ts
 !frontend/src/**/*
 !frontend/public/**/*

.github/workflows/ci.yaml

@@ -12,7 +12,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v3
         with:
           node-version: 20
 
@@ -33,7 +33,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v3
         with:
           node-version: 20
 

.github/workflows/release.yaml

@@ -25,7 +25,7 @@ jobs:
           version: "v0.11.2"
 
       - name: Setup Node.js
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@v3
         with:
           node-version: 20
 

CHANGELOG.md

@@ -1,24 +1,3 @@
-# [1.31.0](https://github.com/apricote/Listory/compare/v1.30.1...v1.31.0) (2023-10-15)
-
-
-### Features
-
-* **frontend:** hide revoked api tokens ([#307](https://github.com/apricote/Listory/issues/307)) ([4cef4f7](https://github.com/apricote/Listory/commit/4cef4f75ace6a38ba19c1d2f93d81389ec2b7cb8))
-
-## [1.30.1](https://github.com/apricote/Listory/compare/v1.30.0...v1.30.1) (2023-10-08)
-
-
-### Bug Fixes
-
-* no listens are being crawled ([#306](https://github.com/apricote/Listory/issues/306)) ([9af3115](https://github.com/apricote/Listory/commit/9af3115cab19cc4ac4a6cd0fb680371154069aa2))
-
-# [1.30.0](https://github.com/apricote/Listory/compare/v1.29.0...v1.30.0) (2023-10-01)
-
-
-### Features
-
-* import listens from spotify extended streaming history ([#305](https://github.com/apricote/Listory/issues/305)) ([7140cb0](https://github.com/apricote/Listory/commit/7140cb0679ec3aac8a2102197d9edb070cf0e6c0))
-
 # [1.29.0](https://github.com/apricote/Listory/compare/v1.28.2...v1.29.0) (2023-09-30)
 
 

Dockerfile

@@ -1,4 +1,9 @@
-# syntax=docker/dockerfile:1.12
+# syntax=docker/dockerfile:1.5
+
+FROM scratch as ignore
+
+WORKDIR /listory
+COPY . /listory/
 
 ##################
 ## common

charts/listory/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 1.31.0
+version: 1.29.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 1.31.0
+appVersion: 1.29.0

docker-compose.prod.yml

@@ -6,7 +6,7 @@ services:
   #####
 
   db:
-    image: postgres:16.6
+    image: postgres:16.0
     restart: unless-stopped
     environment:
       POSTGRES_PASSWORD: listory
@@ -18,7 +18,7 @@ services:
       - db
 
   api:
-    image: apricote/listory:1.31.0
+    image: apricote/listory:1.29.0
     restart: unless-stopped
     environment:
       DB_HOST: db

docker-compose.yml

@@ -12,7 +12,7 @@ services:
   #####
 
   db:
-    image: postgres:16.6
+    image: postgres:16.0
     environment:
       POSTGRES_PASSWORD: listory
       POSTGRES_USER: listory
@@ -37,8 +37,7 @@ services:
       OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4318
     env_file: .env
     volumes:
-      - ./src:/app/src:ro
-      - ./dist:/app/dist # build cache
+      - ./src:/app/src
     ports:
       - 3000 # API
       - "9464:9464" # Metrics
@@ -73,7 +72,7 @@ services:
       - web
 
   proxy:
-    image: traefik:v2.11.15
+    image: traefik:v2.10.4
     command:
       #- --log.level=debug
       #- --accesslog=true
@@ -116,8 +115,7 @@ services:
       OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4318
     env_file: .env
     volumes:
-      - ./src:/app/src:ro
-      - ./dist:/app/dist # build cache
+      - ./src:/app/src
     ports:
       - "9464:9464" # Metrics
     networks:
@@ -131,7 +129,7 @@ services:
 
   prometheus:
     profiles: ["observability"]
-    image: prom/prometheus:v2.55.1
+    image: prom/prometheus:v2.47.0
     volumes:
       - ./observability/prometheus:/etc/prometheus
       - prometheus_data:/prometheus
@@ -148,7 +146,7 @@ services:
 
   loki:
     profiles: ["observability"]
-    image: grafana/loki:2.9.11
+    image: grafana/loki:2.9.1
     command: ["-config.file=/etc/loki/loki.yaml"]
     ports:
       - "3100" # loki needs to be exposed so it receives logs
@@ -159,7 +157,7 @@ services:
 
   promtail:
     profiles: ["observability"]
-    image: grafana/promtail:2.9.11
+    image: grafana/promtail:2.9.1
     command: ["-config.file=/etc/promtail.yaml"]
     volumes:
       - ./observability/promtail/promtail.yaml:/etc/promtail.yaml
@@ -177,7 +175,7 @@ services:
 
   tempo:
     profiles: ["observability"]
-    image: grafana/tempo:2.6.1
+    image: grafana/tempo:2.2.3
     command: ["-config.file=/etc/tempo.yaml"]
     volumes:
       - ./observability/tempo/tempo.yaml:/etc/tempo.yaml
@@ -191,7 +189,7 @@ services:
 
   grafana:
     profiles: ["observability"]
-    image: grafana/grafana-oss:10.4.14
+    image: grafana/grafana-oss:10.1.4
     volumes:
       - ./observability/grafana/provisioning:/etc/grafana/provisioning
     environment:

frontend/package.json

@@ -8,44 +8,43 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@radix-ui/react-avatar": "1.1.2",
-    "@radix-ui/react-dropdown-menu": "2.1.3",
-    "@radix-ui/react-label": "2.1.1",
-    "@radix-ui/react-navigation-menu": "1.2.2",
-    "@radix-ui/react-select": "2.1.3",
-    "@radix-ui/react-slot": "1.1.1",
-    "@testing-library/jest-dom": "6.6.3",
-    "@testing-library/react": "16.1.0",
-    "@testing-library/user-event": "14.5.2",
-    "@types/jest": "29.5.14",
-    "@types/node": "20.17.16",
-    "@types/react": "18.3.18",
-    "@types/react-dom": "18.3.5",
+    "@radix-ui/react-avatar": "1.0.3",
+    "@radix-ui/react-dropdown-menu": "2.0.5",
+    "@radix-ui/react-label": "2.0.2",
+    "@radix-ui/react-navigation-menu": "1.1.3",
+    "@radix-ui/react-select": "1.2.2",
+    "@radix-ui/react-slot": "1.0.2",
+    "@testing-library/jest-dom": "6.1.3",
+    "@testing-library/react": "14.0.0",
+    "@testing-library/user-event": "14.5.1",
+    "@types/jest": "29.5.5",
+    "@types/node": "20.8.0",
+    "@types/react": "18.2.23",
+    "@types/react-dom": "18.2.8",
     "@types/react-router-dom": "5.3.3",
-    "@types/recharts": "1.8.29",
-    "@vitejs/plugin-react": "4.3.4",
-    "autoprefixer": "10.4.20",
-    "axios": "1.7.9",
-    "class-variance-authority": "0.7.1",
-    "clsx": "2.1.1",
+    "@types/recharts": "1.8.25",
+    "@vitejs/plugin-react": "4.1.0",
+    "autoprefixer": "10.4.16",
+    "axios": "1.5.1",
+    "class-variance-authority": "0.7.0",
+    "clsx": "2.0.0",
     "date-fns": "2.30.0",
     "eslint-config-react-app": "7.0.1",
     "jsdom": "22.1.0",
-    "lucide-react": "0.468.0",
+    "lucide-react": "0.279.0",
     "npm-run-all": "4.1.5",
-    "postcss": "8.4.49",
-    "prettier": "3.4.2",
-    "react": "18.3.1",
-    "react-dom": "18.3.1",
-    "react-files": "3.0.3",
-    "react-router-dom": "6.28.0",
-    "recharts": "2.15.0",
+    "postcss": "8.4.31",
+    "prettier": "3.0.3",
+    "react": "18.2.0",
+    "react-dom": "18.2.0",
+    "react-router-dom": "6.16.0",
+    "recharts": "2.8.0",
     "tailwind-merge": "1.14.0",
-    "tailwindcss": "3.4.16",
+    "tailwindcss": "3.3.3",
     "tailwindcss-animate": "1.0.7",
-    "typescript": "5.7.2",
-    "vite": "5.4.12",
-    "vitest": "1.6.0"
+    "typescript": "5.2.2",
+    "vite": "4.4.9",
+    "vitest": "0.34.6"
   },
   "scripts": {
     "format": "prettier --write \"./*.js\" \"src/**/*.(tsx|ts|css)\"",

frontend/src/App.tsx

@@ -2,7 +2,6 @@ import React from "react";
 import { Route, Routes } from "react-router-dom";
 import { AuthApiTokens } from "./components/AuthApiTokens";
 import { Footer } from "./components/Footer";
-import { ImportListens } from "./components/ImportListens";
 import { LoginFailure } from "./components/LoginFailure";
 import { LoginLoading } from "./components/LoginLoading";
 import { NavBar } from "./components/NavBar";
@@ -54,7 +53,6 @@ export function App() {
                   element={<ReportTopGenres />}
                 />
                 <Route path="/auth/api-tokens" element={<AuthApiTokens />} />
-                <Route path="/import" element={<ImportListens />} />
               </Routes>
             )}
             {!user && (

frontend/src/api/api.ts

@@ -14,8 +14,6 @@ import { TopGenresItem } from "./entities/top-genres-item";
 import { TopGenresOptions } from "./entities/top-genres-options";
 import { TopTracksItem } from "./entities/top-tracks-item";
 import { TopTracksOptions } from "./entities/top-tracks-options";
-import { SpotifyExtendedStreamingHistoryItem } from "./entities/spotify-extended-streaming-history-item";
-import { ExtendedStreamingHistoryStatus } from "./entities/extended-streaming-history-status";
 
 export class UnauthenticatedError extends Error {}
 
@@ -278,7 +276,7 @@ export const revokeApiToken = async (
   id: string,
   client: AxiosInstance,
 ): Promise<void> => {
-  const res = await client.delete(`/api/v1/auth/api-tokens/${id}`);
+  const res = await client.delete<NewApiToken>(`/api/v1/auth/api-tokens/${id}`);
 
   switch (res.status) {
     case 200: {
@@ -292,50 +290,3 @@ export const revokeApiToken = async (
     }
   }
 };
-
-export const importExtendedStreamingHistory = async (
-  listens: SpotifyExtendedStreamingHistoryItem[],
-  client: AxiosInstance
-): Promise<void> => {
-  const res = await client.post(`/api/v1/import/extended-streaming-history`, {
-    listens,
-  });
-
-  switch (res.status) {
-    case 201: {
-      break;
-    }
-    case 401: {
-      throw new UnauthenticatedError(`No token or token expired`);
-    }
-    default: {
-      throw new Error(
-        `Unable to importExtendedStreamingHistory: ${res.status}`
-      );
-    }
-  }
-};
-
-export const getExtendedStreamingHistoryStatus = async (
-  client: AxiosInstance
-): Promise<ExtendedStreamingHistoryStatus> => {
-  const res = await client.get<ExtendedStreamingHistoryStatus>(
-    `/api/v1/import/extended-streaming-history/status`
-  );
-
-  switch (res.status) {
-    case 200: {
-      break;
-    }
-    case 401: {
-      throw new UnauthenticatedError(`No token or token expired`);
-    }
-    default: {
-      throw new Error(
-        `Unable to getExtendedStreamingHistoryStatus: ${res.status}`
-      );
-    }
-  }
-
-  return res.data;
-};

frontend/src/api/entities/extended-streaming-history-status.ts

@@ -1,4 +0,0 @@
-export interface ExtendedStreamingHistoryStatus {
-  total: number;
-  imported: number;
-}

frontend/src/api/entities/spotify-extended-streaming-history-item.ts

@@ -1,4 +0,0 @@
-export interface SpotifyExtendedStreamingHistoryItem {
-  ts: string;
-  spotify_track_uri: string;
-}

frontend/src/components/AuthApiTokens.tsx

@@ -2,6 +2,7 @@ import { format, formatDistanceToNow } from "date-fns";
 import React, { FormEvent, useCallback, useMemo, useState } from "react";
 import { ApiToken, NewApiToken } from "../api/entities/api-token";
 import { useApiTokens } from "../hooks/use-api";
+import { useAuthProtection } from "../hooks/use-auth-protection";
 import { SpinnerIcon } from "../icons/Spinner";
 import TrashcanIcon from "../icons/Trashcan";
 import { Spinner } from "./ui/Spinner";
@@ -9,10 +10,7 @@ import { Spinner } from "./ui/Spinner";
 export const AuthApiTokens: React.FC = () => {
   const { apiTokens, isLoading, createToken, revokeToken } = useApiTokens();
   const sortedTokens = useMemo(
-    () =>
-      apiTokens
-        .filter((token) => !token.revokedAt)
-        .sort((a, b) => (a.createdAt > b.createdAt ? -1 : 1)),
+    () => apiTokens.sort((a, b) => (a.createdAt > b.createdAt ? -1 : 1)),
     [apiTokens],
   );
 
@@ -25,7 +23,7 @@ export const AuthApiTokens: React.FC = () => {
         <p className="mb-4">
           You can use API Tokens to access the Listory API directly. You can
           find the API docs{" "}
-          <a href="/api/docs" target="_blank" className={"underline"}>
+          <a href="/api/docs" target="_blank">
             here
           </a>
           .

frontend/src/components/ImportListens.tsx

@@ -1,280 +0,0 @@
-import React, { useCallback, useEffect, useState } from "react";
-import Files from "react-files";
-import type { ReactFile } from "react-files";
-import {
-  useSpotifyImportExtendedStreamingHistory,
-  useSpotifyImportExtendedStreamingHistoryStatus,
-} from "../hooks/use-api";
-import { SpotifyExtendedStreamingHistoryItem } from "../api/entities/spotify-extended-streaming-history-item";
-import { ErrorIcon } from "../icons/Error";
-import { numberToPercent } from "../util/numberToPercent";
-import { Button } from "./ui/button";
-import { Table, TableBody, TableCell, TableRow } from "./ui/table";
-import { Badge } from "./ui/badge";
-import {
-  Card,
-  CardContent,
-  CardDescription,
-  CardFooter,
-  CardHeader,
-  CardTitle,
-} from "./ui/card";
-import { Code } from "./ui/code";
-
-export const ImportListens: React.FC = () => {
-  return (
-    <>
-      <div className="flex justify-between">
-        <p className="text-2xl font-normal">
-          Import Listens from Spotify Extended Streaming History
-        </p>
-      </div>
-      <div className="shadow-xl bg-gray-100 dark:bg-gray-800 rounded p-4 m-2">
-        <p className="my-4">
-          Here you can import your full Spotify Listen history that was exported
-          from the{" "}
-          <a
-            target="blank"
-            href="https://www.spotify.com/us/account/privacy/"
-            className="underline"
-          >
-            Extended streaming history
-          </a>
-          .
-        </p>
-        <p className="my-4">
-          The extended streaming history contains additional personally
-          identifiable data such as the IP address of the listen (which can be
-          linked to locations). To avoid saving this on the server, the data is
-          preprocessed in your web browser and only the necessary data
-          (timestamp & track ID) are sent to the server.
-        </p>
-        <p className="my-4">
-          If an error occurs, you can always retry uploading the file, Listory
-          deduplicates any listens to make sure that everything is saved only
-          once.
-        </p>
-
-        <FileUpload />
-        <ImportProgress />
-      </div>
-    </>
-  );
-};
-
-interface FileData {
-  file: ReactFile;
-  status: Status;
-  error?: Error;
-}
-
-enum Status {
-  Select,
-  Import,
-  Finished,
-  Error,
-}
-
-const FileUpload: React.FC = () => {
-  // Using a map is ... meh, need to wrap all state updates in `new Map()` so react re-renders
-  const [fileMap, setFileMap] = useState<Map<ReactFile["id"], FileData>>(
-    new Map(),
-  );
-
-  const [status, setStatus] = useState<Status>(Status.Select);
-
-  const addFiles = useCallback(
-    (files: ReactFile[]) => {
-      setFileMap((_fileMap) => {
-        files.forEach((file) =>
-          _fileMap.set(file.id, { file, status: Status.Select }),
-        );
-        return new Map(_fileMap);
-      });
-    },
-    [setFileMap],
-  );
-
-  const updateFile = useCallback((data: FileData) => {
-    setFileMap((_fileMap) => new Map(_fileMap.set(data.file.id, data)));
-  }, []);
-
-  const clearFiles = useCallback(() => {
-    setFileMap(new Map());
-  }, [setFileMap]);
-
-  const { importHistory } = useSpotifyImportExtendedStreamingHistory();
-
-  const handleImport = useCallback(async () => {
-    setStatus(Status.Import);
-
-    let errorOccurred = false;
-
-    for (const data of fileMap.values()) {
-      data.status = Status.Import;
-      updateFile(data);
-
-      let items: SpotifyExtendedStreamingHistoryItem[];
-
-      // Scope so these tmp variables can be GC-ed ASAP
-      {
-        const fileContent = await data.file.text();
-
-        const rawItems = JSON.parse(
-          fileContent,
-        ) as SpotifyExtendedStreamingHistoryItem[];
-
-        items = rawItems
-          .filter(({ spotify_track_uri }) => spotify_track_uri !== null)
-          .map(({ ts, spotify_track_uri }) => ({
-            ts,
-            spotify_track_uri,
-          }));
-      }
-
-      try {
-        await importHistory(items);
-
-        data.status = Status.Finished;
-      } catch (err) {
-        data.error = err as Error;
-        data.status = Status.Error;
-
-        errorOccurred = true;
-      }
-      updateFile(data);
-    }
-
-    if (!errorOccurred) {
-      setStatus(Status.Finished);
-    }
-  }, [fileMap, importHistory, updateFile]);
-
-  return (
-    <Card className="mb-5">
-      <CardHeader>
-        <CardTitle>File Upload</CardTitle>
-        <CardDescription>
-          Select <Code>endsong_XY.json</Code> files here and start the import.
-        </CardDescription>
-      </CardHeader>
-      <CardContent>
-        <Files
-          className="shadow-inner bg-gray-200 dark:bg-gray-700 rounded p-4 text-center cursor-pointer"
-          dragActiveClassName=""
-          onChange={addFiles}
-          accepts={["application/json"]}
-          multiple
-          clickable
-        >
-          Drop files here or click to upload
-        </Files>
-        <Table>
-          <TableBody>
-            {Array.from(fileMap.values()).map((data) => (
-              <File key={data.file.id} data={data} />
-            ))}
-          </TableBody>
-        </Table>
-      </CardContent>
-      <CardFooter className="flex gap-x-2">
-        <Button
-          onClick={() =>
-            handleImport().catch((e) => console.error("Import Failed:", e))
-          }
-          variant="secondary"
-          disabled={status !== Status.Select}
-        >
-          Start Import
-        </Button>
-        <Button
-          onClick={clearFiles}
-          variant="secondary"
-          disabled={status !== Status.Select}
-        >
-          Remove All Files
-        </Button>
-      </CardFooter>
-    </Card>
-  );
-};
-
-const File: React.FC<{ data: FileData }> = ({ data }) => {
-  const hasErrors = data.status === Status.Error && data.error;
-
-  return (
-    <TableRow>
-      <TableCell>{data.file.name}</TableCell>
-      <TableCell className="text-sm font-thin">
-        {data.file.sizeReadable}
-      </TableCell>
-      <TableCell className="text-right">
-        {data.status === Status.Select && <Badge>Prepared for import!</Badge>}
-        {data.status === Status.Import && <Badge>Loading!</Badge>}
-        {data.status === Status.Finished && <Badge>Check!</Badge>}
-        {hasErrors && (
-          <Badge variant="destructive">
-            <ErrorIcon />
-            {data.error?.message}
-          </Badge>
-        )}
-      </TableCell>
-    </TableRow>
-  );
-};
-
-const ImportProgress: React.FC = () => {
-  const {
-    importStatus: { total, imported },
-    isLoading,
-    reload,
-  } = useSpotifyImportExtendedStreamingHistoryStatus();
-
-  useEffect(() => {
-    const interval = setInterval(() => {
-      if (!isLoading) {
-        reload();
-      }
-    }, 1000);
-
-    return () => clearInterval(interval);
-  }, [isLoading, reload]);
-
-  return (
-    <Card>
-      <CardHeader>
-        <CardTitle>Import Progress</CardTitle>
-        <CardDescription>
-          Shows how many of the submitted listens are already imported and
-          visible to you. This will take a while, and the process might halt for
-          a few minutes if we hit the Spotify API rate limit. If this is not
-          finished after a few hours, please contact your Listory administrator.
-        </CardDescription>
-      </CardHeader>
-      <CardContent>
-        <div className="flex pb-2">
-          <div className="md:flex w-10/12">
-            <div className={`md:w-full font-bold`}>
-              Imported
-              <br />
-              {imported}
-            </div>
-          </div>
-          <div className="w-2/12 text-right">
-            Total
-            <br />
-            {total}
-          </div>
-        </div>
-        {total > 0 && (
-          <div className="h-2 w-full bg-gradient-to-r from-teal-200/25 via-green-400 to-violet-400 dark:from-teal-700/25 dark:via-green-600/85 dark:to-amber-500 flex flex-row-reverse">
-            <div
-              style={{ width: numberToPercent(1 - imported / total) }}
-              className="h-full bg-gray-100 dark:bg-gray-900"
-            ></div>
-          </div>
-        )}
-      </CardContent>
-    </Card>
-  );
-};

frontend/src/components/NavBar.tsx

@@ -3,7 +3,6 @@ import { Link } from "react-router-dom";
 import { User } from "../api/entities/user";
 import { useAuth } from "../hooks/use-auth";
 import { CogwheelIcon } from "../icons/Cogwheel";
-import { ImportIcon } from "../icons/Import";
 import { SpotifyLogo } from "../icons/Spotify";
 import { Avatar, AvatarImage, AvatarFallback } from "./ui/avatar";
 import {
@@ -189,12 +188,6 @@ const NavUserInfo: React.FC<{ user: User }> = ({ user }) => {
               API Tokens
             </Link>
           </DropdownMenuItem>
-          <DropdownMenuItem asChild>
-            <Link to="/import">
-              <ImportIcon className="w-5 h-5 fill-current pr-2" />
-              Import Listens
-            </Link>
-          </DropdownMenuItem>
         </DropdownMenuGroup>
       </DropdownMenuContent>
     </DropdownMenu>

frontend/src/components/reports/RecentListens.tsx

@@ -50,7 +50,7 @@ export const RecentListens: React.FC = () => {
         )}
         <div>
           {recentListens.length > 0 && (
-            <Table className="table-auto w-full">
+            <Table className="table-auto w-full text-base">
               <TableBody>
                 {recentListens.map((listen) => (
                   <ListenItem listen={listen} key={listen.id} />

frontend/src/components/reports/TopListItem.tsx

@@ -1,5 +1,4 @@
 import React from "react";
-import { numberToPercent } from "../../util/numberToPercent";
 
 export interface TopListItemProps {
   title: string;
@@ -43,3 +42,9 @@ export const TopListItem: React.FC<TopListItemProps> = ({
 
 const isMaxCountValid = (maxCount: number) =>
   !(Number.isNaN(maxCount) || maxCount === 0);
+
+const numberToPercent = (ratio: number) =>
+  ratio.toLocaleString(undefined, {
+    style: "percent",
+    minimumFractionDigits: 2,
+  });

frontend/src/components/ui/badge.tsx

@@ -1,36 +0,0 @@
-import * as React from "react";
-import { cva, type VariantProps } from "class-variance-authority";
-
-import { cn } from "src/lib/utils";
-
-const badgeVariants = cva(
-  "inline-flex items-center rounded-full border border-gray-200 px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-gray-900 focus:ring-offset-2 dark:border-gray-800 dark:focus:ring-gray-300",
-  {
-    variants: {
-      variant: {
-        default:
-          "border-transparent bg-gray-900 text-gray-50 hover:bg-gray-900/80 dark:bg-gray-50 dark:text-gray-900 dark:hover:bg-gray-50/80",
-        secondary:
-          "border-transparent bg-gray-100 text-gray-900 hover:bg-gray-100/80 dark:bg-gray-800 dark:text-gray-50 dark:hover:bg-gray-800/80",
-        destructive:
-          "border-transparent bg-red-500 text-gray-50 hover:bg-red-500/80 dark:bg-red-900 dark:text-gray-50 dark:hover:bg-red-900/80",
-        outline: "text-gray-900 dark:text-gray-50",
-      },
-    },
-    defaultVariants: {
-      variant: "default",
-    },
-  },
-);
-
-export interface BadgeProps
-  extends React.HTMLAttributes<HTMLDivElement>,
-    VariantProps<typeof badgeVariants> {}
-
-function Badge({ className, variant, ...props }: BadgeProps) {
-  return (
-    <div className={cn(badgeVariants({ variant }), className)} {...props} />
-  );
-}
-
-export { Badge, badgeVariants };

frontend/src/components/ui/card.tsx

@@ -1,86 +0,0 @@
-import * as React from "react";
-
-import { cn } from "src/lib/utils";
-
-const Card = React.forwardRef<
-  HTMLDivElement,
-  React.HTMLAttributes<HTMLDivElement>
->(({ className, ...props }, ref) => (
-  <div
-    ref={ref}
-    className={cn(
-      "rounded-lg border border-gray-200 bg-white text-gray-900 shadow-sm dark:border-gray-800 dark:bg-gray-900 dark:text-gray-50",
-      className,
-    )}
-    {...props}
-  />
-));
-Card.displayName = "Card";
-
-const CardHeader = React.forwardRef<
-  HTMLDivElement,
-  React.HTMLAttributes<HTMLDivElement>
->(({ className, ...props }, ref) => (
-  <div
-    ref={ref}
-    className={cn("flex flex-col space-y-1.5 p-6", className)}
-    {...props}
-  />
-));
-CardHeader.displayName = "CardHeader";
-
-const CardTitle = React.forwardRef<
-  HTMLParagraphElement,
-  React.HTMLAttributes<HTMLHeadingElement>
->(({ className, ...props }, ref) => (
-  <h3
-    ref={ref}
-    className={cn(
-      "text-2xl font-semibold leading-none tracking-tight",
-      className,
-    )}
-    {...props}
-  />
-));
-CardTitle.displayName = "CardTitle";
-
-const CardDescription = React.forwardRef<
-  HTMLParagraphElement,
-  React.HTMLAttributes<HTMLParagraphElement>
->(({ className, ...props }, ref) => (
-  <p
-    ref={ref}
-    className={cn("text-sm text-gray-500 dark:text-gray-400", className)}
-    {...props}
-  />
-));
-CardDescription.displayName = "CardDescription";
-
-const CardContent = React.forwardRef<
-  HTMLDivElement,
-  React.HTMLAttributes<HTMLDivElement>
->(({ className, ...props }, ref) => (
-  <div ref={ref} className={cn("p-6 pt-0", className)} {...props} />
-));
-CardContent.displayName = "CardContent";
-
-const CardFooter = React.forwardRef<
-  HTMLDivElement,
-  React.HTMLAttributes<HTMLDivElement>
->(({ className, ...props }, ref) => (
-  <div
-    ref={ref}
-    className={cn("flex items-center p-6 pt-0", className)}
-    {...props}
-  />
-));
-CardFooter.displayName = "CardFooter";
-
-export {
-  Card,
-  CardHeader,
-  CardFooter,
-  CardTitle,
-  CardDescription,
-  CardContent,
-};

frontend/src/components/ui/code.tsx

@@ -1,9 +0,0 @@
-import React, { PropsWithChildren } from "react";
-
-export const Code: React.FC<PropsWithChildren> = ({ children }) => {
-  return (
-    <code className="tracking-wide font-mono bg-gray-200 dark:bg-gray-600 rounded-md px-1">
-      {children}
-    </code>
-  );
-};

frontend/src/components/ui/table.tsx

@@ -1,6 +1,6 @@
-import * as React from "react";
+import * as React from "react"
 
-import { cn } from "src/lib/utils";
+import { cn } from "src/lib/utils"
 
 const Table = React.forwardRef<
   HTMLTableElement,
@@ -9,20 +9,20 @@ const Table = React.forwardRef<
   <div className="relative w-full overflow-auto">
     <table
       ref={ref}
-      className={cn("w-full caption-bottom", className)}
+      className={cn("w-full caption-bottom text-sm", className)}
       {...props}
     />
   </div>
-));
-Table.displayName = "Table";
+))
+Table.displayName = "Table"
 
 const TableHeader = React.forwardRef<
   HTMLTableSectionElement,
   React.HTMLAttributes<HTMLTableSectionElement>
 >(({ className, ...props }, ref) => (
   <thead ref={ref} className={cn("[&_tr]:border-b", className)} {...props} />
-));
-TableHeader.displayName = "TableHeader";
+))
+TableHeader.displayName = "TableHeader"
 
 const TableBody = React.forwardRef<
   HTMLTableSectionElement,
@@ -33,8 +33,8 @@ const TableBody = React.forwardRef<
     className={cn("[&_tr:last-child]:border-0", className)}
     {...props}
   />
-));
-TableBody.displayName = "TableBody";
+))
+TableBody.displayName = "TableBody"
 
 const TableFooter = React.forwardRef<
   HTMLTableSectionElement,
@@ -42,14 +42,11 @@ const TableFooter = React.forwardRef<
 >(({ className, ...props }, ref) => (
   <tfoot
     ref={ref}
-    className={cn(
-      "bg-gray-900 font-medium text-gray-50 dark:bg-gray-50 dark:text-gray-900",
-      className,
-    )}
+    className={cn("bg-gray-900 font-medium text-gray-50 dark:bg-gray-50 dark:text-gray-900", className)}
     {...props}
   />
-));
-TableFooter.displayName = "TableFooter";
+))
+TableFooter.displayName = "TableFooter"
 
 const TableRow = React.forwardRef<
   HTMLTableRowElement,
@@ -59,12 +56,12 @@ const TableRow = React.forwardRef<
     ref={ref}
     className={cn(
       "border-b transition-colors hover:bg-gray-100/50 data-[state=selected]:bg-gray-100 dark:hover:bg-gray-800/50 dark:data-[state=selected]:bg-gray-800",
-      className,
+      className
     )}
     {...props}
   />
-));
-TableRow.displayName = "TableRow";
+))
+TableRow.displayName = "TableRow"
 
 const TableHead = React.forwardRef<
   HTMLTableCellElement,
@@ -74,12 +71,12 @@ const TableHead = React.forwardRef<
     ref={ref}
     className={cn(
       "h-12 px-4 text-left align-middle font-medium text-gray-500 [&:has([role=checkbox])]:pr-0 dark:text-gray-400",
-      className,
+      className
     )}
     {...props}
   />
-));
-TableHead.displayName = "TableHead";
+))
+TableHead.displayName = "TableHead"
 
 const TableCell = React.forwardRef<
   HTMLTableCellElement,
@@ -90,8 +87,8 @@ const TableCell = React.forwardRef<
     className={cn("p-4 align-middle [&:has([role=checkbox])]:pr-0", className)}
     {...props}
   />
-));
-TableCell.displayName = "TableCell";
+))
+TableCell.displayName = "TableCell"
 
 const TableCaption = React.forwardRef<
   HTMLTableCaptionElement,
@@ -102,8 +99,8 @@ const TableCaption = React.forwardRef<
     className={cn("mt-4 text-sm text-gray-500 dark:text-gray-400", className)}
     {...props}
   />
-));
-TableCaption.displayName = "TableCaption";
+))
+TableCaption.displayName = "TableCaption"
 
 export {
   Table,
@@ -114,4 +111,4 @@ export {
   TableRow,
   TableCell,
   TableCaption,
-};
+}

frontend/src/hooks/use-api.tsx

@@ -2,14 +2,12 @@ import { useCallback, useMemo } from "react";
 import {
   createApiToken,
   getApiTokens,
-  getExtendedStreamingHistoryStatus,
   getListensReport,
   getRecentListens,
   getTopAlbums,
   getTopArtists,
   getTopGenres,
   getTopTracks,
-  importExtendedStreamingHistory,
   revokeApiToken,
 } from "../api/api";
 import { ListenReportOptions } from "../api/entities/listen-report-options";
@@ -20,7 +18,6 @@ import { TopGenresOptions } from "../api/entities/top-genres-options";
 import { TopTracksOptions } from "../api/entities/top-tracks-options";
 import { useApiClient } from "./use-api-client";
 import { useAsync } from "./use-async";
-import { SpotifyExtendedStreamingHistoryItem } from "../api/entities/spotify-extended-streaming-history-item";
 
 const INITIAL_EMPTY_ARRAY: [] = [];
 Object.freeze(INITIAL_EMPTY_ARRAY);
@@ -146,7 +143,9 @@ export const useApiTokens = () => {
   const createToken = useCallback(
     async (description: string) => {
       const apiToken = await createApiToken(description, client);
+      console.log("apiToken created", apiToken);
       await reload();
+      console.log("reloaded data");
 
       return apiToken;
     },
@@ -163,38 +162,3 @@ export const useApiTokens = () => {
 
   return { apiTokens, isLoading, error, createToken, revokeToken };
 };
-
-export const useSpotifyImportExtendedStreamingHistory = () => {
-  const { client } = useApiClient();
-
-  const importHistory = useCallback(
-    async (listens: SpotifyExtendedStreamingHistoryItem[]) => {
-      return importExtendedStreamingHistory(listens, client);
-    },
-    [client]
-  );
-
-  const getStatus = useCallback(async () => {
-    return getExtendedStreamingHistoryStatus(client);
-  }, [client]);
-
-  return { importHistory, getStatus };
-};
-
-export const useSpotifyImportExtendedStreamingHistoryStatus = () => {
-  const { client } = useApiClient();
-
-  const fetchData = useMemo(
-    () => () => getExtendedStreamingHistoryStatus(client),
-    [client]
-  );
-
-  const {
-    value: importStatus,
-    pending: isLoading,
-    error,
-    reload,
-  } = useAsync(fetchData, { total: 0, imported: 0 });
-
-  return { importStatus, isLoading, error, reload };
-};

frontend/src/hooks/use-auth-protection.tsx

@@ -0,0 +1,16 @@
+import { useCallback } from "react";
+import { useNavigate } from "react-router-dom";
+import { useAuth } from "./use-auth";
+
+export function useAuthProtection() {
+  const { user } = useAuth();
+  const navigate = useNavigate();
+
+  const requireUser = useCallback(async () => {
+    if (!user) {
+      navigate("/");
+    }
+  }, [user, navigate]);
+
+  return { requireUser };
+}

frontend/src/icons/Error.tsx

@@ -1,14 +0,0 @@
-import React from "react";
-
-export const ErrorIcon: React.FC<React.SVGProps<SVGSVGElement>> = (props) => {
-  return (
-    <svg
-      version="1.1"
-      xmlns="http://www.w3.org/2000/svg"
-      viewBox="0 0 50 50"
-      fill="#D75A4A"
-    >
-      <circle fill="fill" cx="25" cy="25" r="25" />
-    </svg>
-  );
-};

frontend/src/icons/Import.tsx

@@ -1,12 +0,0 @@
-import * as React from "react";
-export const ImportIcon: React.FC<React.SVGProps<SVGSVGElement>> = (props) => (
-  <svg
-    xmlns="http://www.w3.org/2000/svg"
-    xmlSpace="preserve"
-    viewBox="0 0 60.903 60.903"
-    {...props}
-  >
-    <path d="M49.561 16.464H39.45v6h10.111c3.008 0 5.341 1.535 5.341 2.857v26.607c0 1.321-2.333 2.858-5.341 2.858H11.34c-3.007 0-5.34-1.537-5.34-2.858V25.324c0-1.322 2.333-2.858 5.34-2.858h10.11v-6H11.34C4.981 16.466 0 20.357 0 25.324v26.605c0 4.968 4.981 8.857 11.34 8.857h38.223c6.357 0 11.34-3.891 11.34-8.857V25.324c-.001-4.969-4.982-8.86-11.342-8.86z" />
-    <path d="M39.529 29.004a2.99 2.99 0 0 0-2.121.88l-3.756 3.755V3.117a3 3 0 0 0-6 0v30.724l-3.959-3.958a2.992 2.992 0 0 0-4.242 0 2.997 2.997 0 0 0 0 4.241l8.957 8.957a2.988 2.988 0 0 0 2.12.877h.045c.768 0 1.534-.291 2.12-.877l8.957-8.957a2.997 2.997 0 0 0-2.121-5.12z" />
-  </svg>
-);

frontend/src/react-files.d.ts

@@ -1,36 +0,0 @@
-//// <reference types="react" />
-
-declare module "react-files" {
-  declare const Files: React.FC<
-    Partial<{
-      accepts: string[];
-      children: React.ReactNode;
-      className: string;
-      clickable: boolean;
-      dragActiveClassName: string;
-      inputProps: unknown;
-      multiple: boolean;
-      maxFiles: number;
-      maxFileSize: number;
-      minFileSize: number;
-      name: string;
-      onChange: (files: ReactFile[]) => void;
-      onDragEnter: () => void;
-      onDragLeave: () => void;
-      onError: (
-        error: { code: number; message: string },
-        file: ReactFile
-      ) => void;
-      style: object;
-    }>
-  >;
-
-  export type ReactFile = File & {
-    id: string;
-    extension: string;
-    sizeReadable: string;
-    preview: { type: "image"; url: string } | { type: "file" };
-  };
-
-  export default Files;
-}

frontend/src/util/numberToPercent.ts

@@ -1,5 +0,0 @@
-export const numberToPercent = (ratio: number) =>
-  ratio.toLocaleString(undefined, {
-    style: "percent",
-    minimumFractionDigits: 2,
-  });

frontend/tsconfig.json

@@ -1,7 +1,11 @@
 {
   "compilerOptions": {
-    "target": "ES2022",
-    "lib": ["dom", "dom.iterable", "esnext"],
+    "target": "es5",
+    "lib": [
+      "dom",
+      "dom.iterable",
+      "esnext"
+    ],
     "allowJs": true,
     "skipLibCheck": true,
     "esModuleInterop": true,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@listory/api",
-  "version": "1.31.0",
+  "version": "1.29.0",
   "description": "Track your Spotify Listen History",
   "author": {
     "name": "Julian Tölle",
@@ -28,91 +28,91 @@
   "dependencies": {
     "@apricote/nest-pg-boss": "2.1.0",
     "@narando/nest-axios-interceptor": "3.0.0",
-    "@nestjs/axios": "3.1.3",
-    "@nestjs/common": "10.4.15",
-    "@nestjs/config": "3.3.0",
-    "@nestjs/core": "10.4.15",
-    "@nestjs/jwt": "10.2.0",
-    "@nestjs/passport": "10.0.3",
-    "@nestjs/platform-express": "10.4.15",
-    "@nestjs/serve-static": "4.0.2",
-    "@nestjs/swagger": "7.4.2",
-    "@nestjs/terminus": "10.2.3",
-    "@nestjs/typeorm": "10.0.2",
-    "@opentelemetry/api": "1.9.0",
+    "@nestjs/axios": "3.0.0",
+    "@nestjs/common": "10.2.6",
+    "@nestjs/config": "3.1.1",
+    "@nestjs/core": "10.2.6",
+    "@nestjs/jwt": "10.1.1",
+    "@nestjs/passport": "10.0.2",
+    "@nestjs/platform-express": "10.2.6",
+    "@nestjs/serve-static": "4.0.0",
+    "@nestjs/swagger": "7.1.12",
+    "@nestjs/terminus": "10.1.1",
+    "@nestjs/typeorm": "10.0.0",
+    "@opentelemetry/api": "1.6.0",
     "@opentelemetry/api-metrics": "0.33.0",
-    "@opentelemetry/context-async-hooks": "1.29.0",
-    "@opentelemetry/exporter-prometheus": "0.56.0",
-    "@opentelemetry/exporter-trace-otlp-http": "0.56.0",
-    "@opentelemetry/instrumentation": "0.56.0",
-    "@opentelemetry/instrumentation-dns": "0.42.0",
-    "@opentelemetry/instrumentation-express": "0.46.0",
-    "@opentelemetry/instrumentation-http": "0.56.0",
-    "@opentelemetry/instrumentation-nestjs-core": "0.43.0",
-    "@opentelemetry/instrumentation-pg": "0.49.0",
-    "@opentelemetry/instrumentation-pino": "0.45.0",
-    "@opentelemetry/resources": "1.29.0",
-    "@opentelemetry/sdk-metrics": "1.29.0",
-    "@opentelemetry/sdk-node": "0.56.0",
-    "@opentelemetry/sdk-trace-base": "1.29.0",
-    "@opentelemetry/semantic-conventions": "1.28.0",
-    "@sentry/node": "7.120.3",
+    "@opentelemetry/context-async-hooks": "1.17.0",
+    "@opentelemetry/exporter-prometheus": "0.43.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.43.0",
+    "@opentelemetry/instrumentation": "0.43.0",
+    "@opentelemetry/instrumentation-dns": "0.32.2",
+    "@opentelemetry/instrumentation-express": "0.33.1",
+    "@opentelemetry/instrumentation-http": "0.43.0",
+    "@opentelemetry/instrumentation-nestjs-core": "0.33.1",
+    "@opentelemetry/instrumentation-pg": "0.36.1",
+    "@opentelemetry/instrumentation-pino": "0.34.1",
+    "@opentelemetry/resources": "1.17.0",
+    "@opentelemetry/sdk-metrics": "1.17.0",
+    "@opentelemetry/sdk-node": "0.43.0",
+    "@opentelemetry/sdk-trace-base": "1.17.0",
+    "@opentelemetry/semantic-conventions": "1.17.0",
+    "@sentry/node": "7.72.0",
     "class-transformer": "0.5.1",
-    "class-validator": "0.14.1",
-    "cookie-parser": "1.4.7",
+    "class-validator": "0.14.0",
+    "cookie-parser": "1.4.6",
     "date-fns": "2.30.0",
-    "joi": "17.13.3",
+    "joi": "17.10.2",
     "lodash": "4.17.21",
-    "nest-raven": "10.1.0",
+    "nest-raven": "10.0.0",
     "nestjs-otel": "5.1.5",
-    "nestjs-pino": "4.1.0",
+    "nestjs-pino": "3.5.0",
     "nestjs-typeorm-paginate": "4.0.4",
-    "passport": "0.7.0",
+    "passport": "0.6.0",
     "passport-http-bearer": "1.0.1",
     "passport-jwt": "4.0.1",
     "passport-spotify": "2.0.0",
-    "pg": "8.13.1",
+    "pg": "8.11.3",
     "pg-boss": "9.0.3",
-    "pino": "8.21.0",
-    "pino-http": "9.0.0",
-    "reflect-metadata": "0.1.14",
-    "rimraf": "5.0.10",
+    "pino": "8.15.1",
+    "pino-http": "8.5.0",
+    "reflect-metadata": "0.1.13",
+    "rimraf": "5.0.5",
     "rxjs": "7.8.1",
-    "typeorm": "0.3.20"
+    "typeorm": "0.3.17"
   },
   "devDependencies": {
-    "@nestjs/cli": "10.4.9",
-    "@nestjs/schematics": "10.2.3",
-    "@nestjs/testing": "10.4.15",
-    "@types/cookie-parser": "1.4.8",
-    "@types/express": "5.0.0",
-    "@types/jest": "29.5.14",
-    "@types/lodash": "4.17.14",
-    "@types/node": "20.17.16",
-    "@types/passport-http-bearer": "1.0.41",
-    "@types/passport-jwt": "4.0.1",
-    "@types/supertest": "6.0.2",
-    "@typescript-eslint/eslint-plugin": "6.21.0",
-    "@typescript-eslint/parser": "6.21.0",
-    "eslint": "8.57.1",
+    "@nestjs/cli": "10.1.18",
+    "@nestjs/schematics": "10.0.2",
+    "@nestjs/testing": "10.2.6",
+    "@types/cookie-parser": "1.4.4",
+    "@types/express": "4.17.18",
+    "@types/jest": "29.5.5",
+    "@types/lodash": "4.14.199",
+    "@types/node": "20.8.0",
+    "@types/passport-http-bearer": "1.0.38",
+    "@types/passport-jwt": "3.0.10",
+    "@types/supertest": "2.0.13",
+    "@typescript-eslint/eslint-plugin": "6.7.3",
+    "@typescript-eslint/parser": "6.7.3",
+    "eslint": "8.50.0",
     "eslint-config-airbnb-base": "15.0.0",
     "eslint-config-airbnb-typescript": "17.1.0",
-    "eslint-config-prettier": "9.1.0",
-    "eslint-plugin-import": "2.31.0",
-    "eslint-plugin-jsdoc": "48.11.0",
-    "eslint-plugin-jsx-a11y": "6.10.2",
+    "eslint-config-prettier": "9.0.0",
+    "eslint-plugin-import": "2.28.1",
+    "eslint-plugin-jsdoc": "46.8.2",
+    "eslint-plugin-jsx-a11y": "6.7.1",
     "eslint-plugin-prefer-arrow": "1.2.3",
-    "eslint-plugin-react": "7.37.4",
-    "eslint-plugin-react-hooks": "4.6.2",
+    "eslint-plugin-react": "7.33.2",
+    "eslint-plugin-react-hooks": "4.6.0",
     "jest": "29.7.0",
-    "pino-pretty": "10.3.1",
-    "prettier": "3.4.2",
-    "supertest": "6.3.4",
-    "ts-jest": "29.2.5",
-    "ts-loader": "9.5.1",
-    "ts-node": "10.9.2",
+    "pino-pretty": "10.2.0",
+    "prettier": "3.0.3",
+    "supertest": "6.3.3",
+    "ts-jest": "29.1.1",
+    "ts-loader": "9.4.4",
+    "ts-node": "10.9.1",
     "tsconfig-paths": "4.2.0",
-    "typescript": "5.7.2"
+    "typescript": "5.2.2"
   },
   "jest": {
     "moduleFileExtensions": [

renovate.json

@@ -7,8 +7,7 @@
     ":automergeBranch",
     ":automergeLinters",
     ":automergeTesters",
-    ":automergeTypes",
-    ":maintainLockFilesWeekly"
+    ":automergeTypes"
   ],
   "packageRules": [
     {

src/auth/auth.controller.spec.ts

@@ -1,5 +1,5 @@
 import { Test, TestingModule } from "@nestjs/testing";
-import type { Response as ExpressResponse } from "express";
+import type { Response } from "express";
 import { User } from "../users/user.entity";
 import { AuthSession } from "./auth-session.entity";
 import { AuthController } from "./auth.controller";
@@ -27,7 +27,7 @@ describe("AuthController", () => {
 
   describe("spotifyCallback", () => {
     let user: User;
-    let res: ExpressResponse;
+    let res: Response;
     let refreshToken: string;
 
     beforeEach(() => {
@@ -36,7 +36,7 @@ describe("AuthController", () => {
         statusCode: 200,
         cookie: jest.fn(),
         redirect: jest.fn(),
-      } as unknown as ExpressResponse;
+      } as unknown as Response;
 
       refreshToken = "REFRESH_TOKEN";
       authService.createSession = jest.fn().mockResolvedValue({ refreshToken });

src/auth/auth.controller.ts

@@ -1,5 +1,5 @@
 import {
-  Body as NestBody,
+  Body,
   Controller,
   Delete,
   Get,
@@ -10,7 +10,7 @@ import {
   UseGuards,
 } from "@nestjs/common";
 import { ApiBody, ApiTags } from "@nestjs/swagger";
-import type { Response as ExpressResponse } from "express";
+import type { Response } from "express";
 import { User } from "../users/user.entity";
 import { AuthSession } from "./auth-session.entity";
 import { AuthService } from "./auth.service";
@@ -42,7 +42,7 @@ export class AuthController {
   @Get("spotify/callback")
   @UseFilters(SpotifyAuthFilter)
   @UseGuards(SpotifyAuthGuard)
-  async spotifyCallback(@ReqUser() user: User, @Res() res: ExpressResponse) {
+  async spotifyCallback(@ReqUser() user: User, @Res() res: Response) {
     const { refreshToken } = await this.authService.createSession(user);
 
     // Refresh token should not be accessible to frontend to reduce risk
@@ -69,7 +69,7 @@ export class AuthController {
   @AuthAccessToken()
   async createApiToken(
     @ReqUser() user: User,
-    @NestBody("description") description: string,
+    @Body("description") description: string,
   ): Promise<NewApiTokenDto> {
     const apiToken = await this.authService.createApiToken(user, description);
 

src/auth/spotify.filter.ts

@@ -5,14 +5,14 @@ import {
   ForbiddenException,
   Logger,
 } from "@nestjs/common";
-import type { Response as ExpressResponse } from "express";
+import type { Response } from "express";
 
 @Catch()
 export class SpotifyAuthFilter implements ExceptionFilter {
   private readonly logger = new Logger(this.constructor.name);
 
   catch(exception: Error, host: ArgumentsHost) {
-    const response = host.switchToHttp().getResponse<ExpressResponse>();
+    const response = host.switchToHttp().getResponse<Response>();
 
     let reason = "unknown";
 

src/database/database.module.ts

@@ -27,7 +27,7 @@ export const DatabaseModule = TypeOrmModule.forRootAsync({
 
     // Debug/Development Options
     //
-    //logging: true,
+    // logging: true,
     //
     // synchronize: true,
     // migrationsRun: false,

src/database/migrations/09-CreateSpotifyImportTables.ts

@@ -1,68 +0,0 @@
-import {
-  MigrationInterface,
-  QueryRunner,
-  Table,
-  TableIndex,
-  TableForeignKey,
-} from "typeorm";
-import { TableColumnOptions } from "typeorm/schema-builder/options/TableColumnOptions";
-
-const primaryUUIDColumn: TableColumnOptions = {
-  name: "id",
-  type: "uuid",
-  isPrimary: true,
-  isGenerated: true,
-  generationStrategy: "uuid",
-};
-
-export class CreateSpotifyImportTables0000000000009
-  implements MigrationInterface
-{
-  async up(queryRunner: QueryRunner): Promise<void> {
-    await queryRunner.createTable(
-      new Table({
-        name: "spotify_extended_streaming_history_listen",
-        columns: [
-          primaryUUIDColumn,
-          { name: "userId", type: "uuid" },
-          { name: "playedAt", type: "timestamp" },
-          { name: "spotifyTrackUri", type: "varchar" },
-          { name: "trackId", type: "uuid", isNullable: true },
-          { name: "listenId", type: "uuid", isNullable: true },
-        ],
-        indices: [
-          new TableIndex({
-            name: "IDX_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_USER_PLAYED_AT",
-            columnNames: ["userId", "playedAt", "spotifyTrackUri"],
-            isUnique: true,
-          }),
-        ],
-        foreignKeys: [
-          new TableForeignKey({
-            name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_USER_ID",
-            columnNames: ["userId"],
-            referencedColumnNames: ["id"],
-            referencedTableName: "user",
-          }),
-          new TableForeignKey({
-            name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_TRACK_ID",
-            columnNames: ["trackId"],
-            referencedColumnNames: ["id"],
-            referencedTableName: "track",
-          }),
-          new TableForeignKey({
-            name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_LISTEN_ID",
-            columnNames: ["listenId"],
-            referencedColumnNames: ["id"],
-            referencedTableName: "listen",
-          }),
-        ],
-      }),
-      true,
-    );
-  }
-
-  async down(queryRunner: QueryRunner): Promise<void> {
-    await queryRunner.dropTable("spotify_extended_streaming_history_listen");
-  }
-}

src/listens/listen.repository.ts

@@ -3,6 +3,10 @@ import { Repository, SelectQueryBuilder } from "typeorm";
 import { EntityRepository } from "../database/entity-repository";
 import { Interval } from "../reports/interval";
 import { User } from "../users/user.entity";
+import {
+  CreateListenRequestDto,
+  CreateListenResponseDto,
+} from "./dto/create-listen.dto";
 import { Listen } from "./listen.entity";
 
 export class ListenScopes extends SelectQueryBuilder<Listen> {
@@ -33,4 +37,52 @@ export class ListenRepository extends Repository<Listen> {
   get scoped(): ListenScopes {
     return new ListenScopes(this.createQueryBuilder("listen"));
   }
+
+  async insertNoConflict({
+    user,
+    track,
+    playedAt,
+  }: CreateListenRequestDto): Promise<CreateListenResponseDto> {
+    const result = await this.createQueryBuilder()
+      .insert()
+      .values({
+        user,
+        track,
+        playedAt,
+      })
+      .onConflict('("playedAt", "trackId", "userId") DO NOTHING')
+      .execute();
+
+    const [insertedRowIdentifier] = result.identifiers;
+
+    if (!insertedRowIdentifier) {
+      // We did not insert a new listen, it already existed
+      return {
+        listen: await this.findOneBy({ user, track, playedAt }),
+        isDuplicate: true,
+      };
+    }
+
+    return {
+      listen: await this.findOneBy({ id: insertedRowIdentifier.id }),
+      isDuplicate: false,
+    };
+  }
+
+  /**
+   *
+   * @param rows
+   * @returns A list of all new (non-duplicate) listens
+   */
+  async insertsNoConflict(rows: CreateListenRequestDto[]): Promise<Listen[]> {
+    const result = await this.createQueryBuilder()
+      .insert()
+      .values(rows)
+      .orIgnore()
+      .execute();
+
+    return this.findBy(
+      result.identifiers.filter(Boolean).map(({ id }) => ({ id })),
+    );
+  }
 }

src/listens/listens.service.spec.ts

@@ -4,7 +4,9 @@ import {
   paginate,
   PaginationTypeEnum,
 } from "nestjs-typeorm-paginate";
+import { Track } from "../music-library/track.entity";
 import { User } from "../users/user.entity";
+import { CreateListenResponseDto } from "./dto/create-listen.dto";
 import { GetListensDto } from "./dto/get-listens.dto";
 import { Listen } from "./listen.entity";
 import { ListenRepository, ListenScopes } from "./listen.repository";
@@ -33,6 +35,39 @@ describe("ListensService", () => {
     expect(listenRepository).toBeDefined();
   });
 
+  describe("createListen", () => {
+    let user: User;
+    let track: Track;
+    let playedAt: Date;
+    let response: CreateListenResponseDto;
+    beforeEach(() => {
+      user = { id: "USER" } as User;
+      track = { id: "TRACK" } as Track;
+      playedAt = new Date("2021-01-01T00:00:00Z");
+
+      response = {
+        listen: {
+          id: "LISTEN",
+        } as Listen,
+        isDuplicate: true,
+      };
+      listenRepository.insertNoConflict = jest.fn().mockResolvedValue(response);
+    });
+
+    it("creates the listen", async () => {
+      await expect(
+        service.createListen({ user, track, playedAt }),
+      ).resolves.toEqual(response);
+
+      expect(listenRepository.insertNoConflict).toHaveBeenCalledTimes(1);
+      expect(listenRepository.insertNoConflict).toHaveBeenLastCalledWith({
+        user,
+        track,
+        playedAt,
+      });
+    });
+  });
+
   describe("getListens", () => {
     let options: GetListensDto & IPaginationOptions;
     let user: User;

src/listens/listens.service.ts

@@ -1,12 +1,14 @@
 import { Injectable } from "@nestjs/common";
-import { Span } from "nestjs-otel";
 import {
   IPaginationOptions,
   paginate,
   Pagination,
   PaginationTypeEnum,
 } from "nestjs-typeorm-paginate";
-import { CreateListenRequestDto } from "./dto/create-listen.dto";
+import {
+  CreateListenRequestDto,
+  CreateListenResponseDto,
+} from "./dto/create-listen.dto";
 import { GetListensDto } from "./dto/get-listens.dto";
 import { Listen } from "./listen.entity";
 import { ListenRepository, ListenScopes } from "./listen.repository";
@@ -15,7 +17,20 @@ import { ListenRepository, ListenScopes } from "./listen.repository";
 export class ListensService {
   constructor(private readonly listenRepository: ListenRepository) {}
 
-  @Span()
+  async createListen({
+    user,
+    track,
+    playedAt,
+  }: CreateListenRequestDto): Promise<CreateListenResponseDto> {
+    const response = await this.listenRepository.insertNoConflict({
+      user,
+      track,
+      playedAt,
+    });
+
+    return response;
+  }
+
   async createListens(
     listensData: CreateListenRequestDto[],
   ): Promise<Listen[]> {
@@ -31,11 +46,9 @@ export class ListensService {
         ),
     );
 
-    const newListens = await this.listenRepository.save(
+    return this.listenRepository.save(
       missingListens.map((entry) => this.listenRepository.create(entry)),
     );
-
-    return [...existingListens, ...newListens];
   }
 
   async getListens(

src/main.ts

@@ -43,7 +43,6 @@ async function bootstrap() {
 
   const app = await NestFactory.create<NestExpressApplication>(AppModule, {
     bufferLogs: true,
-    rawBody: true,
   });
   app.useLogger(app.get(Logger));
   app.useGlobalPipes(
@@ -52,10 +51,6 @@ async function bootstrap() {
       transformOptions: { enableImplicitConversion: true },
     }),
   );
-  app.useBodyParser("json", {
-    limit:
-      "10mb" /* Need large bodies for Spotify Extended Streaming History */,
-  });
   app.enableShutdownHooks();
 
   const configService = app.get<ConfigService>(ConfigService);

src/music-library/dto/find-track.dto.ts

@@ -1,6 +1,5 @@
 export class FindTrackDto {
   spotify: {
-    id?: string;
-    uri?: string;
+    id: string;
   };
 }

src/music-library/music-library.service.ts

@@ -175,7 +175,9 @@ export class MusicLibraryService {
   }
 
   async findTrack(query: FindTrackDto): Promise<Track | undefined> {
-    return this.trackRepository.findOneBy(query);
+    return this.trackRepository.findOneBy({
+      spotify: { id: query.spotify.id },
+    });
   }
 
   async findTracks(query: FindTrackDto[]): Promise<Track[]> {

src/override.d.ts

@@ -0,0 +1,5 @@
+// Issue with opentelemetry-js: https://github.com/open-telemetry/opentelemetry-js/issues/3580#issuecomment-1701157270
+export {};
+declare global {
+  type BlobPropertyBag = unknown;
+}

src/sources/scheduler.service.ts

@@ -33,7 +33,7 @@ export class SchedulerService implements OnApplicationBootstrap {
   }
 
   private async setupSpotifyCrawlerSupervisor(): Promise<void> {
-    // await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {});
+    await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {});
   }
 
   @Span()

src/sources/spotify/import-extended-streaming-history/dto/extended-streaming-history-status.dto.ts

@@ -1,13 +0,0 @@
-import { ApiProperty } from "@nestjs/swagger";
-
-export class ExtendedStreamingHistoryStatusDto {
-  @ApiProperty({
-    type: Number,
-  })
-  total: number;
-
-  @ApiProperty({
-    type: Number,
-  })
-  imported: number;
-}

src/sources/spotify/import-extended-streaming-history/dto/import-extended-streaming-history.dto.ts

@@ -1,13 +0,0 @@
-import { ApiProperty } from "@nestjs/swagger";
-import { ArrayMaxSize } from "class-validator";
-import { SpotifyExtendedStreamingHistoryItemDto } from "./spotify-extended-streaming-history-item.dto";
-
-export class ImportExtendedStreamingHistoryDto {
-  @ApiProperty({
-    type: SpotifyExtendedStreamingHistoryItemDto,
-    isArray: true,
-    maxItems: 50_000,
-  })
-  @ArrayMaxSize(50_000) // File size is ~16k by default, might need refactoring if Spotify starts exporting larger files
-  listens: SpotifyExtendedStreamingHistoryItemDto[];
-}

src/sources/spotify/import-extended-streaming-history/dto/spotify-extended-streaming-history-item.dto.ts

@@ -1,9 +0,0 @@
-import { ApiProperty } from "@nestjs/swagger";
-
-export class SpotifyExtendedStreamingHistoryItemDto {
-  @ApiProperty({ format: "iso8601", example: "2018-11-30T08:33:33Z" })
-  ts: string;
-
-  @ApiProperty({ example: "spotify:track:6askbS4pEVWbbDnUGEXh3G" })
-  spotify_track_uri: string;
-}

src/sources/spotify/import-extended-streaming-history/import.controller.ts

@@ -1,32 +0,0 @@
-import { Body as NestBody, Controller, Get, Post } from "@nestjs/common";
-import { ApiBody, ApiTags } from "@nestjs/swagger";
-import { AuthAccessToken } from "../../../auth/decorators/auth-access-token.decorator";
-import { ReqUser } from "../../../auth/decorators/req-user.decorator";
-import { User } from "../../../users/user.entity";
-import { ExtendedStreamingHistoryStatusDto } from "./dto/extended-streaming-history-status.dto";
-import { ImportExtendedStreamingHistoryDto } from "./dto/import-extended-streaming-history.dto";
-import { ImportService } from "./import.service";
-
-@ApiTags("import")
-@Controller("api/v1/import")
-export class ImportController {
-  constructor(private readonly importService: ImportService) {}
-
-  @Post("extended-streaming-history")
-  @ApiBody({ type: () => ImportExtendedStreamingHistoryDto })
-  @AuthAccessToken()
-  async importExtendedStreamingHistory(
-    @ReqUser() user: User,
-    @NestBody() data: ImportExtendedStreamingHistoryDto,
-  ): Promise<void> {
-    return this.importService.importExtendedStreamingHistory(user, data);
-  }
-
-  @Get("extended-streaming-history/status")
-  @AuthAccessToken()
-  async getExtendedStreamingHistoryStatus(
-    @ReqUser() user: User,
-  ): Promise<ExtendedStreamingHistoryStatusDto> {
-    return this.importService.getExtendedStreamingHistoryStatus(user);
-  }
-}

src/sources/spotify/import-extended-streaming-history/import.service.ts

@@ -1,177 +0,0 @@
-import { JobService } from "@apricote/nest-pg-boss";
-import { Injectable, Logger } from "@nestjs/common";
-import { uniq } from "lodash";
-import { Span } from "nestjs-otel";
-import type { Job } from "pg-boss";
-import { ListensService } from "../../../listens/listens.service";
-import { User } from "../../../users/user.entity";
-import { SpotifyService } from "../spotify.service";
-import { ExtendedStreamingHistoryStatusDto } from "./dto/extended-streaming-history-status.dto";
-import { ImportExtendedStreamingHistoryDto } from "./dto/import-extended-streaming-history.dto";
-import {
-  IProcessSpotifyExtendedStreamingHistoryListenJob,
-  ProcessSpotifyExtendedStreamingHistoryListenJob,
-} from "./jobs";
-import { SpotifyExtendedStreamingHistoryListenRepository } from "./listen.repository";
-
-@Injectable()
-export class ImportService {
-  private readonly logger = new Logger(this.constructor.name);
-
-  constructor(
-    private readonly importListenRepository: SpotifyExtendedStreamingHistoryListenRepository,
-    @ProcessSpotifyExtendedStreamingHistoryListenJob.Inject()
-    private readonly processListenJobService: JobService<IProcessSpotifyExtendedStreamingHistoryListenJob>,
-    private readonly spotifyService: SpotifyService,
-    private readonly listensService: ListensService,
-  ) {}
-
-  @Span()
-  async importExtendedStreamingHistory(
-    user: User,
-    { listens: importListens }: ImportExtendedStreamingHistoryDto,
-  ): Promise<void> {
-    // IDK what's happening, but my personal data set has entries with duplicate
-    // listens? might be related to offline mode.
-    // Anyway, this cleans it up:
-    const uniqEntries = new Set();
-    const uniqueListens = importListens.filter((listen) => {
-      const key = `${listen.spotify_track_uri}-${listen.ts}`;
-
-      if (!uniqEntries.has(key)) {
-        // New entry
-        uniqEntries.add(key);
-        return true;
-      }
-
-      return false;
-    });
-
-    let listens = uniqueListens.map((listenData) =>
-      this.importListenRepository.create({
-        user,
-        playedAt: new Date(listenData.ts),
-        spotifyTrackUri: listenData.spotify_track_uri,
-      }),
-    );
-
-    // Save listens to import table
-    const insertResult = await this.importListenRepository.upsert(listens, [
-      "user",
-      "playedAt",
-      "spotifyTrackUri",
-    ]);
-
-    const processJobs = insertResult.identifiers.map((listen) => ({
-      data: {
-        id: listen.id,
-      },
-      singletonKey: listen.id,
-      retryLimit: 10,
-      retryDelay: 5,
-      retryBackoff: true,
-    }));
-
-    // Schedule jobs to process imports
-    await this.processListenJobService.insert(processJobs);
-  }
-
-  @ProcessSpotifyExtendedStreamingHistoryListenJob.Handle({
-    // Spotify API "Get Several XY" allows max 50 IDs
-    batchSize: 50,
-    newJobCheckInterval: 500,
-  })
-  @Span()
-  async processListens(
-    jobs: Job<IProcessSpotifyExtendedStreamingHistoryListenJob>[],
-  ): Promise<void> {
-    this.logger.debug(
-      { jobs: jobs.length },
-      "processing extended streaming history listens",
-    );
-    const importListens = await this.importListenRepository.findBy(
-      jobs.map((job) => ({ id: job.data.id })),
-    );
-
-    const listensWithoutTracks = importListens.filter(
-      (importListen) => !importListen.track,
-    );
-    if (listensWithoutTracks.length > 0) {
-      const missingTrackIDs = uniq(
-        listensWithoutTracks.map((importListen) =>
-          importListen.spotifyTrackUri.replace("spotify:track:", ""),
-        ),
-      );
-
-      const tracks = await this.spotifyService.importTracks(missingTrackIDs);
-
-      listensWithoutTracks.forEach((listen) => {
-        listen.track = tracks.find(
-          (track) => listen.spotifyTrackUri === track.spotify.uri,
-        );
-        if (!listen.track) {
-          this.logger.warn(
-            { listen },
-            "could not find track for extended streaming history listen",
-          );
-          throw new Error(
-            `could not find track for extended streaming history listen`,
-          );
-        }
-      });
-
-      // Using upsert instead of save to only do a single query
-      await this.importListenRepository.upsert(listensWithoutTracks, ["id"]);
-    }
-
-    const listensWithoutListen = importListens.filter(
-      (importListen) => !importListen.listen,
-    );
-    if (listensWithoutListen.length > 0) {
-      const listens = await this.listensService.createListens(
-        listensWithoutListen.map((listen) => ({
-          user: listen.user,
-          track: listen.track,
-          playedAt: listen.playedAt,
-        })),
-      );
-
-      listensWithoutListen.forEach((importListen) => {
-        importListen.listen = listens.find(
-          (listen) =>
-            importListen.user.id === listen.user.id &&
-            importListen.track.id === listen.track.id &&
-            importListen.playedAt.getTime() === listen.playedAt.getTime(),
-        );
-        if (!importListen.listen) {
-          this.logger.warn(
-            { listen: importListen, listens: listens },
-            "could not find listen for extended streaming history listen",
-          );
-          throw new Error(
-            `could not find listen for extended streaming history listen`,
-          );
-        }
-      });
-
-      // Using upsert instead of save to only do a single query
-      await this.importListenRepository.upsert(listensWithoutListen, ["id"]);
-    }
-  }
-
-  @Span()
-  async getExtendedStreamingHistoryStatus(
-    user: User,
-  ): Promise<ExtendedStreamingHistoryStatusDto> {
-    const qb = this.importListenRepository
-      .createQueryBuilder("listen")
-      .where("listen.userId = :user", { user: user.id });
-
-    const [total, imported] = await Promise.all([
-      qb.clone().getCount(),
-      qb.clone().andWhere("listen.listenId IS NOT NULL").getCount(),
-    ]);
-
-    return { total, imported };
-  }
-}

src/sources/spotify/import-extended-streaming-history/index.ts

@@ -1,4 +0,0 @@
-export { ImportController } from "./import.controller";
-export { ImportService } from "./import.service";
-export { ProcessSpotifyExtendedStreamingHistoryListenJob } from "./jobs";
-export { SpotifyExtendedStreamingHistoryListenRepository } from "./listen.repository";

src/sources/spotify/import-extended-streaming-history/jobs.ts

@@ -1,7 +0,0 @@
-import { createJob } from "@apricote/nest-pg-boss";
-
-export type IProcessSpotifyExtendedStreamingHistoryListenJob = { id: string };
-export const ProcessSpotifyExtendedStreamingHistoryListenJob =
-  createJob<IProcessSpotifyExtendedStreamingHistoryListenJob>(
-    "process-spotify-extended-streaming-history-listen",
-  );

src/sources/spotify/import-extended-streaming-history/listen.entity.ts

@@ -1,25 +0,0 @@
-import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm";
-import { Track } from "../../../music-library/track.entity";
-import { User } from "../../../users/user.entity";
-import { Listen } from "../../../listens/listen.entity";
-
-@Entity({ name: "spotify_extended_streaming_history_listen" })
-export class SpotifyExtendedStreamingHistoryListen {
-  @PrimaryGeneratedColumn("uuid")
-  id: string;
-
-  @ManyToOne(() => User, { eager: true })
-  user: User;
-
-  @Column({ type: "timestamp" })
-  playedAt: Date;
-
-  @Column()
-  spotifyTrackUri: string;
-
-  @ManyToOne(() => Track, { nullable: true, eager: true })
-  track?: Track;
-
-  @ManyToOne(() => Listen, { nullable: true, eager: true })
-  listen?: Listen;
-}

src/sources/spotify/import-extended-streaming-history/listen.repository.ts

@@ -1,6 +0,0 @@
-import { Repository } from "typeorm";
-import { EntityRepository } from "../../../database/entity-repository";
-import { SpotifyExtendedStreamingHistoryListen } from "./listen.entity";
-
-@EntityRepository(SpotifyExtendedStreamingHistoryListen)
-export class SpotifyExtendedStreamingHistoryListenRepository extends Repository<SpotifyExtendedStreamingHistoryListen> {}

src/sources/spotify/spotify.module.ts

@@ -1,37 +1,25 @@
 import { PGBossModule } from "@apricote/nest-pg-boss";
 import { Module } from "@nestjs/common";
-import { TypeOrmRepositoryModule } from "../../database/entity-repository/typeorm-repository.module";
 import { ListensModule } from "../../listens/listens.module";
 import { MusicLibraryModule } from "../../music-library/music-library.module";
 import { UsersModule } from "../../users/users.module";
 import { ImportSpotifyJob } from "../jobs";
-import {
-  ImportController,
-  ImportService,
-  ProcessSpotifyExtendedStreamingHistoryListenJob,
-  SpotifyExtendedStreamingHistoryListenRepository,
-} from "./import-extended-streaming-history";
 import { SpotifyApiModule } from "./spotify-api/spotify-api.module";
 import { SpotifyAuthModule } from "./spotify-auth/spotify-auth.module";
 import { SpotifyService } from "./spotify.service";
 
 @Module({
   imports: [
-    PGBossModule.forJobs([
-      ImportSpotifyJob,
-      ProcessSpotifyExtendedStreamingHistoryListenJob,
-    ]),
-    TypeOrmRepositoryModule.for([
-      SpotifyExtendedStreamingHistoryListenRepository,
-    ]),
+    PGBossModule.forJobs([ImportSpotifyJob]),
     UsersModule,
     ListensModule,
     MusicLibraryModule,
     SpotifyApiModule,
     SpotifyAuthModule,
   ],
-  providers: [SpotifyService, ImportService],
-  controllers: [ImportController],
+  providers: [SpotifyService],
   exports: [SpotifyService],
 })
-export class SpotifyModule {}
+export class SpotifyModule {
+  constructor(private readonly spotifyService: SpotifyService) {}
+}

tsconfig.json

@@ -5,12 +5,13 @@
     "removeComments": true,
     "emitDecoratorMetadata": true,
     "experimentalDecorators": true,
-    "target": "ES2022",
+    "target": "ES2020",
     "sourceMap": true,
     "outDir": "./dist",
     "baseUrl": "./",
     "incremental": true,
     "paths": {},
+    "lib": ["ES2020"]
   },
   "exclude": ["node_modules", "dist"],
   "include": ["src", "test"]