chore(deps): update dependency eslint-plugin-react to v7.37.4

Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>

.dockerignore

@@ -19,5 +19,6 @@
 !frontend/tsconfig.json
 !frontend/vite.config.js
 !frontend/index.html
+!frontend/*.d.ts
 !frontend/src/**/*
 !frontend/public/**/*

.github/workflows/ci.yaml

@@ -12,7 +12,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 20
 
@@ -33,7 +33,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Setup Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 20
 

.github/workflows/release.yaml

@@ -25,7 +25,7 @@ jobs:
           version: "v0.11.2"
 
       - name: Setup Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 20
 

CHANGELOG.md

@@ -1,3 +1,24 @@
+# [1.31.0](https://github.com/apricote/Listory/compare/v1.30.1...v1.31.0) (2023-10-15)
+
+
+### Features
+
+* **frontend:** hide revoked api tokens ([#307](https://github.com/apricote/Listory/issues/307)) ([4cef4f7](https://github.com/apricote/Listory/commit/4cef4f75ace6a38ba19c1d2f93d81389ec2b7cb8))
+
+## [1.30.1](https://github.com/apricote/Listory/compare/v1.30.0...v1.30.1) (2023-10-08)
+
+
+### Bug Fixes
+
+* no listens are being crawled ([#306](https://github.com/apricote/Listory/issues/306)) ([9af3115](https://github.com/apricote/Listory/commit/9af3115cab19cc4ac4a6cd0fb680371154069aa2))
+
+# [1.30.0](https://github.com/apricote/Listory/compare/v1.29.0...v1.30.0) (2023-10-01)
+
+
+### Features
+
+* import listens from spotify extended streaming history ([#305](https://github.com/apricote/Listory/issues/305)) ([7140cb0](https://github.com/apricote/Listory/commit/7140cb0679ec3aac8a2102197d9edb070cf0e6c0))
+
 # [1.29.0](https://github.com/apricote/Listory/compare/v1.28.2...v1.29.0) (2023-09-30)
 
 

Dockerfile

@@ -1,9 +1,4 @@
-# syntax=docker/dockerfile:1.5
-
-FROM scratch as ignore
-
-WORKDIR /listory
-COPY . /listory/
+# syntax=docker/dockerfile:1.12
 
 ##################
 ## common

charts/listory/Chart.yaml

@@ -14,8 +14,8 @@ type: application
 
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
-version: 1.29.0
+version: 1.31.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 1.29.0
+appVersion: 1.31.0

docker-compose.prod.yml

@@ -6,7 +6,7 @@ services:
   #####
 
   db:
-    image: postgres:16.0
+    image: postgres:16.6
     restart: unless-stopped
     environment:
       POSTGRES_PASSWORD: listory
@@ -18,7 +18,7 @@ services:
       - db
 
   api:
-    image: apricote/listory:1.29.0
+    image: apricote/listory:1.31.0
     restart: unless-stopped
     environment:
       DB_HOST: db

docker-compose.yml

@@ -12,7 +12,7 @@ services:
   #####
 
   db:
-    image: postgres:16.0
+    image: postgres:16.6
     environment:
       POSTGRES_PASSWORD: listory
       POSTGRES_USER: listory
@@ -37,7 +37,8 @@ services:
       OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4318
     env_file: .env
     volumes:
-      - ./src:/app/src
+      - ./src:/app/src:ro
+      - ./dist:/app/dist # build cache
     ports:
       - 3000 # API
       - "9464:9464" # Metrics
@@ -72,7 +73,7 @@ services:
       - web
 
   proxy:
-    image: traefik:v2.10.4
+    image: traefik:v2.11.15
     command:
       #- --log.level=debug
       #- --accesslog=true
@@ -115,7 +116,8 @@ services:
       OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4318
     env_file: .env
     volumes:
-      - ./src:/app/src
+      - ./src:/app/src:ro
+      - ./dist:/app/dist # build cache
     ports:
       - "9464:9464" # Metrics
     networks:
@@ -129,7 +131,7 @@ services:
 
   prometheus:
     profiles: ["observability"]
-    image: prom/prometheus:v2.47.0
+    image: prom/prometheus:v2.55.1
     volumes:
       - ./observability/prometheus:/etc/prometheus
       - prometheus_data:/prometheus
@@ -146,7 +148,7 @@ services:
 
   loki:
     profiles: ["observability"]
-    image: grafana/loki:2.9.1
+    image: grafana/loki:2.9.11
     command: ["-config.file=/etc/loki/loki.yaml"]
     ports:
       - "3100" # loki needs to be exposed so it receives logs
@@ -157,7 +159,7 @@ services:
 
   promtail:
     profiles: ["observability"]
-    image: grafana/promtail:2.9.1
+    image: grafana/promtail:2.9.11
     command: ["-config.file=/etc/promtail.yaml"]
     volumes:
       - ./observability/promtail/promtail.yaml:/etc/promtail.yaml
@@ -175,7 +177,7 @@ services:
 
   tempo:
     profiles: ["observability"]
-    image: grafana/tempo:2.2.3
+    image: grafana/tempo:2.6.1
     command: ["-config.file=/etc/tempo.yaml"]
     volumes:
       - ./observability/tempo/tempo.yaml:/etc/tempo.yaml
@@ -189,7 +191,7 @@ services:
 
   grafana:
     profiles: ["observability"]
-    image: grafana/grafana-oss:10.1.4
+    image: grafana/grafana-oss:10.4.14
     volumes:
       - ./observability/grafana/provisioning:/etc/grafana/provisioning
     environment:

frontend/package.json

@@ -8,43 +8,44 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@radix-ui/react-avatar": "1.0.3",
-    "@radix-ui/react-dropdown-menu": "2.0.5",
-    "@radix-ui/react-label": "2.0.2",
-    "@radix-ui/react-navigation-menu": "1.1.3",
-    "@radix-ui/react-select": "1.2.2",
-    "@radix-ui/react-slot": "1.0.2",
-    "@testing-library/jest-dom": "6.1.3",
-    "@testing-library/react": "14.0.0",
-    "@testing-library/user-event": "14.5.1",
-    "@types/jest": "29.5.5",
-    "@types/node": "20.8.0",
-    "@types/react": "18.2.23",
-    "@types/react-dom": "18.2.8",
+    "@radix-ui/react-avatar": "1.1.2",
+    "@radix-ui/react-dropdown-menu": "2.1.3",
+    "@radix-ui/react-label": "2.1.1",
+    "@radix-ui/react-navigation-menu": "1.2.2",
+    "@radix-ui/react-select": "2.1.3",
+    "@radix-ui/react-slot": "1.1.1",
+    "@testing-library/jest-dom": "6.6.3",
+    "@testing-library/react": "16.1.0",
+    "@testing-library/user-event": "14.5.2",
+    "@types/jest": "29.5.14",
+    "@types/node": "20.17.16",
+    "@types/react": "18.3.18",
+    "@types/react-dom": "18.3.5",
     "@types/react-router-dom": "5.3.3",
-    "@types/recharts": "1.8.25",
-    "@vitejs/plugin-react": "4.1.0",
-    "autoprefixer": "10.4.16",
-    "axios": "1.5.1",
-    "class-variance-authority": "0.7.0",
-    "clsx": "2.0.0",
+    "@types/recharts": "1.8.29",
+    "@vitejs/plugin-react": "4.3.4",
+    "autoprefixer": "10.4.20",
+    "axios": "1.7.9",
+    "class-variance-authority": "0.7.1",
+    "clsx": "2.1.1",
     "date-fns": "2.30.0",
     "eslint-config-react-app": "7.0.1",
     "jsdom": "22.1.0",
-    "lucide-react": "0.279.0",
+    "lucide-react": "0.468.0",
     "npm-run-all": "4.1.5",
-    "postcss": "8.4.31",
-    "prettier": "3.0.3",
-    "react": "18.2.0",
-    "react-dom": "18.2.0",
-    "react-router-dom": "6.16.0",
-    "recharts": "2.8.0",
+    "postcss": "8.4.49",
+    "prettier": "3.4.2",
+    "react": "18.3.1",
+    "react-dom": "18.3.1",
+    "react-files": "3.0.3",
+    "react-router-dom": "6.28.0",
+    "recharts": "2.15.0",
     "tailwind-merge": "1.14.0",
-    "tailwindcss": "3.3.3",
+    "tailwindcss": "3.4.16",
     "tailwindcss-animate": "1.0.7",
-    "typescript": "5.2.2",
-    "vite": "4.4.9",
-    "vitest": "0.34.6"
+    "typescript": "5.7.2",
+    "vite": "5.4.12",
+    "vitest": "1.6.0"
   },
   "scripts": {
     "format": "prettier --write \"./*.js\" \"src/**/*.(tsx|ts|css)\"",

frontend/src/App.tsx

@@ -2,6 +2,7 @@ import React from "react";
 import { Route, Routes } from "react-router-dom";
 import { AuthApiTokens } from "./components/AuthApiTokens";
 import { Footer } from "./components/Footer";
+import { ImportListens } from "./components/ImportListens";
 import { LoginFailure } from "./components/LoginFailure";
 import { LoginLoading } from "./components/LoginLoading";
 import { NavBar } from "./components/NavBar";
@@ -53,6 +54,7 @@ export function App() {
                   element={<ReportTopGenres />}
                 />
                 <Route path="/auth/api-tokens" element={<AuthApiTokens />} />
+                <Route path="/import" element={<ImportListens />} />
               </Routes>
             )}
             {!user && (

frontend/src/api/api.ts

@@ -14,6 +14,8 @@ import { TopGenresItem } from "./entities/top-genres-item";
 import { TopGenresOptions } from "./entities/top-genres-options";
 import { TopTracksItem } from "./entities/top-tracks-item";
 import { TopTracksOptions } from "./entities/top-tracks-options";
+import { SpotifyExtendedStreamingHistoryItem } from "./entities/spotify-extended-streaming-history-item";
+import { ExtendedStreamingHistoryStatus } from "./entities/extended-streaming-history-status";
 
 export class UnauthenticatedError extends Error {}
 
@@ -276,7 +278,7 @@ export const revokeApiToken = async (
   id: string,
   client: AxiosInstance,
 ): Promise<void> => {
-  const res = await client.delete<NewApiToken>(`/api/v1/auth/api-tokens/${id}`);
+  const res = await client.delete(`/api/v1/auth/api-tokens/${id}`);
 
   switch (res.status) {
     case 200: {
@@ -290,3 +292,50 @@ export const revokeApiToken = async (
     }
   }
 };
+
+export const importExtendedStreamingHistory = async (
+  listens: SpotifyExtendedStreamingHistoryItem[],
+  client: AxiosInstance
+): Promise<void> => {
+  const res = await client.post(`/api/v1/import/extended-streaming-history`, {
+    listens,
+  });
+
+  switch (res.status) {
+    case 201: {
+      break;
+    }
+    case 401: {
+      throw new UnauthenticatedError(`No token or token expired`);
+    }
+    default: {
+      throw new Error(
+        `Unable to importExtendedStreamingHistory: ${res.status}`
+      );
+    }
+  }
+};
+
+export const getExtendedStreamingHistoryStatus = async (
+  client: AxiosInstance
+): Promise<ExtendedStreamingHistoryStatus> => {
+  const res = await client.get<ExtendedStreamingHistoryStatus>(
+    `/api/v1/import/extended-streaming-history/status`
+  );
+
+  switch (res.status) {
+    case 200: {
+      break;
+    }
+    case 401: {
+      throw new UnauthenticatedError(`No token or token expired`);
+    }
+    default: {
+      throw new Error(
+        `Unable to getExtendedStreamingHistoryStatus: ${res.status}`
+      );
+    }
+  }
+
+  return res.data;
+};

frontend/src/api/entities/extended-streaming-history-status.ts

@@ -0,0 +1,4 @@
+export interface ExtendedStreamingHistoryStatus {
+  total: number;
+  imported: number;
+}

frontend/src/api/entities/spotify-extended-streaming-history-item.ts

@@ -0,0 +1,4 @@
+export interface SpotifyExtendedStreamingHistoryItem {
+  ts: string;
+  spotify_track_uri: string;
+}

frontend/src/components/AuthApiTokens.tsx

@@ -2,7 +2,6 @@ import { format, formatDistanceToNow } from "date-fns";
 import React, { FormEvent, useCallback, useMemo, useState } from "react";
 import { ApiToken, NewApiToken } from "../api/entities/api-token";
 import { useApiTokens } from "../hooks/use-api";
-import { useAuthProtection } from "../hooks/use-auth-protection";
 import { SpinnerIcon } from "../icons/Spinner";
 import TrashcanIcon from "../icons/Trashcan";
 import { Spinner } from "./ui/Spinner";
@@ -10,7 +9,10 @@ import { Spinner } from "./ui/Spinner";
 export const AuthApiTokens: React.FC = () => {
   const { apiTokens, isLoading, createToken, revokeToken } = useApiTokens();
   const sortedTokens = useMemo(
-    () => apiTokens.sort((a, b) => (a.createdAt > b.createdAt ? -1 : 1)),
+    () =>
+      apiTokens
+        .filter((token) => !token.revokedAt)
+        .sort((a, b) => (a.createdAt > b.createdAt ? -1 : 1)),
     [apiTokens],
   );
 
@@ -23,7 +25,7 @@ export const AuthApiTokens: React.FC = () => {
         <p className="mb-4">
           You can use API Tokens to access the Listory API directly. You can
           find the API docs{" "}
-          <a href="/api/docs" target="_blank">
+          <a href="/api/docs" target="_blank" className={"underline"}>
             here
           </a>
           .

frontend/src/components/ImportListens.tsx

@@ -0,0 +1,280 @@
+import React, { useCallback, useEffect, useState } from "react";
+import Files from "react-files";
+import type { ReactFile } from "react-files";
+import {
+  useSpotifyImportExtendedStreamingHistory,
+  useSpotifyImportExtendedStreamingHistoryStatus,
+} from "../hooks/use-api";
+import { SpotifyExtendedStreamingHistoryItem } from "../api/entities/spotify-extended-streaming-history-item";
+import { ErrorIcon } from "../icons/Error";
+import { numberToPercent } from "../util/numberToPercent";
+import { Button } from "./ui/button";
+import { Table, TableBody, TableCell, TableRow } from "./ui/table";
+import { Badge } from "./ui/badge";
+import {
+  Card,
+  CardContent,
+  CardDescription,
+  CardFooter,
+  CardHeader,
+  CardTitle,
+} from "./ui/card";
+import { Code } from "./ui/code";
+
+export const ImportListens: React.FC = () => {
+  return (
+    <>
+      <div className="flex justify-between">
+        <p className="text-2xl font-normal">
+          Import Listens from Spotify Extended Streaming History
+        </p>
+      </div>
+      <div className="shadow-xl bg-gray-100 dark:bg-gray-800 rounded p-4 m-2">
+        <p className="my-4">
+          Here you can import your full Spotify Listen history that was exported
+          from the{" "}
+          <a
+            target="blank"
+            href="https://www.spotify.com/us/account/privacy/"
+            className="underline"
+          >
+            Extended streaming history
+          </a>
+          .
+        </p>
+        <p className="my-4">
+          The extended streaming history contains additional personally
+          identifiable data such as the IP address of the listen (which can be
+          linked to locations). To avoid saving this on the server, the data is
+          preprocessed in your web browser and only the necessary data
+          (timestamp & track ID) are sent to the server.
+        </p>
+        <p className="my-4">
+          If an error occurs, you can always retry uploading the file, Listory
+          deduplicates any listens to make sure that everything is saved only
+          once.
+        </p>
+
+        <FileUpload />
+        <ImportProgress />
+      </div>
+    </>
+  );
+};
+
+interface FileData {
+  file: ReactFile;
+  status: Status;
+  error?: Error;
+}
+
+enum Status {
+  Select,
+  Import,
+  Finished,
+  Error,
+}
+
+const FileUpload: React.FC = () => {
+  // Using a map is ... meh, need to wrap all state updates in `new Map()` so react re-renders
+  const [fileMap, setFileMap] = useState<Map<ReactFile["id"], FileData>>(
+    new Map(),
+  );
+
+  const [status, setStatus] = useState<Status>(Status.Select);
+
+  const addFiles = useCallback(
+    (files: ReactFile[]) => {
+      setFileMap((_fileMap) => {
+        files.forEach((file) =>
+          _fileMap.set(file.id, { file, status: Status.Select }),
+        );
+        return new Map(_fileMap);
+      });
+    },
+    [setFileMap],
+  );
+
+  const updateFile = useCallback((data: FileData) => {
+    setFileMap((_fileMap) => new Map(_fileMap.set(data.file.id, data)));
+  }, []);
+
+  const clearFiles = useCallback(() => {
+    setFileMap(new Map());
+  }, [setFileMap]);
+
+  const { importHistory } = useSpotifyImportExtendedStreamingHistory();
+
+  const handleImport = useCallback(async () => {
+    setStatus(Status.Import);
+
+    let errorOccurred = false;
+
+    for (const data of fileMap.values()) {
+      data.status = Status.Import;
+      updateFile(data);
+
+      let items: SpotifyExtendedStreamingHistoryItem[];
+
+      // Scope so these tmp variables can be GC-ed ASAP
+      {
+        const fileContent = await data.file.text();
+
+        const rawItems = JSON.parse(
+          fileContent,
+        ) as SpotifyExtendedStreamingHistoryItem[];
+
+        items = rawItems
+          .filter(({ spotify_track_uri }) => spotify_track_uri !== null)
+          .map(({ ts, spotify_track_uri }) => ({
+            ts,
+            spotify_track_uri,
+          }));
+      }
+
+      try {
+        await importHistory(items);
+
+        data.status = Status.Finished;
+      } catch (err) {
+        data.error = err as Error;
+        data.status = Status.Error;
+
+        errorOccurred = true;
+      }
+      updateFile(data);
+    }
+
+    if (!errorOccurred) {
+      setStatus(Status.Finished);
+    }
+  }, [fileMap, importHistory, updateFile]);
+
+  return (
+    <Card className="mb-5">
+      <CardHeader>
+        <CardTitle>File Upload</CardTitle>
+        <CardDescription>
+          Select <Code>endsong_XY.json</Code> files here and start the import.
+        </CardDescription>
+      </CardHeader>
+      <CardContent>
+        <Files
+          className="shadow-inner bg-gray-200 dark:bg-gray-700 rounded p-4 text-center cursor-pointer"
+          dragActiveClassName=""
+          onChange={addFiles}
+          accepts={["application/json"]}
+          multiple
+          clickable
+        >
+          Drop files here or click to upload
+        </Files>
+        <Table>
+          <TableBody>
+            {Array.from(fileMap.values()).map((data) => (
+              <File key={data.file.id} data={data} />
+            ))}
+          </TableBody>
+        </Table>
+      </CardContent>
+      <CardFooter className="flex gap-x-2">
+        <Button
+          onClick={() =>
+            handleImport().catch((e) => console.error("Import Failed:", e))
+          }
+          variant="secondary"
+          disabled={status !== Status.Select}
+        >
+          Start Import
+        </Button>
+        <Button
+          onClick={clearFiles}
+          variant="secondary"
+          disabled={status !== Status.Select}
+        >
+          Remove All Files
+        </Button>
+      </CardFooter>
+    </Card>
+  );
+};
+
+const File: React.FC<{ data: FileData }> = ({ data }) => {
+  const hasErrors = data.status === Status.Error && data.error;
+
+  return (
+    <TableRow>
+      <TableCell>{data.file.name}</TableCell>
+      <TableCell className="text-sm font-thin">
+        {data.file.sizeReadable}
+      </TableCell>
+      <TableCell className="text-right">
+        {data.status === Status.Select && <Badge>Prepared for import!</Badge>}
+        {data.status === Status.Import && <Badge>Loading!</Badge>}
+        {data.status === Status.Finished && <Badge>Check!</Badge>}
+        {hasErrors && (
+          <Badge variant="destructive">
+            <ErrorIcon />
+            {data.error?.message}
+          </Badge>
+        )}
+      </TableCell>
+    </TableRow>
+  );
+};
+
+const ImportProgress: React.FC = () => {
+  const {
+    importStatus: { total, imported },
+    isLoading,
+    reload,
+  } = useSpotifyImportExtendedStreamingHistoryStatus();
+
+  useEffect(() => {
+    const interval = setInterval(() => {
+      if (!isLoading) {
+        reload();
+      }
+    }, 1000);
+
+    return () => clearInterval(interval);
+  }, [isLoading, reload]);
+
+  return (
+    <Card>
+      <CardHeader>
+        <CardTitle>Import Progress</CardTitle>
+        <CardDescription>
+          Shows how many of the submitted listens are already imported and
+          visible to you. This will take a while, and the process might halt for
+          a few minutes if we hit the Spotify API rate limit. If this is not
+          finished after a few hours, please contact your Listory administrator.
+        </CardDescription>
+      </CardHeader>
+      <CardContent>
+        <div className="flex pb-2">
+          <div className="md:flex w-10/12">
+            <div className={`md:w-full font-bold`}>
+              Imported
+              <br />
+              {imported}
+            </div>
+          </div>
+          <div className="w-2/12 text-right">
+            Total
+            <br />
+            {total}
+          </div>
+        </div>
+        {total > 0 && (
+          <div className="h-2 w-full bg-gradient-to-r from-teal-200/25 via-green-400 to-violet-400 dark:from-teal-700/25 dark:via-green-600/85 dark:to-amber-500 flex flex-row-reverse">
+            <div
+              style={{ width: numberToPercent(1 - imported / total) }}
+              className="h-full bg-gray-100 dark:bg-gray-900"
+            ></div>
+          </div>
+        )}
+      </CardContent>
+    </Card>
+  );
+};

frontend/src/components/NavBar.tsx

@@ -3,6 +3,7 @@ import { Link } from "react-router-dom";
 import { User } from "../api/entities/user";
 import { useAuth } from "../hooks/use-auth";
 import { CogwheelIcon } from "../icons/Cogwheel";
+import { ImportIcon } from "../icons/Import";
 import { SpotifyLogo } from "../icons/Spotify";
 import { Avatar, AvatarImage, AvatarFallback } from "./ui/avatar";
 import {
@@ -188,6 +189,12 @@ const NavUserInfo: React.FC<{ user: User }> = ({ user }) => {
               API Tokens
             </Link>
           </DropdownMenuItem>
+          <DropdownMenuItem asChild>
+            <Link to="/import">
+              <ImportIcon className="w-5 h-5 fill-current pr-2" />
+              Import Listens
+            </Link>
+          </DropdownMenuItem>
         </DropdownMenuGroup>
       </DropdownMenuContent>
     </DropdownMenu>

frontend/src/components/reports/RecentListens.tsx

@@ -50,7 +50,7 @@ export const RecentListens: React.FC = () => {
         )}
         <div>
           {recentListens.length > 0 && (
-            <Table className="table-auto w-full text-base">
+            <Table className="table-auto w-full">
               <TableBody>
                 {recentListens.map((listen) => (
                   <ListenItem listen={listen} key={listen.id} />

frontend/src/components/reports/TopListItem.tsx

@@ -1,4 +1,5 @@
 import React from "react";
+import { numberToPercent } from "../../util/numberToPercent";
 
 export interface TopListItemProps {
   title: string;
@@ -42,9 +43,3 @@ export const TopListItem: React.FC<TopListItemProps> = ({
 
 const isMaxCountValid = (maxCount: number) =>
   !(Number.isNaN(maxCount) || maxCount === 0);
-
-const numberToPercent = (ratio: number) =>
-  ratio.toLocaleString(undefined, {
-    style: "percent",
-    minimumFractionDigits: 2,
-  });

frontend/src/components/ui/badge.tsx

@@ -0,0 +1,36 @@
+import * as React from "react";
+import { cva, type VariantProps } from "class-variance-authority";
+
+import { cn } from "src/lib/utils";
+
+const badgeVariants = cva(
+  "inline-flex items-center rounded-full border border-gray-200 px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-gray-900 focus:ring-offset-2 dark:border-gray-800 dark:focus:ring-gray-300",
+  {
+    variants: {
+      variant: {
+        default:
+          "border-transparent bg-gray-900 text-gray-50 hover:bg-gray-900/80 dark:bg-gray-50 dark:text-gray-900 dark:hover:bg-gray-50/80",
+        secondary:
+          "border-transparent bg-gray-100 text-gray-900 hover:bg-gray-100/80 dark:bg-gray-800 dark:text-gray-50 dark:hover:bg-gray-800/80",
+        destructive:
+          "border-transparent bg-red-500 text-gray-50 hover:bg-red-500/80 dark:bg-red-900 dark:text-gray-50 dark:hover:bg-red-900/80",
+        outline: "text-gray-900 dark:text-gray-50",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+    },
+  },
+);
+
+export interface BadgeProps
+  extends React.HTMLAttributes<HTMLDivElement>,
+    VariantProps<typeof badgeVariants> {}
+
+function Badge({ className, variant, ...props }: BadgeProps) {
+  return (
+    <div className={cn(badgeVariants({ variant }), className)} {...props} />
+  );
+}
+
+export { Badge, badgeVariants };

frontend/src/components/ui/card.tsx

@@ -0,0 +1,86 @@
+import * as React from "react";
+
+import { cn } from "src/lib/utils";
+
+const Card = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement>
+>(({ className, ...props }, ref) => (
+  <div
+    ref={ref}
+    className={cn(
+      "rounded-lg border border-gray-200 bg-white text-gray-900 shadow-sm dark:border-gray-800 dark:bg-gray-900 dark:text-gray-50",
+      className,
+    )}
+    {...props}
+  />
+));
+Card.displayName = "Card";
+
+const CardHeader = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement>
+>(({ className, ...props }, ref) => (
+  <div
+    ref={ref}
+    className={cn("flex flex-col space-y-1.5 p-6", className)}
+    {...props}
+  />
+));
+CardHeader.displayName = "CardHeader";
+
+const CardTitle = React.forwardRef<
+  HTMLParagraphElement,
+  React.HTMLAttributes<HTMLHeadingElement>
+>(({ className, ...props }, ref) => (
+  <h3
+    ref={ref}
+    className={cn(
+      "text-2xl font-semibold leading-none tracking-tight",
+      className,
+    )}
+    {...props}
+  />
+));
+CardTitle.displayName = "CardTitle";
+
+const CardDescription = React.forwardRef<
+  HTMLParagraphElement,
+  React.HTMLAttributes<HTMLParagraphElement>
+>(({ className, ...props }, ref) => (
+  <p
+    ref={ref}
+    className={cn("text-sm text-gray-500 dark:text-gray-400", className)}
+    {...props}
+  />
+));
+CardDescription.displayName = "CardDescription";
+
+const CardContent = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement>
+>(({ className, ...props }, ref) => (
+  <div ref={ref} className={cn("p-6 pt-0", className)} {...props} />
+));
+CardContent.displayName = "CardContent";
+
+const CardFooter = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement>
+>(({ className, ...props }, ref) => (
+  <div
+    ref={ref}
+    className={cn("flex items-center p-6 pt-0", className)}
+    {...props}
+  />
+));
+CardFooter.displayName = "CardFooter";
+
+export {
+  Card,
+  CardHeader,
+  CardFooter,
+  CardTitle,
+  CardDescription,
+  CardContent,
+};

frontend/src/components/ui/code.tsx

@@ -0,0 +1,9 @@
+import React, { PropsWithChildren } from "react";
+
+export const Code: React.FC<PropsWithChildren> = ({ children }) => {
+  return (
+    <code className="tracking-wide font-mono bg-gray-200 dark:bg-gray-600 rounded-md px-1">
+      {children}
+    </code>
+  );
+};

frontend/src/components/ui/table.tsx

@@ -1,6 +1,6 @@
-import * as React from "react"
+import * as React from "react";
 
-import { cn } from "src/lib/utils"
+import { cn } from "src/lib/utils";
 
 const Table = React.forwardRef<
   HTMLTableElement,
@@ -9,20 +9,20 @@ const Table = React.forwardRef<
   <div className="relative w-full overflow-auto">
     <table
       ref={ref}
-      className={cn("w-full caption-bottom text-sm", className)}
+      className={cn("w-full caption-bottom", className)}
       {...props}
     />
   </div>
-))
-Table.displayName = "Table"
+));
+Table.displayName = "Table";
 
 const TableHeader = React.forwardRef<
   HTMLTableSectionElement,
   React.HTMLAttributes<HTMLTableSectionElement>
 >(({ className, ...props }, ref) => (
   <thead ref={ref} className={cn("[&_tr]:border-b", className)} {...props} />
-))
-TableHeader.displayName = "TableHeader"
+));
+TableHeader.displayName = "TableHeader";
 
 const TableBody = React.forwardRef<
   HTMLTableSectionElement,
@@ -33,8 +33,8 @@ const TableBody = React.forwardRef<
     className={cn("[&_tr:last-child]:border-0", className)}
     {...props}
   />
-))
-TableBody.displayName = "TableBody"
+));
+TableBody.displayName = "TableBody";
 
 const TableFooter = React.forwardRef<
   HTMLTableSectionElement,
@@ -42,11 +42,14 @@ const TableFooter = React.forwardRef<
 >(({ className, ...props }, ref) => (
   <tfoot
     ref={ref}
-    className={cn("bg-gray-900 font-medium text-gray-50 dark:bg-gray-50 dark:text-gray-900", className)}
+    className={cn(
+      "bg-gray-900 font-medium text-gray-50 dark:bg-gray-50 dark:text-gray-900",
+      className,
+    )}
     {...props}
   />
-))
-TableFooter.displayName = "TableFooter"
+));
+TableFooter.displayName = "TableFooter";
 
 const TableRow = React.forwardRef<
   HTMLTableRowElement,
@@ -56,12 +59,12 @@ const TableRow = React.forwardRef<
     ref={ref}
     className={cn(
       "border-b transition-colors hover:bg-gray-100/50 data-[state=selected]:bg-gray-100 dark:hover:bg-gray-800/50 dark:data-[state=selected]:bg-gray-800",
-      className
+      className,
     )}
     {...props}
   />
-))
-TableRow.displayName = "TableRow"
+));
+TableRow.displayName = "TableRow";
 
 const TableHead = React.forwardRef<
   HTMLTableCellElement,
@@ -71,12 +74,12 @@ const TableHead = React.forwardRef<
     ref={ref}
     className={cn(
       "h-12 px-4 text-left align-middle font-medium text-gray-500 [&:has([role=checkbox])]:pr-0 dark:text-gray-400",
-      className
+      className,
     )}
     {...props}
   />
-))
-TableHead.displayName = "TableHead"
+));
+TableHead.displayName = "TableHead";
 
 const TableCell = React.forwardRef<
   HTMLTableCellElement,
@@ -87,8 +90,8 @@ const TableCell = React.forwardRef<
     className={cn("p-4 align-middle [&:has([role=checkbox])]:pr-0", className)}
     {...props}
   />
-))
-TableCell.displayName = "TableCell"
+));
+TableCell.displayName = "TableCell";
 
 const TableCaption = React.forwardRef<
   HTMLTableCaptionElement,
@@ -99,8 +102,8 @@ const TableCaption = React.forwardRef<
     className={cn("mt-4 text-sm text-gray-500 dark:text-gray-400", className)}
     {...props}
   />
-))
-TableCaption.displayName = "TableCaption"
+));
+TableCaption.displayName = "TableCaption";
 
 export {
   Table,
@@ -111,4 +114,4 @@ export {
   TableRow,
   TableCell,
   TableCaption,
-}
+};

frontend/src/hooks/use-api.tsx

@@ -2,12 +2,14 @@ import { useCallback, useMemo } from "react";
 import {
   createApiToken,
   getApiTokens,
+  getExtendedStreamingHistoryStatus,
   getListensReport,
   getRecentListens,
   getTopAlbums,
   getTopArtists,
   getTopGenres,
   getTopTracks,
+  importExtendedStreamingHistory,
   revokeApiToken,
 } from "../api/api";
 import { ListenReportOptions } from "../api/entities/listen-report-options";
@@ -18,6 +20,7 @@ import { TopGenresOptions } from "../api/entities/top-genres-options";
 import { TopTracksOptions } from "../api/entities/top-tracks-options";
 import { useApiClient } from "./use-api-client";
 import { useAsync } from "./use-async";
+import { SpotifyExtendedStreamingHistoryItem } from "../api/entities/spotify-extended-streaming-history-item";
 
 const INITIAL_EMPTY_ARRAY: [] = [];
 Object.freeze(INITIAL_EMPTY_ARRAY);
@@ -143,9 +146,7 @@ export const useApiTokens = () => {
   const createToken = useCallback(
     async (description: string) => {
       const apiToken = await createApiToken(description, client);
-      console.log("apiToken created", apiToken);
       await reload();
-      console.log("reloaded data");
 
       return apiToken;
     },
@@ -162,3 +163,38 @@ export const useApiTokens = () => {
 
   return { apiTokens, isLoading, error, createToken, revokeToken };
 };
+
+export const useSpotifyImportExtendedStreamingHistory = () => {
+  const { client } = useApiClient();
+
+  const importHistory = useCallback(
+    async (listens: SpotifyExtendedStreamingHistoryItem[]) => {
+      return importExtendedStreamingHistory(listens, client);
+    },
+    [client]
+  );
+
+  const getStatus = useCallback(async () => {
+    return getExtendedStreamingHistoryStatus(client);
+  }, [client]);
+
+  return { importHistory, getStatus };
+};
+
+export const useSpotifyImportExtendedStreamingHistoryStatus = () => {
+  const { client } = useApiClient();
+
+  const fetchData = useMemo(
+    () => () => getExtendedStreamingHistoryStatus(client),
+    [client]
+  );
+
+  const {
+    value: importStatus,
+    pending: isLoading,
+    error,
+    reload,
+  } = useAsync(fetchData, { total: 0, imported: 0 });
+
+  return { importStatus, isLoading, error, reload };
+};

frontend/src/hooks/use-auth-protection.tsx

@@ -1,16 +0,0 @@
-import { useCallback } from "react";
-import { useNavigate } from "react-router-dom";
-import { useAuth } from "./use-auth";
-
-export function useAuthProtection() {
-  const { user } = useAuth();
-  const navigate = useNavigate();
-
-  const requireUser = useCallback(async () => {
-    if (!user) {
-      navigate("/");
-    }
-  }, [user, navigate]);
-
-  return { requireUser };
-}

frontend/src/icons/Error.tsx

@@ -0,0 +1,14 @@
+import React from "react";
+
+export const ErrorIcon: React.FC<React.SVGProps<SVGSVGElement>> = (props) => {
+  return (
+    <svg
+      version="1.1"
+      xmlns="http://www.w3.org/2000/svg"
+      viewBox="0 0 50 50"
+      fill="#D75A4A"
+    >
+      <circle fill="fill" cx="25" cy="25" r="25" />
+    </svg>
+  );
+};

frontend/src/icons/Import.tsx

@@ -0,0 +1,12 @@
+import * as React from "react";
+export const ImportIcon: React.FC<React.SVGProps<SVGSVGElement>> = (props) => (
+  <svg
+    xmlns="http://www.w3.org/2000/svg"
+    xmlSpace="preserve"
+    viewBox="0 0 60.903 60.903"
+    {...props}
+  >
+    <path d="M49.561 16.464H39.45v6h10.111c3.008 0 5.341 1.535 5.341 2.857v26.607c0 1.321-2.333 2.858-5.341 2.858H11.34c-3.007 0-5.34-1.537-5.34-2.858V25.324c0-1.322 2.333-2.858 5.34-2.858h10.11v-6H11.34C4.981 16.466 0 20.357 0 25.324v26.605c0 4.968 4.981 8.857 11.34 8.857h38.223c6.357 0 11.34-3.891 11.34-8.857V25.324c-.001-4.969-4.982-8.86-11.342-8.86z" />
+    <path d="M39.529 29.004a2.99 2.99 0 0 0-2.121.88l-3.756 3.755V3.117a3 3 0 0 0-6 0v30.724l-3.959-3.958a2.992 2.992 0 0 0-4.242 0 2.997 2.997 0 0 0 0 4.241l8.957 8.957a2.988 2.988 0 0 0 2.12.877h.045c.768 0 1.534-.291 2.12-.877l8.957-8.957a2.997 2.997 0 0 0-2.121-5.12z" />
+  </svg>
+);

frontend/src/react-files.d.ts

@@ -0,0 +1,36 @@
+//// <reference types="react" />
+
+declare module "react-files" {
+  declare const Files: React.FC<
+    Partial<{
+      accepts: string[];
+      children: React.ReactNode;
+      className: string;
+      clickable: boolean;
+      dragActiveClassName: string;
+      inputProps: unknown;
+      multiple: boolean;
+      maxFiles: number;
+      maxFileSize: number;
+      minFileSize: number;
+      name: string;
+      onChange: (files: ReactFile[]) => void;
+      onDragEnter: () => void;
+      onDragLeave: () => void;
+      onError: (
+        error: { code: number; message: string },
+        file: ReactFile
+      ) => void;
+      style: object;
+    }>
+  >;
+
+  export type ReactFile = File & {
+    id: string;
+    extension: string;
+    sizeReadable: string;
+    preview: { type: "image"; url: string } | { type: "file" };
+  };
+
+  export default Files;
+}

frontend/src/util/numberToPercent.ts

@@ -0,0 +1,5 @@
+export const numberToPercent = (ratio: number) =>
+  ratio.toLocaleString(undefined, {
+    style: "percent",
+    minimumFractionDigits: 2,
+  });

frontend/tsconfig.json

@@ -1,11 +1,7 @@
 {
   "compilerOptions": {
-    "target": "es5",
-    "lib": [
-      "dom",
-      "dom.iterable",
-      "esnext"
-    ],
+    "target": "ES2022",
+    "lib": ["dom", "dom.iterable", "esnext"],
     "allowJs": true,
     "skipLibCheck": true,
     "esModuleInterop": true,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@listory/api",
-  "version": "1.29.0",
+  "version": "1.31.0",
   "description": "Track your Spotify Listen History",
   "author": {
     "name": "Julian Tölle",
@@ -28,91 +28,91 @@
   "dependencies": {
     "@apricote/nest-pg-boss": "2.1.0",
     "@narando/nest-axios-interceptor": "3.0.0",
-    "@nestjs/axios": "3.0.0",
-    "@nestjs/common": "10.2.6",
-    "@nestjs/config": "3.1.1",
-    "@nestjs/core": "10.2.6",
-    "@nestjs/jwt": "10.1.1",
-    "@nestjs/passport": "10.0.2",
-    "@nestjs/platform-express": "10.2.6",
-    "@nestjs/serve-static": "4.0.0",
-    "@nestjs/swagger": "7.1.12",
-    "@nestjs/terminus": "10.1.1",
-    "@nestjs/typeorm": "10.0.0",
-    "@opentelemetry/api": "1.6.0",
+    "@nestjs/axios": "3.1.3",
+    "@nestjs/common": "10.4.15",
+    "@nestjs/config": "3.3.0",
+    "@nestjs/core": "10.4.15",
+    "@nestjs/jwt": "10.2.0",
+    "@nestjs/passport": "10.0.3",
+    "@nestjs/platform-express": "10.4.15",
+    "@nestjs/serve-static": "4.0.2",
+    "@nestjs/swagger": "7.4.2",
+    "@nestjs/terminus": "10.2.3",
+    "@nestjs/typeorm": "10.0.2",
+    "@opentelemetry/api": "1.9.0",
     "@opentelemetry/api-metrics": "0.33.0",
-    "@opentelemetry/context-async-hooks": "1.17.0",
-    "@opentelemetry/exporter-prometheus": "0.43.0",
-    "@opentelemetry/exporter-trace-otlp-http": "0.43.0",
-    "@opentelemetry/instrumentation": "0.43.0",
-    "@opentelemetry/instrumentation-dns": "0.32.2",
-    "@opentelemetry/instrumentation-express": "0.33.1",
-    "@opentelemetry/instrumentation-http": "0.43.0",
-    "@opentelemetry/instrumentation-nestjs-core": "0.33.1",
-    "@opentelemetry/instrumentation-pg": "0.36.1",
-    "@opentelemetry/instrumentation-pino": "0.34.1",
-    "@opentelemetry/resources": "1.17.0",
-    "@opentelemetry/sdk-metrics": "1.17.0",
-    "@opentelemetry/sdk-node": "0.43.0",
-    "@opentelemetry/sdk-trace-base": "1.17.0",
-    "@opentelemetry/semantic-conventions": "1.17.0",
-    "@sentry/node": "7.72.0",
+    "@opentelemetry/context-async-hooks": "1.29.0",
+    "@opentelemetry/exporter-prometheus": "0.56.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.56.0",
+    "@opentelemetry/instrumentation": "0.56.0",
+    "@opentelemetry/instrumentation-dns": "0.42.0",
+    "@opentelemetry/instrumentation-express": "0.46.0",
+    "@opentelemetry/instrumentation-http": "0.56.0",
+    "@opentelemetry/instrumentation-nestjs-core": "0.43.0",
+    "@opentelemetry/instrumentation-pg": "0.49.0",
+    "@opentelemetry/instrumentation-pino": "0.45.0",
+    "@opentelemetry/resources": "1.29.0",
+    "@opentelemetry/sdk-metrics": "1.29.0",
+    "@opentelemetry/sdk-node": "0.56.0",
+    "@opentelemetry/sdk-trace-base": "1.29.0",
+    "@opentelemetry/semantic-conventions": "1.28.0",
+    "@sentry/node": "7.120.3",
     "class-transformer": "0.5.1",
-    "class-validator": "0.14.0",
-    "cookie-parser": "1.4.6",
+    "class-validator": "0.14.1",
+    "cookie-parser": "1.4.7",
     "date-fns": "2.30.0",
-    "joi": "17.10.2",
+    "joi": "17.13.3",
     "lodash": "4.17.21",
-    "nest-raven": "10.0.0",
+    "nest-raven": "10.1.0",
     "nestjs-otel": "5.1.5",
-    "nestjs-pino": "3.5.0",
+    "nestjs-pino": "4.1.0",
     "nestjs-typeorm-paginate": "4.0.4",
-    "passport": "0.6.0",
+    "passport": "0.7.0",
     "passport-http-bearer": "1.0.1",
     "passport-jwt": "4.0.1",
     "passport-spotify": "2.0.0",
-    "pg": "8.11.3",
+    "pg": "8.13.1",
     "pg-boss": "9.0.3",
-    "pino": "8.15.1",
-    "pino-http": "8.5.0",
-    "reflect-metadata": "0.1.13",
-    "rimraf": "5.0.5",
+    "pino": "8.21.0",
+    "pino-http": "9.0.0",
+    "reflect-metadata": "0.1.14",
+    "rimraf": "5.0.10",
     "rxjs": "7.8.1",
-    "typeorm": "0.3.17"
+    "typeorm": "0.3.20"
   },
   "devDependencies": {
-    "@nestjs/cli": "10.1.18",
-    "@nestjs/schematics": "10.0.2",
-    "@nestjs/testing": "10.2.6",
-    "@types/cookie-parser": "1.4.4",
-    "@types/express": "4.17.18",
-    "@types/jest": "29.5.5",
-    "@types/lodash": "4.14.199",
-    "@types/node": "20.8.0",
-    "@types/passport-http-bearer": "1.0.38",
-    "@types/passport-jwt": "3.0.10",
-    "@types/supertest": "2.0.13",
-    "@typescript-eslint/eslint-plugin": "6.7.3",
-    "@typescript-eslint/parser": "6.7.3",
-    "eslint": "8.50.0",
+    "@nestjs/cli": "10.4.9",
+    "@nestjs/schematics": "10.2.3",
+    "@nestjs/testing": "10.4.15",
+    "@types/cookie-parser": "1.4.8",
+    "@types/express": "5.0.0",
+    "@types/jest": "29.5.14",
+    "@types/lodash": "4.17.14",
+    "@types/node": "20.17.16",
+    "@types/passport-http-bearer": "1.0.41",
+    "@types/passport-jwt": "4.0.1",
+    "@types/supertest": "6.0.2",
+    "@typescript-eslint/eslint-plugin": "6.21.0",
+    "@typescript-eslint/parser": "6.21.0",
+    "eslint": "8.57.1",
     "eslint-config-airbnb-base": "15.0.0",
     "eslint-config-airbnb-typescript": "17.1.0",
-    "eslint-config-prettier": "9.0.0",
-    "eslint-plugin-import": "2.28.1",
-    "eslint-plugin-jsdoc": "46.8.2",
-    "eslint-plugin-jsx-a11y": "6.7.1",
+    "eslint-config-prettier": "9.1.0",
+    "eslint-plugin-import": "2.31.0",
+    "eslint-plugin-jsdoc": "48.11.0",
+    "eslint-plugin-jsx-a11y": "6.10.2",
     "eslint-plugin-prefer-arrow": "1.2.3",
-    "eslint-plugin-react": "7.33.2",
-    "eslint-plugin-react-hooks": "4.6.0",
+    "eslint-plugin-react": "7.37.4",
+    "eslint-plugin-react-hooks": "4.6.2",
     "jest": "29.7.0",
-    "pino-pretty": "10.2.0",
-    "prettier": "3.0.3",
-    "supertest": "6.3.3",
-    "ts-jest": "29.1.1",
-    "ts-loader": "9.4.4",
-    "ts-node": "10.9.1",
+    "pino-pretty": "10.3.1",
+    "prettier": "3.4.2",
+    "supertest": "6.3.4",
+    "ts-jest": "29.2.5",
+    "ts-loader": "9.5.1",
+    "ts-node": "10.9.2",
     "tsconfig-paths": "4.2.0",
-    "typescript": "5.2.2"
+    "typescript": "5.7.2"
   },
   "jest": {
     "moduleFileExtensions": [

renovate.json

@@ -7,7 +7,8 @@
     ":automergeBranch",
     ":automergeLinters",
     ":automergeTesters",
-    ":automergeTypes"
+    ":automergeTypes",
+    ":maintainLockFilesWeekly"
   ],
   "packageRules": [
     {

src/auth/auth.controller.spec.ts

@@ -1,5 +1,5 @@
 import { Test, TestingModule } from "@nestjs/testing";
-import type { Response } from "express";
+import type { Response as ExpressResponse } from "express";
 import { User } from "../users/user.entity";
 import { AuthSession } from "./auth-session.entity";
 import { AuthController } from "./auth.controller";
@@ -27,7 +27,7 @@ describe("AuthController", () => {
 
   describe("spotifyCallback", () => {
     let user: User;
-    let res: Response;
+    let res: ExpressResponse;
     let refreshToken: string;
 
     beforeEach(() => {
@@ -36,7 +36,7 @@ describe("AuthController", () => {
         statusCode: 200,
         cookie: jest.fn(),
         redirect: jest.fn(),
-      } as unknown as Response;
+      } as unknown as ExpressResponse;
 
       refreshToken = "REFRESH_TOKEN";
       authService.createSession = jest.fn().mockResolvedValue({ refreshToken });

src/auth/auth.controller.ts

@@ -1,5 +1,5 @@
 import {
-  Body,
+  Body as NestBody,
   Controller,
   Delete,
   Get,
@@ -10,7 +10,7 @@ import {
   UseGuards,
 } from "@nestjs/common";
 import { ApiBody, ApiTags } from "@nestjs/swagger";
-import type { Response } from "express";
+import type { Response as ExpressResponse } from "express";
 import { User } from "../users/user.entity";
 import { AuthSession } from "./auth-session.entity";
 import { AuthService } from "./auth.service";
@@ -42,7 +42,7 @@ export class AuthController {
   @Get("spotify/callback")
   @UseFilters(SpotifyAuthFilter)
   @UseGuards(SpotifyAuthGuard)
-  async spotifyCallback(@ReqUser() user: User, @Res() res: Response) {
+  async spotifyCallback(@ReqUser() user: User, @Res() res: ExpressResponse) {
     const { refreshToken } = await this.authService.createSession(user);
 
     // Refresh token should not be accessible to frontend to reduce risk
@@ -69,7 +69,7 @@ export class AuthController {
   @AuthAccessToken()
   async createApiToken(
     @ReqUser() user: User,
-    @Body("description") description: string,
+    @NestBody("description") description: string,
   ): Promise<NewApiTokenDto> {
     const apiToken = await this.authService.createApiToken(user, description);
 

src/auth/spotify.filter.ts

@@ -5,14 +5,14 @@ import {
   ForbiddenException,
   Logger,
 } from "@nestjs/common";
-import type { Response } from "express";
+import type { Response as ExpressResponse } from "express";
 
 @Catch()
 export class SpotifyAuthFilter implements ExceptionFilter {
   private readonly logger = new Logger(this.constructor.name);
 
   catch(exception: Error, host: ArgumentsHost) {
-    const response = host.switchToHttp().getResponse<Response>();
+    const response = host.switchToHttp().getResponse<ExpressResponse>();
 
     let reason = "unknown";
 

src/database/database.module.ts

@@ -27,7 +27,7 @@ export const DatabaseModule = TypeOrmModule.forRootAsync({
 
     // Debug/Development Options
     //
-    // logging: true,
+    //logging: true,
     //
     // synchronize: true,
     // migrationsRun: false,

src/database/migrations/09-CreateSpotifyImportTables.ts

@@ -0,0 +1,68 @@
+import {
+  MigrationInterface,
+  QueryRunner,
+  Table,
+  TableIndex,
+  TableForeignKey,
+} from "typeorm";
+import { TableColumnOptions } from "typeorm/schema-builder/options/TableColumnOptions";
+
+const primaryUUIDColumn: TableColumnOptions = {
+  name: "id",
+  type: "uuid",
+  isPrimary: true,
+  isGenerated: true,
+  generationStrategy: "uuid",
+};
+
+export class CreateSpotifyImportTables0000000000009
+  implements MigrationInterface
+{
+  async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.createTable(
+      new Table({
+        name: "spotify_extended_streaming_history_listen",
+        columns: [
+          primaryUUIDColumn,
+          { name: "userId", type: "uuid" },
+          { name: "playedAt", type: "timestamp" },
+          { name: "spotifyTrackUri", type: "varchar" },
+          { name: "trackId", type: "uuid", isNullable: true },
+          { name: "listenId", type: "uuid", isNullable: true },
+        ],
+        indices: [
+          new TableIndex({
+            name: "IDX_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_USER_PLAYED_AT",
+            columnNames: ["userId", "playedAt", "spotifyTrackUri"],
+            isUnique: true,
+          }),
+        ],
+        foreignKeys: [
+          new TableForeignKey({
+            name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_USER_ID",
+            columnNames: ["userId"],
+            referencedColumnNames: ["id"],
+            referencedTableName: "user",
+          }),
+          new TableForeignKey({
+            name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_TRACK_ID",
+            columnNames: ["trackId"],
+            referencedColumnNames: ["id"],
+            referencedTableName: "track",
+          }),
+          new TableForeignKey({
+            name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_LISTEN_ID",
+            columnNames: ["listenId"],
+            referencedColumnNames: ["id"],
+            referencedTableName: "listen",
+          }),
+        ],
+      }),
+      true,
+    );
+  }
+
+  async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.dropTable("spotify_extended_streaming_history_listen");
+  }
+}

src/listens/listen.repository.ts

@@ -3,10 +3,6 @@ import { Repository, SelectQueryBuilder } from "typeorm";
 import { EntityRepository } from "../database/entity-repository";
 import { Interval } from "../reports/interval";
 import { User } from "../users/user.entity";
-import {
-  CreateListenRequestDto,
-  CreateListenResponseDto,
-} from "./dto/create-listen.dto";
 import { Listen } from "./listen.entity";
 
 export class ListenScopes extends SelectQueryBuilder<Listen> {
@@ -37,52 +33,4 @@ export class ListenRepository extends Repository<Listen> {
   get scoped(): ListenScopes {
     return new ListenScopes(this.createQueryBuilder("listen"));
   }
-
-  async insertNoConflict({
-    user,
-    track,
-    playedAt,
-  }: CreateListenRequestDto): Promise<CreateListenResponseDto> {
-    const result = await this.createQueryBuilder()
-      .insert()
-      .values({
-        user,
-        track,
-        playedAt,
-      })
-      .onConflict('("playedAt", "trackId", "userId") DO NOTHING')
-      .execute();
-
-    const [insertedRowIdentifier] = result.identifiers;
-
-    if (!insertedRowIdentifier) {
-      // We did not insert a new listen, it already existed
-      return {
-        listen: await this.findOneBy({ user, track, playedAt }),
-        isDuplicate: true,
-      };
-    }
-
-    return {
-      listen: await this.findOneBy({ id: insertedRowIdentifier.id }),
-      isDuplicate: false,
-    };
-  }
-
-  /**
-   *
-   * @param rows
-   * @returns A list of all new (non-duplicate) listens
-   */
-  async insertsNoConflict(rows: CreateListenRequestDto[]): Promise<Listen[]> {
-    const result = await this.createQueryBuilder()
-      .insert()
-      .values(rows)
-      .orIgnore()
-      .execute();
-
-    return this.findBy(
-      result.identifiers.filter(Boolean).map(({ id }) => ({ id })),
-    );
-  }
 }

src/listens/listens.service.spec.ts

@@ -4,9 +4,7 @@ import {
   paginate,
   PaginationTypeEnum,
 } from "nestjs-typeorm-paginate";
-import { Track } from "../music-library/track.entity";
 import { User } from "../users/user.entity";
-import { CreateListenResponseDto } from "./dto/create-listen.dto";
 import { GetListensDto } from "./dto/get-listens.dto";
 import { Listen } from "./listen.entity";
 import { ListenRepository, ListenScopes } from "./listen.repository";
@@ -35,39 +33,6 @@ describe("ListensService", () => {
     expect(listenRepository).toBeDefined();
   });
 
-  describe("createListen", () => {
-    let user: User;
-    let track: Track;
-    let playedAt: Date;
-    let response: CreateListenResponseDto;
-    beforeEach(() => {
-      user = { id: "USER" } as User;
-      track = { id: "TRACK" } as Track;
-      playedAt = new Date("2021-01-01T00:00:00Z");
-
-      response = {
-        listen: {
-          id: "LISTEN",
-        } as Listen,
-        isDuplicate: true,
-      };
-      listenRepository.insertNoConflict = jest.fn().mockResolvedValue(response);
-    });
-
-    it("creates the listen", async () => {
-      await expect(
-        service.createListen({ user, track, playedAt }),
-      ).resolves.toEqual(response);
-
-      expect(listenRepository.insertNoConflict).toHaveBeenCalledTimes(1);
-      expect(listenRepository.insertNoConflict).toHaveBeenLastCalledWith({
-        user,
-        track,
-        playedAt,
-      });
-    });
-  });
-
   describe("getListens", () => {
     let options: GetListensDto & IPaginationOptions;
     let user: User;

src/listens/listens.service.ts

@@ -1,14 +1,12 @@
 import { Injectable } from "@nestjs/common";
+import { Span } from "nestjs-otel";
 import {
   IPaginationOptions,
   paginate,
   Pagination,
   PaginationTypeEnum,
 } from "nestjs-typeorm-paginate";
-import {
-  CreateListenRequestDto,
-  CreateListenResponseDto,
-} from "./dto/create-listen.dto";
+import { CreateListenRequestDto } from "./dto/create-listen.dto";
 import { GetListensDto } from "./dto/get-listens.dto";
 import { Listen } from "./listen.entity";
 import { ListenRepository, ListenScopes } from "./listen.repository";
@@ -17,20 +15,7 @@ import { ListenRepository, ListenScopes } from "./listen.repository";
 export class ListensService {
   constructor(private readonly listenRepository: ListenRepository) {}
 
-  async createListen({
-    user,
-    track,
-    playedAt,
-  }: CreateListenRequestDto): Promise<CreateListenResponseDto> {
-    const response = await this.listenRepository.insertNoConflict({
-      user,
-      track,
-      playedAt,
-    });
-
-    return response;
-  }
-
+  @Span()
   async createListens(
     listensData: CreateListenRequestDto[],
   ): Promise<Listen[]> {
@@ -46,9 +31,11 @@ export class ListensService {
         ),
     );
 
-    return this.listenRepository.save(
+    const newListens = await this.listenRepository.save(
       missingListens.map((entry) => this.listenRepository.create(entry)),
     );
+
+    return [...existingListens, ...newListens];
   }
 
   async getListens(

src/main.ts

@@ -43,6 +43,7 @@ async function bootstrap() {
 
   const app = await NestFactory.create<NestExpressApplication>(AppModule, {
     bufferLogs: true,
+    rawBody: true,
   });
   app.useLogger(app.get(Logger));
   app.useGlobalPipes(
@@ -51,6 +52,10 @@ async function bootstrap() {
       transformOptions: { enableImplicitConversion: true },
     }),
   );
+  app.useBodyParser("json", {
+    limit:
+      "10mb" /* Need large bodies for Spotify Extended Streaming History */,
+  });
   app.enableShutdownHooks();
 
   const configService = app.get<ConfigService>(ConfigService);

src/music-library/dto/find-track.dto.ts

@@ -1,5 +1,6 @@
 export class FindTrackDto {
   spotify: {
-    id: string;
+    id?: string;
+    uri?: string;
   };
 }

src/music-library/music-library.service.ts

@@ -175,9 +175,7 @@ export class MusicLibraryService {
   }
 
   async findTrack(query: FindTrackDto): Promise<Track | undefined> {
-    return this.trackRepository.findOneBy({
-      spotify: { id: query.spotify.id },
-    });
+    return this.trackRepository.findOneBy(query);
   }
 
   async findTracks(query: FindTrackDto[]): Promise<Track[]> {

src/override.d.ts

@@ -1,5 +0,0 @@
-// Issue with opentelemetry-js: https://github.com/open-telemetry/opentelemetry-js/issues/3580#issuecomment-1701157270
-export {};
-declare global {
-  type BlobPropertyBag = unknown;
-}

src/sources/scheduler.service.ts

@@ -33,7 +33,7 @@ export class SchedulerService implements OnApplicationBootstrap {
   }
 
   private async setupSpotifyCrawlerSupervisor(): Promise<void> {
-    await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {});
+    // await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {});
   }
 
   @Span()

src/sources/spotify/import-extended-streaming-history/dto/extended-streaming-history-status.dto.ts

@@ -0,0 +1,13 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class ExtendedStreamingHistoryStatusDto {
+  @ApiProperty({
+    type: Number,
+  })
+  total: number;
+
+  @ApiProperty({
+    type: Number,
+  })
+  imported: number;
+}

src/sources/spotify/import-extended-streaming-history/dto/import-extended-streaming-history.dto.ts

@@ -0,0 +1,13 @@
+import { ApiProperty } from "@nestjs/swagger";
+import { ArrayMaxSize } from "class-validator";
+import { SpotifyExtendedStreamingHistoryItemDto } from "./spotify-extended-streaming-history-item.dto";
+
+export class ImportExtendedStreamingHistoryDto {
+  @ApiProperty({
+    type: SpotifyExtendedStreamingHistoryItemDto,
+    isArray: true,
+    maxItems: 50_000,
+  })
+  @ArrayMaxSize(50_000) // File size is ~16k by default, might need refactoring if Spotify starts exporting larger files
+  listens: SpotifyExtendedStreamingHistoryItemDto[];
+}

src/sources/spotify/import-extended-streaming-history/dto/spotify-extended-streaming-history-item.dto.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class SpotifyExtendedStreamingHistoryItemDto {
+  @ApiProperty({ format: "iso8601", example: "2018-11-30T08:33:33Z" })
+  ts: string;
+
+  @ApiProperty({ example: "spotify:track:6askbS4pEVWbbDnUGEXh3G" })
+  spotify_track_uri: string;
+}

src/sources/spotify/import-extended-streaming-history/import.controller.ts

@@ -0,0 +1,32 @@
+import { Body as NestBody, Controller, Get, Post } from "@nestjs/common";
+import { ApiBody, ApiTags } from "@nestjs/swagger";
+import { AuthAccessToken } from "../../../auth/decorators/auth-access-token.decorator";
+import { ReqUser } from "../../../auth/decorators/req-user.decorator";
+import { User } from "../../../users/user.entity";
+import { ExtendedStreamingHistoryStatusDto } from "./dto/extended-streaming-history-status.dto";
+import { ImportExtendedStreamingHistoryDto } from "./dto/import-extended-streaming-history.dto";
+import { ImportService } from "./import.service";
+
+@ApiTags("import")
+@Controller("api/v1/import")
+export class ImportController {
+  constructor(private readonly importService: ImportService) {}
+
+  @Post("extended-streaming-history")
+  @ApiBody({ type: () => ImportExtendedStreamingHistoryDto })
+  @AuthAccessToken()
+  async importExtendedStreamingHistory(
+    @ReqUser() user: User,
+    @NestBody() data: ImportExtendedStreamingHistoryDto,
+  ): Promise<void> {
+    return this.importService.importExtendedStreamingHistory(user, data);
+  }
+
+  @Get("extended-streaming-history/status")
+  @AuthAccessToken()
+  async getExtendedStreamingHistoryStatus(
+    @ReqUser() user: User,
+  ): Promise<ExtendedStreamingHistoryStatusDto> {
+    return this.importService.getExtendedStreamingHistoryStatus(user);
+  }
+}

src/sources/spotify/import-extended-streaming-history/import.service.ts

@@ -0,0 +1,177 @@
+import { JobService } from "@apricote/nest-pg-boss";
+import { Injectable, Logger } from "@nestjs/common";
+import { uniq } from "lodash";
+import { Span } from "nestjs-otel";
+import type { Job } from "pg-boss";
+import { ListensService } from "../../../listens/listens.service";
+import { User } from "../../../users/user.entity";
+import { SpotifyService } from "../spotify.service";
+import { ExtendedStreamingHistoryStatusDto } from "./dto/extended-streaming-history-status.dto";
+import { ImportExtendedStreamingHistoryDto } from "./dto/import-extended-streaming-history.dto";
+import {
+  IProcessSpotifyExtendedStreamingHistoryListenJob,
+  ProcessSpotifyExtendedStreamingHistoryListenJob,
+} from "./jobs";
+import { SpotifyExtendedStreamingHistoryListenRepository } from "./listen.repository";
+
+@Injectable()
+export class ImportService {
+  private readonly logger = new Logger(this.constructor.name);
+
+  constructor(
+    private readonly importListenRepository: SpotifyExtendedStreamingHistoryListenRepository,
+    @ProcessSpotifyExtendedStreamingHistoryListenJob.Inject()
+    private readonly processListenJobService: JobService<IProcessSpotifyExtendedStreamingHistoryListenJob>,
+    private readonly spotifyService: SpotifyService,
+    private readonly listensService: ListensService,
+  ) {}
+
+  @Span()
+  async importExtendedStreamingHistory(
+    user: User,
+    { listens: importListens }: ImportExtendedStreamingHistoryDto,
+  ): Promise<void> {
+    // IDK what's happening, but my personal data set has entries with duplicate
+    // listens? might be related to offline mode.
+    // Anyway, this cleans it up:
+    const uniqEntries = new Set();
+    const uniqueListens = importListens.filter((listen) => {
+      const key = `${listen.spotify_track_uri}-${listen.ts}`;
+
+      if (!uniqEntries.has(key)) {
+        // New entry
+        uniqEntries.add(key);
+        return true;
+      }
+
+      return false;
+    });
+
+    let listens = uniqueListens.map((listenData) =>
+      this.importListenRepository.create({
+        user,
+        playedAt: new Date(listenData.ts),
+        spotifyTrackUri: listenData.spotify_track_uri,
+      }),
+    );
+
+    // Save listens to import table
+    const insertResult = await this.importListenRepository.upsert(listens, [
+      "user",
+      "playedAt",
+      "spotifyTrackUri",
+    ]);
+
+    const processJobs = insertResult.identifiers.map((listen) => ({
+      data: {
+        id: listen.id,
+      },
+      singletonKey: listen.id,
+      retryLimit: 10,
+      retryDelay: 5,
+      retryBackoff: true,
+    }));
+
+    // Schedule jobs to process imports
+    await this.processListenJobService.insert(processJobs);
+  }
+
+  @ProcessSpotifyExtendedStreamingHistoryListenJob.Handle({
+    // Spotify API "Get Several XY" allows max 50 IDs
+    batchSize: 50,
+    newJobCheckInterval: 500,
+  })
+  @Span()
+  async processListens(
+    jobs: Job<IProcessSpotifyExtendedStreamingHistoryListenJob>[],
+  ): Promise<void> {
+    this.logger.debug(
+      { jobs: jobs.length },
+      "processing extended streaming history listens",
+    );
+    const importListens = await this.importListenRepository.findBy(
+      jobs.map((job) => ({ id: job.data.id })),
+    );
+
+    const listensWithoutTracks = importListens.filter(
+      (importListen) => !importListen.track,
+    );
+    if (listensWithoutTracks.length > 0) {
+      const missingTrackIDs = uniq(
+        listensWithoutTracks.map((importListen) =>
+          importListen.spotifyTrackUri.replace("spotify:track:", ""),
+        ),
+      );
+
+      const tracks = await this.spotifyService.importTracks(missingTrackIDs);
+
+      listensWithoutTracks.forEach((listen) => {
+        listen.track = tracks.find(
+          (track) => listen.spotifyTrackUri === track.spotify.uri,
+        );
+        if (!listen.track) {
+          this.logger.warn(
+            { listen },
+            "could not find track for extended streaming history listen",
+          );
+          throw new Error(
+            `could not find track for extended streaming history listen`,
+          );
+        }
+      });
+
+      // Using upsert instead of save to only do a single query
+      await this.importListenRepository.upsert(listensWithoutTracks, ["id"]);
+    }
+
+    const listensWithoutListen = importListens.filter(
+      (importListen) => !importListen.listen,
+    );
+    if (listensWithoutListen.length > 0) {
+      const listens = await this.listensService.createListens(
+        listensWithoutListen.map((listen) => ({
+          user: listen.user,
+          track: listen.track,
+          playedAt: listen.playedAt,
+        })),
+      );
+
+      listensWithoutListen.forEach((importListen) => {
+        importListen.listen = listens.find(
+          (listen) =>
+            importListen.user.id === listen.user.id &&
+            importListen.track.id === listen.track.id &&
+            importListen.playedAt.getTime() === listen.playedAt.getTime(),
+        );
+        if (!importListen.listen) {
+          this.logger.warn(
+            { listen: importListen, listens: listens },
+            "could not find listen for extended streaming history listen",
+          );
+          throw new Error(
+            `could not find listen for extended streaming history listen`,
+          );
+        }
+      });
+
+      // Using upsert instead of save to only do a single query
+      await this.importListenRepository.upsert(listensWithoutListen, ["id"]);
+    }
+  }
+
+  @Span()
+  async getExtendedStreamingHistoryStatus(
+    user: User,
+  ): Promise<ExtendedStreamingHistoryStatusDto> {
+    const qb = this.importListenRepository
+      .createQueryBuilder("listen")
+      .where("listen.userId = :user", { user: user.id });
+
+    const [total, imported] = await Promise.all([
+      qb.clone().getCount(),
+      qb.clone().andWhere("listen.listenId IS NOT NULL").getCount(),
+    ]);
+
+    return { total, imported };
+  }
+}

src/sources/spotify/import-extended-streaming-history/index.ts

@@ -0,0 +1,4 @@
+export { ImportController } from "./import.controller";
+export { ImportService } from "./import.service";
+export { ProcessSpotifyExtendedStreamingHistoryListenJob } from "./jobs";
+export { SpotifyExtendedStreamingHistoryListenRepository } from "./listen.repository";

src/sources/spotify/import-extended-streaming-history/jobs.ts

@@ -0,0 +1,7 @@
+import { createJob } from "@apricote/nest-pg-boss";
+
+export type IProcessSpotifyExtendedStreamingHistoryListenJob = { id: string };
+export const ProcessSpotifyExtendedStreamingHistoryListenJob =
+  createJob<IProcessSpotifyExtendedStreamingHistoryListenJob>(
+    "process-spotify-extended-streaming-history-listen",
+  );

src/sources/spotify/import-extended-streaming-history/listen.entity.ts

@@ -0,0 +1,25 @@
+import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm";
+import { Track } from "../../../music-library/track.entity";
+import { User } from "../../../users/user.entity";
+import { Listen } from "../../../listens/listen.entity";
+
+@Entity({ name: "spotify_extended_streaming_history_listen" })
+export class SpotifyExtendedStreamingHistoryListen {
+  @PrimaryGeneratedColumn("uuid")
+  id: string;
+
+  @ManyToOne(() => User, { eager: true })
+  user: User;
+
+  @Column({ type: "timestamp" })
+  playedAt: Date;
+
+  @Column()
+  spotifyTrackUri: string;
+
+  @ManyToOne(() => Track, { nullable: true, eager: true })
+  track?: Track;
+
+  @ManyToOne(() => Listen, { nullable: true, eager: true })
+  listen?: Listen;
+}

src/sources/spotify/import-extended-streaming-history/listen.repository.ts

@@ -0,0 +1,6 @@
+import { Repository } from "typeorm";
+import { EntityRepository } from "../../../database/entity-repository";
+import { SpotifyExtendedStreamingHistoryListen } from "./listen.entity";
+
+@EntityRepository(SpotifyExtendedStreamingHistoryListen)
+export class SpotifyExtendedStreamingHistoryListenRepository extends Repository<SpotifyExtendedStreamingHistoryListen> {}

src/sources/spotify/spotify.module.ts

@@ -1,25 +1,37 @@
 import { PGBossModule } from "@apricote/nest-pg-boss";
 import { Module } from "@nestjs/common";
+import { TypeOrmRepositoryModule } from "../../database/entity-repository/typeorm-repository.module";
 import { ListensModule } from "../../listens/listens.module";
 import { MusicLibraryModule } from "../../music-library/music-library.module";
 import { UsersModule } from "../../users/users.module";
 import { ImportSpotifyJob } from "../jobs";
+import {
+  ImportController,
+  ImportService,
+  ProcessSpotifyExtendedStreamingHistoryListenJob,
+  SpotifyExtendedStreamingHistoryListenRepository,
+} from "./import-extended-streaming-history";
 import { SpotifyApiModule } from "./spotify-api/spotify-api.module";
 import { SpotifyAuthModule } from "./spotify-auth/spotify-auth.module";
 import { SpotifyService } from "./spotify.service";
 
 @Module({
   imports: [
-    PGBossModule.forJobs([ImportSpotifyJob]),
+    PGBossModule.forJobs([
+      ImportSpotifyJob,
+      ProcessSpotifyExtendedStreamingHistoryListenJob,
+    ]),
+    TypeOrmRepositoryModule.for([
+      SpotifyExtendedStreamingHistoryListenRepository,
+    ]),
     UsersModule,
     ListensModule,
     MusicLibraryModule,
     SpotifyApiModule,
     SpotifyAuthModule,
   ],
-  providers: [SpotifyService],
+  providers: [SpotifyService, ImportService],
+  controllers: [ImportController],
   exports: [SpotifyService],
 })
-export class SpotifyModule {
-  constructor(private readonly spotifyService: SpotifyService) {}
-}
+export class SpotifyModule {}

tsconfig.json

@@ -5,13 +5,12 @@
     "removeComments": true,
     "emitDecoratorMetadata": true,
     "experimentalDecorators": true,
-    "target": "ES2020",
+    "target": "ES2022",
     "sourceMap": true,
     "outDir": "./dist",
     "baseUrl": "./",
     "incremental": true,
     "paths": {},
-    "lib": ["ES2020"]
   },
   "exclude": ["node_modules", "dist"],
   "include": ["src", "test"]