feat: import listens from spotify extended streaming history (#305)

.dockerignore

@@ -19,5 +19,6 @@
 !frontend/tsconfig.json
 !frontend/vite.config.js
 !frontend/index.html
+!frontend/*.d.ts
 !frontend/src/**/*
 !frontend/public/**/*

Dockerfile

@@ -1,10 +1,5 @@
 # syntax=docker/dockerfile:1.5
 
-FROM scratch as ignore
-
-WORKDIR /listory
-COPY . /listory/
-
 ##################
 ## common
 ##################

docker-compose.yml

@@ -37,7 +37,8 @@ services:
       OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4318
     env_file: .env
     volumes:
-      - ./src:/app/src
+      - ./src:/app/src:ro
+      - ./dist:/app/dist # build cache
     ports:
       - 3000 # API
       - "9464:9464" # Metrics
@@ -115,7 +116,8 @@ services:
       OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4318
     env_file: .env
     volumes:
-      - ./src:/app/src
+      - ./src:/app/src:ro
+      - ./dist:/app/dist # build cache
     ports:
       - "9464:9464" # Metrics
     networks:

frontend/package-lock.json

@@ -38,6 +38,7 @@
         "prettier": "3.0.3",
         "react": "18.2.0",
         "react-dom": "18.2.0",
+        "react-files": "3.0.0",
         "react-router-dom": "6.16.0",
         "recharts": "2.8.0",
         "tailwind-merge": "1.14.0",
@@ -8143,6 +8144,15 @@
         "react": "^18.2.0"
       }
     },
+    "node_modules/react-files": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/react-files/-/react-files-3.0.0.tgz",
+      "integrity": "sha512-/Zz7S98vZFYxHw3RVSZcf3dD+xO714ZQd/jEhIp8q+MofBgydXWlHdw05TA4jradL7XpZFPvJaIvM6Z6I5nIHw==",
+      "peerDependencies": {
+        "react": ">=16.8",
+        "react-dom": ">=16.8"
+      }
+    },
     "node_modules/react-is": {
       "version": "17.0.2",
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz",
@@ -15363,6 +15373,12 @@
         "scheduler": "^0.23.0"
       }
     },
+    "react-files": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/react-files/-/react-files-3.0.0.tgz",
+      "integrity": "sha512-/Zz7S98vZFYxHw3RVSZcf3dD+xO714ZQd/jEhIp8q+MofBgydXWlHdw05TA4jradL7XpZFPvJaIvM6Z6I5nIHw==",
+      "requires": {}
+    },
     "react-is": {
       "version": "17.0.2",
       "resolved": "https://registry.npmjs.org/react-is/-/react-is-17.0.2.tgz",

frontend/package.json

@@ -37,6 +37,7 @@
     "prettier": "3.0.3",
     "react": "18.2.0",
     "react-dom": "18.2.0",
+    "react-files": "3.0.0",
     "react-router-dom": "6.16.0",
     "recharts": "2.8.0",
     "tailwind-merge": "1.14.0",

frontend/src/App.tsx

@@ -2,6 +2,7 @@ import React from "react";
 import { Route, Routes } from "react-router-dom";
 import { AuthApiTokens } from "./components/AuthApiTokens";
 import { Footer } from "./components/Footer";
+import { ImportListens } from "./components/ImportListens";
 import { LoginFailure } from "./components/LoginFailure";
 import { LoginLoading } from "./components/LoginLoading";
 import { NavBar } from "./components/NavBar";
@@ -53,6 +54,7 @@ export function App() {
                   element={<ReportTopGenres />}
                 />
                 <Route path="/auth/api-tokens" element={<AuthApiTokens />} />
+                <Route path="/import" element={<ImportListens />} />
               </Routes>
             )}
             {!user && (

frontend/src/api/api.ts

@@ -14,6 +14,8 @@ import { TopGenresItem } from "./entities/top-genres-item";
 import { TopGenresOptions } from "./entities/top-genres-options";
 import { TopTracksItem } from "./entities/top-tracks-item";
 import { TopTracksOptions } from "./entities/top-tracks-options";
+import { SpotifyExtendedStreamingHistoryItem } from "./entities/spotify-extended-streaming-history-item";
+import { ExtendedStreamingHistoryStatus } from "./entities/extended-streaming-history-status";
 
 export class UnauthenticatedError extends Error {}
 
@@ -276,7 +278,7 @@ export const revokeApiToken = async (
   id: string,
   client: AxiosInstance,
 ): Promise<void> => {
-  const res = await client.delete<NewApiToken>(`/api/v1/auth/api-tokens/${id}`);
+  const res = await client.delete(`/api/v1/auth/api-tokens/${id}`);
 
   switch (res.status) {
     case 200: {
@@ -290,3 +292,50 @@ export const revokeApiToken = async (
     }
   }
 };
+
+export const importExtendedStreamingHistory = async (
+  listens: SpotifyExtendedStreamingHistoryItem[],
+  client: AxiosInstance
+): Promise<void> => {
+  const res = await client.post(`/api/v1/import/extended-streaming-history`, {
+    listens,
+  });
+
+  switch (res.status) {
+    case 201: {
+      break;
+    }
+    case 401: {
+      throw new UnauthenticatedError(`No token or token expired`);
+    }
+    default: {
+      throw new Error(
+        `Unable to importExtendedStreamingHistory: ${res.status}`
+      );
+    }
+  }
+};
+
+export const getExtendedStreamingHistoryStatus = async (
+  client: AxiosInstance
+): Promise<ExtendedStreamingHistoryStatus> => {
+  const res = await client.get<ExtendedStreamingHistoryStatus>(
+    `/api/v1/import/extended-streaming-history/status`
+  );
+
+  switch (res.status) {
+    case 200: {
+      break;
+    }
+    case 401: {
+      throw new UnauthenticatedError(`No token or token expired`);
+    }
+    default: {
+      throw new Error(
+        `Unable to getExtendedStreamingHistoryStatus: ${res.status}`
+      );
+    }
+  }
+
+  return res.data;
+};

frontend/src/api/entities/extended-streaming-history-status.ts

@@ -0,0 +1,4 @@
+export interface ExtendedStreamingHistoryStatus {
+  total: number;
+  imported: number;
+}

frontend/src/api/entities/spotify-extended-streaming-history-item.ts

@@ -0,0 +1,4 @@
+export interface SpotifyExtendedStreamingHistoryItem {
+  ts: string;
+  spotify_track_uri: string;
+}

frontend/src/components/AuthApiTokens.tsx

@@ -2,7 +2,6 @@ import { format, formatDistanceToNow } from "date-fns";
 import React, { FormEvent, useCallback, useMemo, useState } from "react";
 import { ApiToken, NewApiToken } from "../api/entities/api-token";
 import { useApiTokens } from "../hooks/use-api";
-import { useAuthProtection } from "../hooks/use-auth-protection";
 import { SpinnerIcon } from "../icons/Spinner";
 import TrashcanIcon from "../icons/Trashcan";
 import { Spinner } from "./ui/Spinner";

frontend/src/components/ImportListens.tsx

@@ -0,0 +1,280 @@
+import React, { useCallback, useEffect, useState } from "react";
+import Files from "react-files";
+import type { ReactFile } from "react-files";
+import {
+  useSpotifyImportExtendedStreamingHistory,
+  useSpotifyImportExtendedStreamingHistoryStatus,
+} from "../hooks/use-api";
+import { SpotifyExtendedStreamingHistoryItem } from "../api/entities/spotify-extended-streaming-history-item";
+import { ErrorIcon } from "../icons/Error";
+import { numberToPercent } from "../util/numberToPercent";
+import { Button } from "./ui/button";
+import { Table, TableBody, TableCell, TableRow } from "./ui/table";
+import { Badge } from "./ui/badge";
+import {
+  Card,
+  CardContent,
+  CardDescription,
+  CardFooter,
+  CardHeader,
+  CardTitle,
+} from "./ui/card";
+import { Code } from "./ui/code";
+
+export const ImportListens: React.FC = () => {
+  return (
+    <>
+      <div className="flex justify-between">
+        <p className="text-2xl font-normal">
+          Import Listens from Spotify Extended Streaming History
+        </p>
+      </div>
+      <div className="shadow-xl bg-gray-100 dark:bg-gray-800 rounded p-4 m-2">
+        <p className="my-4">
+          Here you can import your full Spotify Listen history that was exported
+          from the{" "}
+          <a
+            target="blank"
+            href="https://www.spotify.com/us/account/privacy/"
+            className="underline"
+          >
+            Extended streaming history
+          </a>
+          .
+        </p>
+        <p className="my-4">
+          The extended streaming history contains additional personally
+          identifiable data such as the IP address of the listen (which can be
+          linked to locations). To avoid saving this on the server, the data is
+          preprocessed in your web browser and only the necessary data
+          (timestamp & track ID) are sent to the server.
+        </p>
+        <p className="my-4">
+          If an error occurs, you can always retry uploading the file, Listory
+          deduplicates any listens to make sure that everything is saved only
+          once.
+        </p>
+
+        <FileUpload />
+        <ImportProgress />
+      </div>
+    </>
+  );
+};
+
+interface FileData {
+  file: ReactFile;
+  status: Status;
+  error?: Error;
+}
+
+enum Status {
+  Select,
+  Import,
+  Finished,
+  Error,
+}
+
+const FileUpload: React.FC = () => {
+  // Using a map is ... meh, need to wrap all state updates in `new Map()` so react re-renders
+  const [fileMap, setFileMap] = useState<Map<ReactFile["id"], FileData>>(
+    new Map(),
+  );
+
+  const [status, setStatus] = useState<Status>(Status.Select);
+
+  const addFiles = useCallback(
+    (files: ReactFile[]) => {
+      setFileMap((_fileMap) => {
+        files.forEach((file) =>
+          _fileMap.set(file.id, { file, status: Status.Select }),
+        );
+        return new Map(_fileMap);
+      });
+    },
+    [setFileMap],
+  );
+
+  const updateFile = useCallback((data: FileData) => {
+    setFileMap((_fileMap) => new Map(_fileMap.set(data.file.id, data)));
+  }, []);
+
+  const clearFiles = useCallback(() => {
+    setFileMap(new Map());
+  }, [setFileMap]);
+
+  const { importHistory } = useSpotifyImportExtendedStreamingHistory();
+
+  const handleImport = useCallback(async () => {
+    setStatus(Status.Import);
+
+    let errorOccurred = false;
+
+    for (const data of fileMap.values()) {
+      data.status = Status.Import;
+      updateFile(data);
+
+      let items: SpotifyExtendedStreamingHistoryItem[];
+
+      // Scope so these tmp variables can be GC-ed ASAP
+      {
+        const fileContent = await data.file.text();
+
+        const rawItems = JSON.parse(
+          fileContent,
+        ) as SpotifyExtendedStreamingHistoryItem[];
+
+        items = rawItems
+          .filter(({ spotify_track_uri }) => spotify_track_uri !== null)
+          .map(({ ts, spotify_track_uri }) => ({
+            ts,
+            spotify_track_uri,
+          }));
+      }
+
+      try {
+        await importHistory(items);
+
+        data.status = Status.Finished;
+      } catch (err) {
+        data.error = err as Error;
+        data.status = Status.Error;
+
+        errorOccurred = true;
+      }
+      updateFile(data);
+    }
+
+    if (!errorOccurred) {
+      setStatus(Status.Finished);
+    }
+  }, [fileMap, importHistory, updateFile]);
+
+  return (
+    <Card className="mb-5">
+      <CardHeader>
+        <CardTitle>File Upload</CardTitle>
+        <CardDescription>
+          Select <Code>endsong_XY.json</Code> files here and start the import.
+        </CardDescription>
+      </CardHeader>
+      <CardContent>
+        <Files
+          className="shadow-inner bg-gray-200 dark:bg-gray-700 rounded p-4 text-center cursor-pointer"
+          dragActiveClassName=""
+          onChange={addFiles}
+          accepts={["application/json"]}
+          multiple
+          clickable
+        >
+          Drop files here or click to upload
+        </Files>
+        <Table>
+          <TableBody>
+            {Array.from(fileMap.values()).map((data) => (
+              <File key={data.file.id} data={data} />
+            ))}
+          </TableBody>
+        </Table>
+      </CardContent>
+      <CardFooter className="flex gap-x-2">
+        <Button
+          onClick={() =>
+            handleImport().catch((e) => console.error("Import Failed:", e))
+          }
+          variant="secondary"
+          disabled={status !== Status.Select}
+        >
+          Start Import
+        </Button>
+        <Button
+          onClick={clearFiles}
+          variant="secondary"
+          disabled={status !== Status.Select}
+        >
+          Remove All Files
+        </Button>
+      </CardFooter>
+    </Card>
+  );
+};
+
+const File: React.FC<{ data: FileData }> = ({ data }) => {
+  const hasErrors = data.status === Status.Error && data.error;
+
+  return (
+    <TableRow>
+      <TableCell>{data.file.name}</TableCell>
+      <TableCell className="text-sm font-thin">
+        {data.file.sizeReadable}
+      </TableCell>
+      <TableCell className="text-right">
+        {data.status === Status.Select && <Badge>Prepared for import!</Badge>}
+        {data.status === Status.Import && <Badge>Loading!</Badge>}
+        {data.status === Status.Finished && <Badge>Check!</Badge>}
+        {hasErrors && (
+          <Badge variant="destructive">
+            <ErrorIcon />
+            {data.error?.message}
+          </Badge>
+        )}
+      </TableCell>
+    </TableRow>
+  );
+};
+
+const ImportProgress: React.FC = () => {
+  const {
+    importStatus: { total, imported },
+    isLoading,
+    reload,
+  } = useSpotifyImportExtendedStreamingHistoryStatus();
+
+  useEffect(() => {
+    const interval = setInterval(() => {
+      if (!isLoading) {
+        reload();
+      }
+    }, 1000);
+
+    return () => clearInterval(interval);
+  }, [isLoading, reload]);
+
+  return (
+    <Card>
+      <CardHeader>
+        <CardTitle>Import Progress</CardTitle>
+        <CardDescription>
+          Shows how many of the submitted listens are already imported and
+          visible to you. This will take a while, and the process might halt for
+          a few minutes if we hit the Spotify API rate limit. If this is not
+          finished after a few hours, please contact your Listory administrator.
+        </CardDescription>
+      </CardHeader>
+      <CardContent>
+        <div className="flex pb-2">
+          <div className="md:flex w-10/12">
+            <div className={`md:w-full font-bold`}>
+              Imported
+              <br />
+              {imported}
+            </div>
+          </div>
+          <div className="w-2/12 text-right">
+            Total
+            <br />
+            {total}
+          </div>
+        </div>
+        {total > 0 && (
+          <div className="h-2 w-full bg-gradient-to-r from-teal-200/25 via-green-400 to-violet-400 dark:from-teal-700/25 dark:via-green-600/85 dark:to-amber-500 flex flex-row-reverse">
+            <div
+              style={{ width: numberToPercent(1 - imported / total) }}
+              className="h-full bg-gray-100 dark:bg-gray-900"
+            ></div>
+          </div>
+        )}
+      </CardContent>
+    </Card>
+  );
+};

frontend/src/components/NavBar.tsx

@@ -3,6 +3,7 @@ import { Link } from "react-router-dom";
 import { User } from "../api/entities/user";
 import { useAuth } from "../hooks/use-auth";
 import { CogwheelIcon } from "../icons/Cogwheel";
+import { ImportIcon } from "../icons/Import";
 import { SpotifyLogo } from "../icons/Spotify";
 import { Avatar, AvatarImage, AvatarFallback } from "./ui/avatar";
 import {
@@ -188,6 +189,12 @@ const NavUserInfo: React.FC<{ user: User }> = ({ user }) => {
               API Tokens
             </Link>
           </DropdownMenuItem>
+          <DropdownMenuItem asChild>
+            <Link to="/import">
+              <ImportIcon className="w-5 h-5 fill-current pr-2" />
+              Import Listens
+            </Link>
+          </DropdownMenuItem>
         </DropdownMenuGroup>
       </DropdownMenuContent>
     </DropdownMenu>

frontend/src/components/reports/RecentListens.tsx

@@ -50,7 +50,7 @@ export const RecentListens: React.FC = () => {
         )}
         <div>
           {recentListens.length > 0 && (
-            <Table className="table-auto w-full text-base">
+            <Table className="table-auto w-full">
               <TableBody>
                 {recentListens.map((listen) => (
                   <ListenItem listen={listen} key={listen.id} />

frontend/src/components/reports/TopListItem.tsx

@@ -1,4 +1,5 @@
 import React from "react";
+import { numberToPercent } from "../../util/numberToPercent";
 
 export interface TopListItemProps {
   title: string;
@@ -42,9 +43,3 @@ export const TopListItem: React.FC<TopListItemProps> = ({
 
 const isMaxCountValid = (maxCount: number) =>
   !(Number.isNaN(maxCount) || maxCount === 0);
-
-const numberToPercent = (ratio: number) =>
-  ratio.toLocaleString(undefined, {
-    style: "percent",
-    minimumFractionDigits: 2,
-  });

frontend/src/components/ui/badge.tsx

@@ -0,0 +1,36 @@
+import * as React from "react";
+import { cva, type VariantProps } from "class-variance-authority";
+
+import { cn } from "src/lib/utils";
+
+const badgeVariants = cva(
+  "inline-flex items-center rounded-full border border-gray-200 px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-gray-900 focus:ring-offset-2 dark:border-gray-800 dark:focus:ring-gray-300",
+  {
+    variants: {
+      variant: {
+        default:
+          "border-transparent bg-gray-900 text-gray-50 hover:bg-gray-900/80 dark:bg-gray-50 dark:text-gray-900 dark:hover:bg-gray-50/80",
+        secondary:
+          "border-transparent bg-gray-100 text-gray-900 hover:bg-gray-100/80 dark:bg-gray-800 dark:text-gray-50 dark:hover:bg-gray-800/80",
+        destructive:
+          "border-transparent bg-red-500 text-gray-50 hover:bg-red-500/80 dark:bg-red-900 dark:text-gray-50 dark:hover:bg-red-900/80",
+        outline: "text-gray-900 dark:text-gray-50",
+      },
+    },
+    defaultVariants: {
+      variant: "default",
+    },
+  },
+);
+
+export interface BadgeProps
+  extends React.HTMLAttributes<HTMLDivElement>,
+    VariantProps<typeof badgeVariants> {}
+
+function Badge({ className, variant, ...props }: BadgeProps) {
+  return (
+    <div className={cn(badgeVariants({ variant }), className)} {...props} />
+  );
+}
+
+export { Badge, badgeVariants };

frontend/src/components/ui/card.tsx

@@ -0,0 +1,86 @@
+import * as React from "react";
+
+import { cn } from "src/lib/utils";
+
+const Card = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement>
+>(({ className, ...props }, ref) => (
+  <div
+    ref={ref}
+    className={cn(
+      "rounded-lg border border-gray-200 bg-white text-gray-900 shadow-sm dark:border-gray-800 dark:bg-gray-900 dark:text-gray-50",
+      className,
+    )}
+    {...props}
+  />
+));
+Card.displayName = "Card";
+
+const CardHeader = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement>
+>(({ className, ...props }, ref) => (
+  <div
+    ref={ref}
+    className={cn("flex flex-col space-y-1.5 p-6", className)}
+    {...props}
+  />
+));
+CardHeader.displayName = "CardHeader";
+
+const CardTitle = React.forwardRef<
+  HTMLParagraphElement,
+  React.HTMLAttributes<HTMLHeadingElement>
+>(({ className, ...props }, ref) => (
+  <h3
+    ref={ref}
+    className={cn(
+      "text-2xl font-semibold leading-none tracking-tight",
+      className,
+    )}
+    {...props}
+  />
+));
+CardTitle.displayName = "CardTitle";
+
+const CardDescription = React.forwardRef<
+  HTMLParagraphElement,
+  React.HTMLAttributes<HTMLParagraphElement>
+>(({ className, ...props }, ref) => (
+  <p
+    ref={ref}
+    className={cn("text-sm text-gray-500 dark:text-gray-400", className)}
+    {...props}
+  />
+));
+CardDescription.displayName = "CardDescription";
+
+const CardContent = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement>
+>(({ className, ...props }, ref) => (
+  <div ref={ref} className={cn("p-6 pt-0", className)} {...props} />
+));
+CardContent.displayName = "CardContent";
+
+const CardFooter = React.forwardRef<
+  HTMLDivElement,
+  React.HTMLAttributes<HTMLDivElement>
+>(({ className, ...props }, ref) => (
+  <div
+    ref={ref}
+    className={cn("flex items-center p-6 pt-0", className)}
+    {...props}
+  />
+));
+CardFooter.displayName = "CardFooter";
+
+export {
+  Card,
+  CardHeader,
+  CardFooter,
+  CardTitle,
+  CardDescription,
+  CardContent,
+};

frontend/src/components/ui/code.tsx

@@ -0,0 +1,9 @@
+import React, { PropsWithChildren } from "react";
+
+export const Code: React.FC<PropsWithChildren> = ({ children }) => {
+  return (
+    <code className="tracking-wide font-mono bg-gray-200 dark:bg-gray-600 rounded-md px-1">
+      {children}
+    </code>
+  );
+};

frontend/src/components/ui/table.tsx

@@ -1,6 +1,6 @@
-import * as React from "react"
+import * as React from "react";
 
-import { cn } from "src/lib/utils"
+import { cn } from "src/lib/utils";
 
 const Table = React.forwardRef<
   HTMLTableElement,
@@ -9,20 +9,20 @@ const Table = React.forwardRef<
   <div className="relative w-full overflow-auto">
     <table
       ref={ref}
-      className={cn("w-full caption-bottom text-sm", className)}
+      className={cn("w-full caption-bottom", className)}
       {...props}
     />
   </div>
-))
+));
-Table.displayName = "Table"
+Table.displayName = "Table";
 
 const TableHeader = React.forwardRef<
   HTMLTableSectionElement,
   React.HTMLAttributes<HTMLTableSectionElement>
 >(({ className, ...props }, ref) => (
   <thead ref={ref} className={cn("[&_tr]:border-b", className)} {...props} />
-))
+));
-TableHeader.displayName = "TableHeader"
+TableHeader.displayName = "TableHeader";
 
 const TableBody = React.forwardRef<
   HTMLTableSectionElement,
@@ -33,8 +33,8 @@ const TableBody = React.forwardRef<
     className={cn("[&_tr:last-child]:border-0", className)}
     {...props}
   />
-))
+));
-TableBody.displayName = "TableBody"
+TableBody.displayName = "TableBody";
 
 const TableFooter = React.forwardRef<
   HTMLTableSectionElement,
@@ -42,11 +42,14 @@ const TableFooter = React.forwardRef<
 >(({ className, ...props }, ref) => (
   <tfoot
     ref={ref}
-    className={cn("bg-gray-900 font-medium text-gray-50 dark:bg-gray-50 dark:text-gray-900", className)}
+    className={cn(
+      "bg-gray-900 font-medium text-gray-50 dark:bg-gray-50 dark:text-gray-900",
+      className,
+    )}
     {...props}
   />
-))
+));
-TableFooter.displayName = "TableFooter"
+TableFooter.displayName = "TableFooter";
 
 const TableRow = React.forwardRef<
   HTMLTableRowElement,
@@ -56,12 +59,12 @@ const TableRow = React.forwardRef<
     ref={ref}
     className={cn(
       "border-b transition-colors hover:bg-gray-100/50 data-[state=selected]:bg-gray-100 dark:hover:bg-gray-800/50 dark:data-[state=selected]:bg-gray-800",
-      className
+      className,
     )}
     {...props}
   />
-))
+));
-TableRow.displayName = "TableRow"
+TableRow.displayName = "TableRow";
 
 const TableHead = React.forwardRef<
   HTMLTableCellElement,
@@ -71,12 +74,12 @@ const TableHead = React.forwardRef<
     ref={ref}
     className={cn(
       "h-12 px-4 text-left align-middle font-medium text-gray-500 [&:has([role=checkbox])]:pr-0 dark:text-gray-400",
-      className
+      className,
     )}
     {...props}
   />
-))
+));
-TableHead.displayName = "TableHead"
+TableHead.displayName = "TableHead";
 
 const TableCell = React.forwardRef<
   HTMLTableCellElement,
@@ -87,8 +90,8 @@ const TableCell = React.forwardRef<
     className={cn("p-4 align-middle [&:has([role=checkbox])]:pr-0", className)}
     {...props}
   />
-))
+));
-TableCell.displayName = "TableCell"
+TableCell.displayName = "TableCell";
 
 const TableCaption = React.forwardRef<
   HTMLTableCaptionElement,
@@ -99,8 +102,8 @@ const TableCaption = React.forwardRef<
     className={cn("mt-4 text-sm text-gray-500 dark:text-gray-400", className)}
     {...props}
   />
-))
+));
-TableCaption.displayName = "TableCaption"
+TableCaption.displayName = "TableCaption";
 
 export {
   Table,
@@ -111,4 +114,4 @@ export {
   TableRow,
   TableCell,
   TableCaption,
-}
+};

frontend/src/hooks/use-api.tsx

@@ -2,12 +2,14 @@ import { useCallback, useMemo } from "react";
 import {
   createApiToken,
   getApiTokens,
+  getExtendedStreamingHistoryStatus,
   getListensReport,
   getRecentListens,
   getTopAlbums,
   getTopArtists,
   getTopGenres,
   getTopTracks,
+  importExtendedStreamingHistory,
   revokeApiToken,
 } from "../api/api";
 import { ListenReportOptions } from "../api/entities/listen-report-options";
@@ -18,6 +20,7 @@ import { TopGenresOptions } from "../api/entities/top-genres-options";
 import { TopTracksOptions } from "../api/entities/top-tracks-options";
 import { useApiClient } from "./use-api-client";
 import { useAsync } from "./use-async";
+import { SpotifyExtendedStreamingHistoryItem } from "../api/entities/spotify-extended-streaming-history-item";
 
 const INITIAL_EMPTY_ARRAY: [] = [];
 Object.freeze(INITIAL_EMPTY_ARRAY);
@@ -143,9 +146,7 @@ export const useApiTokens = () => {
   const createToken = useCallback(
     async (description: string) => {
       const apiToken = await createApiToken(description, client);
-      console.log("apiToken created", apiToken);
       await reload();
-      console.log("reloaded data");
 
       return apiToken;
     },
@@ -162,3 +163,38 @@ export const useApiTokens = () => {
 
   return { apiTokens, isLoading, error, createToken, revokeToken };
 };
+
+export const useSpotifyImportExtendedStreamingHistory = () => {
+  const { client } = useApiClient();
+
+  const importHistory = useCallback(
+    async (listens: SpotifyExtendedStreamingHistoryItem[]) => {
+      return importExtendedStreamingHistory(listens, client);
+    },
+    [client]
+  );
+
+  const getStatus = useCallback(async () => {
+    return getExtendedStreamingHistoryStatus(client);
+  }, [client]);
+
+  return { importHistory, getStatus };
+};
+
+export const useSpotifyImportExtendedStreamingHistoryStatus = () => {
+  const { client } = useApiClient();
+
+  const fetchData = useMemo(
+    () => () => getExtendedStreamingHistoryStatus(client),
+    [client]
+  );
+
+  const {
+    value: importStatus,
+    pending: isLoading,
+    error,
+    reload,
+  } = useAsync(fetchData, { total: 0, imported: 0 });
+
+  return { importStatus, isLoading, error, reload };
+};

frontend/src/hooks/use-auth-protection.tsx

@@ -1,16 +0,0 @@
-import { useCallback } from "react";
-import { useNavigate } from "react-router-dom";
-import { useAuth } from "./use-auth";
-
-export function useAuthProtection() {
-  const { user } = useAuth();
-  const navigate = useNavigate();
-
-  const requireUser = useCallback(async () => {
-    if (!user) {
-      navigate("/");
-    }
-  }, [user, navigate]);
-
-  return { requireUser };
-}

frontend/src/icons/Error.tsx

@@ -0,0 +1,14 @@
+import React from "react";
+
+export const ErrorIcon: React.FC<React.SVGProps<SVGSVGElement>> = (props) => {
+  return (
+    <svg
+      version="1.1"
+      xmlns="http://www.w3.org/2000/svg"
+      viewBox="0 0 50 50"
+      fill="#D75A4A"
+    >
+      <circle fill="fill" cx="25" cy="25" r="25" />
+    </svg>
+  );
+};

frontend/src/icons/Import.tsx

@@ -0,0 +1,12 @@
+import * as React from "react";
+export const ImportIcon: React.FC<React.SVGProps<SVGSVGElement>> = (props) => (
+  <svg
+    xmlns="http://www.w3.org/2000/svg"
+    xmlSpace="preserve"
+    viewBox="0 0 60.903 60.903"
+    {...props}
+  >
+    <path d="M49.561 16.464H39.45v6h10.111c3.008 0 5.341 1.535 5.341 2.857v26.607c0 1.321-2.333 2.858-5.341 2.858H11.34c-3.007 0-5.34-1.537-5.34-2.858V25.324c0-1.322 2.333-2.858 5.34-2.858h10.11v-6H11.34C4.981 16.466 0 20.357 0 25.324v26.605c0 4.968 4.981 8.857 11.34 8.857h38.223c6.357 0 11.34-3.891 11.34-8.857V25.324c-.001-4.969-4.982-8.86-11.342-8.86z" />
+    <path d="M39.529 29.004a2.99 2.99 0 0 0-2.121.88l-3.756 3.755V3.117a3 3 0 0 0-6 0v30.724l-3.959-3.958a2.992 2.992 0 0 0-4.242 0 2.997 2.997 0 0 0 0 4.241l8.957 8.957a2.988 2.988 0 0 0 2.12.877h.045c.768 0 1.534-.291 2.12-.877l8.957-8.957a2.997 2.997 0 0 0-2.121-5.12z" />
+  </svg>
+);

frontend/src/react-files.d.ts

@@ -0,0 +1,36 @@
+//// <reference types="react" />
+
+declare module "react-files" {
+  declare const Files: React.FC<
+    Partial<{
+      accepts: string[];
+      children: React.ReactNode;
+      className: string;
+      clickable: boolean;
+      dragActiveClassName: string;
+      inputProps: unknown;
+      multiple: boolean;
+      maxFiles: number;
+      maxFileSize: number;
+      minFileSize: number;
+      name: string;
+      onChange: (files: ReactFile[]) => void;
+      onDragEnter: () => void;
+      onDragLeave: () => void;
+      onError: (
+        error: { code: number; message: string },
+        file: ReactFile
+      ) => void;
+      style: object;
+    }>
+  >;
+
+  export type ReactFile = File & {
+    id: string;
+    extension: string;
+    sizeReadable: string;
+    preview: { type: "image"; url: string } | { type: "file" };
+  };
+
+  export default Files;
+}

frontend/src/util/numberToPercent.ts

@@ -0,0 +1,5 @@
+export const numberToPercent = (ratio: number) =>
+  ratio.toLocaleString(undefined, {
+    style: "percent",
+    minimumFractionDigits: 2,
+  });

frontend/tsconfig.json

@@ -1,11 +1,7 @@
 {
   "compilerOptions": {
-    "target": "es5",
+    "target": "ES2022",
-    "lib": [
+    "lib": ["dom", "dom.iterable", "esnext"],
-      "dom",
-      "dom.iterable",
-      "esnext"
-    ],
     "allowJs": true,
     "skipLibCheck": true,
     "esModuleInterop": true,

package-lock.json

@@ -7082,9 +7082,9 @@
       "dev": true
     },
     "node_modules/globals": {
-      "version": "13.21.0",
+      "version": "13.22.0",
-      "resolved": "https://registry.npmjs.org/globals/-/globals-13.21.0.tgz",
+      "resolved": "https://registry.npmjs.org/globals/-/globals-13.22.0.tgz",
-      "integrity": "sha512-ybyme3s4yy/t/3s35bewwXKOf7cvzfreG2lH0lZl0JB7I4GxRP2ghxOK/Nb9EkRXdbBXZLfq/p/0W2JUONB/Gg==",
+      "integrity": "sha512-H1Ddc/PbZHTDVJSnj8kWptIRSD6AM3pK+mKytuIVF4uoBV7rshFlhhvA58ceJ5wp3Er58w6zj7bykMpYXt3ETw==",
       "dev": true,
       "dependencies": {
         "type-fest": "^0.20.2"
@@ -9961,9 +9961,9 @@
       }
     },
     "node_modules/path-scurry/node_modules/minipass": {
-      "version": "7.0.3",
+      "version": "7.0.4",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
-      "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==",
+      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
       "engines": {
         "node": ">=16 || 14 >=14.17"
       }
@@ -10939,9 +10939,9 @@
       }
     },
     "node_modules/rimraf/node_modules/minipass": {
-      "version": "7.0.3",
+      "version": "7.0.4",
-      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
-      "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg==",
+      "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==",
       "engines": {
         "node": ">=16 || 14 >=14.17"
       }
@@ -18278,9 +18278,9 @@
       "dev": true
     },
     "globals": {
-      "version": "13.21.0",
+      "version": "13.22.0",
-      "resolved": "https://registry.npmjs.org/globals/-/globals-13.21.0.tgz",
+      "resolved": "https://registry.npmjs.org/globals/-/globals-13.22.0.tgz",
-      "integrity": "sha512-ybyme3s4yy/t/3s35bewwXKOf7cvzfreG2lH0lZl0JB7I4GxRP2ghxOK/Nb9EkRXdbBXZLfq/p/0W2JUONB/Gg==",
+      "integrity": "sha512-H1Ddc/PbZHTDVJSnj8kWptIRSD6AM3pK+mKytuIVF4uoBV7rshFlhhvA58ceJ5wp3Er58w6zj7bykMpYXt3ETw==",
       "dev": true,
       "requires": {
         "type-fest": "^0.20.2"
@@ -20388,9 +20388,9 @@
           "integrity": "sha512-IJ4uwUTi2qCccrioU6g9g/5rvvVl13bsdczUUcqbciD9iLr095yj8DQKdObriEvuNSx325N1rV1O0sJFszx75g=="
         },
         "minipass": {
-          "version": "7.0.3",
+          "version": "7.0.4",
-          "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz",
+          "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
-          "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg=="
+          "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ=="
         }
       }
     },
@@ -21112,9 +21112,9 @@
           }
         },
         "minipass": {
-          "version": "7.0.3",
+          "version": "7.0.4",
-          "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.3.tgz",
+          "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz",
-          "integrity": "sha512-LhbbwCfz3vsb12j/WkWQPZfKTsgqIe1Nf/ti1pKjYESGLHIVjWU96G9/ljLH4F9mWNVhlQOm0VySdAWzf05dpg=="
+          "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ=="
         }
       }
     },

src/auth/auth.controller.spec.ts

@@ -1,5 +1,5 @@
 import { Test, TestingModule } from "@nestjs/testing";
-import type { Response } from "express";
+import type { Response as ExpressResponse } from "express";
 import { User } from "../users/user.entity";
 import { AuthSession } from "./auth-session.entity";
 import { AuthController } from "./auth.controller";
@@ -27,7 +27,7 @@ describe("AuthController", () => {
 
   describe("spotifyCallback", () => {
     let user: User;
-    let res: Response;
+    let res: ExpressResponse;
     let refreshToken: string;
 
     beforeEach(() => {
@@ -36,7 +36,7 @@ describe("AuthController", () => {
         statusCode: 200,
         cookie: jest.fn(),
         redirect: jest.fn(),
-      } as unknown as Response;
+      } as unknown as ExpressResponse;
 
       refreshToken = "REFRESH_TOKEN";
       authService.createSession = jest.fn().mockResolvedValue({ refreshToken });

src/auth/auth.controller.ts

@@ -1,5 +1,5 @@
 import {
-  Body,
+  Body as NestBody,
   Controller,
   Delete,
   Get,
@@ -10,7 +10,7 @@ import {
   UseGuards,
 } from "@nestjs/common";
 import { ApiBody, ApiTags } from "@nestjs/swagger";
-import type { Response } from "express";
+import type { Response as ExpressResponse } from "express";
 import { User } from "../users/user.entity";
 import { AuthSession } from "./auth-session.entity";
 import { AuthService } from "./auth.service";
@@ -42,7 +42,7 @@ export class AuthController {
   @Get("spotify/callback")
   @UseFilters(SpotifyAuthFilter)
   @UseGuards(SpotifyAuthGuard)
-  async spotifyCallback(@ReqUser() user: User, @Res() res: Response) {
+  async spotifyCallback(@ReqUser() user: User, @Res() res: ExpressResponse) {
     const { refreshToken } = await this.authService.createSession(user);
 
     // Refresh token should not be accessible to frontend to reduce risk
@@ -69,7 +69,7 @@ export class AuthController {
   @AuthAccessToken()
   async createApiToken(
     @ReqUser() user: User,
-    @Body("description") description: string,
+    @NestBody("description") description: string,
   ): Promise<NewApiTokenDto> {
     const apiToken = await this.authService.createApiToken(user, description);
 

src/auth/spotify.filter.ts

@@ -5,14 +5,14 @@ import {
   ForbiddenException,
   Logger,
 } from "@nestjs/common";
-import type { Response } from "express";
+import type { Response as ExpressResponse } from "express";
 
 @Catch()
 export class SpotifyAuthFilter implements ExceptionFilter {
   private readonly logger = new Logger(this.constructor.name);
 
   catch(exception: Error, host: ArgumentsHost) {
-    const response = host.switchToHttp().getResponse<Response>();
+    const response = host.switchToHttp().getResponse<ExpressResponse>();
 
     let reason = "unknown";
 

src/database/database.module.ts

@@ -27,7 +27,7 @@ export const DatabaseModule = TypeOrmModule.forRootAsync({
 
     // Debug/Development Options
     //
-    // logging: true,
+    //logging: true,
     //
     // synchronize: true,
     // migrationsRun: false,

src/database/migrations/09-CreateSpotifyImportTables.ts

@@ -0,0 +1,68 @@
+import {
+  MigrationInterface,
+  QueryRunner,
+  Table,
+  TableIndex,
+  TableForeignKey,
+} from "typeorm";
+import { TableColumnOptions } from "typeorm/schema-builder/options/TableColumnOptions";
+
+const primaryUUIDColumn: TableColumnOptions = {
+  name: "id",
+  type: "uuid",
+  isPrimary: true,
+  isGenerated: true,
+  generationStrategy: "uuid",
+};
+
+export class CreateSpotifyImportTables0000000000009
+  implements MigrationInterface
+{
+  async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.createTable(
+      new Table({
+        name: "spotify_extended_streaming_history_listen",
+        columns: [
+          primaryUUIDColumn,
+          { name: "userId", type: "uuid" },
+          { name: "playedAt", type: "timestamp" },
+          { name: "spotifyTrackUri", type: "varchar" },
+          { name: "trackId", type: "uuid", isNullable: true },
+          { name: "listenId", type: "uuid", isNullable: true },
+        ],
+        indices: [
+          new TableIndex({
+            name: "IDX_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_USER_PLAYED_AT",
+            columnNames: ["userId", "playedAt", "spotifyTrackUri"],
+            isUnique: true,
+          }),
+        ],
+        foreignKeys: [
+          new TableForeignKey({
+            name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_USER_ID",
+            columnNames: ["userId"],
+            referencedColumnNames: ["id"],
+            referencedTableName: "user",
+          }),
+          new TableForeignKey({
+            name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_TRACK_ID",
+            columnNames: ["trackId"],
+            referencedColumnNames: ["id"],
+            referencedTableName: "track",
+          }),
+          new TableForeignKey({
+            name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_LISTEN_ID",
+            columnNames: ["listenId"],
+            referencedColumnNames: ["id"],
+            referencedTableName: "listen",
+          }),
+        ],
+      }),
+      true,
+    );
+  }
+
+  async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.dropTable("spotify_extended_streaming_history_listen");
+  }
+}

src/listens/listen.repository.ts

@@ -3,10 +3,6 @@ import { Repository, SelectQueryBuilder } from "typeorm";
 import { EntityRepository } from "../database/entity-repository";
 import { Interval } from "../reports/interval";
 import { User } from "../users/user.entity";
-import {
-  CreateListenRequestDto,
-  CreateListenResponseDto,
-} from "./dto/create-listen.dto";
 import { Listen } from "./listen.entity";
 
 export class ListenScopes extends SelectQueryBuilder<Listen> {
@@ -37,52 +33,4 @@ export class ListenRepository extends Repository<Listen> {
   get scoped(): ListenScopes {
     return new ListenScopes(this.createQueryBuilder("listen"));
   }
-
-  async insertNoConflict({
-    user,
-    track,
-    playedAt,
-  }: CreateListenRequestDto): Promise<CreateListenResponseDto> {
-    const result = await this.createQueryBuilder()
-      .insert()
-      .values({
-        user,
-        track,
-        playedAt,
-      })
-      .onConflict('("playedAt", "trackId", "userId") DO NOTHING')
-      .execute();
-
-    const [insertedRowIdentifier] = result.identifiers;
-
-    if (!insertedRowIdentifier) {
-      // We did not insert a new listen, it already existed
-      return {
-        listen: await this.findOneBy({ user, track, playedAt }),
-        isDuplicate: true,
-      };
-    }
-
-    return {
-      listen: await this.findOneBy({ id: insertedRowIdentifier.id }),
-      isDuplicate: false,
-    };
-  }
-
-  /**
-   *
-   * @param rows
-   * @returns A list of all new (non-duplicate) listens
-   */
-  async insertsNoConflict(rows: CreateListenRequestDto[]): Promise<Listen[]> {
-    const result = await this.createQueryBuilder()
-      .insert()
-      .values(rows)
-      .orIgnore()
-      .execute();
-
-    return this.findBy(
-      result.identifiers.filter(Boolean).map(({ id }) => ({ id })),
-    );
-  }
 }

src/listens/listens.service.spec.ts

@@ -4,9 +4,7 @@ import {
   paginate,
   PaginationTypeEnum,
 } from "nestjs-typeorm-paginate";
-import { Track } from "../music-library/track.entity";
 import { User } from "../users/user.entity";
-import { CreateListenResponseDto } from "./dto/create-listen.dto";
 import { GetListensDto } from "./dto/get-listens.dto";
 import { Listen } from "./listen.entity";
 import { ListenRepository, ListenScopes } from "./listen.repository";
@@ -35,39 +33,6 @@ describe("ListensService", () => {
     expect(listenRepository).toBeDefined();
   });
 
-  describe("createListen", () => {
-    let user: User;
-    let track: Track;
-    let playedAt: Date;
-    let response: CreateListenResponseDto;
-    beforeEach(() => {
-      user = { id: "USER" } as User;
-      track = { id: "TRACK" } as Track;
-      playedAt = new Date("2021-01-01T00:00:00Z");
-
-      response = {
-        listen: {
-          id: "LISTEN",
-        } as Listen,
-        isDuplicate: true,
-      };
-      listenRepository.insertNoConflict = jest.fn().mockResolvedValue(response);
-    });
-
-    it("creates the listen", async () => {
-      await expect(
-        service.createListen({ user, track, playedAt }),
-      ).resolves.toEqual(response);
-
-      expect(listenRepository.insertNoConflict).toHaveBeenCalledTimes(1);
-      expect(listenRepository.insertNoConflict).toHaveBeenLastCalledWith({
-        user,
-        track,
-        playedAt,
-      });
-    });
-  });
-
   describe("getListens", () => {
     let options: GetListensDto & IPaginationOptions;
     let user: User;

src/listens/listens.service.ts

@@ -1,14 +1,12 @@
 import { Injectable } from "@nestjs/common";
+import { Span } from "nestjs-otel";
 import {
   IPaginationOptions,
   paginate,
   Pagination,
   PaginationTypeEnum,
 } from "nestjs-typeorm-paginate";
-import {
+import { CreateListenRequestDto } from "./dto/create-listen.dto";
-  CreateListenRequestDto,
-  CreateListenResponseDto,
-} from "./dto/create-listen.dto";
 import { GetListensDto } from "./dto/get-listens.dto";
 import { Listen } from "./listen.entity";
 import { ListenRepository, ListenScopes } from "./listen.repository";
@@ -17,20 +15,7 @@ import { ListenRepository, ListenScopes } from "./listen.repository";
 export class ListensService {
   constructor(private readonly listenRepository: ListenRepository) {}
 
-  async createListen({
+  @Span()
-    user,
-    track,
-    playedAt,
-  }: CreateListenRequestDto): Promise<CreateListenResponseDto> {
-    const response = await this.listenRepository.insertNoConflict({
-      user,
-      track,
-      playedAt,
-    });
-
-    return response;
-  }
-
   async createListens(
     listensData: CreateListenRequestDto[],
   ): Promise<Listen[]> {
@@ -46,9 +31,11 @@ export class ListensService {
         ),
     );
 
-    return this.listenRepository.save(
+    const newListens = await this.listenRepository.save(
       missingListens.map((entry) => this.listenRepository.create(entry)),
     );
+
+    return [...existingListens, ...newListens];
   }
 
   async getListens(

src/main.ts

@@ -43,6 +43,7 @@ async function bootstrap() {
 
   const app = await NestFactory.create<NestExpressApplication>(AppModule, {
     bufferLogs: true,
+    rawBody: true,
   });
   app.useLogger(app.get(Logger));
   app.useGlobalPipes(
@@ -51,6 +52,10 @@ async function bootstrap() {
       transformOptions: { enableImplicitConversion: true },
     }),
   );
+  app.useBodyParser("json", {
+    limit:
+      "10mb" /* Need large bodies for Spotify Extended Streaming History */,
+  });
   app.enableShutdownHooks();
 
   const configService = app.get<ConfigService>(ConfigService);

src/music-library/dto/find-track.dto.ts

@@ -1,5 +1,6 @@
 export class FindTrackDto {
   spotify: {
-    id: string;
+    id?: string;
+    uri?: string;
   };
 }

src/music-library/music-library.service.ts

@@ -175,9 +175,7 @@ export class MusicLibraryService {
   }
 
   async findTrack(query: FindTrackDto): Promise<Track | undefined> {
-    return this.trackRepository.findOneBy({
+    return this.trackRepository.findOneBy(query);
-      spotify: { id: query.spotify.id },
-    });
   }
 
   async findTracks(query: FindTrackDto[]): Promise<Track[]> {

src/override.d.ts

@@ -1,5 +0,0 @@
-// Issue with opentelemetry-js: https://github.com/open-telemetry/opentelemetry-js/issues/3580#issuecomment-1701157270
-export {};
-declare global {
-  type BlobPropertyBag = unknown;
-}

src/sources/scheduler.service.ts

@@ -33,11 +33,11 @@ export class SchedulerService implements OnApplicationBootstrap {
   }
 
   private async setupSpotifyCrawlerSupervisor(): Promise<void> {
-    await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {});
+    // await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {});
   }
 
   @Span()
-  @CrawlerSupervisorJob.Handle()
+  // @CrawlerSupervisorJob.Handle()
   async superviseImportJobs(): Promise<void> {
     this.logger.log("Starting crawler jobs");
     const userInfo = await this.spotifyService.getCrawlableUserInfo();

src/sources/spotify/import-extended-streaming-history/dto/extended-streaming-history-status.dto.ts

@@ -0,0 +1,13 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class ExtendedStreamingHistoryStatusDto {
+  @ApiProperty({
+    type: Number,
+  })
+  total: number;
+
+  @ApiProperty({
+    type: Number,
+  })
+  imported: number;
+}

src/sources/spotify/import-extended-streaming-history/dto/import-extended-streaming-history.dto.ts

@@ -0,0 +1,13 @@
+import { ApiProperty } from "@nestjs/swagger";
+import { ArrayMaxSize } from "class-validator";
+import { SpotifyExtendedStreamingHistoryItemDto } from "./spotify-extended-streaming-history-item.dto";
+
+export class ImportExtendedStreamingHistoryDto {
+  @ApiProperty({
+    type: SpotifyExtendedStreamingHistoryItemDto,
+    isArray: true,
+    maxItems: 50_000,
+  })
+  @ArrayMaxSize(50_000) // File size is ~16k by default, might need refactoring if Spotify starts exporting larger files
+  listens: SpotifyExtendedStreamingHistoryItemDto[];
+}

src/sources/spotify/import-extended-streaming-history/dto/spotify-extended-streaming-history-item.dto.ts

@@ -0,0 +1,9 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class SpotifyExtendedStreamingHistoryItemDto {
+  @ApiProperty({ format: "iso8601", example: "2018-11-30T08:33:33Z" })
+  ts: string;
+
+  @ApiProperty({ example: "spotify:track:6askbS4pEVWbbDnUGEXh3G" })
+  spotify_track_uri: string;
+}

src/sources/spotify/import-extended-streaming-history/import.controller.ts

@@ -0,0 +1,32 @@
+import { Body as NestBody, Controller, Get, Post } from "@nestjs/common";
+import { ApiBody, ApiTags } from "@nestjs/swagger";
+import { AuthAccessToken } from "../../../auth/decorators/auth-access-token.decorator";
+import { ReqUser } from "../../../auth/decorators/req-user.decorator";
+import { User } from "../../../users/user.entity";
+import { ExtendedStreamingHistoryStatusDto } from "./dto/extended-streaming-history-status.dto";
+import { ImportExtendedStreamingHistoryDto } from "./dto/import-extended-streaming-history.dto";
+import { ImportService } from "./import.service";
+
+@ApiTags("import")
+@Controller("api/v1/import")
+export class ImportController {
+  constructor(private readonly importService: ImportService) {}
+
+  @Post("extended-streaming-history")
+  @ApiBody({ type: () => ImportExtendedStreamingHistoryDto })
+  @AuthAccessToken()
+  async importExtendedStreamingHistory(
+    @ReqUser() user: User,
+    @NestBody() data: ImportExtendedStreamingHistoryDto,
+  ): Promise<void> {
+    return this.importService.importExtendedStreamingHistory(user, data);
+  }
+
+  @Get("extended-streaming-history/status")
+  @AuthAccessToken()
+  async getExtendedStreamingHistoryStatus(
+    @ReqUser() user: User,
+  ): Promise<ExtendedStreamingHistoryStatusDto> {
+    return this.importService.getExtendedStreamingHistoryStatus(user);
+  }
+}

src/sources/spotify/import-extended-streaming-history/import.service.ts

@@ -0,0 +1,177 @@
+import { JobService } from "@apricote/nest-pg-boss";
+import { Injectable, Logger } from "@nestjs/common";
+import { uniq } from "lodash";
+import { Span } from "nestjs-otel";
+import type { Job } from "pg-boss";
+import { ListensService } from "../../../listens/listens.service";
+import { User } from "../../../users/user.entity";
+import { SpotifyService } from "../spotify.service";
+import { ExtendedStreamingHistoryStatusDto } from "./dto/extended-streaming-history-status.dto";
+import { ImportExtendedStreamingHistoryDto } from "./dto/import-extended-streaming-history.dto";
+import {
+  IProcessSpotifyExtendedStreamingHistoryListenJob,
+  ProcessSpotifyExtendedStreamingHistoryListenJob,
+} from "./jobs";
+import { SpotifyExtendedStreamingHistoryListenRepository } from "./listen.repository";
+
+@Injectable()
+export class ImportService {
+  private readonly logger = new Logger(this.constructor.name);
+
+  constructor(
+    private readonly importListenRepository: SpotifyExtendedStreamingHistoryListenRepository,
+    @ProcessSpotifyExtendedStreamingHistoryListenJob.Inject()
+    private readonly processListenJobService: JobService<IProcessSpotifyExtendedStreamingHistoryListenJob>,
+    private readonly spotifyService: SpotifyService,
+    private readonly listensService: ListensService,
+  ) {}
+
+  @Span()
+  async importExtendedStreamingHistory(
+    user: User,
+    { listens: importListens }: ImportExtendedStreamingHistoryDto,
+  ): Promise<void> {
+    // IDK what's happening, but my personal data set has entries with duplicate
+    // listens? might be related to offline mode.
+    // Anyway, this cleans it up:
+    const uniqEntries = new Set();
+    const uniqueListens = importListens.filter((listen) => {
+      const key = `${listen.spotify_track_uri}-${listen.ts}`;
+
+      if (!uniqEntries.has(key)) {
+        // New entry
+        uniqEntries.add(key);
+        return true;
+      }
+
+      return false;
+    });
+
+    let listens = uniqueListens.map((listenData) =>
+      this.importListenRepository.create({
+        user,
+        playedAt: new Date(listenData.ts),
+        spotifyTrackUri: listenData.spotify_track_uri,
+      }),
+    );
+
+    // Save listens to import table
+    const insertResult = await this.importListenRepository.upsert(listens, [
+      "user",
+      "playedAt",
+      "spotifyTrackUri",
+    ]);
+
+    const processJobs = insertResult.identifiers.map((listen) => ({
+      data: {
+        id: listen.id,
+      },
+      singletonKey: listen.id,
+      retryLimit: 10,
+      retryDelay: 5,
+      retryBackoff: true,
+    }));
+
+    // Schedule jobs to process imports
+    await this.processListenJobService.insert(processJobs);
+  }
+
+  @ProcessSpotifyExtendedStreamingHistoryListenJob.Handle({
+    // Spotify API "Get Several XY" allows max 50 IDs
+    batchSize: 50,
+    newJobCheckInterval: 500,
+  })
+  @Span()
+  async processListens(
+    jobs: Job<IProcessSpotifyExtendedStreamingHistoryListenJob>[],
+  ): Promise<void> {
+    this.logger.debug(
+      { jobs: jobs.length },
+      "processing extended streaming history listens",
+    );
+    const importListens = await this.importListenRepository.findBy(
+      jobs.map((job) => ({ id: job.data.id })),
+    );
+
+    const listensWithoutTracks = importListens.filter(
+      (importListen) => !importListen.track,
+    );
+    if (listensWithoutTracks.length > 0) {
+      const missingTrackIDs = uniq(
+        listensWithoutTracks.map((importListen) =>
+          importListen.spotifyTrackUri.replace("spotify:track:", ""),
+        ),
+      );
+
+      const tracks = await this.spotifyService.importTracks(missingTrackIDs);
+
+      listensWithoutTracks.forEach((listen) => {
+        listen.track = tracks.find(
+          (track) => listen.spotifyTrackUri === track.spotify.uri,
+        );
+        if (!listen.track) {
+          this.logger.warn(
+            { listen },
+            "could not find track for extended streaming history listen",
+          );
+          throw new Error(
+            `could not find track for extended streaming history listen`,
+          );
+        }
+      });
+
+      // Using upsert instead of save to only do a single query
+      await this.importListenRepository.upsert(listensWithoutTracks, ["id"]);
+    }
+
+    const listensWithoutListen = importListens.filter(
+      (importListen) => !importListen.listen,
+    );
+    if (listensWithoutListen.length > 0) {
+      const listens = await this.listensService.createListens(
+        listensWithoutListen.map((listen) => ({
+          user: listen.user,
+          track: listen.track,
+          playedAt: listen.playedAt,
+        })),
+      );
+
+      listensWithoutListen.forEach((importListen) => {
+        importListen.listen = listens.find(
+          (listen) =>
+            importListen.user.id === listen.user.id &&
+            importListen.track.id === listen.track.id &&
+            importListen.playedAt.getTime() === listen.playedAt.getTime(),
+        );
+        if (!importListen.listen) {
+          this.logger.warn(
+            { listen: importListen, listens: listens },
+            "could not find listen for extended streaming history listen",
+          );
+          throw new Error(
+            `could not find listen for extended streaming history listen`,
+          );
+        }
+      });
+
+      // Using upsert instead of save to only do a single query
+      await this.importListenRepository.upsert(listensWithoutListen, ["id"]);
+    }
+  }
+
+  @Span()
+  async getExtendedStreamingHistoryStatus(
+    user: User,
+  ): Promise<ExtendedStreamingHistoryStatusDto> {
+    const qb = this.importListenRepository
+      .createQueryBuilder("listen")
+      .where("listen.userId = :user", { user: user.id });
+
+    const [total, imported] = await Promise.all([
+      qb.clone().getCount(),
+      qb.clone().andWhere("listen.listenId IS NOT NULL").getCount(),
+    ]);
+
+    return { total, imported };
+  }
+}

src/sources/spotify/import-extended-streaming-history/index.ts

@@ -0,0 +1,4 @@
+export { ImportController } from "./import.controller";
+export { ImportService } from "./import.service";
+export { ProcessSpotifyExtendedStreamingHistoryListenJob } from "./jobs";
+export { SpotifyExtendedStreamingHistoryListenRepository } from "./listen.repository";

src/sources/spotify/import-extended-streaming-history/jobs.ts

@@ -0,0 +1,7 @@
+import { createJob } from "@apricote/nest-pg-boss";
+
+export type IProcessSpotifyExtendedStreamingHistoryListenJob = { id: string };
+export const ProcessSpotifyExtendedStreamingHistoryListenJob =
+  createJob<IProcessSpotifyExtendedStreamingHistoryListenJob>(
+    "process-spotify-extended-streaming-history-listen",
+  );

src/sources/spotify/import-extended-streaming-history/listen.entity.ts

@@ -0,0 +1,25 @@
+import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm";
+import { Track } from "../../../music-library/track.entity";
+import { User } from "../../../users/user.entity";
+import { Listen } from "../../../listens/listen.entity";
+
+@Entity({ name: "spotify_extended_streaming_history_listen" })
+export class SpotifyExtendedStreamingHistoryListen {
+  @PrimaryGeneratedColumn("uuid")
+  id: string;
+
+  @ManyToOne(() => User, { eager: true })
+  user: User;
+
+  @Column({ type: "timestamp" })
+  playedAt: Date;
+
+  @Column()
+  spotifyTrackUri: string;
+
+  @ManyToOne(() => Track, { nullable: true, eager: true })
+  track?: Track;
+
+  @ManyToOne(() => Listen, { nullable: true, eager: true })
+  listen?: Listen;
+}

src/sources/spotify/import-extended-streaming-history/listen.repository.ts

@@ -0,0 +1,6 @@
+import { Repository } from "typeorm";
+import { EntityRepository } from "../../../database/entity-repository";
+import { SpotifyExtendedStreamingHistoryListen } from "./listen.entity";
+
+@EntityRepository(SpotifyExtendedStreamingHistoryListen)
+export class SpotifyExtendedStreamingHistoryListenRepository extends Repository<SpotifyExtendedStreamingHistoryListen> {}

src/sources/spotify/spotify.module.ts

@@ -1,25 +1,37 @@
 import { PGBossModule } from "@apricote/nest-pg-boss";
 import { Module } from "@nestjs/common";
+import { TypeOrmRepositoryModule } from "../../database/entity-repository/typeorm-repository.module";
 import { ListensModule } from "../../listens/listens.module";
 import { MusicLibraryModule } from "../../music-library/music-library.module";
 import { UsersModule } from "../../users/users.module";
 import { ImportSpotifyJob } from "../jobs";
+import {
+  ImportController,
+  ImportService,
+  ProcessSpotifyExtendedStreamingHistoryListenJob,
+  SpotifyExtendedStreamingHistoryListenRepository,
+} from "./import-extended-streaming-history";
 import { SpotifyApiModule } from "./spotify-api/spotify-api.module";
 import { SpotifyAuthModule } from "./spotify-auth/spotify-auth.module";
 import { SpotifyService } from "./spotify.service";
 
 @Module({
   imports: [
-    PGBossModule.forJobs([ImportSpotifyJob]),
+    PGBossModule.forJobs([
+      ImportSpotifyJob,
+      ProcessSpotifyExtendedStreamingHistoryListenJob,
+    ]),
+    TypeOrmRepositoryModule.for([
+      SpotifyExtendedStreamingHistoryListenRepository,
+    ]),
     UsersModule,
     ListensModule,
     MusicLibraryModule,
     SpotifyApiModule,
     SpotifyAuthModule,
   ],
-  providers: [SpotifyService],
+  providers: [SpotifyService, ImportService],
+  controllers: [ImportController],
   exports: [SpotifyService],
 })
-export class SpotifyModule {
+export class SpotifyModule {}
-  constructor(private readonly spotifyService: SpotifyService) {}
-}

tsconfig.json

@@ -5,13 +5,12 @@
     "removeComments": true,
     "emitDecoratorMetadata": true,
     "experimentalDecorators": true,
-    "target": "ES2020",
+    "target": "ES2022",
     "sourceMap": true,
     "outDir": "./dist",
     "baseUrl": "./",
     "incremental": true,
     "paths": {},
-    "lib": ["ES2020"]
   },
   "exclude": ["node_modules", "dist"],
   "include": ["src", "test"]