(
+ `/api/v1/import/extended-streaming-history/status`
+ );
+
+ switch (res.status) {
+ case 200: {
+ break;
+ }
+ case 401: {
+ throw new UnauthenticatedError(`No token or token expired`);
+ }
+ default: {
+ throw new Error(
+ `Unable to getExtendedStreamingHistoryStatus: ${res.status}`
+ );
+ }
+ }
+
+ return res.data;
+};
diff --git a/frontend/src/api/entities/extended-streaming-history-status.ts b/frontend/src/api/entities/extended-streaming-history-status.ts
new file mode 100644
index 0000000..d6fa488
--- /dev/null
+++ b/frontend/src/api/entities/extended-streaming-history-status.ts
@@ -0,0 +1,4 @@
+export interface ExtendedStreamingHistoryStatus {
+ total: number;
+ imported: number;
+}
diff --git a/frontend/src/api/entities/spotify-extended-streaming-history-item.ts b/frontend/src/api/entities/spotify-extended-streaming-history-item.ts
new file mode 100644
index 0000000..cc91d3c
--- /dev/null
+++ b/frontend/src/api/entities/spotify-extended-streaming-history-item.ts
@@ -0,0 +1,4 @@
+export interface SpotifyExtendedStreamingHistoryItem {
+ ts: string;
+ spotify_track_uri: string;
+}
diff --git a/frontend/src/components/AuthApiTokens.tsx b/frontend/src/components/AuthApiTokens.tsx
index d73b755..0e50589 100644
--- a/frontend/src/components/AuthApiTokens.tsx
+++ b/frontend/src/components/AuthApiTokens.tsx
@@ -2,7 +2,6 @@ import { format, formatDistanceToNow } from "date-fns";
import React, { FormEvent, useCallback, useMemo, useState } from "react";
import { ApiToken, NewApiToken } from "../api/entities/api-token";
import { useApiTokens } from "../hooks/use-api";
-import { useAuthProtection } from "../hooks/use-auth-protection";
import { SpinnerIcon } from "../icons/Spinner";
import TrashcanIcon from "../icons/Trashcan";
import { Spinner } from "./ui/Spinner";
@@ -10,7 +9,10 @@ import { Spinner } from "./ui/Spinner";
export const AuthApiTokens: React.FC = () => {
const { apiTokens, isLoading, createToken, revokeToken } = useApiTokens();
const sortedTokens = useMemo(
- () => apiTokens.sort((a, b) => (a.createdAt > b.createdAt ? -1 : 1)),
+ () =>
+ apiTokens
+ .filter((token) => !token.revokedAt)
+ .sort((a, b) => (a.createdAt > b.createdAt ? -1 : 1)),
[apiTokens],
);
@@ -23,7 +25,7 @@ export const AuthApiTokens: React.FC = () => {
You can use API Tokens to access the Listory API directly. You can
find the API docs{" "}
-
+
here
.
diff --git a/frontend/src/components/ImportListens.tsx b/frontend/src/components/ImportListens.tsx
new file mode 100644
index 0000000..f0699b6
--- /dev/null
+++ b/frontend/src/components/ImportListens.tsx
@@ -0,0 +1,280 @@
+import React, { useCallback, useEffect, useState } from "react";
+import Files from "react-files";
+import type { ReactFile } from "react-files";
+import {
+ useSpotifyImportExtendedStreamingHistory,
+ useSpotifyImportExtendedStreamingHistoryStatus,
+} from "../hooks/use-api";
+import { SpotifyExtendedStreamingHistoryItem } from "../api/entities/spotify-extended-streaming-history-item";
+import { ErrorIcon } from "../icons/Error";
+import { numberToPercent } from "../util/numberToPercent";
+import { Button } from "./ui/button";
+import { Table, TableBody, TableCell, TableRow } from "./ui/table";
+import { Badge } from "./ui/badge";
+import {
+ Card,
+ CardContent,
+ CardDescription,
+ CardFooter,
+ CardHeader,
+ CardTitle,
+} from "./ui/card";
+import { Code } from "./ui/code";
+
+export const ImportListens: React.FC = () => {
+ return (
+ <>
+
+
+ Import Listens from Spotify Extended Streaming History
+
+
+
+ Here you can import your full Spotify Listen history that was exported
+ from the{" "}
+
+ Extended streaming history
+
+ .
+
+
+ The extended streaming history contains additional personally
+ identifiable data such as the IP address of the listen (which can be
+ linked to locations). To avoid saving this on the server, the data is
+ preprocessed in your web browser and only the necessary data
+ (timestamp & track ID) are sent to the server.
+
+
+ If an error occurs, you can always retry uploading the file, Listory
+ deduplicates any listens to make sure that everything is saved only
+ once.
+
+
+
+
+
>(
+ new Map(),
+ );
+
+ const [status, setStatus] = useState(Status.Select);
+
+ const addFiles = useCallback(
+ (files: ReactFile[]) => {
+ setFileMap((_fileMap) => {
+ files.forEach((file) =>
+ _fileMap.set(file.id, { file, status: Status.Select }),
+ );
+ return new Map(_fileMap);
+ });
+ },
+ [setFileMap],
+ );
+
+ const updateFile = useCallback((data: FileData) => {
+ setFileMap((_fileMap) => new Map(_fileMap.set(data.file.id, data)));
+ }, []);
+
+ const clearFiles = useCallback(() => {
+ setFileMap(new Map());
+ }, [setFileMap]);
+
+ const { importHistory } = useSpotifyImportExtendedStreamingHistory();
+
+ const handleImport = useCallback(async () => {
+ setStatus(Status.Import);
+
+ let errorOccurred = false;
+
+ for (const data of fileMap.values()) {
+ data.status = Status.Import;
+ updateFile(data);
+
+ let items: SpotifyExtendedStreamingHistoryItem[];
+
+ // Scope so these tmp variables can be GC-ed ASAP
+ {
+ const fileContent = await data.file.text();
+
+ const rawItems = JSON.parse(
+ fileContent,
+ ) as SpotifyExtendedStreamingHistoryItem[];
+
+ items = rawItems
+ .filter(({ spotify_track_uri }) => spotify_track_uri !== null)
+ .map(({ ts, spotify_track_uri }) => ({
+ ts,
+ spotify_track_uri,
+ }));
+ }
+
+ try {
+ await importHistory(items);
+
+ data.status = Status.Finished;
+ } catch (err) {
+ data.error = err as Error;
+ data.status = Status.Error;
+
+ errorOccurred = true;
+ }
+ updateFile(data);
+ }
+
+ if (!errorOccurred) {
+ setStatus(Status.Finished);
+ }
+ }, [fileMap, importHistory, updateFile]);
+
+ return (
+
+
+ File Upload
+
+ Select endsong_XY.json files here and start the import.
+
+
+
+
+ Drop files here or click to upload
+
+
+
+ {Array.from(fileMap.values()).map((data) => (
+
+
+
+
+
+ handleImport().catch((e) => console.error("Import Failed:", e))
+ }
+ variant="secondary"
+ disabled={status !== Status.Select}
+ >
+ Start Import
+
+
+ Remove All Files
+
+
+
+ );
+};
+
+const File: React.FC<{ data: FileData }> = ({ data }) => {
+ const hasErrors = data.status === Status.Error && data.error;
+
+ return (
+
+ {data.file.name}
+
+ {data.file.sizeReadable}
+
+
+ {data.status === Status.Select && Prepared for import! }
+ {data.status === Status.Import && Loading! }
+ {data.status === Status.Finished && Check! }
+ {hasErrors && (
+
+
+ )}
+
+
+ );
+};
+
+const ImportProgress: React.FC = () => {
+ const {
+ importStatus: { total, imported },
+ isLoading,
+ reload,
+ } = useSpotifyImportExtendedStreamingHistoryStatus();
+
+ useEffect(() => {
+ const interval = setInterval(() => {
+ if (!isLoading) {
+ reload();
+ }
+ }, 1000);
+
+ return () => clearInterval(interval);
+ }, [isLoading, reload]);
+
+ return (
+
+
+ Import Progress
+
+ Shows how many of the submitted listens are already imported and
+ visible to you. This will take a while, and the process might halt for
+ a few minutes if we hit the Spotify API rate limit. If this is not
+ finished after a few hours, please contact your Listory administrator.
+
+
+
+
+
+
+ Imported
+
+
+
+ Total
+
+
+
+ );
+};
diff --git a/frontend/src/components/NavBar.tsx b/frontend/src/components/NavBar.tsx
index 6f8a738..1af0f3b 100644
--- a/frontend/src/components/NavBar.tsx
+++ b/frontend/src/components/NavBar.tsx
@@ -3,6 +3,7 @@ import { Link } from "react-router-dom";
import { User } from "../api/entities/user";
import { useAuth } from "../hooks/use-auth";
import { CogwheelIcon } from "../icons/Cogwheel";
+import { ImportIcon } from "../icons/Import";
import { SpotifyLogo } from "../icons/Spotify";
import { Avatar, AvatarImage, AvatarFallback } from "./ui/avatar";
import {
@@ -188,6 +189,12 @@ const NavUserInfo: React.FC<{ user: User }> = ({ user }) => {
API Tokens
+
+
diff --git a/frontend/src/components/reports/RecentListens.tsx b/frontend/src/components/reports/RecentListens.tsx
index 1b7669b..860e413 100644
--- a/frontend/src/components/reports/RecentListens.tsx
+++ b/frontend/src/components/reports/RecentListens.tsx
@@ -50,7 +50,7 @@ export const RecentListens: React.FC = () => {
)}
{recentListens.length > 0 && (
-
+
{recentListens.map((listen) => (
= ({
const isMaxCountValid = (maxCount: number) =>
!(Number.isNaN(maxCount) || maxCount === 0);
-
-const numberToPercent = (ratio: number) =>
- ratio.toLocaleString(undefined, {
- style: "percent",
- minimumFractionDigits: 2,
- });
diff --git a/frontend/src/components/ui/badge.tsx b/frontend/src/components/ui/badge.tsx
new file mode 100644
index 0000000..f37c22b
--- /dev/null
+++ b/frontend/src/components/ui/badge.tsx
@@ -0,0 +1,36 @@
+import * as React from "react";
+import { cva, type VariantProps } from "class-variance-authority";
+
+import { cn } from "src/lib/utils";
+
+const badgeVariants = cva(
+ "inline-flex items-center rounded-full border border-gray-200 px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-gray-900 focus:ring-offset-2 dark:border-gray-800 dark:focus:ring-gray-300",
+ {
+ variants: {
+ variant: {
+ default:
+ "border-transparent bg-gray-900 text-gray-50 hover:bg-gray-900/80 dark:bg-gray-50 dark:text-gray-900 dark:hover:bg-gray-50/80",
+ secondary:
+ "border-transparent bg-gray-100 text-gray-900 hover:bg-gray-100/80 dark:bg-gray-800 dark:text-gray-50 dark:hover:bg-gray-800/80",
+ destructive:
+ "border-transparent bg-red-500 text-gray-50 hover:bg-red-500/80 dark:bg-red-900 dark:text-gray-50 dark:hover:bg-red-900/80",
+ outline: "text-gray-900 dark:text-gray-50",
+ },
+ },
+ defaultVariants: {
+ variant: "default",
+ },
+ },
+);
+
+export interface BadgeProps
+ extends React.HTMLAttributes,
+ VariantProps {}
+
+function Badge({ className, variant, ...props }: BadgeProps) {
+ return (
+
+ );
+}
+
+export { Badge, badgeVariants };
diff --git a/frontend/src/components/ui/card.tsx b/frontend/src/components/ui/card.tsx
new file mode 100644
index 0000000..d50d8d3
--- /dev/null
+++ b/frontend/src/components/ui/card.tsx
@@ -0,0 +1,86 @@
+import * as React from "react";
+
+import { cn } from "src/lib/utils";
+
+const Card = React.forwardRef<
+ HTMLDivElement,
+ React.HTMLAttributes
+>(({ className, ...props }, ref) => (
+
+));
+Card.displayName = "Card";
+
+const CardHeader = React.forwardRef<
+ HTMLDivElement,
+ React.HTMLAttributes
+>(({ className, ...props }, ref) => (
+
+));
+CardHeader.displayName = "CardHeader";
+
+const CardTitle = React.forwardRef<
+ HTMLParagraphElement,
+ React.HTMLAttributes
+>(({ className, ...props }, ref) => (
+
+>(({ className, ...props }, ref) => (
+
+));
+CardDescription.displayName = "CardDescription";
+
+const CardContent = React.forwardRef<
+ HTMLDivElement,
+ React.HTMLAttributes
+>(({ className, ...props }, ref) => (
+
+));
+CardContent.displayName = "CardContent";
+
+const CardFooter = React.forwardRef<
+ HTMLDivElement,
+ React.HTMLAttributes
+>(({ className, ...props }, ref) => (
+
+));
+CardFooter.displayName = "CardFooter";
+
+export {
+ Card,
+ CardHeader,
+ CardFooter,
+ CardTitle,
+ CardDescription,
+ CardContent,
+};
diff --git a/frontend/src/components/ui/code.tsx b/frontend/src/components/ui/code.tsx
new file mode 100644
index 0000000..f98a725
--- /dev/null
+++ b/frontend/src/components/ui/code.tsx
@@ -0,0 +1,9 @@
+import React, { PropsWithChildren } from "react";
+
+export const Code: React.FC = ({ children }) => {
+ return (
+
+ {children}
+
+ );
+};
diff --git a/frontend/src/components/ui/table.tsx b/frontend/src/components/ui/table.tsx
index 2eed8a5..a7efabf 100644
--- a/frontend/src/components/ui/table.tsx
+++ b/frontend/src/components/ui/table.tsx
@@ -1,6 +1,6 @@
-import * as React from "react"
+import * as React from "react";
-import { cn } from "src/lib/utils"
+import { cn } from "src/lib/utils";
const Table = React.forwardRef<
HTMLTableElement,
@@ -9,20 +9,20 @@ const Table = React.forwardRef<
-))
-Table.displayName = "Table"
+));
+Table.displayName = "Table";
const TableHeader = React.forwardRef<
HTMLTableSectionElement,
React.HTMLAttributes
>(({ className, ...props }, ref) => (
> = (props) => {
+ return (
+
+
+ );
+};
diff --git a/frontend/src/icons/Import.tsx b/frontend/src/icons/Import.tsx
new file mode 100644
index 0000000..2abd107
--- /dev/null
+++ b/frontend/src/icons/Import.tsx
@@ -0,0 +1,12 @@
+import * as React from "react";
+export const ImportIcon: React.FC> = (props) => (
+
+
+);
diff --git a/frontend/src/react-files.d.ts b/frontend/src/react-files.d.ts
new file mode 100644
index 0000000..77fbd22
--- /dev/null
+++ b/frontend/src/react-files.d.ts
@@ -0,0 +1,36 @@
+//// {
const apiToken = await this.authService.createApiToken(user, description);
diff --git a/src/auth/spotify.filter.ts b/src/auth/spotify.filter.ts
index aeca9ce..ec6502f 100644
--- a/src/auth/spotify.filter.ts
+++ b/src/auth/spotify.filter.ts
@@ -5,14 +5,14 @@ import {
ForbiddenException,
Logger,
} from "@nestjs/common";
-import type { Response } from "express";
+import type { Response as ExpressResponse } from "express";
@Catch()
export class SpotifyAuthFilter implements ExceptionFilter {
private readonly logger = new Logger(this.constructor.name);
catch(exception: Error, host: ArgumentsHost) {
- const response = host.switchToHttp().getResponse();
+ const response = host.switchToHttp().getResponse();
let reason = "unknown";
diff --git a/src/database/database.module.ts b/src/database/database.module.ts
index 06ac587..e60493a 100644
--- a/src/database/database.module.ts
+++ b/src/database/database.module.ts
@@ -27,7 +27,7 @@ export const DatabaseModule = TypeOrmModule.forRootAsync({
// Debug/Development Options
//
- // logging: true,
+ //logging: true,
//
// synchronize: true,
// migrationsRun: false,
diff --git a/src/database/migrations/09-CreateSpotifyImportTables.ts b/src/database/migrations/09-CreateSpotifyImportTables.ts
new file mode 100644
index 0000000..1d967a3
--- /dev/null
+++ b/src/database/migrations/09-CreateSpotifyImportTables.ts
@@ -0,0 +1,68 @@
+import {
+ MigrationInterface,
+ QueryRunner,
+ Table,
+ TableIndex,
+ TableForeignKey,
+} from "typeorm";
+import { TableColumnOptions } from "typeorm/schema-builder/options/TableColumnOptions";
+
+const primaryUUIDColumn: TableColumnOptions = {
+ name: "id",
+ type: "uuid",
+ isPrimary: true,
+ isGenerated: true,
+ generationStrategy: "uuid",
+};
+
+export class CreateSpotifyImportTables0000000000009
+ implements MigrationInterface
+{
+ async up(queryRunner: QueryRunner): Promise {
+ await queryRunner.createTable(
+ new Table({
+ name: "spotify_extended_streaming_history_listen",
+ columns: [
+ primaryUUIDColumn,
+ { name: "userId", type: "uuid" },
+ { name: "playedAt", type: "timestamp" },
+ { name: "spotifyTrackUri", type: "varchar" },
+ { name: "trackId", type: "uuid", isNullable: true },
+ { name: "listenId", type: "uuid", isNullable: true },
+ ],
+ indices: [
+ new TableIndex({
+ name: "IDX_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_USER_PLAYED_AT",
+ columnNames: ["userId", "playedAt", "spotifyTrackUri"],
+ isUnique: true,
+ }),
+ ],
+ foreignKeys: [
+ new TableForeignKey({
+ name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_USER_ID",
+ columnNames: ["userId"],
+ referencedColumnNames: ["id"],
+ referencedTableName: "user",
+ }),
+ new TableForeignKey({
+ name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_TRACK_ID",
+ columnNames: ["trackId"],
+ referencedColumnNames: ["id"],
+ referencedTableName: "track",
+ }),
+ new TableForeignKey({
+ name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_LISTEN_ID",
+ columnNames: ["listenId"],
+ referencedColumnNames: ["id"],
+ referencedTableName: "listen",
+ }),
+ ],
+ }),
+ true,
+ );
+ }
+
+ async down(queryRunner: QueryRunner): Promise {
+ await queryRunner.dropTable("spotify_extended_streaming_history_listen");
+ }
+}
diff --git a/src/listens/listen.repository.ts b/src/listens/listen.repository.ts
index 22c22d4..f12d8fa 100644
--- a/src/listens/listen.repository.ts
+++ b/src/listens/listen.repository.ts
@@ -3,10 +3,6 @@ import { Repository, SelectQueryBuilder } from "typeorm";
import { EntityRepository } from "../database/entity-repository";
import { Interval } from "../reports/interval";
import { User } from "../users/user.entity";
-import {
- CreateListenRequestDto,
- CreateListenResponseDto,
-} from "./dto/create-listen.dto";
import { Listen } from "./listen.entity";
export class ListenScopes extends SelectQueryBuilder {
@@ -37,52 +33,4 @@ export class ListenRepository extends Repository {
get scoped(): ListenScopes {
return new ListenScopes(this.createQueryBuilder("listen"));
}
-
- async insertNoConflict({
- user,
- track,
- playedAt,
- }: CreateListenRequestDto): Promise {
- const result = await this.createQueryBuilder()
- .insert()
- .values({
- user,
- track,
- playedAt,
- })
- .onConflict('("playedAt", "trackId", "userId") DO NOTHING')
- .execute();
-
- const [insertedRowIdentifier] = result.identifiers;
-
- if (!insertedRowIdentifier) {
- // We did not insert a new listen, it already existed
- return {
- listen: await this.findOneBy({ user, track, playedAt }),
- isDuplicate: true,
- };
- }
-
- return {
- listen: await this.findOneBy({ id: insertedRowIdentifier.id }),
- isDuplicate: false,
- };
- }
-
- /**
- *
- * @param rows
- * @returns A list of all new (non-duplicate) listens
- */
- async insertsNoConflict(rows: CreateListenRequestDto[]): Promise {
- const result = await this.createQueryBuilder()
- .insert()
- .values(rows)
- .orIgnore()
- .execute();
-
- return this.findBy(
- result.identifiers.filter(Boolean).map(({ id }) => ({ id })),
- );
- }
}
diff --git a/src/listens/listens.service.spec.ts b/src/listens/listens.service.spec.ts
index a943d6f..db35054 100644
--- a/src/listens/listens.service.spec.ts
+++ b/src/listens/listens.service.spec.ts
@@ -4,9 +4,7 @@ import {
paginate,
PaginationTypeEnum,
} from "nestjs-typeorm-paginate";
-import { Track } from "../music-library/track.entity";
import { User } from "../users/user.entity";
-import { CreateListenResponseDto } from "./dto/create-listen.dto";
import { GetListensDto } from "./dto/get-listens.dto";
import { Listen } from "./listen.entity";
import { ListenRepository, ListenScopes } from "./listen.repository";
@@ -35,39 +33,6 @@ describe("ListensService", () => {
expect(listenRepository).toBeDefined();
});
- describe("createListen", () => {
- let user: User;
- let track: Track;
- let playedAt: Date;
- let response: CreateListenResponseDto;
- beforeEach(() => {
- user = { id: "USER" } as User;
- track = { id: "TRACK" } as Track;
- playedAt = new Date("2021-01-01T00:00:00Z");
-
- response = {
- listen: {
- id: "LISTEN",
- } as Listen,
- isDuplicate: true,
- };
- listenRepository.insertNoConflict = jest.fn().mockResolvedValue(response);
- });
-
- it("creates the listen", async () => {
- await expect(
- service.createListen({ user, track, playedAt }),
- ).resolves.toEqual(response);
-
- expect(listenRepository.insertNoConflict).toHaveBeenCalledTimes(1);
- expect(listenRepository.insertNoConflict).toHaveBeenLastCalledWith({
- user,
- track,
- playedAt,
- });
- });
- });
-
describe("getListens", () => {
let options: GetListensDto & IPaginationOptions;
let user: User;
diff --git a/src/listens/listens.service.ts b/src/listens/listens.service.ts
index fea7e90..0e144ef 100644
--- a/src/listens/listens.service.ts
+++ b/src/listens/listens.service.ts
@@ -1,14 +1,12 @@
import { Injectable } from "@nestjs/common";
+import { Span } from "nestjs-otel";
import {
IPaginationOptions,
paginate,
Pagination,
PaginationTypeEnum,
} from "nestjs-typeorm-paginate";
-import {
- CreateListenRequestDto,
- CreateListenResponseDto,
-} from "./dto/create-listen.dto";
+import { CreateListenRequestDto } from "./dto/create-listen.dto";
import { GetListensDto } from "./dto/get-listens.dto";
import { Listen } from "./listen.entity";
import { ListenRepository, ListenScopes } from "./listen.repository";
@@ -17,20 +15,7 @@ import { ListenRepository, ListenScopes } from "./listen.repository";
export class ListensService {
constructor(private readonly listenRepository: ListenRepository) {}
- async createListen({
- user,
- track,
- playedAt,
- }: CreateListenRequestDto): Promise {
- const response = await this.listenRepository.insertNoConflict({
- user,
- track,
- playedAt,
- });
-
- return response;
- }
-
+ @Span()
async createListens(
listensData: CreateListenRequestDto[],
): Promise {
@@ -46,9 +31,11 @@ export class ListensService {
),
);
- return this.listenRepository.save(
+ const newListens = await this.listenRepository.save(
missingListens.map((entry) => this.listenRepository.create(entry)),
);
+
+ return [...existingListens, ...newListens];
}
async getListens(
diff --git a/src/main.ts b/src/main.ts
index 5e47205..2524d67 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -43,6 +43,7 @@ async function bootstrap() {
const app = await NestFactory.create(AppModule, {
bufferLogs: true,
+ rawBody: true,
});
app.useLogger(app.get(Logger));
app.useGlobalPipes(
@@ -51,6 +52,10 @@ async function bootstrap() {
transformOptions: { enableImplicitConversion: true },
}),
);
+ app.useBodyParser("json", {
+ limit:
+ "10mb" /* Need large bodies for Spotify Extended Streaming History */,
+ });
app.enableShutdownHooks();
const configService = app.get(ConfigService);
diff --git a/src/music-library/dto/find-track.dto.ts b/src/music-library/dto/find-track.dto.ts
index 7ffddc5..bfdf4ef 100644
--- a/src/music-library/dto/find-track.dto.ts
+++ b/src/music-library/dto/find-track.dto.ts
@@ -1,5 +1,6 @@
export class FindTrackDto {
spotify: {
- id: string;
+ id?: string;
+ uri?: string;
};
}
diff --git a/src/music-library/music-library.service.ts b/src/music-library/music-library.service.ts
index 58040d6..15ce5b5 100644
--- a/src/music-library/music-library.service.ts
+++ b/src/music-library/music-library.service.ts
@@ -175,9 +175,7 @@ export class MusicLibraryService {
}
async findTrack(query: FindTrackDto): Promise {
- return this.trackRepository.findOneBy({
- spotify: { id: query.spotify.id },
- });
+ return this.trackRepository.findOneBy(query);
}
async findTracks(query: FindTrackDto[]): Promise {
diff --git a/src/override.d.ts b/src/override.d.ts
deleted file mode 100644
index f8f5209..0000000
--- a/src/override.d.ts
+++ /dev/null
@@ -1,5 +0,0 @@
-// Issue with opentelemetry-js: https://github.com/open-telemetry/opentelemetry-js/issues/3580#issuecomment-1701157270
-export {};
-declare global {
- type BlobPropertyBag = unknown;
-}
diff --git a/src/sources/scheduler.service.ts b/src/sources/scheduler.service.ts
index 041395b..3d0fac9 100644
--- a/src/sources/scheduler.service.ts
+++ b/src/sources/scheduler.service.ts
@@ -33,7 +33,7 @@ export class SchedulerService implements OnApplicationBootstrap {
}
private async setupSpotifyCrawlerSupervisor(): Promise {
- await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {});
+ // await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {});
}
@Span()
diff --git a/src/sources/spotify/import-extended-streaming-history/dto/extended-streaming-history-status.dto.ts b/src/sources/spotify/import-extended-streaming-history/dto/extended-streaming-history-status.dto.ts
new file mode 100644
index 0000000..43b67d7
--- /dev/null
+++ b/src/sources/spotify/import-extended-streaming-history/dto/extended-streaming-history-status.dto.ts
@@ -0,0 +1,13 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class ExtendedStreamingHistoryStatusDto {
+ @ApiProperty({
+ type: Number,
+ })
+ total: number;
+
+ @ApiProperty({
+ type: Number,
+ })
+ imported: number;
+}
diff --git a/src/sources/spotify/import-extended-streaming-history/dto/import-extended-streaming-history.dto.ts b/src/sources/spotify/import-extended-streaming-history/dto/import-extended-streaming-history.dto.ts
new file mode 100644
index 0000000..419a246
--- /dev/null
+++ b/src/sources/spotify/import-extended-streaming-history/dto/import-extended-streaming-history.dto.ts
@@ -0,0 +1,13 @@
+import { ApiProperty } from "@nestjs/swagger";
+import { ArrayMaxSize } from "class-validator";
+import { SpotifyExtendedStreamingHistoryItemDto } from "./spotify-extended-streaming-history-item.dto";
+
+export class ImportExtendedStreamingHistoryDto {
+ @ApiProperty({
+ type: SpotifyExtendedStreamingHistoryItemDto,
+ isArray: true,
+ maxItems: 50_000,
+ })
+ @ArrayMaxSize(50_000) // File size is ~16k by default, might need refactoring if Spotify starts exporting larger files
+ listens: SpotifyExtendedStreamingHistoryItemDto[];
+}
diff --git a/src/sources/spotify/import-extended-streaming-history/dto/spotify-extended-streaming-history-item.dto.ts b/src/sources/spotify/import-extended-streaming-history/dto/spotify-extended-streaming-history-item.dto.ts
new file mode 100644
index 0000000..ed7f333
--- /dev/null
+++ b/src/sources/spotify/import-extended-streaming-history/dto/spotify-extended-streaming-history-item.dto.ts
@@ -0,0 +1,9 @@
+import { ApiProperty } from "@nestjs/swagger";
+
+export class SpotifyExtendedStreamingHistoryItemDto {
+ @ApiProperty({ format: "iso8601", example: "2018-11-30T08:33:33Z" })
+ ts: string;
+
+ @ApiProperty({ example: "spotify:track:6askbS4pEVWbbDnUGEXh3G" })
+ spotify_track_uri: string;
+}
diff --git a/src/sources/spotify/import-extended-streaming-history/import.controller.ts b/src/sources/spotify/import-extended-streaming-history/import.controller.ts
new file mode 100644
index 0000000..b6764a5
--- /dev/null
+++ b/src/sources/spotify/import-extended-streaming-history/import.controller.ts
@@ -0,0 +1,32 @@
+import { Body as NestBody, Controller, Get, Post } from "@nestjs/common";
+import { ApiBody, ApiTags } from "@nestjs/swagger";
+import { AuthAccessToken } from "../../../auth/decorators/auth-access-token.decorator";
+import { ReqUser } from "../../../auth/decorators/req-user.decorator";
+import { User } from "../../../users/user.entity";
+import { ExtendedStreamingHistoryStatusDto } from "./dto/extended-streaming-history-status.dto";
+import { ImportExtendedStreamingHistoryDto } from "./dto/import-extended-streaming-history.dto";
+import { ImportService } from "./import.service";
+
+@ApiTags("import")
+@Controller("api/v1/import")
+export class ImportController {
+ constructor(private readonly importService: ImportService) {}
+
+ @Post("extended-streaming-history")
+ @ApiBody({ type: () => ImportExtendedStreamingHistoryDto })
+ @AuthAccessToken()
+ async importExtendedStreamingHistory(
+ @ReqUser() user: User,
+ @NestBody() data: ImportExtendedStreamingHistoryDto,
+ ): Promise {
+ return this.importService.importExtendedStreamingHistory(user, data);
+ }
+
+ @Get("extended-streaming-history/status")
+ @AuthAccessToken()
+ async getExtendedStreamingHistoryStatus(
+ @ReqUser() user: User,
+ ): Promise {
+ return this.importService.getExtendedStreamingHistoryStatus(user);
+ }
+}
diff --git a/src/sources/spotify/import-extended-streaming-history/import.service.ts b/src/sources/spotify/import-extended-streaming-history/import.service.ts
new file mode 100644
index 0000000..d26fe99
--- /dev/null
+++ b/src/sources/spotify/import-extended-streaming-history/import.service.ts
@@ -0,0 +1,177 @@
+import { JobService } from "@apricote/nest-pg-boss";
+import { Injectable, Logger } from "@nestjs/common";
+import { uniq } from "lodash";
+import { Span } from "nestjs-otel";
+import type { Job } from "pg-boss";
+import { ListensService } from "../../../listens/listens.service";
+import { User } from "../../../users/user.entity";
+import { SpotifyService } from "../spotify.service";
+import { ExtendedStreamingHistoryStatusDto } from "./dto/extended-streaming-history-status.dto";
+import { ImportExtendedStreamingHistoryDto } from "./dto/import-extended-streaming-history.dto";
+import {
+ IProcessSpotifyExtendedStreamingHistoryListenJob,
+ ProcessSpotifyExtendedStreamingHistoryListenJob,
+} from "./jobs";
+import { SpotifyExtendedStreamingHistoryListenRepository } from "./listen.repository";
+
+@Injectable()
+export class ImportService {
+ private readonly logger = new Logger(this.constructor.name);
+
+ constructor(
+ private readonly importListenRepository: SpotifyExtendedStreamingHistoryListenRepository,
+ @ProcessSpotifyExtendedStreamingHistoryListenJob.Inject()
+ private readonly processListenJobService: JobService,
+ private readonly spotifyService: SpotifyService,
+ private readonly listensService: ListensService,
+ ) {}
+
+ @Span()
+ async importExtendedStreamingHistory(
+ user: User,
+ { listens: importListens }: ImportExtendedStreamingHistoryDto,
+ ): Promise {
+ // IDK what's happening, but my personal data set has entries with duplicate
+ // listens? might be related to offline mode.
+ // Anyway, this cleans it up:
+ const uniqEntries = new Set();
+ const uniqueListens = importListens.filter((listen) => {
+ const key = `${listen.spotify_track_uri}-${listen.ts}`;
+
+ if (!uniqEntries.has(key)) {
+ // New entry
+ uniqEntries.add(key);
+ return true;
+ }
+
+ return false;
+ });
+
+ let listens = uniqueListens.map((listenData) =>
+ this.importListenRepository.create({
+ user,
+ playedAt: new Date(listenData.ts),
+ spotifyTrackUri: listenData.spotify_track_uri,
+ }),
+ );
+
+ // Save listens to import table
+ const insertResult = await this.importListenRepository.upsert(listens, [
+ "user",
+ "playedAt",
+ "spotifyTrackUri",
+ ]);
+
+ const processJobs = insertResult.identifiers.map((listen) => ({
+ data: {
+ id: listen.id,
+ },
+ singletonKey: listen.id,
+ retryLimit: 10,
+ retryDelay: 5,
+ retryBackoff: true,
+ }));
+
+ // Schedule jobs to process imports
+ await this.processListenJobService.insert(processJobs);
+ }
+
+ @ProcessSpotifyExtendedStreamingHistoryListenJob.Handle({
+ // Spotify API "Get Several XY" allows max 50 IDs
+ batchSize: 50,
+ newJobCheckInterval: 500,
+ })
+ @Span()
+ async processListens(
+ jobs: Job[],
+ ): Promise {
+ this.logger.debug(
+ { jobs: jobs.length },
+ "processing extended streaming history listens",
+ );
+ const importListens = await this.importListenRepository.findBy(
+ jobs.map((job) => ({ id: job.data.id })),
+ );
+
+ const listensWithoutTracks = importListens.filter(
+ (importListen) => !importListen.track,
+ );
+ if (listensWithoutTracks.length > 0) {
+ const missingTrackIDs = uniq(
+ listensWithoutTracks.map((importListen) =>
+ importListen.spotifyTrackUri.replace("spotify:track:", ""),
+ ),
+ );
+
+ const tracks = await this.spotifyService.importTracks(missingTrackIDs);
+
+ listensWithoutTracks.forEach((listen) => {
+ listen.track = tracks.find(
+ (track) => listen.spotifyTrackUri === track.spotify.uri,
+ );
+ if (!listen.track) {
+ this.logger.warn(
+ { listen },
+ "could not find track for extended streaming history listen",
+ );
+ throw new Error(
+ `could not find track for extended streaming history listen`,
+ );
+ }
+ });
+
+ // Using upsert instead of save to only do a single query
+ await this.importListenRepository.upsert(listensWithoutTracks, ["id"]);
+ }
+
+ const listensWithoutListen = importListens.filter(
+ (importListen) => !importListen.listen,
+ );
+ if (listensWithoutListen.length > 0) {
+ const listens = await this.listensService.createListens(
+ listensWithoutListen.map((listen) => ({
+ user: listen.user,
+ track: listen.track,
+ playedAt: listen.playedAt,
+ })),
+ );
+
+ listensWithoutListen.forEach((importListen) => {
+ importListen.listen = listens.find(
+ (listen) =>
+ importListen.user.id === listen.user.id &&
+ importListen.track.id === listen.track.id &&
+ importListen.playedAt.getTime() === listen.playedAt.getTime(),
+ );
+ if (!importListen.listen) {
+ this.logger.warn(
+ { listen: importListen, listens: listens },
+ "could not find listen for extended streaming history listen",
+ );
+ throw new Error(
+ `could not find listen for extended streaming history listen`,
+ );
+ }
+ });
+
+ // Using upsert instead of save to only do a single query
+ await this.importListenRepository.upsert(listensWithoutListen, ["id"]);
+ }
+ }
+
+ @Span()
+ async getExtendedStreamingHistoryStatus(
+ user: User,
+ ): Promise {
+ const qb = this.importListenRepository
+ .createQueryBuilder("listen")
+ .where("listen.userId = :user", { user: user.id });
+
+ const [total, imported] = await Promise.all([
+ qb.clone().getCount(),
+ qb.clone().andWhere("listen.listenId IS NOT NULL").getCount(),
+ ]);
+
+ return { total, imported };
+ }
+}
diff --git a/src/sources/spotify/import-extended-streaming-history/index.ts b/src/sources/spotify/import-extended-streaming-history/index.ts
new file mode 100644
index 0000000..b5db823
--- /dev/null
+++ b/src/sources/spotify/import-extended-streaming-history/index.ts
@@ -0,0 +1,4 @@
+export { ImportController } from "./import.controller";
+export { ImportService } from "./import.service";
+export { ProcessSpotifyExtendedStreamingHistoryListenJob } from "./jobs";
+export { SpotifyExtendedStreamingHistoryListenRepository } from "./listen.repository";
diff --git a/src/sources/spotify/import-extended-streaming-history/jobs.ts b/src/sources/spotify/import-extended-streaming-history/jobs.ts
new file mode 100644
index 0000000..38c55c8
--- /dev/null
+++ b/src/sources/spotify/import-extended-streaming-history/jobs.ts
@@ -0,0 +1,7 @@
+import { createJob } from "@apricote/nest-pg-boss";
+
+export type IProcessSpotifyExtendedStreamingHistoryListenJob = { id: string };
+export const ProcessSpotifyExtendedStreamingHistoryListenJob =
+ createJob(
+ "process-spotify-extended-streaming-history-listen",
+ );
diff --git a/src/sources/spotify/import-extended-streaming-history/listen.entity.ts b/src/sources/spotify/import-extended-streaming-history/listen.entity.ts
new file mode 100644
index 0000000..4fa8982
--- /dev/null
+++ b/src/sources/spotify/import-extended-streaming-history/listen.entity.ts
@@ -0,0 +1,25 @@
+import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm";
+import { Track } from "../../../music-library/track.entity";
+import { User } from "../../../users/user.entity";
+import { Listen } from "../../../listens/listen.entity";
+
+@Entity({ name: "spotify_extended_streaming_history_listen" })
+export class SpotifyExtendedStreamingHistoryListen {
+ @PrimaryGeneratedColumn("uuid")
+ id: string;
+
+ @ManyToOne(() => User, { eager: true })
+ user: User;
+
+ @Column({ type: "timestamp" })
+ playedAt: Date;
+
+ @Column()
+ spotifyTrackUri: string;
+
+ @ManyToOne(() => Track, { nullable: true, eager: true })
+ track?: Track;
+
+ @ManyToOne(() => Listen, { nullable: true, eager: true })
+ listen?: Listen;
+}
diff --git a/src/sources/spotify/import-extended-streaming-history/listen.repository.ts b/src/sources/spotify/import-extended-streaming-history/listen.repository.ts
new file mode 100644
index 0000000..7c40359
--- /dev/null
+++ b/src/sources/spotify/import-extended-streaming-history/listen.repository.ts
@@ -0,0 +1,6 @@
+import { Repository } from "typeorm";
+import { EntityRepository } from "../../../database/entity-repository";
+import { SpotifyExtendedStreamingHistoryListen } from "./listen.entity";
+
+@EntityRepository(SpotifyExtendedStreamingHistoryListen)
+export class SpotifyExtendedStreamingHistoryListenRepository extends Repository {}
diff --git a/src/sources/spotify/spotify.module.ts b/src/sources/spotify/spotify.module.ts
index 1df6bbf..021ffec 100644
--- a/src/sources/spotify/spotify.module.ts
+++ b/src/sources/spotify/spotify.module.ts
@@ -1,25 +1,37 @@
import { PGBossModule } from "@apricote/nest-pg-boss";
import { Module } from "@nestjs/common";
+import { TypeOrmRepositoryModule } from "../../database/entity-repository/typeorm-repository.module";
import { ListensModule } from "../../listens/listens.module";
import { MusicLibraryModule } from "../../music-library/music-library.module";
import { UsersModule } from "../../users/users.module";
import { ImportSpotifyJob } from "../jobs";
+import {
+ ImportController,
+ ImportService,
+ ProcessSpotifyExtendedStreamingHistoryListenJob,
+ SpotifyExtendedStreamingHistoryListenRepository,
+} from "./import-extended-streaming-history";
import { SpotifyApiModule } from "./spotify-api/spotify-api.module";
import { SpotifyAuthModule } from "./spotify-auth/spotify-auth.module";
import { SpotifyService } from "./spotify.service";
@Module({
imports: [
- PGBossModule.forJobs([ImportSpotifyJob]),
+ PGBossModule.forJobs([
+ ImportSpotifyJob,
+ ProcessSpotifyExtendedStreamingHistoryListenJob,
+ ]),
+ TypeOrmRepositoryModule.for([
+ SpotifyExtendedStreamingHistoryListenRepository,
+ ]),
UsersModule,
ListensModule,
MusicLibraryModule,
SpotifyApiModule,
SpotifyAuthModule,
],
- providers: [SpotifyService],
+ providers: [SpotifyService, ImportService],
+ controllers: [ImportController],
exports: [SpotifyService],
})
-export class SpotifyModule {
- constructor(private readonly spotifyService: SpotifyService) {}
-}
+export class SpotifyModule {}
diff --git a/tsconfig.json b/tsconfig.json
index cce4f57..78c9382 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -5,13 +5,12 @@
"removeComments": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
- "target": "ES2020",
+ "target": "ES2022",
"sourceMap": true,
"outDir": "./dist",
"baseUrl": "./",
"incremental": true,
"paths": {},
- "lib": ["ES2020"]
},
"exclude": ["node_modules", "dist"],
"include": ["src", "test"]