(
- `/api/v1/import/extended-streaming-history/status`
- );
-
- switch (res.status) {
- case 200: {
- break;
- }
- case 401: {
- throw new UnauthenticatedError(`No token or token expired`);
- }
- default: {
- throw new Error(
- `Unable to getExtendedStreamingHistoryStatus: ${res.status}`
- );
- }
- }
-
- return res.data;
-};
diff --git a/frontend/src/api/entities/extended-streaming-history-status.ts b/frontend/src/api/entities/extended-streaming-history-status.ts
deleted file mode 100644
index d6fa488..0000000
--- a/frontend/src/api/entities/extended-streaming-history-status.ts
+++ /dev/null
@@ -1,4 +0,0 @@
-export interface ExtendedStreamingHistoryStatus {
- total: number;
- imported: number;
-}
diff --git a/frontend/src/api/entities/spotify-extended-streaming-history-item.ts b/frontend/src/api/entities/spotify-extended-streaming-history-item.ts
deleted file mode 100644
index cc91d3c..0000000
--- a/frontend/src/api/entities/spotify-extended-streaming-history-item.ts
+++ /dev/null
@@ -1,4 +0,0 @@
-export interface SpotifyExtendedStreamingHistoryItem {
- ts: string;
- spotify_track_uri: string;
-}
diff --git a/frontend/src/components/AuthApiTokens.tsx b/frontend/src/components/AuthApiTokens.tsx
index 0e50589..d73b755 100644
--- a/frontend/src/components/AuthApiTokens.tsx
+++ b/frontend/src/components/AuthApiTokens.tsx
@@ -2,6 +2,7 @@ import { format, formatDistanceToNow } from "date-fns";
import React, { FormEvent, useCallback, useMemo, useState } from "react";
import { ApiToken, NewApiToken } from "../api/entities/api-token";
import { useApiTokens } from "../hooks/use-api";
+import { useAuthProtection } from "../hooks/use-auth-protection";
import { SpinnerIcon } from "../icons/Spinner";
import TrashcanIcon from "../icons/Trashcan";
import { Spinner } from "./ui/Spinner";
@@ -9,10 +10,7 @@ import { Spinner } from "./ui/Spinner";
export const AuthApiTokens: React.FC = () => {
const { apiTokens, isLoading, createToken, revokeToken } = useApiTokens();
const sortedTokens = useMemo(
- () =>
- apiTokens
- .filter((token) => !token.revokedAt)
- .sort((a, b) => (a.createdAt > b.createdAt ? -1 : 1)),
+ () => apiTokens.sort((a, b) => (a.createdAt > b.createdAt ? -1 : 1)),
[apiTokens],
);
@@ -25,7 +23,7 @@ export const AuthApiTokens: React.FC = () => {
You can use API Tokens to access the Listory API directly. You can
find the API docs{" "}
-
+
here
.
diff --git a/frontend/src/components/ImportListens.tsx b/frontend/src/components/ImportListens.tsx
deleted file mode 100644
index f0699b6..0000000
--- a/frontend/src/components/ImportListens.tsx
+++ /dev/null
@@ -1,280 +0,0 @@
-import React, { useCallback, useEffect, useState } from "react";
-import Files from "react-files";
-import type { ReactFile } from "react-files";
-import {
- useSpotifyImportExtendedStreamingHistory,
- useSpotifyImportExtendedStreamingHistoryStatus,
-} from "../hooks/use-api";
-import { SpotifyExtendedStreamingHistoryItem } from "../api/entities/spotify-extended-streaming-history-item";
-import { ErrorIcon } from "../icons/Error";
-import { numberToPercent } from "../util/numberToPercent";
-import { Button } from "./ui/button";
-import { Table, TableBody, TableCell, TableRow } from "./ui/table";
-import { Badge } from "./ui/badge";
-import {
- Card,
- CardContent,
- CardDescription,
- CardFooter,
- CardHeader,
- CardTitle,
-} from "./ui/card";
-import { Code } from "./ui/code";
-
-export const ImportListens: React.FC = () => {
- return (
- <>
-
-
- Import Listens from Spotify Extended Streaming History
-
-
-
- Here you can import your full Spotify Listen history that was exported
- from the{" "}
-
- Extended streaming history
-
- .
-
-
- The extended streaming history contains additional personally
- identifiable data such as the IP address of the listen (which can be
- linked to locations). To avoid saving this on the server, the data is
- preprocessed in your web browser and only the necessary data
- (timestamp & track ID) are sent to the server.
-
-
- If an error occurs, you can always retry uploading the file, Listory
- deduplicates any listens to make sure that everything is saved only
- once.
-
-
-
-
-
>(
- new Map(),
- );
-
- const [status, setStatus] = useState(Status.Select);
-
- const addFiles = useCallback(
- (files: ReactFile[]) => {
- setFileMap((_fileMap) => {
- files.forEach((file) =>
- _fileMap.set(file.id, { file, status: Status.Select }),
- );
- return new Map(_fileMap);
- });
- },
- [setFileMap],
- );
-
- const updateFile = useCallback((data: FileData) => {
- setFileMap((_fileMap) => new Map(_fileMap.set(data.file.id, data)));
- }, []);
-
- const clearFiles = useCallback(() => {
- setFileMap(new Map());
- }, [setFileMap]);
-
- const { importHistory } = useSpotifyImportExtendedStreamingHistory();
-
- const handleImport = useCallback(async () => {
- setStatus(Status.Import);
-
- let errorOccurred = false;
-
- for (const data of fileMap.values()) {
- data.status = Status.Import;
- updateFile(data);
-
- let items: SpotifyExtendedStreamingHistoryItem[];
-
- // Scope so these tmp variables can be GC-ed ASAP
- {
- const fileContent = await data.file.text();
-
- const rawItems = JSON.parse(
- fileContent,
- ) as SpotifyExtendedStreamingHistoryItem[];
-
- items = rawItems
- .filter(({ spotify_track_uri }) => spotify_track_uri !== null)
- .map(({ ts, spotify_track_uri }) => ({
- ts,
- spotify_track_uri,
- }));
- }
-
- try {
- await importHistory(items);
-
- data.status = Status.Finished;
- } catch (err) {
- data.error = err as Error;
- data.status = Status.Error;
-
- errorOccurred = true;
- }
- updateFile(data);
- }
-
- if (!errorOccurred) {
- setStatus(Status.Finished);
- }
- }, [fileMap, importHistory, updateFile]);
-
- return (
-
-
- File Upload
-
- Select endsong_XY.json files here and start the import.
-
-
-
-
- Drop files here or click to upload
-
-
-
- {Array.from(fileMap.values()).map((data) => (
-
-
-
-
-
- handleImport().catch((e) => console.error("Import Failed:", e))
- }
- variant="secondary"
- disabled={status !== Status.Select}
- >
- Start Import
-
-
- Remove All Files
-
-
-
- );
-};
-
-const File: React.FC<{ data: FileData }> = ({ data }) => {
- const hasErrors = data.status === Status.Error && data.error;
-
- return (
-
- {data.file.name}
-
- {data.file.sizeReadable}
-
-
- {data.status === Status.Select && Prepared for import! }
- {data.status === Status.Import && Loading! }
- {data.status === Status.Finished && Check! }
- {hasErrors && (
-
-
- )}
-
-
- );
-};
-
-const ImportProgress: React.FC = () => {
- const {
- importStatus: { total, imported },
- isLoading,
- reload,
- } = useSpotifyImportExtendedStreamingHistoryStatus();
-
- useEffect(() => {
- const interval = setInterval(() => {
- if (!isLoading) {
- reload();
- }
- }, 1000);
-
- return () => clearInterval(interval);
- }, [isLoading, reload]);
-
- return (
-
-
- Import Progress
-
- Shows how many of the submitted listens are already imported and
- visible to you. This will take a while, and the process might halt for
- a few minutes if we hit the Spotify API rate limit. If this is not
- finished after a few hours, please contact your Listory administrator.
-
-
-
-
-
-
- Imported
-
-
-
- Total
-
-
-
- );
-};
diff --git a/frontend/src/components/NavBar.tsx b/frontend/src/components/NavBar.tsx
index 1af0f3b..6f8a738 100644
--- a/frontend/src/components/NavBar.tsx
+++ b/frontend/src/components/NavBar.tsx
@@ -3,7 +3,6 @@ import { Link } from "react-router-dom";
import { User } from "../api/entities/user";
import { useAuth } from "../hooks/use-auth";
import { CogwheelIcon } from "../icons/Cogwheel";
-import { ImportIcon } from "../icons/Import";
import { SpotifyLogo } from "../icons/Spotify";
import { Avatar, AvatarImage, AvatarFallback } from "./ui/avatar";
import {
@@ -189,12 +188,6 @@ const NavUserInfo: React.FC<{ user: User }> = ({ user }) => {
API Tokens
-
-
diff --git a/frontend/src/components/reports/RecentListens.tsx b/frontend/src/components/reports/RecentListens.tsx
index 860e413..1b7669b 100644
--- a/frontend/src/components/reports/RecentListens.tsx
+++ b/frontend/src/components/reports/RecentListens.tsx
@@ -50,7 +50,7 @@ export const RecentListens: React.FC = () => {
)}
{recentListens.length > 0 && (
-
+
{recentListens.map((listen) => (
= ({
const isMaxCountValid = (maxCount: number) =>
!(Number.isNaN(maxCount) || maxCount === 0);
+
+const numberToPercent = (ratio: number) =>
+ ratio.toLocaleString(undefined, {
+ style: "percent",
+ minimumFractionDigits: 2,
+ });
diff --git a/frontend/src/components/ui/badge.tsx b/frontend/src/components/ui/badge.tsx
deleted file mode 100644
index f37c22b..0000000
--- a/frontend/src/components/ui/badge.tsx
+++ /dev/null
@@ -1,36 +0,0 @@
-import * as React from "react";
-import { cva, type VariantProps } from "class-variance-authority";
-
-import { cn } from "src/lib/utils";
-
-const badgeVariants = cva(
- "inline-flex items-center rounded-full border border-gray-200 px-2.5 py-0.5 text-xs font-semibold transition-colors focus:outline-none focus:ring-2 focus:ring-gray-900 focus:ring-offset-2 dark:border-gray-800 dark:focus:ring-gray-300",
- {
- variants: {
- variant: {
- default:
- "border-transparent bg-gray-900 text-gray-50 hover:bg-gray-900/80 dark:bg-gray-50 dark:text-gray-900 dark:hover:bg-gray-50/80",
- secondary:
- "border-transparent bg-gray-100 text-gray-900 hover:bg-gray-100/80 dark:bg-gray-800 dark:text-gray-50 dark:hover:bg-gray-800/80",
- destructive:
- "border-transparent bg-red-500 text-gray-50 hover:bg-red-500/80 dark:bg-red-900 dark:text-gray-50 dark:hover:bg-red-900/80",
- outline: "text-gray-900 dark:text-gray-50",
- },
- },
- defaultVariants: {
- variant: "default",
- },
- },
-);
-
-export interface BadgeProps
- extends React.HTMLAttributes,
- VariantProps {}
-
-function Badge({ className, variant, ...props }: BadgeProps) {
- return (
-
- );
-}
-
-export { Badge, badgeVariants };
diff --git a/frontend/src/components/ui/card.tsx b/frontend/src/components/ui/card.tsx
deleted file mode 100644
index d50d8d3..0000000
--- a/frontend/src/components/ui/card.tsx
+++ /dev/null
@@ -1,86 +0,0 @@
-import * as React from "react";
-
-import { cn } from "src/lib/utils";
-
-const Card = React.forwardRef<
- HTMLDivElement,
- React.HTMLAttributes
->(({ className, ...props }, ref) => (
-
-));
-Card.displayName = "Card";
-
-const CardHeader = React.forwardRef<
- HTMLDivElement,
- React.HTMLAttributes
->(({ className, ...props }, ref) => (
-
-));
-CardHeader.displayName = "CardHeader";
-
-const CardTitle = React.forwardRef<
- HTMLParagraphElement,
- React.HTMLAttributes
->(({ className, ...props }, ref) => (
-
->(({ className, ...props }, ref) => (
-
-));
-CardDescription.displayName = "CardDescription";
-
-const CardContent = React.forwardRef<
- HTMLDivElement,
- React.HTMLAttributes
->(({ className, ...props }, ref) => (
-
-));
-CardContent.displayName = "CardContent";
-
-const CardFooter = React.forwardRef<
- HTMLDivElement,
- React.HTMLAttributes
->(({ className, ...props }, ref) => (
-
-));
-CardFooter.displayName = "CardFooter";
-
-export {
- Card,
- CardHeader,
- CardFooter,
- CardTitle,
- CardDescription,
- CardContent,
-};
diff --git a/frontend/src/components/ui/code.tsx b/frontend/src/components/ui/code.tsx
deleted file mode 100644
index f98a725..0000000
--- a/frontend/src/components/ui/code.tsx
+++ /dev/null
@@ -1,9 +0,0 @@
-import React, { PropsWithChildren } from "react";
-
-export const Code: React.FC = ({ children }) => {
- return (
-
- {children}
-
- );
-};
diff --git a/frontend/src/components/ui/table.tsx b/frontend/src/components/ui/table.tsx
index a7efabf..2eed8a5 100644
--- a/frontend/src/components/ui/table.tsx
+++ b/frontend/src/components/ui/table.tsx
@@ -1,6 +1,6 @@
-import * as React from "react";
+import * as React from "react"
-import { cn } from "src/lib/utils";
+import { cn } from "src/lib/utils"
const Table = React.forwardRef<
HTMLTableElement,
@@ -9,20 +9,20 @@ const Table = React.forwardRef<
-));
-Table.displayName = "Table";
+))
+Table.displayName = "Table"
const TableHeader = React.forwardRef<
HTMLTableSectionElement,
React.HTMLAttributes
>(({ className, ...props }, ref) => (
> = (props) => {
- return (
-
-
- );
-};
diff --git a/frontend/src/icons/Import.tsx b/frontend/src/icons/Import.tsx
deleted file mode 100644
index 2abd107..0000000
--- a/frontend/src/icons/Import.tsx
+++ /dev/null
@@ -1,12 +0,0 @@
-import * as React from "react";
-export const ImportIcon: React.FC> = (props) => (
-
-
-);
diff --git a/frontend/src/react-files.d.ts b/frontend/src/react-files.d.ts
deleted file mode 100644
index 77fbd22..0000000
--- a/frontend/src/react-files.d.ts
+++ /dev/null
@@ -1,36 +0,0 @@
-//// {
const apiToken = await this.authService.createApiToken(user, description);
diff --git a/src/auth/spotify.filter.ts b/src/auth/spotify.filter.ts
index ec6502f..aeca9ce 100644
--- a/src/auth/spotify.filter.ts
+++ b/src/auth/spotify.filter.ts
@@ -5,14 +5,14 @@ import {
ForbiddenException,
Logger,
} from "@nestjs/common";
-import type { Response as ExpressResponse } from "express";
+import type { Response } from "express";
@Catch()
export class SpotifyAuthFilter implements ExceptionFilter {
private readonly logger = new Logger(this.constructor.name);
catch(exception: Error, host: ArgumentsHost) {
- const response = host.switchToHttp().getResponse();
+ const response = host.switchToHttp().getResponse();
let reason = "unknown";
diff --git a/src/database/database.module.ts b/src/database/database.module.ts
index e60493a..06ac587 100644
--- a/src/database/database.module.ts
+++ b/src/database/database.module.ts
@@ -27,7 +27,7 @@ export const DatabaseModule = TypeOrmModule.forRootAsync({
// Debug/Development Options
//
- //logging: true,
+ // logging: true,
//
// synchronize: true,
// migrationsRun: false,
diff --git a/src/database/migrations/09-CreateSpotifyImportTables.ts b/src/database/migrations/09-CreateSpotifyImportTables.ts
deleted file mode 100644
index 1d967a3..0000000
--- a/src/database/migrations/09-CreateSpotifyImportTables.ts
+++ /dev/null
@@ -1,68 +0,0 @@
-import {
- MigrationInterface,
- QueryRunner,
- Table,
- TableIndex,
- TableForeignKey,
-} from "typeorm";
-import { TableColumnOptions } from "typeorm/schema-builder/options/TableColumnOptions";
-
-const primaryUUIDColumn: TableColumnOptions = {
- name: "id",
- type: "uuid",
- isPrimary: true,
- isGenerated: true,
- generationStrategy: "uuid",
-};
-
-export class CreateSpotifyImportTables0000000000009
- implements MigrationInterface
-{
- async up(queryRunner: QueryRunner): Promise {
- await queryRunner.createTable(
- new Table({
- name: "spotify_extended_streaming_history_listen",
- columns: [
- primaryUUIDColumn,
- { name: "userId", type: "uuid" },
- { name: "playedAt", type: "timestamp" },
- { name: "spotifyTrackUri", type: "varchar" },
- { name: "trackId", type: "uuid", isNullable: true },
- { name: "listenId", type: "uuid", isNullable: true },
- ],
- indices: [
- new TableIndex({
- name: "IDX_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_USER_PLAYED_AT",
- columnNames: ["userId", "playedAt", "spotifyTrackUri"],
- isUnique: true,
- }),
- ],
- foreignKeys: [
- new TableForeignKey({
- name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_USER_ID",
- columnNames: ["userId"],
- referencedColumnNames: ["id"],
- referencedTableName: "user",
- }),
- new TableForeignKey({
- name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_TRACK_ID",
- columnNames: ["trackId"],
- referencedColumnNames: ["id"],
- referencedTableName: "track",
- }),
- new TableForeignKey({
- name: "FK_SPOTIFY_EXTENDED_STREAMING_HISTORY_LISTEN_LISTEN_ID",
- columnNames: ["listenId"],
- referencedColumnNames: ["id"],
- referencedTableName: "listen",
- }),
- ],
- }),
- true,
- );
- }
-
- async down(queryRunner: QueryRunner): Promise {
- await queryRunner.dropTable("spotify_extended_streaming_history_listen");
- }
-}
diff --git a/src/listens/listen.repository.ts b/src/listens/listen.repository.ts
index f12d8fa..22c22d4 100644
--- a/src/listens/listen.repository.ts
+++ b/src/listens/listen.repository.ts
@@ -3,6 +3,10 @@ import { Repository, SelectQueryBuilder } from "typeorm";
import { EntityRepository } from "../database/entity-repository";
import { Interval } from "../reports/interval";
import { User } from "../users/user.entity";
+import {
+ CreateListenRequestDto,
+ CreateListenResponseDto,
+} from "./dto/create-listen.dto";
import { Listen } from "./listen.entity";
export class ListenScopes extends SelectQueryBuilder {
@@ -33,4 +37,52 @@ export class ListenRepository extends Repository {
get scoped(): ListenScopes {
return new ListenScopes(this.createQueryBuilder("listen"));
}
+
+ async insertNoConflict({
+ user,
+ track,
+ playedAt,
+ }: CreateListenRequestDto): Promise {
+ const result = await this.createQueryBuilder()
+ .insert()
+ .values({
+ user,
+ track,
+ playedAt,
+ })
+ .onConflict('("playedAt", "trackId", "userId") DO NOTHING')
+ .execute();
+
+ const [insertedRowIdentifier] = result.identifiers;
+
+ if (!insertedRowIdentifier) {
+ // We did not insert a new listen, it already existed
+ return {
+ listen: await this.findOneBy({ user, track, playedAt }),
+ isDuplicate: true,
+ };
+ }
+
+ return {
+ listen: await this.findOneBy({ id: insertedRowIdentifier.id }),
+ isDuplicate: false,
+ };
+ }
+
+ /**
+ *
+ * @param rows
+ * @returns A list of all new (non-duplicate) listens
+ */
+ async insertsNoConflict(rows: CreateListenRequestDto[]): Promise {
+ const result = await this.createQueryBuilder()
+ .insert()
+ .values(rows)
+ .orIgnore()
+ .execute();
+
+ return this.findBy(
+ result.identifiers.filter(Boolean).map(({ id }) => ({ id })),
+ );
+ }
}
diff --git a/src/listens/listens.service.spec.ts b/src/listens/listens.service.spec.ts
index db35054..a943d6f 100644
--- a/src/listens/listens.service.spec.ts
+++ b/src/listens/listens.service.spec.ts
@@ -4,7 +4,9 @@ import {
paginate,
PaginationTypeEnum,
} from "nestjs-typeorm-paginate";
+import { Track } from "../music-library/track.entity";
import { User } from "../users/user.entity";
+import { CreateListenResponseDto } from "./dto/create-listen.dto";
import { GetListensDto } from "./dto/get-listens.dto";
import { Listen } from "./listen.entity";
import { ListenRepository, ListenScopes } from "./listen.repository";
@@ -33,6 +35,39 @@ describe("ListensService", () => {
expect(listenRepository).toBeDefined();
});
+ describe("createListen", () => {
+ let user: User;
+ let track: Track;
+ let playedAt: Date;
+ let response: CreateListenResponseDto;
+ beforeEach(() => {
+ user = { id: "USER" } as User;
+ track = { id: "TRACK" } as Track;
+ playedAt = new Date("2021-01-01T00:00:00Z");
+
+ response = {
+ listen: {
+ id: "LISTEN",
+ } as Listen,
+ isDuplicate: true,
+ };
+ listenRepository.insertNoConflict = jest.fn().mockResolvedValue(response);
+ });
+
+ it("creates the listen", async () => {
+ await expect(
+ service.createListen({ user, track, playedAt }),
+ ).resolves.toEqual(response);
+
+ expect(listenRepository.insertNoConflict).toHaveBeenCalledTimes(1);
+ expect(listenRepository.insertNoConflict).toHaveBeenLastCalledWith({
+ user,
+ track,
+ playedAt,
+ });
+ });
+ });
+
describe("getListens", () => {
let options: GetListensDto & IPaginationOptions;
let user: User;
diff --git a/src/listens/listens.service.ts b/src/listens/listens.service.ts
index 0e144ef..fea7e90 100644
--- a/src/listens/listens.service.ts
+++ b/src/listens/listens.service.ts
@@ -1,12 +1,14 @@
import { Injectable } from "@nestjs/common";
-import { Span } from "nestjs-otel";
import {
IPaginationOptions,
paginate,
Pagination,
PaginationTypeEnum,
} from "nestjs-typeorm-paginate";
-import { CreateListenRequestDto } from "./dto/create-listen.dto";
+import {
+ CreateListenRequestDto,
+ CreateListenResponseDto,
+} from "./dto/create-listen.dto";
import { GetListensDto } from "./dto/get-listens.dto";
import { Listen } from "./listen.entity";
import { ListenRepository, ListenScopes } from "./listen.repository";
@@ -15,7 +17,20 @@ import { ListenRepository, ListenScopes } from "./listen.repository";
export class ListensService {
constructor(private readonly listenRepository: ListenRepository) {}
- @Span()
+ async createListen({
+ user,
+ track,
+ playedAt,
+ }: CreateListenRequestDto): Promise {
+ const response = await this.listenRepository.insertNoConflict({
+ user,
+ track,
+ playedAt,
+ });
+
+ return response;
+ }
+
async createListens(
listensData: CreateListenRequestDto[],
): Promise {
@@ -31,11 +46,9 @@ export class ListensService {
),
);
- const newListens = await this.listenRepository.save(
+ return this.listenRepository.save(
missingListens.map((entry) => this.listenRepository.create(entry)),
);
-
- return [...existingListens, ...newListens];
}
async getListens(
diff --git a/src/main.ts b/src/main.ts
index 2524d67..5e47205 100644
--- a/src/main.ts
+++ b/src/main.ts
@@ -43,7 +43,6 @@ async function bootstrap() {
const app = await NestFactory.create(AppModule, {
bufferLogs: true,
- rawBody: true,
});
app.useLogger(app.get(Logger));
app.useGlobalPipes(
@@ -52,10 +51,6 @@ async function bootstrap() {
transformOptions: { enableImplicitConversion: true },
}),
);
- app.useBodyParser("json", {
- limit:
- "10mb" /* Need large bodies for Spotify Extended Streaming History */,
- });
app.enableShutdownHooks();
const configService = app.get(ConfigService);
diff --git a/src/music-library/dto/find-track.dto.ts b/src/music-library/dto/find-track.dto.ts
index bfdf4ef..7ffddc5 100644
--- a/src/music-library/dto/find-track.dto.ts
+++ b/src/music-library/dto/find-track.dto.ts
@@ -1,6 +1,5 @@
export class FindTrackDto {
spotify: {
- id?: string;
- uri?: string;
+ id: string;
};
}
diff --git a/src/music-library/music-library.service.ts b/src/music-library/music-library.service.ts
index 15ce5b5..58040d6 100644
--- a/src/music-library/music-library.service.ts
+++ b/src/music-library/music-library.service.ts
@@ -175,7 +175,9 @@ export class MusicLibraryService {
}
async findTrack(query: FindTrackDto): Promise {
- return this.trackRepository.findOneBy(query);
+ return this.trackRepository.findOneBy({
+ spotify: { id: query.spotify.id },
+ });
}
async findTracks(query: FindTrackDto[]): Promise {
diff --git a/src/override.d.ts b/src/override.d.ts
new file mode 100644
index 0000000..f8f5209
--- /dev/null
+++ b/src/override.d.ts
@@ -0,0 +1,5 @@
+// Issue with opentelemetry-js: https://github.com/open-telemetry/opentelemetry-js/issues/3580#issuecomment-1701157270
+export {};
+declare global {
+ type BlobPropertyBag = unknown;
+}
diff --git a/src/sources/scheduler.service.ts b/src/sources/scheduler.service.ts
index 3d0fac9..041395b 100644
--- a/src/sources/scheduler.service.ts
+++ b/src/sources/scheduler.service.ts
@@ -33,7 +33,7 @@ export class SchedulerService implements OnApplicationBootstrap {
}
private async setupSpotifyCrawlerSupervisor(): Promise {
- // await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {});
+ await this.superviseImportJobsJobService.schedule("*/1 * * * *", {}, {});
}
@Span()
diff --git a/src/sources/spotify/import-extended-streaming-history/dto/extended-streaming-history-status.dto.ts b/src/sources/spotify/import-extended-streaming-history/dto/extended-streaming-history-status.dto.ts
deleted file mode 100644
index 43b67d7..0000000
--- a/src/sources/spotify/import-extended-streaming-history/dto/extended-streaming-history-status.dto.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import { ApiProperty } from "@nestjs/swagger";
-
-export class ExtendedStreamingHistoryStatusDto {
- @ApiProperty({
- type: Number,
- })
- total: number;
-
- @ApiProperty({
- type: Number,
- })
- imported: number;
-}
diff --git a/src/sources/spotify/import-extended-streaming-history/dto/import-extended-streaming-history.dto.ts b/src/sources/spotify/import-extended-streaming-history/dto/import-extended-streaming-history.dto.ts
deleted file mode 100644
index 419a246..0000000
--- a/src/sources/spotify/import-extended-streaming-history/dto/import-extended-streaming-history.dto.ts
+++ /dev/null
@@ -1,13 +0,0 @@
-import { ApiProperty } from "@nestjs/swagger";
-import { ArrayMaxSize } from "class-validator";
-import { SpotifyExtendedStreamingHistoryItemDto } from "./spotify-extended-streaming-history-item.dto";
-
-export class ImportExtendedStreamingHistoryDto {
- @ApiProperty({
- type: SpotifyExtendedStreamingHistoryItemDto,
- isArray: true,
- maxItems: 50_000,
- })
- @ArrayMaxSize(50_000) // File size is ~16k by default, might need refactoring if Spotify starts exporting larger files
- listens: SpotifyExtendedStreamingHistoryItemDto[];
-}
diff --git a/src/sources/spotify/import-extended-streaming-history/dto/spotify-extended-streaming-history-item.dto.ts b/src/sources/spotify/import-extended-streaming-history/dto/spotify-extended-streaming-history-item.dto.ts
deleted file mode 100644
index ed7f333..0000000
--- a/src/sources/spotify/import-extended-streaming-history/dto/spotify-extended-streaming-history-item.dto.ts
+++ /dev/null
@@ -1,9 +0,0 @@
-import { ApiProperty } from "@nestjs/swagger";
-
-export class SpotifyExtendedStreamingHistoryItemDto {
- @ApiProperty({ format: "iso8601", example: "2018-11-30T08:33:33Z" })
- ts: string;
-
- @ApiProperty({ example: "spotify:track:6askbS4pEVWbbDnUGEXh3G" })
- spotify_track_uri: string;
-}
diff --git a/src/sources/spotify/import-extended-streaming-history/import.controller.ts b/src/sources/spotify/import-extended-streaming-history/import.controller.ts
deleted file mode 100644
index b6764a5..0000000
--- a/src/sources/spotify/import-extended-streaming-history/import.controller.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-import { Body as NestBody, Controller, Get, Post } from "@nestjs/common";
-import { ApiBody, ApiTags } from "@nestjs/swagger";
-import { AuthAccessToken } from "../../../auth/decorators/auth-access-token.decorator";
-import { ReqUser } from "../../../auth/decorators/req-user.decorator";
-import { User } from "../../../users/user.entity";
-import { ExtendedStreamingHistoryStatusDto } from "./dto/extended-streaming-history-status.dto";
-import { ImportExtendedStreamingHistoryDto } from "./dto/import-extended-streaming-history.dto";
-import { ImportService } from "./import.service";
-
-@ApiTags("import")
-@Controller("api/v1/import")
-export class ImportController {
- constructor(private readonly importService: ImportService) {}
-
- @Post("extended-streaming-history")
- @ApiBody({ type: () => ImportExtendedStreamingHistoryDto })
- @AuthAccessToken()
- async importExtendedStreamingHistory(
- @ReqUser() user: User,
- @NestBody() data: ImportExtendedStreamingHistoryDto,
- ): Promise {
- return this.importService.importExtendedStreamingHistory(user, data);
- }
-
- @Get("extended-streaming-history/status")
- @AuthAccessToken()
- async getExtendedStreamingHistoryStatus(
- @ReqUser() user: User,
- ): Promise {
- return this.importService.getExtendedStreamingHistoryStatus(user);
- }
-}
diff --git a/src/sources/spotify/import-extended-streaming-history/import.service.ts b/src/sources/spotify/import-extended-streaming-history/import.service.ts
deleted file mode 100644
index d26fe99..0000000
--- a/src/sources/spotify/import-extended-streaming-history/import.service.ts
+++ /dev/null
@@ -1,177 +0,0 @@
-import { JobService } from "@apricote/nest-pg-boss";
-import { Injectable, Logger } from "@nestjs/common";
-import { uniq } from "lodash";
-import { Span } from "nestjs-otel";
-import type { Job } from "pg-boss";
-import { ListensService } from "../../../listens/listens.service";
-import { User } from "../../../users/user.entity";
-import { SpotifyService } from "../spotify.service";
-import { ExtendedStreamingHistoryStatusDto } from "./dto/extended-streaming-history-status.dto";
-import { ImportExtendedStreamingHistoryDto } from "./dto/import-extended-streaming-history.dto";
-import {
- IProcessSpotifyExtendedStreamingHistoryListenJob,
- ProcessSpotifyExtendedStreamingHistoryListenJob,
-} from "./jobs";
-import { SpotifyExtendedStreamingHistoryListenRepository } from "./listen.repository";
-
-@Injectable()
-export class ImportService {
- private readonly logger = new Logger(this.constructor.name);
-
- constructor(
- private readonly importListenRepository: SpotifyExtendedStreamingHistoryListenRepository,
- @ProcessSpotifyExtendedStreamingHistoryListenJob.Inject()
- private readonly processListenJobService: JobService,
- private readonly spotifyService: SpotifyService,
- private readonly listensService: ListensService,
- ) {}
-
- @Span()
- async importExtendedStreamingHistory(
- user: User,
- { listens: importListens }: ImportExtendedStreamingHistoryDto,
- ): Promise {
- // IDK what's happening, but my personal data set has entries with duplicate
- // listens? might be related to offline mode.
- // Anyway, this cleans it up:
- const uniqEntries = new Set();
- const uniqueListens = importListens.filter((listen) => {
- const key = `${listen.spotify_track_uri}-${listen.ts}`;
-
- if (!uniqEntries.has(key)) {
- // New entry
- uniqEntries.add(key);
- return true;
- }
-
- return false;
- });
-
- let listens = uniqueListens.map((listenData) =>
- this.importListenRepository.create({
- user,
- playedAt: new Date(listenData.ts),
- spotifyTrackUri: listenData.spotify_track_uri,
- }),
- );
-
- // Save listens to import table
- const insertResult = await this.importListenRepository.upsert(listens, [
- "user",
- "playedAt",
- "spotifyTrackUri",
- ]);
-
- const processJobs = insertResult.identifiers.map((listen) => ({
- data: {
- id: listen.id,
- },
- singletonKey: listen.id,
- retryLimit: 10,
- retryDelay: 5,
- retryBackoff: true,
- }));
-
- // Schedule jobs to process imports
- await this.processListenJobService.insert(processJobs);
- }
-
- @ProcessSpotifyExtendedStreamingHistoryListenJob.Handle({
- // Spotify API "Get Several XY" allows max 50 IDs
- batchSize: 50,
- newJobCheckInterval: 500,
- })
- @Span()
- async processListens(
- jobs: Job[],
- ): Promise {
- this.logger.debug(
- { jobs: jobs.length },
- "processing extended streaming history listens",
- );
- const importListens = await this.importListenRepository.findBy(
- jobs.map((job) => ({ id: job.data.id })),
- );
-
- const listensWithoutTracks = importListens.filter(
- (importListen) => !importListen.track,
- );
- if (listensWithoutTracks.length > 0) {
- const missingTrackIDs = uniq(
- listensWithoutTracks.map((importListen) =>
- importListen.spotifyTrackUri.replace("spotify:track:", ""),
- ),
- );
-
- const tracks = await this.spotifyService.importTracks(missingTrackIDs);
-
- listensWithoutTracks.forEach((listen) => {
- listen.track = tracks.find(
- (track) => listen.spotifyTrackUri === track.spotify.uri,
- );
- if (!listen.track) {
- this.logger.warn(
- { listen },
- "could not find track for extended streaming history listen",
- );
- throw new Error(
- `could not find track for extended streaming history listen`,
- );
- }
- });
-
- // Using upsert instead of save to only do a single query
- await this.importListenRepository.upsert(listensWithoutTracks, ["id"]);
- }
-
- const listensWithoutListen = importListens.filter(
- (importListen) => !importListen.listen,
- );
- if (listensWithoutListen.length > 0) {
- const listens = await this.listensService.createListens(
- listensWithoutListen.map((listen) => ({
- user: listen.user,
- track: listen.track,
- playedAt: listen.playedAt,
- })),
- );
-
- listensWithoutListen.forEach((importListen) => {
- importListen.listen = listens.find(
- (listen) =>
- importListen.user.id === listen.user.id &&
- importListen.track.id === listen.track.id &&
- importListen.playedAt.getTime() === listen.playedAt.getTime(),
- );
- if (!importListen.listen) {
- this.logger.warn(
- { listen: importListen, listens: listens },
- "could not find listen for extended streaming history listen",
- );
- throw new Error(
- `could not find listen for extended streaming history listen`,
- );
- }
- });
-
- // Using upsert instead of save to only do a single query
- await this.importListenRepository.upsert(listensWithoutListen, ["id"]);
- }
- }
-
- @Span()
- async getExtendedStreamingHistoryStatus(
- user: User,
- ): Promise {
- const qb = this.importListenRepository
- .createQueryBuilder("listen")
- .where("listen.userId = :user", { user: user.id });
-
- const [total, imported] = await Promise.all([
- qb.clone().getCount(),
- qb.clone().andWhere("listen.listenId IS NOT NULL").getCount(),
- ]);
-
- return { total, imported };
- }
-}
diff --git a/src/sources/spotify/import-extended-streaming-history/index.ts b/src/sources/spotify/import-extended-streaming-history/index.ts
deleted file mode 100644
index b5db823..0000000
--- a/src/sources/spotify/import-extended-streaming-history/index.ts
+++ /dev/null
@@ -1,4 +0,0 @@
-export { ImportController } from "./import.controller";
-export { ImportService } from "./import.service";
-export { ProcessSpotifyExtendedStreamingHistoryListenJob } from "./jobs";
-export { SpotifyExtendedStreamingHistoryListenRepository } from "./listen.repository";
diff --git a/src/sources/spotify/import-extended-streaming-history/jobs.ts b/src/sources/spotify/import-extended-streaming-history/jobs.ts
deleted file mode 100644
index 38c55c8..0000000
--- a/src/sources/spotify/import-extended-streaming-history/jobs.ts
+++ /dev/null
@@ -1,7 +0,0 @@
-import { createJob } from "@apricote/nest-pg-boss";
-
-export type IProcessSpotifyExtendedStreamingHistoryListenJob = { id: string };
-export const ProcessSpotifyExtendedStreamingHistoryListenJob =
- createJob(
- "process-spotify-extended-streaming-history-listen",
- );
diff --git a/src/sources/spotify/import-extended-streaming-history/listen.entity.ts b/src/sources/spotify/import-extended-streaming-history/listen.entity.ts
deleted file mode 100644
index 4fa8982..0000000
--- a/src/sources/spotify/import-extended-streaming-history/listen.entity.ts
+++ /dev/null
@@ -1,25 +0,0 @@
-import { Column, Entity, ManyToOne, PrimaryGeneratedColumn } from "typeorm";
-import { Track } from "../../../music-library/track.entity";
-import { User } from "../../../users/user.entity";
-import { Listen } from "../../../listens/listen.entity";
-
-@Entity({ name: "spotify_extended_streaming_history_listen" })
-export class SpotifyExtendedStreamingHistoryListen {
- @PrimaryGeneratedColumn("uuid")
- id: string;
-
- @ManyToOne(() => User, { eager: true })
- user: User;
-
- @Column({ type: "timestamp" })
- playedAt: Date;
-
- @Column()
- spotifyTrackUri: string;
-
- @ManyToOne(() => Track, { nullable: true, eager: true })
- track?: Track;
-
- @ManyToOne(() => Listen, { nullable: true, eager: true })
- listen?: Listen;
-}
diff --git a/src/sources/spotify/import-extended-streaming-history/listen.repository.ts b/src/sources/spotify/import-extended-streaming-history/listen.repository.ts
deleted file mode 100644
index 7c40359..0000000
--- a/src/sources/spotify/import-extended-streaming-history/listen.repository.ts
+++ /dev/null
@@ -1,6 +0,0 @@
-import { Repository } from "typeorm";
-import { EntityRepository } from "../../../database/entity-repository";
-import { SpotifyExtendedStreamingHistoryListen } from "./listen.entity";
-
-@EntityRepository(SpotifyExtendedStreamingHistoryListen)
-export class SpotifyExtendedStreamingHistoryListenRepository extends Repository {}
diff --git a/src/sources/spotify/spotify.module.ts b/src/sources/spotify/spotify.module.ts
index 021ffec..1df6bbf 100644
--- a/src/sources/spotify/spotify.module.ts
+++ b/src/sources/spotify/spotify.module.ts
@@ -1,37 +1,25 @@
import { PGBossModule } from "@apricote/nest-pg-boss";
import { Module } from "@nestjs/common";
-import { TypeOrmRepositoryModule } from "../../database/entity-repository/typeorm-repository.module";
import { ListensModule } from "../../listens/listens.module";
import { MusicLibraryModule } from "../../music-library/music-library.module";
import { UsersModule } from "../../users/users.module";
import { ImportSpotifyJob } from "../jobs";
-import {
- ImportController,
- ImportService,
- ProcessSpotifyExtendedStreamingHistoryListenJob,
- SpotifyExtendedStreamingHistoryListenRepository,
-} from "./import-extended-streaming-history";
import { SpotifyApiModule } from "./spotify-api/spotify-api.module";
import { SpotifyAuthModule } from "./spotify-auth/spotify-auth.module";
import { SpotifyService } from "./spotify.service";
@Module({
imports: [
- PGBossModule.forJobs([
- ImportSpotifyJob,
- ProcessSpotifyExtendedStreamingHistoryListenJob,
- ]),
- TypeOrmRepositoryModule.for([
- SpotifyExtendedStreamingHistoryListenRepository,
- ]),
+ PGBossModule.forJobs([ImportSpotifyJob]),
UsersModule,
ListensModule,
MusicLibraryModule,
SpotifyApiModule,
SpotifyAuthModule,
],
- providers: [SpotifyService, ImportService],
- controllers: [ImportController],
+ providers: [SpotifyService],
exports: [SpotifyService],
})
-export class SpotifyModule {}
+export class SpotifyModule {
+ constructor(private readonly spotifyService: SpotifyService) {}
+}
diff --git a/tsconfig.json b/tsconfig.json
index 78c9382..cce4f57 100644
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -5,12 +5,13 @@
"removeComments": true,
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
- "target": "ES2022",
+ "target": "ES2020",
"sourceMap": true,
"outDir": "./dist",
"baseUrl": "./",
"incremental": true,
"paths": {},
+ "lib": ["ES2020"]
},
"exclude": ["node_modules", "dist"],
"include": ["src", "test"]