-
Notifications
You must be signed in to change notification settings - Fork 29
feat(Melon): Implement Melon provider #234
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| export interface MelonAlbumInfoResponse { | ||
| response: { | ||
| ALBUMINFO: MelonAlbumInfo; | ||
| /** The type of the album, e.g. "정규", "EP", "싱글" */ | ||
| ALBUMTYPE?: string; | ||
| /** The agency/label name (기획사) */ | ||
| PLANCNPY?: string; | ||
| /** The distributor/publisher (발매사) */ | ||
| SELLCNPY?: string; | ||
| }; | ||
| } | ||
|
|
||
| export interface MelonAlbumInfo { | ||
| ALBUMID: string; | ||
| ALBUMNAME: string; | ||
| /** The album's release date, formatted as YYYY.MM.DD */ | ||
| ISSUEDATE: string; | ||
| /** 500px thumbnail CDN URL with query string */ | ||
| ALBUMIMG: string; | ||
| /** 1000px CDN URL with query string */ | ||
| ALBUMIMGLARGE: string; | ||
| ISSERVICE: boolean; | ||
| ARTISTLIST: MelonArtist[]; | ||
| } | ||
|
|
||
| export interface MelonArtist { | ||
| ARTISTID: string; | ||
| ARTISTNAME: string; | ||
| } | ||
|
|
||
| export interface MelonSongListResponse { | ||
| response: { | ||
| CDLIST: MelonDisc[]; | ||
| }; | ||
| } | ||
|
|
||
| export interface MelonDisc { | ||
| /** Disc number as a string, e.g. "1" */ | ||
| CDNO: string; | ||
| SONGLIST: MelonSong[]; | ||
| } | ||
|
|
||
| export interface MelonSong { | ||
| SONGID: string; | ||
| SONGNAME: string; | ||
| ARTISTLIST: MelonArtist[]; | ||
| /** Duration in seconds as a string */ | ||
| PLAYTIME: string; | ||
| /** Track number as a string */ | ||
| TRACKNO: string; | ||
| ISSERVICE: boolean; | ||
| /** Whether the song is marked as a title song, which is usually actively promoted, has an MV, and/or was previously released as a single. */ | ||
| ISTITLESONG: boolean; | ||
| ISHOLDBACK: boolean; | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| import type { ReleaseOptions } from '@/harmonizer/types.ts'; | ||
| import { describeProvider, makeProviderOptions } from '@/providers/test_spec.ts'; | ||
| import { stubProviderLookups } from '@/providers/test_stubs.ts'; | ||
| import { afterAll, describe } from '@std/testing/bdd'; | ||
| import { assertSnapshot } from '@std/testing/snapshot'; | ||
| import { assert, assertEquals } from 'std/assert/mod.ts'; | ||
|
|
||
| import MelonProvider from './mod.ts'; | ||
|
|
||
| describe('Melon provider', () => { | ||
| const melon = new MelonProvider(makeProviderOptions()); | ||
| const stubs = [stubProviderLookups(melon)]; | ||
|
|
||
| const releaseOptions: ReleaseOptions = { | ||
| withISRC: false, | ||
| withAllTrackArtists: true, | ||
| }; | ||
|
|
||
| describeProvider(melon, { | ||
| urls: [{ | ||
| description: 'album page', | ||
| url: new URL('https://www.melon.com/album/detail.htm?albumId=13693526'), | ||
| id: { type: 'album', id: '13693526' }, | ||
| isCanonical: true, | ||
| }, { | ||
| description: 'artist page', | ||
| url: new URL('https://www.melon.com/artist/detail.htm?artistId=1273010'), | ||
| id: { type: 'artist', id: '1273010' }, | ||
| isCanonical: true, | ||
| }, { | ||
| description: 'song page', | ||
| url: new URL('https://www.melon.com/song/detail.htm?songId=602111505'), | ||
| id: { type: 'song', id: '602111505' }, | ||
| isCanonical: true, | ||
| }, { | ||
| description: 'search page (unsupported)', | ||
| url: new URL('https://www.melon.com/search/total/index.htm?q=test'), | ||
| id: undefined, | ||
| }], | ||
| invalidIds: ['abc', 'not-a-number', '123abc'], | ||
| releaseLookup: [{ | ||
| description: 'Regular (정규) album with unavailable CD-only tracks', | ||
| release: new URL('https://www.melon.com/album/detail.htm?albumId=13691104'), | ||
| options: releaseOptions, | ||
| assert: async (release, ctx) => { | ||
| await assertSnapshot(ctx, release); | ||
| assertEquals(release.media.length, 1, 'Should have one disc'); | ||
| assertEquals(release.media[0].tracklist.length, 11, 'Should have 11 tracks (without the two CD-only tracks)'); | ||
| assert(release.types?.includes('Album'), 'Should be classified as Album'); | ||
| }, | ||
| }, { | ||
| description: 'Regular (정규) album with multiple discs and multiple artists', | ||
| release: new URL('https://www.melon.com/album/detail.htm?albumId=10000727'), | ||
| options: releaseOptions, | ||
| assert: async (release, ctx) => { | ||
| await assertSnapshot(ctx, release); | ||
| assertEquals(release.media.length, 2, 'Should have two discs'); | ||
| assertEquals(release.media[0].tracklist.length, 10, 'Disc 1 should have 10 tracks'); | ||
| assertEquals(release.media[1].tracklist.length, 7, 'Disc 2 should have 7 tracks'); | ||
| assert(release.types?.includes('Album'), 'Should be classified as Album'); | ||
| const lastTrack = release.media[1].tracklist.at(-1)!; | ||
| assertEquals( | ||
| lastTrack.artists?.map((a) => a.name), | ||
| ['웨이 (크레용팝)', '초아 (크레용팝)'], | ||
| 'Track with multiple artists should have correct artists', | ||
| ); | ||
| }, | ||
| }, { | ||
| description: 'EP (미니) album', | ||
| release: new URL('https://www.melon.com/album/detail.htm?albumId=13693526'), | ||
| options: releaseOptions, | ||
| assert: async (release, ctx) => { | ||
| await assertSnapshot(ctx, release); | ||
| assertEquals(release.media.length, 1, 'Should have one disc'); | ||
| assertEquals(release.media[0].tracklist.length, 7, 'Should have 7 tracks'); | ||
| assert(release.types?.includes('EP'), 'Should be classified as EP'); | ||
| }, | ||
| }, { | ||
| description: 'Single (싱글) album', | ||
| release: new URL('https://www.melon.com/album/detail.htm?albumId=13732020'), | ||
| options: releaseOptions, | ||
| assert: async (release, ctx) => { | ||
| await assertSnapshot(ctx, release); | ||
| assertEquals(release.media.length, 1, 'Should have one disc'); | ||
| assertEquals(release.media[0].tracklist.length, 1, 'Should have 1 track'); | ||
| assert(release.types?.includes('Single'), 'Should be classified as Single'); | ||
| }, | ||
| }], | ||
| }); | ||
|
|
||
| afterAll(() => { | ||
| stubs.forEach((s) => s.restore()); | ||
| }); | ||
| }); |
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
| @@ -0,0 +1,231 @@ | ||||||
| import { type ApiQueryOptions, type CacheEntry, MetadataApiProvider, ReleaseApiLookup } from '@/providers/base.ts'; | ||||||
| import { DurationPrecision, FeatureQuality, type FeatureQualityMap } from '@/providers/features.ts'; | ||||||
| import type { ProviderCategory } from '@/providers/categories.ts'; | ||||||
| import type { PartialDate } from '@/utils/date.ts'; | ||||||
| import { ProviderError } from '@/utils/errors.ts'; | ||||||
| import type { | ||||||
| ArtistCreditName, | ||||||
| Artwork, | ||||||
| EntityId, | ||||||
| HarmonyMedium, | ||||||
| HarmonyRelease, | ||||||
| HarmonyTrack, | ||||||
| Label, | ||||||
| LinkType, | ||||||
| ReleaseGroupType, | ||||||
| } from '@/harmonizer/types.ts'; | ||||||
| import type { | ||||||
| MelonAlbumInfo, | ||||||
| MelonAlbumInfoResponse, | ||||||
| MelonArtist, | ||||||
| MelonDisc, | ||||||
| MelonSong, | ||||||
| MelonSongListResponse, | ||||||
| } from './api_types.ts'; | ||||||
|
|
||||||
| const API_BASE = 'https://m2.melon.com/m6'; | ||||||
|
Owner
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
allows us to use the two argument URL constructor. |
||||||
|
|
||||||
| export default class MelonProvider extends MetadataApiProvider { | ||||||
| readonly name = 'Melon'; | ||||||
|
|
||||||
| readonly supportedUrls = new URLPattern({ | ||||||
| hostname: 'www.melon.com', | ||||||
| pathname: String.raw`/:type(album|song|artist)/detail.htm\?\1Id=:id(\d+)`, | ||||||
|
Owner
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Interesting, I wouldn't have thought that this works, the search query is a separate property of the URLPattern usually. I have an unpublished provider which extracts the ID from the query that way, but this surely wouldn't allow us to have a regex back reference. |
||||||
| }); | ||||||
|
|
||||||
| override readonly categories = new Set<ProviderCategory>(['digital']); | ||||||
|
|
||||||
| override readonly features: FeatureQualityMap = { | ||||||
| 'cover size': 2000, | ||||||
| 'duration precision': DurationPrecision.SECONDS, | ||||||
| 'GTIN lookup': FeatureQuality.MISSING, | ||||||
| 'MBID resolving': FeatureQuality.PRESENT, | ||||||
| 'release label': FeatureQuality.PRESENT, | ||||||
| }; | ||||||
|
|
||||||
| readonly entityTypeMap = { | ||||||
| artist: 'artist', | ||||||
| release: 'album', | ||||||
| recording: 'song', | ||||||
| }; | ||||||
|
|
||||||
| override readonly availableRegions = new Set(['KR']); | ||||||
|
|
||||||
| readonly releaseLookup = MelonReleaseLookup; | ||||||
|
|
||||||
| override readonly launchDate: PartialDate = { | ||||||
| year: 2004, | ||||||
| month: 11, | ||||||
| }; | ||||||
|
|
||||||
| override extractEntityFromUrl(url: URL): EntityId | undefined { | ||||||
| if (!this.supportsDomain(url)) return undefined; | ||||||
|
|
||||||
| const albumId = url.searchParams.get('albumId'); | ||||||
| if (albumId && url.pathname.startsWith('/album/')) { | ||||||
| return { type: 'album', id: albumId }; | ||||||
| } | ||||||
|
|
||||||
| const artistId = url.searchParams.get('artistId'); | ||||||
| if (artistId && url.pathname.startsWith('/artist/')) { | ||||||
| return { type: 'artist', id: artistId }; | ||||||
| } | ||||||
|
|
||||||
| const songId = url.searchParams.get('songId'); | ||||||
| if (songId && url.pathname.startsWith('/song/')) { | ||||||
| return { type: 'song', id: songId }; | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| constructUrl(entity: EntityId): URL { | ||||||
| return new URL(`https://www.melon.com/${entity.type}/detail.htm?${entity.type}Id=${entity.id}`); | ||||||
| } | ||||||
|
|
||||||
| override getLinkTypesForEntity(): LinkType[] { | ||||||
| return ['paid streaming', 'paid download']; | ||||||
| } | ||||||
|
|
||||||
| query<Data>(apiUrl: URL, options?: ApiQueryOptions): Promise<CacheEntry<Data>> { | ||||||
| return this.fetchJSON<Data>(apiUrl, { | ||||||
| policy: { maxTimestamp: options?.snapshotMaxTimestamp }, | ||||||
| }); | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| export class MelonReleaseLookup extends ReleaseApiLookup<MelonProvider, MelonRawRelease> { | ||||||
| constructReleaseApiUrl(): URL { | ||||||
| const url = new URL(`${API_BASE}/v3/album/info.json`); | ||||||
|
Owner
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
| url.searchParams.set('albumId', this.lookup.value); | ||||||
| return url; | ||||||
| } | ||||||
|
|
||||||
| protected async getRawRelease(): Promise<MelonRawRelease> { | ||||||
| if (this.lookup.method === 'gtin') { | ||||||
| throw new ProviderError(this.provider.name, 'GTIN lookups are not supported'); | ||||||
| } | ||||||
|
|
||||||
| const albumId = this.lookup.value; | ||||||
|
|
||||||
| const infoUrl = this.constructReleaseApiUrl(); | ||||||
| const songListUrl = new URL(`${API_BASE}/v2/album/song/list.json`); | ||||||
|
Owner
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
| songListUrl.searchParams.set('albumId', albumId); | ||||||
|
|
||||||
| const snapshotMaxTimestamp = this.options.snapshotMaxTimestamp; | ||||||
| const [infoEntry, songListEntry] = await Promise.all([ | ||||||
| this.provider.query<MelonAlbumInfoResponse>(infoUrl, { snapshotMaxTimestamp }), | ||||||
| this.provider.query<MelonSongListResponse>(songListUrl, { snapshotMaxTimestamp }), | ||||||
| ]); | ||||||
|
|
||||||
| this.updateCacheTime(infoEntry.timestamp); | ||||||
| this.updateCacheTime(songListEntry.timestamp); | ||||||
|
|
||||||
| const albumInfo = infoEntry.content.response.ALBUMINFO; | ||||||
| const albumType = infoEntry.content.response.ALBUMTYPE; | ||||||
| const planCnpy = infoEntry.content.response.PLANCNPY; | ||||||
| const cdList = songListEntry.content.response.CDLIST; | ||||||
|
|
||||||
| return { | ||||||
| albumInfo, | ||||||
| albumType, | ||||||
| planCnpy, | ||||||
| cdList, | ||||||
| }; | ||||||
| } | ||||||
|
|
||||||
| protected convertRawRelease(raw: MelonRawRelease): HarmonyRelease { | ||||||
| this.entity = { type: 'album', id: raw.albumInfo.ALBUMID }; | ||||||
|
|
||||||
| return { | ||||||
| title: raw.albumInfo.ALBUMNAME, | ||||||
| artists: raw.albumInfo.ARTISTLIST.map((a) => this.convertRawArtist(a)), | ||||||
| releaseDate: this.convertReleaseDate(parseISSUEDATE(raw.albumInfo.ISSUEDATE)), | ||||||
| labels: this.extractLabels(raw.planCnpy), | ||||||
| images: [this.coverArtwork(raw.albumInfo.ALBUMIMGLARGE)], | ||||||
| types: mapReleaseType(raw.albumType), | ||||||
| availableIn: ['KR'], | ||||||
| status: 'Official', | ||||||
| packaging: 'None', | ||||||
| media: this.buildMedia(raw.cdList), | ||||||
| // TODO: check if we can check for download availability | ||||||
|
Owner
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I see you already had the same question as me 😀 |
||||||
| externalLinks: raw.albumInfo.ISSERVICE | ||||||
| ? [{ url: this.provider.constructUrl(this.entity).toString(), types: ['paid streaming'] }] | ||||||
| : [], | ||||||
| info: this.generateReleaseInfo(), | ||||||
| }; | ||||||
| } | ||||||
|
|
||||||
| private convertRawArtist(artist: MelonArtist): ArtistCreditName { | ||||||
| return { | ||||||
| name: artist.ARTISTNAME, | ||||||
| creditedName: artist.ARTISTNAME, | ||||||
| externalIds: this.provider.makeExternalIds({ type: 'artist', id: artist.ARTISTID }), | ||||||
| }; | ||||||
| } | ||||||
|
|
||||||
| private extractLabels(planCnpy?: string): Label[] { | ||||||
| if (!planCnpy) return []; | ||||||
| return planCnpy.split(', ').map((name) => ({ name: name.trim() })); | ||||||
|
Owner
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We already have a utility function |
||||||
| } | ||||||
|
|
||||||
| private coverArtwork(largeUrl: string): Artwork { | ||||||
| const originalUrl = largeUrl.replace( | ||||||
| /(images\/.*\/[^/_]*)((_[^/.]*)_)?(_?[^/._]*)?(\.[^/.?]*)(?:[?/].*)?$/, | ||||||
| '$1$3_org$5', | ||||||
| ); | ||||||
| return { | ||||||
| url: originalUrl, | ||||||
| thumbUrl: largeUrl, | ||||||
|
Owner
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I would consider using the 500px image for the thumbnail, most of the other provider thumbnails are even only half of that.. |
||||||
| types: ['front'], | ||||||
| }; | ||||||
| } | ||||||
|
|
||||||
| private buildMedia(cdList: MelonDisc[]): HarmonyMedium[] { | ||||||
| return cdList.map((disc) => ({ | ||||||
| number: parseInt(disc.CDNO), | ||||||
| format: 'Digital Media', | ||||||
| tracklist: disc.SONGLIST | ||||||
| .filter((song) => song.ISSERVICE) | ||||||
| .map((song) => this.convertRawTrack(song)), | ||||||
| })); | ||||||
| } | ||||||
|
|
||||||
| private convertRawTrack(song: MelonSong): HarmonyTrack { | ||||||
| return { | ||||||
| title: song.SONGNAME, | ||||||
| artists: song.ARTISTLIST.map((a) => this.convertRawArtist(a)), | ||||||
| number: parseInt(song.TRACKNO), | ||||||
| length: parseInt(song.PLAYTIME) * 1000, | ||||||
| recording: { | ||||||
| externalIds: this.provider.makeExternalIds({ type: 'song', id: song.SONGID }), | ||||||
| }, | ||||||
| }; | ||||||
| } | ||||||
| } | ||||||
|
|
||||||
| interface MelonRawRelease { | ||||||
| albumInfo: MelonAlbumInfo; | ||||||
| albumType?: string; | ||||||
| planCnpy?: string; | ||||||
| cdList: MelonDisc[]; | ||||||
| } | ||||||
|
|
||||||
| function parseISSUEDATE(date: string): PartialDate { | ||||||
|
Owner
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. We should revisit this helper as part of #169 one day 😇 |
||||||
| const match = date.match(/^(\d{4})\.(\d{2})\.(\d{2})$/); | ||||||
| if (!match) return {}; | ||||||
| return { | ||||||
| year: parseInt(match[1]), | ||||||
| month: parseInt(match[2]), | ||||||
| day: parseInt(match[3]), | ||||||
| }; | ||||||
| } | ||||||
|
|
||||||
| function mapReleaseType(albumType?: string): ReleaseGroupType[] | undefined { | ||||||
| if (!albumType) return undefined; | ||||||
| if (albumType === '싱글') return ['Single']; | ||||||
| if (albumType === '정규') return ['Album']; | ||||||
| if (albumType === 'EP') return ['EP']; | ||||||
| if (albumType === '옴니버스') return ['Compilation']; | ||||||
| if (albumType === 'OST') return ['Soundtrack']; | ||||||
| if (albumType === '리믹스') return ['Single', 'Remix']; | ||||||
|
Owner
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Could you confirm the theory that this is only used for remix singles and not remix albums? |
||||||
| return undefined; | ||||||
| } | ||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Have you seen examples where ISSERVICE and ISHOLDBACK indicate unavailable tracks/releases or something else which is of interest?
There is also an ISFREE attribute in the testdata which you haven't used here.