Multiple Lyrics Providers (spotDL#1397)

Co-authored-by: Silverarmor <[email protected]>
Co-authored-by: Jakub Kot <[email protected]>

Author: 3 people

spotdl/console/__init__.py

@@ -83,6 +83,7 @@ def graceful_exit(signal, frame):
             arguments.output_format,
             arguments.use_youtube,
             arguments.generate_m3u,
+            arguments.lyrics_provider,
             arguments.search_threads,
         )
 

spotdl/download/embed_metadata.py

@@ -153,6 +153,9 @@ def _embed_mp3_cover(audio_file, song_object, converted_file_path):
 def _embed_mp3_lyrics(audio_file, song_object):
     # ! setting the lyrics
     lyrics = song_object.lyrics
+    if not lyrics:
+        return audio_file
+
     USLTOutput = USLT(encoding=3, lang=u"eng", desc=u"desc", text=lyrics)
     audio_file["USLT::'eng'"] = USLTOutput
 

spotdl/parsers/argument_parser.py

@@ -113,6 +113,15 @@ def parse_arguments():
         "--use-youtube", help="Use youtube instead of YTM", action="store_true"
     )
 
+    # Option to select a lyrics provider
+    parser.add_argument(
+        "--lyrics-provider",
+        help="Select a lyrics provider",
+        type=str,
+        choices=["genius", "musixmatch"],
+        default="musixmatch",
+    )
+
     # Option to specify path to local ffmpeg
     parser.add_argument("-f", "--ffmpeg", help="Path to ffmpeg", dest="ffmpeg")
 

spotdl/parsers/query_parser.py

@@ -2,11 +2,11 @@
 from pathlib import Path
 
 from spotdl.search import SongObject, song_gatherer
-from spotdl.providers import provider_utils, metadata_provider
+from spotdl.providers import lyrics_providers, metadata_provider
 
 
 def parse_query(
-    query: List[str], format, use_youtube, generate_m3u, threads
+    query: List[str], format, use_youtube, generate_m3u, lyrics_provider, threads
 ) -> List[SongObject]:
     """
     Parse query and return list containing song object
@@ -20,7 +20,9 @@ def parse_query(
             continue
 
         songs_list.extend(
-            parse_request(request, format, use_youtube, generate_m3u, threads)
+            parse_request(
+                request, format, use_youtube, generate_m3u, lyrics_provider, threads
+            )
         )
 
         # linefeed to visually separate output for each query
@@ -42,6 +44,7 @@ def parse_request(
     output_format: str = None,
     use_youtube: bool = False,
     generate_m3u: bool = False,
+    lyrics_provider: str = None,
     threads: int = 1,
 ) -> List[SongObject]:
     song_list: List[SongObject] = []
@@ -59,39 +62,47 @@ def parse_request(
             print("Fetching YouTube video with spotify metadata")
             song_list = [
                 song
-                for song in [get_youtube_meta_track(urls[0], urls[1], output_format)]
+                for song in [
+                    get_youtube_meta_track(
+                        urls[0], urls[1], output_format, lyrics_provider
+                    )
+                ]
                 if song is not None
             ]
     elif "open.spotify.com" in request and "track" in request:
         print("Fetching Song...")
-        song = song_gatherer.from_spotify_url(request, output_format, use_youtube)
+        song = song_gatherer.from_spotify_url(
+            request, output_format, use_youtube, lyrics_provider
+        )
         try:
             song_list = [song] if song.youtube_link is not None else []
         except (OSError, ValueError, LookupError):
             song_list = []
     elif "open.spotify.com" in request and "album" in request:
         print("Fetching Album...")
         song_list = song_gatherer.from_album(
-            request, output_format, use_youtube, generate_m3u, threads
+            request, output_format, use_youtube, lyrics_provider, generate_m3u, threads
         )
     elif "open.spotify.com" in request and "playlist" in request:
         print("Fetching Playlist...")
         song_list = song_gatherer.from_playlist(
-            request, output_format, use_youtube, generate_m3u, threads
+            request, output_format, use_youtube, lyrics_provider, generate_m3u, threads
         )
     elif "open.spotify.com" in request and "artist" in request:
         print("Fetching artist...")
         song_list = song_gatherer.from_artist(
-            request, output_format, use_youtube, threads
+            request, output_format, use_youtube, lyrics_provider, threads
         )
     elif request == "saved":
         print("Fetching Saved Songs...")
-        song_list = song_gatherer.from_saved_tracks(output_format, use_youtube, threads)
+        song_list = song_gatherer.from_saved_tracks(
+            output_format, use_youtube, lyrics_provider, threads
+        )
     else:
         print('Searching Spotify for song named "%s"...' % request)
         try:
             song_list = song_gatherer.from_search_term(
-                request, output_format, use_youtube
+                request, output_format, use_youtube, lyrics_provider
             )
         except Exception as e:
             print(e)
@@ -103,7 +114,10 @@ def parse_request(
 
 
 def get_youtube_meta_track(
-    youtube_url: str, spotify_url: str, output_format: str = None
+    youtube_url: str,
+    spotify_url: str,
+    output_format: str = None,
+    lyrics_provider: str = None,
 ):
     # check if URL is a playlist, user, artist or album, if yes raise an Exception,
     # else procede
@@ -129,8 +143,11 @@ def get_youtube_meta_track(
         print(f'Skipping "{converted_file_name}" as it\'s already downloaded')
         return None
 
-    # (try to) Get lyrics from Genius
-    lyrics = provider_utils._get_song_lyrics(song_name, contributing_artist)
+    # (try to) Get lyrics from musixmatch/genius
+    if lyrics_provider == "genius":
+        lyrics = lyrics_providers.get_lyrics_genius(song_name, contributing_artist)
+    elif lyrics_provider == "musixmatch":
+        lyrics = lyrics_providers.get_lyrics_musixmatch(song_name, contributing_artist)
 
     return SongObject(
         raw_track_meta, raw_album_meta, raw_artist_meta, youtube_url, lyrics

spotdl/providers/__init__.py

@@ -1,8 +1,8 @@
 from spotdl.providers.metadata_provider import from_url
 from spotdl.providers import ytm_provider as youtube_music
+from spotdl.providers import lyrics_providers
 from spotdl.providers.provider_utils import (
     _create_song_title,
-    _get_song_lyrics,
     _match_percentage,
     _parse_duration,
 )

spotdl/providers/lyrics_providers.py

@@ -0,0 +1,115 @@
+from requests import get
+from bs4 import BeautifulSoup
+from typing import List
+from urllib.parse import quote
+
+user_agent = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \
+(KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36"
+}
+
+
+def get_lyrics_genius(song_name: str, song_artists: List[str]) -> str:
+    """
+    `str` `song_name` : name of song
+    `list<str>` `song_artists` : list containing name of contributing artists
+    RETURNS `str`: Lyrics of the song.
+    Gets the lyrics of the song.
+    """
+    headers = {
+        "Authorization": "Bearer alXXDbPZtK1m2RrZ8I4k2Hn8Ahsd0Gh_o076HYvcdlBvmc0ULL1H8Z8xRlew5qaG",
+    }
+    headers.update(user_agent)
+
+    api_search_url = "https://api.genius.com/search"
+    artist_str = ", ".join(
+        artist for artist in song_artists if artist.lower() not in song_name.lower()
+    )
+    search_query = f"{song_name} {artist_str}"
+
+    api_response = get(api_search_url, params={"q": search_query}, headers=headers)
+    if not api_response.ok:
+        return ""
+    api_json = api_response.json()
+
+    try:
+        song_id = api_json["response"]["hits"][0]["result"]["id"]
+    except (IndexError, KeyError):
+        return ""
+
+    song_api_url = f"https://api.genius.com/songs/{song_id}"
+    api_response = get(song_api_url, headers=headers)
+    if not api_response.ok:
+        return ""
+    api_json = api_response.json()
+
+    song_url = api_json["response"]["song"]["url"]
+    genius_page = get(song_url, headers=user_agent)
+    if not genius_page.ok:
+        return ""
+
+    soup = BeautifulSoup(genius_page.text.replace("<br/>", "\n"), "html.parser")
+    lyrics_div = soup.select_one("div.lyrics")
+
+    if lyrics_div is not None:
+        return lyrics_div.get_text().strip()
+
+    lyrics_containers = soup.select("div[class^=Lyrics__Container]")
+    lyrics = "\n".join(con.get_text() for con in lyrics_containers)
+    return lyrics.strip()
+
+
+def get_lyrics_musixmatch(
+    song_name: str, song_artists: List[str], track_search=False
+) -> str:
+    """
+    `str` `song_name` : Name of song
+    `list<str>` `song_artists` : list containing name of contributing artists
+    `bool` `track_search` : if `True`, search the musixmatch tracks page.
+    RETURNS `str`: Lyrics of the song.
+    Gets the lyrics of the song.
+    """
+    # remove artist names that are already in the song_name
+    # we do not use SongObject.create_file_name beacause it
+    # removes '/' etc from the artist and song names.
+    artists_str = ", ".join(
+        artist for artist in song_artists if artist.lower() not in song_name.lower()
+    )
+
+    # quote the query so that it's safe to use in a url
+    # e.g "Au/Ra" -> "Au%2FRa"
+    query = quote(f"{song_name} - {artists_str}", safe="")
+
+    # search the `tracks page` if track_search is True
+    if track_search:
+        query += "/tracks"
+
+    search_url = f"https://www.musixmatch.com/search/{query}"
+    search_resp = get(search_url, headers=user_agent)
+    if not search_resp.ok:
+        return ""
+
+    search_soup = BeautifulSoup(search_resp.text, "html.parser")
+    song_url_tag = search_soup.select_one("a[href^='/lyrics/']")
+
+    # song_url_tag being None means no results were found on the
+    # All Results page, therefore, we use `track_search` to
+    # search the tracks page.
+    if song_url_tag is None:
+        # track_serach being True means we are already searching the tracks page.
+        if track_search:
+            return ""
+
+        lyrics = get_lyrics_musixmatch(song_name, song_artists, track_search=True)
+        return lyrics
+
+    song_url = "https://www.musixmatch.com" + song_url_tag.get("href")
+    lyrics_resp = get(song_url, headers=user_agent)
+    if not lyrics_resp.ok:
+        return ""
+
+    lyrics_soup = BeautifulSoup(lyrics_resp.text, "html.parser")
+    lyrics_paragraphs = lyrics_soup.select("p.mxm-lyrics__content")
+    lyrics = "\n".join(i.get_text() for i in lyrics_paragraphs)
+
+    return lyrics

spotdl/providers/provider_utils.py

@@ -1,9 +1,7 @@
-import requests
-
+from pathlib import Path
 from typing import List
+
 from thefuzz import fuzz
-from bs4 import BeautifulSoup
-from pathlib import Path
 
 
 def _match_percentage(str1: str, str2: str, score_cutoff: float = 0) -> float:
@@ -71,47 +69,6 @@ def _create_song_title(song_name: str, song_artists: List[str]) -> str:
     return f"{joined_artists} - {song_name}"
 
 
-def _get_song_lyrics(song_name: str, song_artists: List[str]) -> str:
-    """
-    `str` `song_name` : name of song
-
-    `list<str>` `song_artists` : list containing name of contributing artists
-
-    RETURNS `str`: Lyrics of the song.
-
-    Gets the metadata of the song.
-    """
-
-    headers = {
-        "Authorization": "Bearer alXXDbPZtK1m2RrZ8I4k2Hn8Ahsd0Gh_o076HYvcdlBvmc0ULL1H8Z8xRlew5qaG",
-    }
-    api_search_url = "https://api.genius.com/search"
-    search_query = f'{song_name} {", ".join(song_artists)}'
-
-    try:
-        api_response = requests.get(
-            api_search_url, params={"q": search_query}, headers=headers
-        ).json()
-
-        song_id = api_response["response"]["hits"][0]["result"]["id"]
-        song_api_url = f"https://api.genius.com/songs/{song_id}"
-
-        api_response = requests.get(song_api_url, headers=headers).json()
-
-        song_url = api_response["response"]["song"]["url"]
-
-        genius_page = requests.get(song_url)
-        soup = BeautifulSoup(genius_page.text, "html.parser")
-        lyrics_div = soup.select_one("div.lyrics")
-
-        if lyrics_div is not None:
-            return lyrics_div.get_text().strip()
-
-        return ""
-    except:  # noqa: E722
-        return ""
-
-
 def _sanitize_filename(input_str: str) -> str:
     output = input_str