Aprendiendo japonés con música: analizando las letras de Ichiko Aoba

# Using a fork of LyricsGenius with a bug fix: https://github.com/xathon/LyricsGenius
# pip install git+https://github.com/xathon/LyricsGenius.git
from collections import Counter
from pathlib import Path

import dango
import deepl
import lyricsgenius
import requests
from janome.tokenizer import Tokenizer
from wordcloud import WordCloud


token = "my_genius_api_token"  # Create an account and visit https://genius.com/api-clients
genius = lyricsgenius.Genius(token)

# Configuration.
genius.remove_section_headers = True
genius.excluded_terms = ["(English Translation)"]


def get_lyrics_filename(album_name):
    return f"lyrics_{album_name}.txt"


artist_name = "青葉市子 (Ichiko Aoba)"

# All her albums except a soundtrack (Amiko) and a field recording album (鮎川のしづく [Ayukawa no shizuku]).
albums = [
    "剃刀乙女 (Kamisori otome)",  # 2010
    "檻髪 (Origami)",  # 2011
    "うたびこ (Utabiko)",  # 2012
    "0",  # 2013
    "マホロボシヤ (Mahoroboshiya)",  # 2017
    "qp",  # 2018
    "アダンの風 (Windswept Adan)",  # 2020
]

for album in albums:
    filename = get_lyrics_filename(album)
    # Avoid re-downloading.
    if Path(filename).is_file():
        continue
    album = genius.search_album(album, artist_name)
    album.save_lyrics(extension="txt", sanitize=False, filename=filename)

def read_lyrics_from_file(filename):
    with open(filename, "r", encoding="utf-8") as file:
        album_lyrics = file.read()
    return album_lyrics


def analyse_lyrics(text):
    # docs: https://mocobeta.github.io/janome/api/janome.html#janome.tokenizer.Token
    # Each Token object has the following attributes:
    # - surface: the word as it appears in the text
    # - part_of_speech: the part of speech of the word, which can be a compound value like "動詞,自立,*,*"
    # - infl_type: the type of inflection of the word (e.g., "五段・ラ行" for a verb)
    # - infl_form: the form of inflection of the word (e.g., "連用形" for a verb in the continuous form)
    # - base_form: the word in its dictionary/base form (e.g., "行く" for the verb "行った")
    # - reading: the reading of the word in katakana
    # - phonetic: the phonetic representation of the word in katakana
    excluded_pos = [
        "助詞",  # particles
        "記号",  # symbols
        "助動詞",  # auxiliary verbs
        "接尾",  # suffixes
        "接頭詞",  # prefixes
        "非自立",  # dependent words
    ]
    words = Tokenizer().tokenize(text)
    tokens = [
        w.base_form
        for w in words
        if w.surface == clean_text(w.surface)
        and all(pos not in w.part_of_speech.split(",") for pos in excluded_pos)
    ]
    return Counter(tokens)


# Function to remove non-word characters (space, comma, newline…)
def clean_text(text):
    return "".join([c for c in text if c.isalpha()])


total_frequencies = Counter()
album_frequencies = {}

for album in albums:
    filename = get_lyrics_filename(album)
    album_lyrics = read_lyrics_from_file(filename)
    dictionary_form_counter = analyse_lyrics(album_lyrics, tokenizer="janome")
    album_frequencies[album] = dictionary_form_counter
    total_frequencies += dictionary_form_counter

def generate_wordcloud(
    counter, album_name, font="NotoSansJP-Regular", output_dir="img/wordclouds"
):
    width = 3000
    height = 3000

    wordcloud = WordCloud(
        font_path=font,
        background_color=None,
        mode="RGBA",
        margin=0,
        width=width,
        height=height,
        color_func=lambda *args, **kwargs: "black",
    ).generate_from_frequencies(counter)

    output_filename = f"{output_dir}/{album_name} WordCloud mask.svg"
    with open(output_filename, "w") as f:
        f.write(wordcloud.to_svg())


for album in albums:
    generate_wordcloud(album_frequencies[album], album)

# Overall cloud.
generate_wordcloud(
    counter=total_frequencies,
    album_name="total",
)

# It's translation time!
# Jisho provided too much context for these, or not the right meaning.
manual_overrides = {
    "ここ": "here",
    "そこ": "there",
    "いる": "to be",
    "マホロボシヤ": "Mahoroboshiya",
    "アダン": "Adan",
    "星": "star",
    "Venus": "Venus",
    "Earth": "Earth",
    "Mars": "Mars",
    "Jupiter": "Jupiter",
    "Saturnus": "Saturnus",
    "Uranus": "Uranus",
    "Neptunus": "Neptunus",
    "Mercurius": "Mercurius",
    "髪": "hair",
    "I": "I",
    "pod": "pod",
    "前": "before",
    "m": "am",
    "am": "am",
    "水": "water",
    "抱く": "to embrace",
    "手のひら": "palm",
    "踊る": "to dance",
    "降る": "to fall",
    "どれ": "which",
    "瞬き": "blink",
    "そば": "near",
    "交わす": "to exchange",
    "開ける": "to open",
    "眠れる": "to sleep",
}


def fetch_translation(word):
    print(f"Fetching translation for {word}…")
    if word in manual_overrides:
        print(f"Manual override: {word} = {manual_overrides[word]}")
        return manual_overrides[word]
    url = f"https://jisho.org/api/v1/search/words?keyword={word}"
    response = requests.get(url)
    if response.status_code == 200:
        data = response.json()
        if data["data"]:
            first_entry = data["data"][0]
            first_sense = first_entry["senses"][0]
            first_translation = first_sense["english_definitions"][0]
            print(f"Translation: {word} = {first_translation}")
            return first_translation
    print(f"Translation not found for {word}.")
    return None


def translate_counter(counter, translation_map):
    translated_counter = Counter()
    for word, frequency in counter.items():
        translation = translation_map.get(word)
        if translation:
            if translation in translated_counter:
                # Multiple words can have the same translation (e.g. "僕" & "私" = "I").
                translated_counter[translation] += frequency
            else:
                translated_counter[translation] = frequency
    return translated_counter


# Fetch translations for all words.
translation_map = {}
for word in total_frequencies.keys():
    translation = fetch_translation(word)
    if translation:
        translation_map[word] = translation

translated_total_frequencies = translate_counter(total_frequencies, translation_map)

translated_album_frequencies = {
    album: translate_counter(freq, translation_map)
    for album, freq in album_frequencies.items()
}

# Translated word clouds.
for album in albums:
    generate_wordcloud(
        counter=translated_album_frequencies[album],
        album_name=album + " translated",
        font="Georgia",
        output_dir="img/wordclouds/masks",
    )

generate_wordcloud(
    counter=translated_total_frequencies,
    album_name="total translated",
    output_dir="img/wordclouds/masks",
    font="Georgia",
)

# Note: I used the SVG masks to complete the word clouds with the album covers in Photoshop.
# I got the covers from https://ichikoaoba.com/discography/.

def translate_to_spanish(english_concepts):
    auth_key = "my_auth_key"
    translator = deepl.Translator(auth_key)
    spanish_translations = {}

    manual_overrides = {
        "you": "tú",
        "which": "cuál",
        "that": "ese",
        "who": "quién",
        "nonexistent": "inexistente",
        "to permit": "permitir",
        "to permit": "permitir",
        "to embrace": "abrazar",
    }

    for concept in english_concepts:
        if concept in manual_overrides:
            print(f"Skipping DeepL for {concept}, using {manual_overrides[concept]}.")
            spanish_translations[concept] = manual_overrides[word]
        else:
            print(f"Translating {concept}…")
            # Some words have context in parentheses. We use the entire input as context, but ask only to translate the word.
            result = translator.translate_text(
                concept,
                source_lang="EN",
                target_lang="ES",
                formality="prefer_less",
            )
            print(f"Translation: {word} = {result.text}")
            spanish_translations[word] = result.text
    return spanish_translations


spanish_translation_map = translate_to_spanish(translated_total_frequencies.keys())

translated_total_frequencies_es = translate_counter(
    translated_total_frequencies, spanish_translation_map
)

translated_album_frequencies_es = {
    album: translate_counter(freq, spanish_translation_map)
    for album, freq in translated_album_frequencies.items()
}

for album in album_frequencies.keys():
    generate_wordcloud(
        counter=translated_album_frequencies_es[album],
        album_name="(ES) " + album,
        output_dir="img/wordclouds/masks",
        font="Georgia",
    )

generate_wordcloud(
    counter=translated_total_frequencies_es,
    album_name="(ES) total",
    output_dir="img/wordclouds/masks",
    font="Georgia",
)