Overview#

This notebook gives a general overview of the features included in the dataset.

Hide imports 
%load_ext autoreload
%autoreload 2
import os

import dimcat as dc
import pandas as pd
import plotly.express as px
from dimcat import filters, plotting
from IPython.display import display

import utils

RESULTS_PATH = os.path.abspath(os.path.join(utils.OUTPUT_FOLDER, "overview"))
os.makedirs(RESULTS_PATH, exist_ok=True)


def make_output_path(
    filename: str,
    extension=None,
    path=RESULTS_PATH,
) -> str:
    return utils.make_output_path(filename=filename, extension=extension, path=path)


def save_figure_as(
    fig, filename, formats=("png", "pdf"), directory=RESULTS_PATH, **kwargs
):
    if formats is not None:
        for fmt in formats:
            plotting.write_image(fig, filename, directory, format=fmt, **kwargs)
    else:
        plotting.write_image(fig, filename, directory, **kwargs)

Loading data

D = utils.get_dataset("liszt_pelerinage", corpus_release="v2.3")
package = D.inputs.get_package()
package_info = package._package.custom
git_tag = package_info.get("git_tag")
utils.print_heading("Data and software versions")
print("Franz Liszt – Années de Pèlerinage version v2.3")
print(f"Datapackage '{package.package_name}' @ {git_tag}")
print(f"dimcat version {dc.__version__}\n")
D

Data and software versions
--------------------------

Franz Liszt – Années de Pèlerinage version v2.3
Datapackage 'liszt_pelerinage' @ v2.3
dimcat version 3.4.0

Dataset
=======
{'inputs': {'basepath': None,
            'packages': {'liszt_pelerinage': ["'liszt_pelerinage.measures' (MuseScoreFacetName.MuseScoreMeasures)",
                                              "'liszt_pelerinage.notes' (MuseScoreFacetName.MuseScoreNotes)",
                                              "'liszt_pelerinage.expanded' (MuseScoreFacetName.MuseScoreHarmonies)",
                                              "'liszt_pelerinage.chords' (MuseScoreFacetName.MuseScoreChords)",
                                              "'liszt_pelerinage.metadata' (FeatureName.Metadata)"]}},
 'outputs': {'basepath': None, 'packages': {}},
 'pipeline': []}

filtered_D = filters.HasHarmonyLabelsFilter(keep_values=[True]).process(D)
all_metadata = filtered_D.get_metadata()
assert len(all_metadata) > 0, "No pieces selected for analysis."
all_metadata
TimeSig KeySig last_mc last_mn length_qb last_mc_unfolded last_mn_unfolded length_qb_unfolded volta_mcs all_notes_qb ... originalFormat typesetter staff_1_ambitus staff_1_instrument staff_2_ambitus staff_2_instrument staff_3_ambitus staff_3_instrument staff_4_ambitus staff_4_instrument
corpus piece
liszt_pelerinage 160.01_Chapelle_de_Guillaume_Tell {1: '4/4'} {1: 0} 97 97 388.00 97 97 388.00 () 1902.42 ... xml <NA> 40-96 (E2-C7) Piano 24-79 (C1-G5) Piano <NA> <NA> <NA> <NA>
160.02_Au_Lac_de_Wallenstadt {1: '3/8', 102: '2/4'} {1: -4} 112 112 173.50 112 112 173.50 () 512.13 ... mxl https://musescore.com/user/2749876 61-92 (Db4-Ab6) Piano 44-65 (Ab2-F4) Piano <NA> <NA> <NA> <NA>
160.03_Pastorale {1: '12/8', 12: '6/8', 25: '12/8', 35: '6/8'} {1: 4} 49 48 206.00 49 48 206.00 () 705.00 ... mxl https://musescore.com/user/2749876 59-87 (B3-D#6) Piano 40-64 (E2-E4) Piano <NA> <NA> <NA> <NA>
160.04_Au_Bord_dUne_Source {1: '12/8'} {1: -4, 17: 4, 19: -4} 66 66 396.00 66 66 396.00 () 1169.75 ... mxl https://musescore.com/user/2749876 39-95 (Eb2-B6) Piano 32-92 (Ab1-Ab6) Piano <NA> <NA> <NA> <NA>
160.05_Orage {1: '2/2'} {1: -3, 39: 0, 57: 6, 75: 0, 94: -3} 163 160 732.00 163 160 732.00 () 2811.25 ... xml <NA> 47-95 (B2-B6) Piano 23-94 (B0-Bb6) Piano <NA> <NA> <NA> <NA>
160.06_Vallee_dObermann {1: '4/4'} {1: 1, 78: 0, 123: 1, 174: 4} 221 216 899.25 221 216 899.25 () 3505.17 ... mxl https://musescore.com/user/2749876 33-95 (A1-B6) Piano 23-89 (B0-F6) Piano <NA> <NA> <NA> <NA>
160.07_Eglogue {1: '4/4'} {1: -4} 118 117 469.00 118 117 469.00 () 1927.67 ... mxl https://musescore.com/user/2749876 56-92 (Ab3-Ab6) Piano 32-77 (Ab1-F5) Piano <NA> <NA> <NA> <NA>
160.08_Le_Mal_du_Pays_(Heimweh) {1: '4/4', 20: '6/8', 27: '4/4', 47: '6/8', 66... {1: 1, 20: 5, 28: -2, 47: 2} 70 70 254.00 70 70 254.00 () 723.00 ... xml <NA> 47-95 (B2-B6) Piano 28-90 (E1-F#6) Piano <NA> <NA> <NA> <NA>
160.09_Les_Cloches_de_Geneve_(Nocturne) {1: '6/8', 46: '2/4'} {1: 5} 188 188 453.25 188 188 453.25 () 1146.25 ... mxl <NA> 42-94 (F#2-A#6) Piano 25-92 (C#1-G#6) Piano <NA> <NA> <NA> <NA>
161.01_Sposalizio {1: '6/4', 27: '4/4', 29: '6/4'} {1: 4, 38: 1, 68: 4} 133 133 794.00 133 133 794.00 () 3323.00 ... xml https://musescore.com/user/2749876 47-90 (B2-F#6) Piano 28-86 (E1-D6) Piano <NA> <NA> <NA> <NA>
161.02_Il_Pensieroso {1: '4/4'} {1: 4} 49 48 193.00 49 48 193.00 () 798.00 ... xml Tom Schreyer 49-70 (C#3-Bb4) Piano (2) 25-57 (C#1-A3) Piano (2) <NA> <NA> <NA> <NA>
161.03_Canzonetta_del_Salvator_Rosa {1: '4/4'} {1: 3} 75 75 300.00 75 75 300.00 () 948.25 ... xml Tom Schreyer 49-88 (C#3-E6) Piano (2) 33-90 (A1-F#6) Piano (2) <NA> <NA> <NA> <NA>
161.04_Sonetto_47_del_Petrarca {1: '4/4', 12: '6/4', 35: '4/4', 36: '6/4', 59... {1: 0, 6: -5, 32: 1, 55: 4, 69: -5} 95 95 563.00 95 95 563.00 () 1624.75 ... xml Tom Schreyer 52-97 (E3-C#7) Piano (2) 26-89 (D1-E#6) Piano (2) 57-81 (A3-A5) Piano (2) 43-78 (F##2-F#5) Piano (2)
161.05_Sonetto_104_del_Petrarca {1: '4/4', 7: '14/4', 8: '4/4', 36: '10/4', 37... {1: 0, 6: 4} 80 79 358.50 80 79 358.50 () 984.98 ... xml Tom Schreyer 52-100 (E3-E7) Piano (2) 24-83 (B#0-B5) Piano (2) <NA> <NA> <NA> <NA>
161.06_Sonetto_123_del_Petrarca {1: '4/4', 60: '3/2', 61: '4/4', 67: '11/4', 6... {1: -4, 41: 0, 49: -4} 84 84 345.00 84 84 345.00 () 936.17 ... xml Tom Schreyer 40-94 (E2-Bb6) Piano (2) 35-88 (Cb2-E6) Piano (2) <NA> <NA> <NA> <NA>
161.07_Apres_une_lecture_du_Dante {1: '4/4', 76: '2/4', 77: '4/4', 156: '6/4', 1... {1: -1, 77: 3, 103: 6, 145: 0, 157: 6, 190: 0,... 374 373 1505.25 374 373 1505.25 () 5885.89 ... xml Tom Schreyer 41-102 (F2-F#7) Piano (2) 21-90 (A0-F#6) Piano (2) 24-42 (C1-F#2) Piano (2) <NA> <NA>
162.01_Gondoliera {1: '6/8'} {1: 3, 17: 6} 125 125 385.62 125 125 385.62 () 1212.33 ... <NA> <NA> <NA> Piano <NA> Piano 32-97 (G#1-C#7) Piano 30-85 (F#1-C#6) Piano
162.02_Canzone {1: '2/4'} {1: -6} 60 60 120.00 60 60 120.00 () 289.50 ... xml https://www.custommusictranscription.com/ 34-94 (Bb1-Bb6) Piano (2) 22-80 (Bb0-Ab5) Piano (2) <NA> <NA> <NA> <NA>
162.03_Tarantella_da_Guillaume_Louis_Cottrau._Presto_e_canzone_napolitana {1: '6/8', 201: '2/4'} {1: -2, 201: -3, 261: 4, 267: -3, 336: 4, 345: 1} 481 479 1173.88 481 479 1173.88 () 3096.75 ... xml <NA> 33-92 (A1-Ab6) Piano 26-85 (D1-Db6) Piano <NA> <NA> <NA> <NA>

19 rows × 62 columns

mean_composition_years = utils.corpus_mean_composition_years(all_metadata)
chronological_order = mean_composition_years.index.to_list()
corpus_colors = dict(zip(chronological_order, utils.CORPUS_COLOR_SCALE))
corpus_names = {
    corp: utils.get_corpus_display_name(corp) for corp in chronological_order
}
chronological_corpus_names = list(corpus_names.values())
corpus_name_colors = {
    corpus_names[corp]: color for corp, color in corpus_colors.items()
}

mean_composition_years

corpus
liszt_pelerinage    1851.552632
Name: mean_composition_year, dtype: float64

Composition dates#

This section relies on the dataset’s metadata.

valid_composed_start = pd.to_numeric(all_metadata.composed_start, errors="coerce")
valid_composed_end = pd.to_numeric(all_metadata.composed_end, errors="coerce")
print(
    f"Composition dates range from {int(valid_composed_start.min())} {valid_composed_start.idxmin()} "
    f"to {int(valid_composed_end.max())} {valid_composed_end.idxmax()}."
)

Composition dates range from 1838 ('liszt_pelerinage', '161.01_Sposalizio') to 1861 ('liszt_pelerinage', '162.01_Gondoliera').

Mean composition years per corpus#

def make_summary(metadata_df):
    piece_is_annotated = metadata_df.label_count > 0
    return metadata_df[piece_is_annotated].copy()
Hide source 
summary = make_summary(all_metadata)
bar_data = pd.concat(
    [
        mean_composition_years.rename("year"),
        summary.groupby(level="corpus").size().rename("pieces"),
    ],
    axis=1,
).reset_index()

N = len(summary)
fig = px.bar(
    bar_data,
    x="year",
    y="pieces",
    color="corpus",
    color_discrete_map=corpus_colors,
    title=f"Temporal coverage of the {N} annotated pieces in the Distant Listening Corpus",
)
fig.update_traces(width=5)
fig.update_layout(**utils.STD_LAYOUT)
fig.update_traces(width=5)
save_figure_as(fig, "pieces_timeline_bars")
fig.show()

summary
TimeSig KeySig last_mc last_mn length_qb last_mc_unfolded last_mn_unfolded length_qb_unfolded volta_mcs all_notes_qb ... originalFormat typesetter staff_1_ambitus staff_1_instrument staff_2_ambitus staff_2_instrument staff_3_ambitus staff_3_instrument staff_4_ambitus staff_4_instrument
corpus piece
liszt_pelerinage 160.01_Chapelle_de_Guillaume_Tell {1: '4/4'} {1: 0} 97 97 388.00 97 97 388.00 () 1902.42 ... xml <NA> 40-96 (E2-C7) Piano 24-79 (C1-G5) Piano <NA> <NA> <NA> <NA>
160.02_Au_Lac_de_Wallenstadt {1: '3/8', 102: '2/4'} {1: -4} 112 112 173.50 112 112 173.50 () 512.13 ... mxl https://musescore.com/user/2749876 61-92 (Db4-Ab6) Piano 44-65 (Ab2-F4) Piano <NA> <NA> <NA> <NA>
160.03_Pastorale {1: '12/8', 12: '6/8', 25: '12/8', 35: '6/8'} {1: 4} 49 48 206.00 49 48 206.00 () 705.00 ... mxl https://musescore.com/user/2749876 59-87 (B3-D#6) Piano 40-64 (E2-E4) Piano <NA> <NA> <NA> <NA>
160.04_Au_Bord_dUne_Source {1: '12/8'} {1: -4, 17: 4, 19: -4} 66 66 396.00 66 66 396.00 () 1169.75 ... mxl https://musescore.com/user/2749876 39-95 (Eb2-B6) Piano 32-92 (Ab1-Ab6) Piano <NA> <NA> <NA> <NA>
160.05_Orage {1: '2/2'} {1: -3, 39: 0, 57: 6, 75: 0, 94: -3} 163 160 732.00 163 160 732.00 () 2811.25 ... xml <NA> 47-95 (B2-B6) Piano 23-94 (B0-Bb6) Piano <NA> <NA> <NA> <NA>
160.06_Vallee_dObermann {1: '4/4'} {1: 1, 78: 0, 123: 1, 174: 4} 221 216 899.25 221 216 899.25 () 3505.17 ... mxl https://musescore.com/user/2749876 33-95 (A1-B6) Piano 23-89 (B0-F6) Piano <NA> <NA> <NA> <NA>
160.07_Eglogue {1: '4/4'} {1: -4} 118 117 469.00 118 117 469.00 () 1927.67 ... mxl https://musescore.com/user/2749876 56-92 (Ab3-Ab6) Piano 32-77 (Ab1-F5) Piano <NA> <NA> <NA> <NA>
160.08_Le_Mal_du_Pays_(Heimweh) {1: '4/4', 20: '6/8', 27: '4/4', 47: '6/8', 66... {1: 1, 20: 5, 28: -2, 47: 2} 70 70 254.00 70 70 254.00 () 723.00 ... xml <NA> 47-95 (B2-B6) Piano 28-90 (E1-F#6) Piano <NA> <NA> <NA> <NA>
160.09_Les_Cloches_de_Geneve_(Nocturne) {1: '6/8', 46: '2/4'} {1: 5} 188 188 453.25 188 188 453.25 () 1146.25 ... mxl <NA> 42-94 (F#2-A#6) Piano 25-92 (C#1-G#6) Piano <NA> <NA> <NA> <NA>
161.01_Sposalizio {1: '6/4', 27: '4/4', 29: '6/4'} {1: 4, 38: 1, 68: 4} 133 133 794.00 133 133 794.00 () 3323.00 ... xml https://musescore.com/user/2749876 47-90 (B2-F#6) Piano 28-86 (E1-D6) Piano <NA> <NA> <NA> <NA>
161.02_Il_Pensieroso {1: '4/4'} {1: 4} 49 48 193.00 49 48 193.00 () 798.00 ... xml Tom Schreyer 49-70 (C#3-Bb4) Piano (2) 25-57 (C#1-A3) Piano (2) <NA> <NA> <NA> <NA>
161.03_Canzonetta_del_Salvator_Rosa {1: '4/4'} {1: 3} 75 75 300.00 75 75 300.00 () 948.25 ... xml Tom Schreyer 49-88 (C#3-E6) Piano (2) 33-90 (A1-F#6) Piano (2) <NA> <NA> <NA> <NA>
161.04_Sonetto_47_del_Petrarca {1: '4/4', 12: '6/4', 35: '4/4', 36: '6/4', 59... {1: 0, 6: -5, 32: 1, 55: 4, 69: -5} 95 95 563.00 95 95 563.00 () 1624.75 ... xml Tom Schreyer 52-97 (E3-C#7) Piano (2) 26-89 (D1-E#6) Piano (2) 57-81 (A3-A5) Piano (2) 43-78 (F##2-F#5) Piano (2)
161.05_Sonetto_104_del_Petrarca {1: '4/4', 7: '14/4', 8: '4/4', 36: '10/4', 37... {1: 0, 6: 4} 80 79 358.50 80 79 358.50 () 984.98 ... xml Tom Schreyer 52-100 (E3-E7) Piano (2) 24-83 (B#0-B5) Piano (2) <NA> <NA> <NA> <NA>
161.06_Sonetto_123_del_Petrarca {1: '4/4', 60: '3/2', 61: '4/4', 67: '11/4', 6... {1: -4, 41: 0, 49: -4} 84 84 345.00 84 84 345.00 () 936.17 ... xml Tom Schreyer 40-94 (E2-Bb6) Piano (2) 35-88 (Cb2-E6) Piano (2) <NA> <NA> <NA> <NA>
161.07_Apres_une_lecture_du_Dante {1: '4/4', 76: '2/4', 77: '4/4', 156: '6/4', 1... {1: -1, 77: 3, 103: 6, 145: 0, 157: 6, 190: 0,... 374 373 1505.25 374 373 1505.25 () 5885.89 ... xml Tom Schreyer 41-102 (F2-F#7) Piano (2) 21-90 (A0-F#6) Piano (2) 24-42 (C1-F#2) Piano (2) <NA> <NA>
162.01_Gondoliera {1: '6/8'} {1: 3, 17: 6} 125 125 385.62 125 125 385.62 () 1212.33 ... <NA> <NA> <NA> Piano <NA> Piano 32-97 (G#1-C#7) Piano 30-85 (F#1-C#6) Piano
162.02_Canzone {1: '2/4'} {1: -6} 60 60 120.00 60 60 120.00 () 289.50 ... xml https://www.custommusictranscription.com/ 34-94 (Bb1-Bb6) Piano (2) 22-80 (Bb0-Ab5) Piano (2) <NA> <NA> <NA> <NA>
162.03_Tarantella_da_Guillaume_Louis_Cottrau._Presto_e_canzone_napolitana {1: '6/8', 201: '2/4'} {1: -2, 201: -3, 261: 4, 267: -3, 336: 4, 345: 1} 481 479 1173.88 481 479 1173.88 () 3096.75 ... xml <NA> 33-92 (A1-Ab6) Piano 26-85 (D1-Db6) Piano <NA> <NA> <NA> <NA>

19 rows × 62 columns

Composition years histogram#

Hide source 
hist_data = summary.reset_index()
hist_data.corpus = hist_data.corpus.map(corpus_names)
fig = px.histogram(
    hist_data,
    x="composed_end",
    color="corpus",
    labels=dict(
        composed_end="decade",
        count="pieces",
    ),
    color_discrete_map=corpus_name_colors,
    title=f"Temporal coverage of the {N} annotated pieces in the Distant Listening Corpus",
)
fig.update_traces(xbins=dict(size=10))
fig.update_layout(**utils.STD_LAYOUT)
fig.update_legends(font=dict(size=16))
save_figure_as(fig, "pieces_timeline_histogram", height=1250)
fig.show()

Dimensions#

Overview#

def make_overview_table(groupby, group_name="pieces"):
    n_groups = groupby.size().rename(group_name)
    absolute_numbers = dict(
        measures=groupby.last_mn.sum(),
        length=groupby.length_qb.sum(),
        notes=groupby.n_onsets.sum(),
        labels=groupby.label_count.sum(),
    )
    absolute = pd.DataFrame.from_dict(absolute_numbers)
    absolute = pd.concat([n_groups, absolute], axis=1)
    sum_row = pd.DataFrame(absolute.sum(), columns=["sum"]).T
    absolute = pd.concat([absolute, sum_row])
    return absolute


absolute = make_overview_table(summary.groupby("workTitle"))
# print(absolute.astype(int).to_markdown())
absolute.astype(int)
pieces measures length notes labels
Années de Pèlerinage, Deuxième année: Italie 7 887 4058 21662 1655
Années de Pèlerinage, Première année: Suisse 9 1074 3971 24337 2480
Années de Pèlerinage, Supplément: Venezia e Napoli 3 664 1679 12554 935
sum 19 2625 9709 58553 5070 
def summarize_dataset(D):
    all_metadata = D.get_metadata()
    summary = make_summary(all_metadata)
    return make_overview_table(summary.groupby(level=0))


corpus_summary = summarize_dataset(D)
print(corpus_summary.astype(int).to_markdown())

|                  |   pieces |   measures |   length |   notes |   labels |
|:-----------------|---------:|-----------:|---------:|--------:|---------:|
| liszt_pelerinage |       19 |       2625 |     9709 |   58553 |     5070 |
| sum              |       19 |       2625 |     9709 |   58553 |     5070 |

Measures#

all_measures = D.get_feature("measures")
print(
    f"{len(all_measures.index)} measures over {len(all_measures.groupby(level=[0,1]))} files."
)
all_measures.head()

2640 measures over 19 files.
mc mn quarterbeats duration_qb keysig timesig act_dur mc_offset numbering_offset dont_count barline breaks repeats next markers jump_bwd jump_fwd play_until
corpus piece i
liszt_pelerinage 160.01_Chapelle_de_Guillaume_Tell 0 1 1 0 4.0 0 4/4 1 0 <NA> <NA> <NA> <NA> firstMeasure (2,) <NA> <NA> <NA> <NA>
1 2 2 4 4.0 0 4/4 1 0 <NA> <NA> <NA> <NA> <NA> (3,) <NA> <NA> <NA> <NA>
2 3 3 8 4.0 0 4/4 1 0 <NA> <NA> double <NA> <NA> (4,) <NA> <NA> <NA> <NA>
3 4 4 12 4.0 0 4/4 1 0 <NA> <NA> <NA> <NA> <NA> (5,) <NA> <NA> <NA> <NA>
4 5 5 16 4.0 0 4/4 1 0 <NA> <NA> <NA> line <NA> (6,) <NA> <NA> <NA> <NA> 
all_measures.get_default_analysis().plot_grouped()

Harmony labels#

All symbols, independent of the local key (the mode of which changes their semantics).

try:
    all_annotations = D.get_feature("harmonylabels").df
except Exception:
    all_annotations = pd.DataFrame()
n_annotations = len(all_annotations.index)
includes_annotations = n_annotations > 0
if includes_annotations:
    display(all_annotations.head())
    print(f"Concatenated annotation tables contains {all_annotations.shape[0]} rows.")
    no_chord = all_annotations.root.isna()
    if no_chord.sum() > 0:
        print(
            f"{no_chord.sum()} of them are not chords. Their values are:"
            f" {all_annotations.label[no_chord].value_counts(dropna=False).to_dict()}"
        )
    all_chords = all_annotations[~no_chord].copy()
    print(
        f"Dataset contains {all_chords.shape[0]} tokens and {len(all_chords.chord.unique())} types over "
        f"{len(all_chords.groupby(level=[0,1]))} documents."
    )
    all_annotations["corpus_name"] = all_annotations.index.get_level_values(0).map(
        utils.get_corpus_display_name
    )
    all_chords["corpus_name"] = all_chords.index.get_level_values(0).map(
        utils.get_corpus_display_name
    )
else:
    print("Dataset contains no annotations.")
mc mn quarterbeats quarterbeats_all_endings duration_qb mc_onset mn_onset timesig staff voice ... numeral_or_applied_to_numeral intervals_over_bass intervals_over_root scale_degrees scale_degrees_and_mode scale_degrees_major scale_degrees_minor globalkey localkey chord
corpus piece i
liszt_pelerinage 160.01_Chapelle_de_Guillaume_Tell 0 1 1 1 1 1.00 1/4 1/4 4/4 2 1 ... V (M3, P5) (M3, P5) (5, 7, 2) (5, 7, 2), major (5, 7, 2) (5, #7, 2) C I V
1 1 1 2 2 1.75 1/2 1/2 4/4 2 1 ... IV (M3, P5) (M3, P5) (4, 6, 1) (4, 6, 1), major (4, 6, 1) (4, #6, 1) C I IV
2 1 1 15/4 15/4 6.25 15/16 15/16 4/4 2 1 ... iii (m3, P5) (m3, P5) (3, 5, 7) (3, 5, 7), major (3, 5, 7) (#3, 5, #7) C I iii
3 3 3 10 10 1.75 1/2 1/2 4/4 2 1 ... ii (m3, P5) (m3, P5) (2, 4, 6) (2, 4, 6), major (2, 4, 6) (2, 4, #6) C I ii
4 3 3 47/4 47/4 2.25 15/16 15/16 4/4 2 1 ... I (M3, P5) (M3, P5) (1, 3, 5) (1, 3, 5), major (1, 3, 5) (1, #3, 5) C I I

5 rows × 52 columns

Concatenated annotation tables contains 5037 rows.
Dataset contains 5037 tokens and 755 types over 19 documents.