Overview#

This notebook gives a general overview of the features included in the dataset.

Hide imports 
%load_ext autoreload
%autoreload 2
import os

import dimcat as dc
import pandas as pd
import plotly.express as px
from dimcat import filters, plotting
from IPython.display import display

import utils

RESULTS_PATH = os.path.abspath(os.path.join(utils.OUTPUT_FOLDER, "overview"))
os.makedirs(RESULTS_PATH, exist_ok=True)


def make_output_path(
    filename: str,
    extension=None,
    path=RESULTS_PATH,
) -> str:
    return utils.make_output_path(filename=filename, extension=extension, path=path)


def save_figure_as(
    fig, filename, formats=("png", "pdf"), directory=RESULTS_PATH, **kwargs
):
    if formats is not None:
        for fmt in formats:
            plotting.write_image(fig, filename, directory, format=fmt, **kwargs)
    else:
        plotting.write_image(fig, filename, directory, **kwargs)

Loading data

D = utils.get_dataset("jc_bach_sonatas", corpus_release="v2.4")
package = D.inputs.get_package()
package_info = package._package.custom
git_tag = package_info.get("git_tag")
utils.print_heading("Data and software versions")
print("Johann Christian Bach – Keyboard Sonatas version v2.4")
print(f"Datapackage '{package.package_name}' @ {git_tag}")
print(f"dimcat version {dc.__version__}\n")
D

Data and software versions
--------------------------

Johann Christian Bach – Keyboard Sonatas version v2.4
Datapackage 'jc_bach_sonatas' @ v2.4
dimcat version 3.4.0

Dataset
=======
{'inputs': {'basepath': None,
            'packages': {'jc_bach_sonatas': ["'jc_bach_sonatas.measures' (MuseScoreFacetName.MuseScoreMeasures)",
                                             "'jc_bach_sonatas.notes' (MuseScoreFacetName.MuseScoreNotes)",
                                             "'jc_bach_sonatas.expanded' (MuseScoreFacetName.MuseScoreHarmonies)",
                                             "'jc_bach_sonatas.chords' (MuseScoreFacetName.MuseScoreChords)",
                                             "'jc_bach_sonatas.metadata' (FeatureName.Metadata)"]}},
 'outputs': {'basepath': None, 'packages': {}},
 'pipeline': []}

filtered_D = filters.HasHarmonyLabelsFilter(keep_values=[True]).process(D)
all_metadata = filtered_D.get_metadata()
assert len(all_metadata) > 0, "No pieces selected for analysis."
all_metadata
TimeSig KeySig last_mc last_mn length_qb last_mc_unfolded last_mn_unfolded length_qb_unfolded volta_mcs all_notes_qb ... extension originalFormat musicbrainz viaf imslp pdf staff_1_ambitus staff_1_instrument staff_2_ambitus staff_2_instrument
corpus piece
jc_bach_sonatas wa01op05no1a_Allegretto {1: '2/4'} {1: -2} 84 82 164.0 168 164 328.0 () 313.00 ... <NA> musicxml https://musicbrainz.org/work/89edcc1a-3519-40d... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 59-86 (B3-D6) Piano 39-69 (Eb2-A4) Piano
wa01op05no1b_Tempo_di_Minuetto {1: '3/4'} {1: -2} 74 74 222.0 148 148 444.0 () 445.33 ... <NA> <NA> https://musicbrainz.org/work/52888da6-abdd-469... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 48-86 (C3-D6) Piano 38-72 (D2-C5) Piano
wa02op05no2a_Allegro_di_molto {1: '4/4'} {1: 2} 111 111 444.0 222 222 888.0 () 1043.00 ... <NA> <NA> https://musicbrainz.org/work/8b4b4a8c-8463-3ce... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 54-88 (F#3-E6) Piano 35-71 (B1-B4) Piano
wa02op05no2b_Andante_di_molto {1: '2/4'} {1: 1} 52 50 100.0 104 100 200.0 () 226.00 ... <NA> <NA> https://musicbrainz.org/work/abeecfd0-129a-388... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 62-86 (D4-D6) Piano 33-69 (A1-A4) Piano
wa02op05no2c_Minuetto {1: '3/4'} {1: 2} 28 28 84.0 56 56 168.0 () 201.25 ... <NA> <NA> https://musicbrainz.org/work/bd70cb27-30a6-3ed... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 57-83 (A3-B5) Piano 38-62 (D2-D4) Piano
wa02op05no2d_Minore {1: '3/4'} {1: -1} 24 24 72.0 48 48 144.0 () 164.00 ... <NA> <NA> https://musicbrainz.org/work/bd70cb27-30a6-3ed... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 61-84 (C#4-C6) Piano 38-62 (D2-D4) Piano
wa03op05no3a_Allegro {1: '4/4'} {1: 1} 83 81 325.0 166 162 650.0 () 605.50 ... <NA> <NA> https://musicbrainz.org/work/5c32db9e-a26a-367... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 54-88 (F#3-E6) Piano 35-78 (B1-F#5) Piano
wa03op05no3b_Allegretto {1: '2/4'} {1: 1} 80 80 160.0 160 160 320.0 () 317.58 ... <NA> <NA> https://musicbrainz.org/work/c6c3f763-9340-362... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 55-84 (G3-C6) Piano 36-62 (C2-D4) Piano
wa04op05no4a_Allegro {1: '4/4'} {1: -3} 117 117 468.0 234 234 936.0 () 968.00 ... <NA> <NA> https://musicbrainz.org/work/c30f0740-f172-333... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 50-87 (D3-Eb6) Piano 32-80 (Ab1-Ab5) Piano
wa04op05no4b_Rondeaux {1: '3/4'} {1: -3} 69 69 207.0 138 138 414.0 () 419.25 ... <NA> <NA> https://musicbrainz.org/work/341b752e-a999-334... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 58-87 (Bb3-Eb6) Piano 38-84 (D2-C6) Piano
wa05op05no5a_Allegro_Assai {1: '4/4'} {1: 4} 99 99 396.0 198 198 792.0 () 781.00 ... <NA> <NA> https://musicbrainz.org/work/3c366e5d-9d98-40a... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 47-88 (B2-E6) Piano 35-76 (B1-E5) Piano
wa05op05no5b_Adagio {1: '3/4'} {1: 3} 55 55 165.0 110 110 330.0 () 372.42 ... <NA> <NA> https://musicbrainz.org/work/e68e8946-704b-4f5... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 59-86 (B3-D6) Piano 40-66 (E2-F#4) Piano
wa05op05no5c_Prestissimo {1: '2/4'} {1: 4, 40: 1, 59: 4} 104 102 205.0 208 204 410.0 () 427.50 ... <NA> <NA> https://musicbrainz.org/work/e284e901-4255-4d3... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 59-88 (B3-E6) Piano 38-75 (D2-D#5) Piano
wa06op05no6a_Grave {1: '3/4'} {1: -3} 62 62 184.0 62 62 184.0 () 568.25 ... <NA> <NA> https://musicbrainz.org/work/bd4ca65a-ed09-48f... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 55-84 (G3-C6) Piano 36-70 (C2-Bb4) Piano
wa06op05no6b_Allegro_Moderato {1: '4/4'} {1: -3} 77 77 308.0 77 77 308.0 () 905.00 ... <NA> <NA> https://musicbrainz.org/work/a6cd0c31-22e5-43c... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 58-84 (Bb3-C6) Piano 36-68 (C2-Ab4) Piano
wa06op05no6c_Allegretto {1: '2/2'} {1: -3} 49 46 186.0 95 90 362.0 () 514.00 ... <NA> <NA> https://musicbrainz.org/work/b3606a40-f3ff-46c... https://viaf.org/viaf/177812565/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/2... 55-86 (G3-D6) Piano 36-65 (C2-F4) Piano
wa07op17no1a_Minuetto_Con_Variatione {1: '3/4'} {1: 1} 108 18 324.0 216 216 648.0 () 748.00 ... <NA> <NA> <NA> https://viaf.org/viaf/1458152744515027850009/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 55-84 (G3-C6) Piano 38-67 (D2-G4) Piano
wa08op17no2a_Allegro {1: '2/4'} {1: -3} 123 121 242.0 246 242 484.0 () 518.75 ... <NA> <NA> https://musicbrainz.org/work/1469ebf9-65d8-4ea... https://viaf.org/viaf/179238149/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 51-84 (Eb3-C6) Piano 36-73 (C2-Db5) Piano
wa08op17no2b_Andante {1: '4/4'} {1: -3} 71 71 283.0 71 71 283.0 () 707.25 ... <NA> <NA> https://musicbrainz.org/work/4603d18a-b17d-427... https://viaf.org/viaf/179238149/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 55-87 (G3-Eb6) Piano 39-72 (Eb2-C5) Piano
wa08op17no2c_Prestissimo {1: '12/8'} {1: -3} 103 101 606.0 206 202 1212.0 () 1182.50 ... <NA> <NA> https://musicbrainz.org/work/a88e1148-b3bb-4cf... https://viaf.org/viaf/179238149/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 50-87 (D3-Eb6) Piano 33-75 (A1-Eb5) Piano
wa09op17no3a_Allegro_Assai {1: '4/4'} {1: -3} 114 114 456.0 228 228 912.0 () 1185.00 ... <NA> <NA> <NA> https://viaf.org/viaf/185279678/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 46-87 (Bb2-Eb6) Piano 34-75 (Bb1-Eb5) Piano
wa09op17no3b_Allegro {1: '3/8'} {1: -3} 111 111 166.5 222 222 333.0 () 394.00 ... <NA> <NA> <NA> https://viaf.org/viaf/185279678/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 55-89 (G3-F6) Piano 38-77 (D2-F5) Piano
wa10op17no4a_Allegro {1: '4/4'} {1: 1} 100 98 393.0 200 196 786.0 () 769.50 ... <NA> <NA> <NA> https://viaf.org/viaf/184178383/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 57-86 (A3-D6) Piano 34-67 (A#1-G4) Piano
wa10op17no4b_Presto_Assai {1: '3/8'} {1: 1} 99 99 148.5 198 198 297.0 () 311.75 ... <NA> <NA> <NA> https://viaf.org/viaf/184178383/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 59-86 (B3-D6) Piano 38-67 (D2-G4) Piano
wa11op17no5a_Allegro {1: '4/4'} {1: 3} 103 102 409.0 206 204 818.0 () 797.42 ... <NA> <NA> https://musicbrainz.org/work/8183198c-d5ac-4e1... https://viaf.org/viaf/173992927/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 50-88 (D3-E6) Piano 38-76 (D2-E5) Piano
wa11op17no5b_Presto {1: '3/8'} {1: 3} 127 127 190.5 254 254 381.0 () 363.38 ... <NA> <NA> https://musicbrainz.org/work/3af0eddd-8802-4aa... https://viaf.org/viaf/173992927/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 56-85 (G#3-C#6) Piano 33-71 (A1-B4) Piano
wa12op17no6a_Allegro {1: '4/4'} {1: -2} 120 118 472.0 240 236 944.0 () 970.00 ... <NA> <NA> <NA> https://viaf.org/viaf/260415139/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 55-89 (G3-F6) Piano 34-75 (Bb1-Eb5) Piano
wa12op17no6b_Andante {1: '4/4'} {1: -3} 74 74 296.0 148 148 592.0 () 835.00 ... <NA> <NA> <NA> https://viaf.org/viaf/260415139/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 51-89 (Eb3-F6) Piano 34-68 (Bb1-Ab4) Piano
wa12op17no6c_Prestissimo {1: '12/8'} {1: -2} 107 105 630.0 214 210 1260.0 () 1203.00 ... <NA> <NA> <NA> https://viaf.org/viaf/260415139/ https://imslp.org/wiki/6_Keyboard_Sonatas,_Op.... https://imslp.org/wiki/Special:ReverseLookup/9... 38-89 (D2-F6) Piano 29-72 (F1-C5) Piano

29 rows × 56 columns

mean_composition_years = utils.corpus_mean_composition_years(all_metadata)
chronological_order = mean_composition_years.index.to_list()
corpus_colors = dict(zip(chronological_order, utils.CORPUS_COLOR_SCALE))
corpus_names = {
    corp: utils.get_corpus_display_name(corp) for corp in chronological_order
}
chronological_corpus_names = list(corpus_names.values())
corpus_name_colors = {
    corpus_names[corp]: color for corp, color in corpus_colors.items()
}

mean_composition_years

corpus
jc_bach_sonatas    1771.724138
Name: mean_composition_year, dtype: float64

Composition dates#

This section relies on the dataset’s metadata.

valid_composed_start = pd.to_numeric(all_metadata.composed_start, errors="coerce")
valid_composed_end = pd.to_numeric(all_metadata.composed_end, errors="coerce")
print(
    f"Composition dates range from {int(valid_composed_start.min())} {valid_composed_start.idxmin()} "
    f"to {int(valid_composed_end.max())} {valid_composed_end.idxmax()}."
)

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[7], line 4
      1 valid_composed_start = pd.to_numeric(all_metadata.composed_start, errors="coerce")
      2 valid_composed_end = pd.to_numeric(all_metadata.composed_end, errors="coerce")
      3 print(
----> 4     f"Composition dates range from {int(valid_composed_start.min())} {valid_composed_start.idxmin()} "
      5     f"to {int(valid_composed_end.max())} {valid_composed_end.idxmax()}."
      6 )

ValueError: cannot convert float NaN to integer

Mean composition years per corpus#

def make_summary(metadata_df):
    piece_is_annotated = metadata_df.label_count > 0
    return metadata_df[piece_is_annotated].copy()
Hide source 
summary = make_summary(all_metadata)
bar_data = pd.concat(
    [
        mean_composition_years.rename("year"),
        summary.groupby(level="corpus").size().rename("pieces"),
    ],
    axis=1,
).reset_index()

N = len(summary)
fig = px.bar(
    bar_data,
    x="year",
    y="pieces",
    color="corpus",
    color_discrete_map=corpus_colors,
    title=f"Temporal coverage of the {N} annotated pieces in the Distant Listening Corpus",
)
fig.update_traces(width=5)
fig.update_layout(**utils.STD_LAYOUT)
fig.update_traces(width=5)
save_figure_as(fig, "pieces_timeline_bars")
fig.show()

summary

Composition years histogram#

Hide source 
hist_data = summary.reset_index()
hist_data.corpus = hist_data.corpus.map(corpus_names)
fig = px.histogram(
    hist_data,
    x="composed_end",
    color="corpus",
    labels=dict(
        composed_end="decade",
        count="pieces",
    ),
    color_discrete_map=corpus_name_colors,
    title=f"Temporal coverage of the {N} annotated pieces in the Distant Listening Corpus",
)
fig.update_traces(xbins=dict(size=10))
fig.update_layout(**utils.STD_LAYOUT)
fig.update_legends(font=dict(size=16))
save_figure_as(fig, "pieces_timeline_histogram", height=1250)
fig.show()

Dimensions#

Overview#

def make_overview_table(groupby, group_name="pieces"):
    n_groups = groupby.size().rename(group_name)
    absolute_numbers = dict(
        measures=groupby.last_mn.sum(),
        length=groupby.length_qb.sum(),
        notes=groupby.n_onsets.sum(),
        labels=groupby.label_count.sum(),
    )
    absolute = pd.DataFrame.from_dict(absolute_numbers)
    absolute = pd.concat([n_groups, absolute], axis=1)
    sum_row = pd.DataFrame(absolute.sum(), columns=["sum"]).T
    absolute = pd.concat([absolute, sum_row])
    return absolute


absolute = make_overview_table(summary.groupby("workTitle"))
# print(absolute.astype(int).to_markdown())
absolute.astype(int)

def summarize_dataset(D):
    all_metadata = D.get_metadata()
    summary = make_summary(all_metadata)
    return make_overview_table(summary.groupby(level=0))


corpus_summary = summarize_dataset(D)
print(corpus_summary.astype(int).to_markdown())

Measures#

all_measures = D.get_feature("measures")
print(
    f"{len(all_measures.index)} measures over {len(all_measures.groupby(level=[0,1]))} files."
)
all_measures.head()

all_measures.get_default_analysis().plot_grouped()

Harmony labels#

All symbols, independent of the local key (the mode of which changes their semantics).

try:
    all_annotations = D.get_feature("harmonylabels").df
except Exception:
    all_annotations = pd.DataFrame()
n_annotations = len(all_annotations.index)
includes_annotations = n_annotations > 0
if includes_annotations:
    display(all_annotations.head())
    print(f"Concatenated annotation tables contains {all_annotations.shape[0]} rows.")
    no_chord = all_annotations.root.isna()
    if no_chord.sum() > 0:
        print(
            f"{no_chord.sum()} of them are not chords. Their values are:"
            f" {all_annotations.label[no_chord].value_counts(dropna=False).to_dict()}"
        )
    all_chords = all_annotations[~no_chord].copy()
    print(
        f"Dataset contains {all_chords.shape[0]} tokens and {len(all_chords.chord.unique())} types over "
        f"{len(all_chords.groupby(level=[0,1]))} documents."
    )
    all_annotations["corpus_name"] = all_annotations.index.get_level_values(0).map(
        utils.get_corpus_display_name
    )
    all_chords["corpus_name"] = all_chords.index.get_level_values(0).map(
        utils.get_corpus_display_name
    )
else:
    print("Dataset contains no annotations.")