Overview#

This notebook gives a general overview of the features included in the dataset.

Hide imports
%load_ext autoreload
%autoreload 2
import os

import dimcat as dc
import pandas as pd
import plotly.express as px
from dimcat import filters, plotting
from IPython.display import display

import utils
RESULTS_PATH = os.path.abspath(os.path.join(utils.OUTPUT_FOLDER, "overview"))
os.makedirs(RESULTS_PATH, exist_ok=True)


def make_output_path(
    filename: str,
    extension=None,
    path=RESULTS_PATH,
) -> str:
    return utils.make_output_path(filename=filename, extension=extension, path=path)


def save_figure_as(
    fig, filename, formats=("png", "pdf"), directory=RESULTS_PATH, **kwargs
):
    if formats is not None:
        for fmt in formats:
            plotting.write_image(fig, filename, directory, format=fmt, **kwargs)
    else:
        plotting.write_image(fig, filename, directory, **kwargs)

Loading data

D = utils.get_dataset("schumann_kinderszenen", corpus_release="v2.3")
package = D.inputs.get_package()
package_info = package._package.custom
git_tag = package_info.get("git_tag")
utils.print_heading("Data and software versions")
print("Robert Schumann – Kinderszenen version v2.3")
print(f"Datapackage '{package.package_name}' @ {git_tag}")
print(f"dimcat version {dc.__version__}\n")
D
Data and software versions
--------------------------

Robert Schumann – Kinderszenen version v2.3
Datapackage 'schumann_kinderszenen' @ v2.3
dimcat version 3.4.0
Dataset
=======
{'inputs': {'basepath': None,
            'packages': {'schumann_kinderszenen': ["'schumann_kinderszenen.measures' "
                                                   '(MuseScoreFacetName.MuseScoreMeasures)',
                                                   "'schumann_kinderszenen.notes' (MuseScoreFacetName.MuseScoreNotes)",
                                                   "'schumann_kinderszenen.expanded' "
                                                   '(MuseScoreFacetName.MuseScoreHarmonies)',
                                                   "'schumann_kinderszenen.chords' "
                                                   '(MuseScoreFacetName.MuseScoreChords)',
                                                   "'schumann_kinderszenen.metadata' (FeatureName.Metadata)"]}},
 'outputs': {'basepath': None, 'packages': {}},
 'pipeline': []}
filtered_D = filters.HasHarmonyLabelsFilter(keep_values=[True]).process(D)
all_metadata = filtered_D.get_metadata()
assert len(all_metadata) > 0, "No pieces selected for analysis."
all_metadata
TimeSig KeySig last_mc last_mn length_qb last_mc_unfolded last_mn_unfolded length_qb_unfolded volta_mcs all_notes_qb ... imslp musicbrainz viaf wikidata comments pdf staff_1_ambitus staff_1_instrument staff_2_ambitus staff_2_instrument
corpus piece
schumann_kinderszenen n01 {1: '2/4'} {1: 1} 22 22 44.00 44 44 88.00 () 134.33 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/04bf8808-7a43-30e... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 62-79 (D4-G5) Piano 42-69 (F#2-A4) Piano
n02 {1: '3/4'} {1: 2} 43 40 120.00 43 40 120.00 () 471.00 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/aafe4a73-09ed-391... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 60-81 (B#3-A5) Piano 45-73 (A2-C#5) Piano
n03 {1: '2/4'} {1: 2} 32 31 64.00 32 31 64.00 () 177.50 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/7a8d4ff2-f178-39c... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 48-86 (C3-D6) Piano 36-71 (C2-B4) Piano
n04 {1: '2/4'} {1: 2} 17 17 34.00 17 17 34.00 () 106.00 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/47da16ab-9b11-308... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 49-79 (C#3-G5) Piano 38-71 (D2-B4) Piano
n05 {1: '2/4'} {1: 2} 19 16 36.50 42 40 81.00 (9, 10], [11, 12) 154.25 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/73bc48de-e12b-326... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 measurcount differs from pdf � second house no... https://imslp.org/wiki/Special:ReverseLookup/6... 54-81 (F#3-A5) Piano 33-62 (A1-D4) Piano
n06 {1: '3/4'} {1: 3} 27 24 72.00 36 32 96.00 () 365.50 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/ff539270-fcb9-399... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 47-81 (B2-A5) Piano 25-64 (C#1-E4) Piano
n07 {1: '4/4'} {1: -1} 25 24 97.00 33 32 129.00 () 428.50 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/437982b9-be3c-3b1... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 53-82 (F3-Bb5) Piano 34-76 (Bb1-E5) Piano
n08 {1: '2/4'} {1: -1} 34 32 66.50 49 48 96.50 (25], [26) 272.50 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/2fe296f5-9409-3e9... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 measurcount differs from pdf � second house no... https://imslp.org/wiki/Special:ReverseLookup/6... 48-81 (C3-A5) Piano 29-71 (F1-B4) Piano
n09 {1: '3/4'} {1: 0} 24 24 72.00 48 48 144.00 () 308.50 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/856d54fc-8fe3-35f... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 60-81 (C4-A5) Piano 31-67 (G1-G4) Piano
n10 {1: '2/8'} {1: 5} 58 57 58.25 58 57 58.25 () 173.00 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/7586187c-8d78-351... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 58-80 (A#3-G#5) Piano 32-71 (G#1-B4) Piano
n11 {1: '2/4'} {1: 1} 48 48 96.00 64 64 128.00 () 341.00 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/5e1e2af7-25f2-344... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 55-79 (G3-G5) Piano 31-67 (G1-G4) Piano
n12 {1: '2/4'} {1: 1, 9: 4, 25: 1} 32 32 64.00 32 32 64.00 () 305.50 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/193d0122-bb3e-308... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 51-83 (D#3-B5) Piano 28-71 (E1-B4) Piano
n13 {1: '4/4'} {1: 1} 25 25 115.75 25 25 115.75 () 387.08 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/6a5cf298-971f-395... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 45-91 (A2-G6) Piano 31-71 (G1-B4) Piano

13 rows × 55 columns

mean_composition_years = utils.corpus_mean_composition_years(all_metadata)
chronological_order = mean_composition_years.index.to_list()
corpus_colors = dict(zip(chronological_order, utils.CORPUS_COLOR_SCALE))
corpus_names = {
    corp: utils.get_corpus_display_name(corp) for corp in chronological_order
}
chronological_corpus_names = list(corpus_names.values())
corpus_name_colors = {
    corpus_names[corp]: color for corp, color in corpus_colors.items()
}
mean_composition_years
corpus
schumann_kinderszenen    1838.5
Name: mean_composition_year, dtype: float64

Composition dates#

This section relies on the dataset’s metadata.

valid_composed_start = pd.to_numeric(all_metadata.composed_start, errors="coerce")
valid_composed_end = pd.to_numeric(all_metadata.composed_end, errors="coerce")
print(
    f"Composition dates range from {int(valid_composed_start.min())} {valid_composed_start.idxmin()} "
    f"to {int(valid_composed_end.max())} {valid_composed_end.idxmax()}."
)
Composition dates range from 1838 ('schumann_kinderszenen', 'n01') to 1839 ('schumann_kinderszenen', 'n01').

Mean composition years per corpus#

def make_summary(metadata_df):
    piece_is_annotated = metadata_df.label_count > 0
    return metadata_df[piece_is_annotated].copy()
Hide source
summary = make_summary(all_metadata)
bar_data = pd.concat(
    [
        mean_composition_years.rename("year"),
        summary.groupby(level="corpus").size().rename("pieces"),
    ],
    axis=1,
).reset_index()

N = len(summary)
fig = px.bar(
    bar_data,
    x="year",
    y="pieces",
    color="corpus",
    color_discrete_map=corpus_colors,
    title=f"Temporal coverage of the {N} annotated pieces in the Distant Listening Corpus",
)
fig.update_traces(width=5)
fig.update_layout(**utils.STD_LAYOUT)
fig.update_traces(width=5)
save_figure_as(fig, "pieces_timeline_bars")
fig.show()
summary
TimeSig KeySig last_mc last_mn length_qb last_mc_unfolded last_mn_unfolded length_qb_unfolded volta_mcs all_notes_qb ... imslp musicbrainz viaf wikidata comments pdf staff_1_ambitus staff_1_instrument staff_2_ambitus staff_2_instrument
corpus piece
schumann_kinderszenen n01 {1: '2/4'} {1: 1} 22 22 44.00 44 44 88.00 () 134.33 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/04bf8808-7a43-30e... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 62-79 (D4-G5) Piano 42-69 (F#2-A4) Piano
n02 {1: '3/4'} {1: 2} 43 40 120.00 43 40 120.00 () 471.00 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/aafe4a73-09ed-391... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 60-81 (B#3-A5) Piano 45-73 (A2-C#5) Piano
n03 {1: '2/4'} {1: 2} 32 31 64.00 32 31 64.00 () 177.50 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/7a8d4ff2-f178-39c... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 48-86 (C3-D6) Piano 36-71 (C2-B4) Piano
n04 {1: '2/4'} {1: 2} 17 17 34.00 17 17 34.00 () 106.00 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/47da16ab-9b11-308... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 49-79 (C#3-G5) Piano 38-71 (D2-B4) Piano
n05 {1: '2/4'} {1: 2} 19 16 36.50 42 40 81.00 (9, 10], [11, 12) 154.25 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/73bc48de-e12b-326... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 measurcount differs from pdf � second house no... https://imslp.org/wiki/Special:ReverseLookup/6... 54-81 (F#3-A5) Piano 33-62 (A1-D4) Piano
n06 {1: '3/4'} {1: 3} 27 24 72.00 36 32 96.00 () 365.50 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/ff539270-fcb9-399... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 47-81 (B2-A5) Piano 25-64 (C#1-E4) Piano
n07 {1: '4/4'} {1: -1} 25 24 97.00 33 32 129.00 () 428.50 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/437982b9-be3c-3b1... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 53-82 (F3-Bb5) Piano 34-76 (Bb1-E5) Piano
n08 {1: '2/4'} {1: -1} 34 32 66.50 49 48 96.50 (25], [26) 272.50 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/2fe296f5-9409-3e9... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 measurcount differs from pdf � second house no... https://imslp.org/wiki/Special:ReverseLookup/6... 48-81 (C3-A5) Piano 29-71 (F1-B4) Piano
n09 {1: '3/4'} {1: 0} 24 24 72.00 48 48 144.00 () 308.50 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/856d54fc-8fe3-35f... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 60-81 (C4-A5) Piano 31-67 (G1-G4) Piano
n10 {1: '2/8'} {1: 5} 58 57 58.25 58 57 58.25 () 173.00 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/7586187c-8d78-351... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 58-80 (A#3-G#5) Piano 32-71 (G#1-B4) Piano
n11 {1: '2/4'} {1: 1} 48 48 96.00 64 64 128.00 () 341.00 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/5e1e2af7-25f2-344... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 55-79 (G3-G5) Piano 31-67 (G1-G4) Piano
n12 {1: '2/4'} {1: 1, 9: 4, 25: 1} 32 32 64.00 32 32 64.00 () 305.50 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/193d0122-bb3e-308... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 51-83 (D#3-B5) Piano 28-71 (E1-B4) Piano
n13 {1: '4/4'} {1: 1} 25 25 115.75 25 25 115.75 () 387.08 ... https://imslp.org/wiki/Kinderszenen,_Op.15_(Sc... https://musicbrainz.org/work/6a5cf298-971f-395... https://viaf.org/viaf/174865068/ https://www.wikidata.org/wiki/Q1569982 <NA> https://imslp.org/wiki/Special:ReverseLookup/6... 45-91 (A2-G6) Piano 31-71 (G1-B4) Piano

13 rows × 55 columns

Composition years histogram#

Hide source
hist_data = summary.reset_index()
hist_data.corpus = hist_data.corpus.map(corpus_names)
fig = px.histogram(
    hist_data,
    x="composed_end",
    color="corpus",
    labels=dict(
        composed_end="decade",
        count="pieces",
    ),
    color_discrete_map=corpus_name_colors,
    title=f"Temporal coverage of the {N} annotated pieces in the Distant Listening Corpus",
)
fig.update_traces(xbins=dict(size=10))
fig.update_layout(**utils.STD_LAYOUT)
fig.update_legends(font=dict(size=16))
save_figure_as(fig, "pieces_timeline_histogram", height=1250)
fig.show()

Dimensions#

Overview#

def make_overview_table(groupby, group_name="pieces"):
    n_groups = groupby.size().rename(group_name)
    absolute_numbers = dict(
        measures=groupby.last_mn.sum(),
        length=groupby.length_qb.sum(),
        notes=groupby.n_onsets.sum(),
        labels=groupby.label_count.sum(),
    )
    absolute = pd.DataFrame.from_dict(absolute_numbers)
    absolute = pd.concat([n_groups, absolute], axis=1)
    sum_row = pd.DataFrame(absolute.sum(), columns=["sum"]).T
    absolute = pd.concat([absolute, sum_row])
    return absolute


absolute = make_overview_table(summary.groupby("workTitle"))
# print(absolute.astype(int).to_markdown())
absolute.astype(int)
pieces measures length notes labels
Kinderszenen 13 392 940 5012 948
sum 13 392 940 5012 948
def summarize_dataset(D):
    all_metadata = D.get_metadata()
    summary = make_summary(all_metadata)
    return make_overview_table(summary.groupby(level=0))


corpus_summary = summarize_dataset(D)
print(corpus_summary.astype(int).to_markdown())
|                       |   pieces |   measures |   length |   notes |   labels |
|:----------------------|---------:|-----------:|---------:|--------:|---------:|
| schumann_kinderszenen |       13 |        392 |      940 |    5012 |      948 |
| sum                   |       13 |        392 |      940 |    5012 |      948 |

Measures#

all_measures = D.get_feature("measures")
print(
    f"{len(all_measures.index)} measures over {len(all_measures.groupby(level=[0,1]))} files."
)
all_measures.head()
403 measures over 13 files.
mc mn quarterbeats duration_qb keysig timesig act_dur mc_offset numbering_offset dont_count barline breaks repeats next volta markers jump_bwd jump_fwd play_until
corpus piece i
schumann_kinderszenen n01 0 1 1 0 2.0 1 2/4 1/2 0 <NA> <NA> <NA> <NA> firstMeasure (2,) <NA> <NA> <NA> <NA> <NA>
1 2 2 2 2.0 1 2/4 1/2 0 <NA> <NA> <NA> <NA> <NA> (3,) <NA> <NA> <NA> <NA> <NA>
2 3 3 4 2.0 1 2/4 1/2 0 <NA> <NA> <NA> <NA> <NA> (4,) <NA> <NA> <NA> <NA> <NA>
3 4 4 6 2.0 1 2/4 1/2 0 <NA> <NA> <NA> line <NA> (5,) <NA> <NA> <NA> <NA> <NA>
4 5 5 8 2.0 1 2/4 1/2 0 <NA> <NA> <NA> <NA> <NA> (6,) <NA> <NA> <NA> <NA> <NA>
all_measures.get_default_analysis().plot_grouped()

Harmony labels#

All symbols, independent of the local key (the mode of which changes their semantics).

try:
    all_annotations = D.get_feature("harmonylabels").df
except Exception:
    all_annotations = pd.DataFrame()
n_annotations = len(all_annotations.index)
includes_annotations = n_annotations > 0
if includes_annotations:
    display(all_annotations.head())
    print(f"Concatenated annotation tables contains {all_annotations.shape[0]} rows.")
    no_chord = all_annotations.root.isna()
    if no_chord.sum() > 0:
        print(
            f"{no_chord.sum()} of them are not chords. Their values are:"
            f" {all_annotations.label[no_chord].value_counts(dropna=False).to_dict()}"
        )
    all_chords = all_annotations[~no_chord].copy()
    print(
        f"Dataset contains {all_chords.shape[0]} tokens and {len(all_chords.chord.unique())} types over "
        f"{len(all_chords.groupby(level=[0,1]))} documents."
    )
    all_annotations["corpus_name"] = all_annotations.index.get_level_values(0).map(
        utils.get_corpus_display_name
    )
    all_chords["corpus_name"] = all_chords.index.get_level_values(0).map(
        utils.get_corpus_display_name
    )
else:
    print("Dataset contains no annotations.")
mc mn quarterbeats duration_qb mc_onset mn_onset timesig staff voice volta ... numeral_or_applied_to_numeral intervals_over_bass intervals_over_root scale_degrees scale_degrees_and_mode scale_degrees_major scale_degrees_minor globalkey localkey chord
corpus piece i
schumann_kinderszenen n01 0 1 1 0 1.0 0 0 2/4 2 1 <NA> ... I (M3, P5) (M3, P5) (1, 3, 5) (1, 3, 5), major (1, 3, 5) (1, #3, 5) G I I
1 1 1 1 1.0 1/4 1/4 2/4 2 1 <NA> ... V (m3, d5, d7) (m3, d5, d7) (#4, 6, 1, b3) (#4, 6, 1, b3), major (#4, 6, 1, b3) (#4, #6, 1, 3) G I viio7/V
2 2 2 2 1.0 0 0 2/4 2 1 <NA> ... V (M3, P5) (M3, P5) (5, 7, 2) (5, 7, 2), major (5, 7, 2) (5, #7, 2) G I V
3 2 2 3 1.0 1/4 1/4 2/4 2 1 <NA> ... V (m3, d5, m6) (M3, P5, m7) (7, 2, 4, 5) (7, 2, 4, 5), major (7, 2, 4, 5) (#7, 2, 4, 5) G I V65
4 3 3 4 1.0 0 0 2/4 2 1 <NA> ... I (M3, P5) (M3, P5) (1, 3, 5) (1, 3, 5), major (1, 3, 5) (1, #3, 5) G I I

5 rows × 52 columns

Concatenated annotation tables contains 911 rows.
Dataset contains 911 tokens and 152 types over 13 documents.