Overview#

This notebook gives a general overview of the features included in the dataset.

Hide imports
%load_ext autoreload
%autoreload 2
import os

import dimcat as dc
import pandas as pd
import plotly.express as px
from dimcat import filters, plotting
from IPython.display import display

import utils
RESULTS_PATH = os.path.abspath(os.path.join(utils.OUTPUT_FOLDER, "overview"))
os.makedirs(RESULTS_PATH, exist_ok=True)


def make_output_path(
    filename: str,
    extension=None,
    path=RESULTS_PATH,
) -> str:
    return utils.make_output_path(filename=filename, extension=extension, path=path)


def save_figure_as(
    fig, filename, formats=("png", "pdf"), directory=RESULTS_PATH, **kwargs
):
    if formats is not None:
        for fmt in formats:
            plotting.write_image(fig, filename, directory, format=fmt, **kwargs)
    else:
        plotting.write_image(fig, filename, directory, **kwargs)

Loading data

D = utils.get_dataset("monteverdi_madrigals", corpus_release="v2.3")
package = D.inputs.get_package()
package_info = package._package.custom
git_tag = package_info.get("git_tag")
utils.print_heading("Data and software versions")
print("Claudio Monteverdi – Madrigals version v2.3")
print(f"Datapackage '{package.package_name}' @ {git_tag}")
print(f"dimcat version {dc.__version__}\n")
D
Data and software versions
--------------------------

Claudio Monteverdi – Madrigals version v2.3
Datapackage 'monteverdi_madrigals' @ v2.3
dimcat version 3.4.0
Dataset
=======
{'inputs': {'basepath': None,
            'packages': {'monteverdi_madrigals': ["'monteverdi_madrigals.measures' "
                                                  '(MuseScoreFacetName.MuseScoreMeasures)',
                                                  "'monteverdi_madrigals.notes' (MuseScoreFacetName.MuseScoreNotes)",
                                                  "'monteverdi_madrigals.expanded' "
                                                  '(MuseScoreFacetName.MuseScoreHarmonies)',
                                                  "'monteverdi_madrigals.chords' (MuseScoreFacetName.MuseScoreChords)",
                                                  "'monteverdi_madrigals.metadata' (FeatureName.Metadata)"]}},
 'outputs': {'basepath': None, 'packages': {}},
 'pipeline': []}
filtered_D = filters.HasHarmonyLabelsFilter(keep_values=[True]).process(D)
all_metadata = filtered_D.get_metadata()
assert len(all_metadata) > 0, "No pieces selected for analysis."
all_metadata
TimeSig KeySig last_mc last_mn length_qb last_mc_unfolded last_mn_unfolded length_qb_unfolded all_notes_qb n_onsets ... staff_1_instrument staff_1_ambitus staff_2_instrument staff_2_ambitus staff_3_instrument staff_3_ambitus staff_4_instrument staff_4_ambitus staff_5_instrument staff_5_ambitus
corpus piece
monteverdi_madrigals 2-12 {1: '4/4'} {1: -1} 93 93 372.0 93 93 372.0 1374.00 1011 ... Soprano I 60-77 (C4-F5) Soprano II 60-74 (C4-D5) Alto 53-69 (F3-A4) Tenor 48-65 (C3-F4) Bass 41-58 (F2-Bb3)
3-09 {1: '4/4'} {1: -1} 84 84 680.0 84 84 680.0 2534.00 886 ... Soprano I 60-77 (C4-F5) Soprano II 57-74 (A3-D5) Alto 53-72 (F3-C5) Tenor 48-67 (C3-G4) Bass 41-58 (F2-Bb3)
4-19 {1: '4/4'} {1: 0} 111 111 448.0 111 111 448.0 1726.00 956 ... Soprano I 57-76 (A3-E5) Soprano II 57-76 (A3-E5) Alto 53-69 (F3-A4) Tenor 48-66 (C3-F#4) Bass 43-57 (G2-A3)
5-01 {1: '4/4'} {1: 0} 67 67 268.0 67 67 268.0 1088.50 661 ... Soprano I 64-81 (E4-A5) Soprano II 59-77 (B3-F5) Alto 52-69 (E3-A4) Tenor 55-69 (G3-A4) Bass 45-62 (A2-D4)
5-03 {1: '4/4'} {1: -1} 74 74 296.0 74 74 296.0 1073.00 652 ... Soprano I 62-77 (D4-F5) Soprano II 61-74 (C#4-D5) Alto 50-70 (D3-Bb4) Tenor 49-65 (C#3-F4) Bass 43-58 (G2-Bb3)
5-04a {1: '4/4'} {1: -1} 82 82 328.0 82 82 328.0 1401.00 939 ... Soprano I 65-82 (F4-Bb5) Soprano II 64-81 (E4-A5) Alto 55-74 (G3-D5) Tenor 53-69 (F3-A4) Bass 46-63 (Bb2-Eb4)
5-04c {1: '4/4'} {1: -1} 53 53 212.0 53 53 212.0 826.00 589 ... Soprano I 66-81 (F#4-A5) Soprano II 65-79 (F4-G5) Alto 55-74 (G3-D5) Tenor 55-69 (G3-A4) Bass 46-63 (Bb2-Eb4)
5-04d {1: '4/4'} {1: -1} 59 59 236.0 59 59 236.0 950.00 715 ... Soprano I 65-82 (F4-Bb5) Soprano II 62-81 (D4-A5) Alto 55-74 (G3-D5) Tenor 53-69 (F3-A4) Bass 45-65 (A2-F4)
5-04e {1: '4/4'} {1: -1} 95 95 380.0 95 95 380.0 1530.50 1213 ... Soprano I 65-81 (F4-A5) Soprano II 62-81 (D4-A5) Alto 60-74 (C4-D5) Tenor 50-69 (D3-A4) Bass 43-62 (G2-D4)
5-05b {1: '4/4'} {1: -1} 58 58 232.0 58 58 232.0 930.00 685 ... Soprano I 62-79 (D4-G5) Soprano II 58-77 (Bb3-F5) Alto 55-70 (G3-Bb4) Tenor 62-74 (D4-D5) Bass 41-58 (F2-Bb3)
5-05c {1: '4/4'} {1: -1, 39: 0} 75 75 300.0 75 75 300.0 1258.00 819 ... Soprano I 57-77 (A3-F5) Soprano II 59-76 (B3-E5) Alto 54-69 (F#3-A4) Tenor 50-67 (D3-G4) Bass 43-62 (G2-D4)
5-08 {1: '4/4'} {1: 0} 91 91 364.0 91 91 364.0 1229.00 793 ... Soprano I 60-77 (C4-F5) Soprano II 57-79 (A3-G5) Alto 50-69 (D3-A4) Tenor 48-64 (C3-E4) Bass 41-57 (F2-A3)
5-09 {1: '4/4'} {1: 0} 84 84 336.0 84 84 336.0 1207.50 1238 ... Soprano I 60-77 (C4-F5) Soprano II 55-70 (G3-Bb4) Alto 62-78 (D4-F#5) Tenor 57-75 (A3-Eb5) Bass 41-57 (F2-A3)
5-11 {1: '4/4'} {1: 0} 57 57 228.0 57 57 228.0 724.50 598 ... Soprano I 62-76 (D4-E5) Soprano II 61-74 (C#4-D5) Alto 67-79 (G4-G5) Tenor 62-76 (D4-E5) Bass 38-60 (D2-C4)
6-01a {1: '4/4'} {1: 0} 34 34 136.0 34 34 136.0 513.00 353 ... Soprano I 62-74 (D4-D5) Soprano II 61-70 (C#4-Bb4) Alto 52-67 (E3-G4) Tenor 50-62 (D3-D4) Bass 43-57 (G2-A3)
8-18 {1: '4/4', 28: '3/1', 97: '4/4'} {1: 0} 108 106 1797.0 108 106 1797.0 4714.00 1055 ... Soprano 62-77 (D4-F5) Tenor I 36-55 (C2-G3) Tenor II 36-57 (C2-A3) Bass 40-60 (E2-C4) Harpsichord 40-60 (E2-C4)
8-19 {1: '3/1', 21: '4/4', 34: '3/1', 40: '4/4', 89... {1: 0} 142 142 1008.0 142 142 1008.0 3065.50 1300 ... Alto 57-69 (A3-A4) Tenor 48-65 (C3-F4) Bass I 40-60 (E2-C4) Bass II 40-67 (E2-G4) <NA> <NA>
9-12 {1: '3/2', 10: '2/4', 11: '3/2', 18: '2/4'} {1: 0} 26 26 116.0 26 26 116.0 423.25 254 ... Tenor I 45-55 (A2-G3) Tenor II 41-53 (F2-F3) Bass 43-60 (G2-C4) Harpsichord 31-48 (G1-C3) <NA> <NA>
laudate_pueri_dominum {1: '4/4', 126: '3/2', 138: '4/4'} {1: -1} 177 177 808.0 177 177 808.0 3492.00 2182 ... Soprano 65-81 (F4-A5) Alto I 58-74 (Bb3-D5) Alto II 53-70 (F3-Bb4) Tenor 53-70 (F3-Bb4) Bass 31-53 (G1-F3)

19 rows × 57 columns

mean_composition_years = utils.corpus_mean_composition_years(all_metadata)
chronological_order = mean_composition_years.index.to_list()
corpus_colors = dict(zip(chronological_order, utils.CORPUS_COLOR_SCALE))
corpus_names = {
    corp: utils.get_corpus_display_name(corp) for corp in chronological_order
}
chronological_corpus_names = list(corpus_names.values())
corpus_name_colors = {
    corpus_names[corp]: color for corp, color in corpus_colors.items()
}
mean_composition_years
corpus
monteverdi_madrigals    1612.157895
Name: mean_composition_year, dtype: float64

Composition dates#

This section relies on the dataset’s metadata.

valid_composed_start = pd.to_numeric(all_metadata.composed_start, errors="coerce")
valid_composed_end = pd.to_numeric(all_metadata.composed_end, errors="coerce")
print(
    f"Composition dates range from {int(valid_composed_start.min())} {valid_composed_start.idxmin()} "
    f"to {int(valid_composed_end.max())} {valid_composed_end.idxmax()}."
)
Composition dates range from 1590 ('monteverdi_madrigals', '2-12') to 1651 ('monteverdi_madrigals', '9-12').

Mean composition years per corpus#

def make_summary(metadata_df):
    piece_is_annotated = metadata_df.label_count > 0
    return metadata_df[piece_is_annotated].copy()
Hide source
summary = make_summary(all_metadata)
bar_data = pd.concat(
    [
        mean_composition_years.rename("year"),
        summary.groupby(level="corpus").size().rename("pieces"),
    ],
    axis=1,
).reset_index()

N = len(summary)
fig = px.bar(
    bar_data,
    x="year",
    y="pieces",
    color="corpus",
    color_discrete_map=corpus_colors,
    title=f"Temporal coverage of the {N} annotated pieces in the Distant Listening Corpus",
)
fig.update_traces(width=5)
fig.update_layout(**utils.STD_LAYOUT)
fig.update_traces(width=5)
save_figure_as(fig, "pieces_timeline_bars")
fig.show()
summary
TimeSig KeySig last_mc last_mn length_qb last_mc_unfolded last_mn_unfolded length_qb_unfolded all_notes_qb n_onsets ... staff_1_instrument staff_1_ambitus staff_2_instrument staff_2_ambitus staff_3_instrument staff_3_ambitus staff_4_instrument staff_4_ambitus staff_5_instrument staff_5_ambitus
corpus piece
monteverdi_madrigals 2-12 {1: '4/4'} {1: -1} 93 93 372.0 93 93 372.0 1374.00 1011 ... Soprano I 60-77 (C4-F5) Soprano II 60-74 (C4-D5) Alto 53-69 (F3-A4) Tenor 48-65 (C3-F4) Bass 41-58 (F2-Bb3)
3-09 {1: '4/4'} {1: -1} 84 84 680.0 84 84 680.0 2534.00 886 ... Soprano I 60-77 (C4-F5) Soprano II 57-74 (A3-D5) Alto 53-72 (F3-C5) Tenor 48-67 (C3-G4) Bass 41-58 (F2-Bb3)
4-19 {1: '4/4'} {1: 0} 111 111 448.0 111 111 448.0 1726.00 956 ... Soprano I 57-76 (A3-E5) Soprano II 57-76 (A3-E5) Alto 53-69 (F3-A4) Tenor 48-66 (C3-F#4) Bass 43-57 (G2-A3)
5-01 {1: '4/4'} {1: 0} 67 67 268.0 67 67 268.0 1088.50 661 ... Soprano I 64-81 (E4-A5) Soprano II 59-77 (B3-F5) Alto 52-69 (E3-A4) Tenor 55-69 (G3-A4) Bass 45-62 (A2-D4)
5-03 {1: '4/4'} {1: -1} 74 74 296.0 74 74 296.0 1073.00 652 ... Soprano I 62-77 (D4-F5) Soprano II 61-74 (C#4-D5) Alto 50-70 (D3-Bb4) Tenor 49-65 (C#3-F4) Bass 43-58 (G2-Bb3)
5-04a {1: '4/4'} {1: -1} 82 82 328.0 82 82 328.0 1401.00 939 ... Soprano I 65-82 (F4-Bb5) Soprano II 64-81 (E4-A5) Alto 55-74 (G3-D5) Tenor 53-69 (F3-A4) Bass 46-63 (Bb2-Eb4)
5-04c {1: '4/4'} {1: -1} 53 53 212.0 53 53 212.0 826.00 589 ... Soprano I 66-81 (F#4-A5) Soprano II 65-79 (F4-G5) Alto 55-74 (G3-D5) Tenor 55-69 (G3-A4) Bass 46-63 (Bb2-Eb4)
5-04d {1: '4/4'} {1: -1} 59 59 236.0 59 59 236.0 950.00 715 ... Soprano I 65-82 (F4-Bb5) Soprano II 62-81 (D4-A5) Alto 55-74 (G3-D5) Tenor 53-69 (F3-A4) Bass 45-65 (A2-F4)
5-04e {1: '4/4'} {1: -1} 95 95 380.0 95 95 380.0 1530.50 1213 ... Soprano I 65-81 (F4-A5) Soprano II 62-81 (D4-A5) Alto 60-74 (C4-D5) Tenor 50-69 (D3-A4) Bass 43-62 (G2-D4)
5-05b {1: '4/4'} {1: -1} 58 58 232.0 58 58 232.0 930.00 685 ... Soprano I 62-79 (D4-G5) Soprano II 58-77 (Bb3-F5) Alto 55-70 (G3-Bb4) Tenor 62-74 (D4-D5) Bass 41-58 (F2-Bb3)
5-05c {1: '4/4'} {1: -1, 39: 0} 75 75 300.0 75 75 300.0 1258.00 819 ... Soprano I 57-77 (A3-F5) Soprano II 59-76 (B3-E5) Alto 54-69 (F#3-A4) Tenor 50-67 (D3-G4) Bass 43-62 (G2-D4)
5-08 {1: '4/4'} {1: 0} 91 91 364.0 91 91 364.0 1229.00 793 ... Soprano I 60-77 (C4-F5) Soprano II 57-79 (A3-G5) Alto 50-69 (D3-A4) Tenor 48-64 (C3-E4) Bass 41-57 (F2-A3)
5-09 {1: '4/4'} {1: 0} 84 84 336.0 84 84 336.0 1207.50 1238 ... Soprano I 60-77 (C4-F5) Soprano II 55-70 (G3-Bb4) Alto 62-78 (D4-F#5) Tenor 57-75 (A3-Eb5) Bass 41-57 (F2-A3)
5-11 {1: '4/4'} {1: 0} 57 57 228.0 57 57 228.0 724.50 598 ... Soprano I 62-76 (D4-E5) Soprano II 61-74 (C#4-D5) Alto 67-79 (G4-G5) Tenor 62-76 (D4-E5) Bass 38-60 (D2-C4)
6-01a {1: '4/4'} {1: 0} 34 34 136.0 34 34 136.0 513.00 353 ... Soprano I 62-74 (D4-D5) Soprano II 61-70 (C#4-Bb4) Alto 52-67 (E3-G4) Tenor 50-62 (D3-D4) Bass 43-57 (G2-A3)
8-18 {1: '4/4', 28: '3/1', 97: '4/4'} {1: 0} 108 106 1797.0 108 106 1797.0 4714.00 1055 ... Soprano 62-77 (D4-F5) Tenor I 36-55 (C2-G3) Tenor II 36-57 (C2-A3) Bass 40-60 (E2-C4) Harpsichord 40-60 (E2-C4)
8-19 {1: '3/1', 21: '4/4', 34: '3/1', 40: '4/4', 89... {1: 0} 142 142 1008.0 142 142 1008.0 3065.50 1300 ... Alto 57-69 (A3-A4) Tenor 48-65 (C3-F4) Bass I 40-60 (E2-C4) Bass II 40-67 (E2-G4) <NA> <NA>
9-12 {1: '3/2', 10: '2/4', 11: '3/2', 18: '2/4'} {1: 0} 26 26 116.0 26 26 116.0 423.25 254 ... Tenor I 45-55 (A2-G3) Tenor II 41-53 (F2-F3) Bass 43-60 (G2-C4) Harpsichord 31-48 (G1-C3) <NA> <NA>
laudate_pueri_dominum {1: '4/4', 126: '3/2', 138: '4/4'} {1: -1} 177 177 808.0 177 177 808.0 3492.00 2182 ... Soprano 65-81 (F4-A5) Alto I 58-74 (Bb3-D5) Alto II 53-70 (F3-Bb4) Tenor 53-70 (F3-Bb4) Bass 31-53 (G1-F3)

19 rows × 57 columns

Composition years histogram#

Hide source
hist_data = summary.reset_index()
hist_data.corpus = hist_data.corpus.map(corpus_names)
fig = px.histogram(
    hist_data,
    x="composed_end",
    color="corpus",
    labels=dict(
        composed_end="decade",
        count="pieces",
    ),
    color_discrete_map=corpus_name_colors,
    title=f"Temporal coverage of the {N} annotated pieces in the Distant Listening Corpus",
)
fig.update_traces(xbins=dict(size=10))
fig.update_layout(**utils.STD_LAYOUT)
fig.update_legends(font=dict(size=16))
save_figure_as(fig, "pieces_timeline_histogram", height=1250)
fig.show()

Dimensions#

Overview#

def make_overview_table(groupby, group_name="pieces"):
    n_groups = groupby.size().rename(group_name)
    absolute_numbers = dict(
        measures=groupby.last_mn.sum(),
        length=groupby.length_qb.sum(),
        notes=groupby.n_onsets.sum(),
        labels=groupby.label_count.sum(),
    )
    absolute = pd.DataFrame.from_dict(absolute_numbers)
    absolute = pd.concat([n_groups, absolute], axis=1)
    sum_row = pd.DataFrame(absolute.sum(), columns=["sum"]).T
    absolute = pd.concat([absolute, sum_row])
    return absolute


absolute = make_overview_table(summary.groupby("workTitle"))
# print(absolute.astype(int).to_markdown())
absolute.astype(int)
pieces measures length notes labels
Il quarto libro de madrigali 1 111 448 956 185
Il quinto libro de madrigali 11 795 3180 8902 1592
Il secondo libro de madrigali 1 93 372 1011 225
Il sesto libro de madrigali 1 34 136 353 83
Il terzo libro de madrigali 1 84 680 886 190
Madrigali e canzonette...Libro nono 1 26 116 254 52
Madrigali guerriri, et amorosi...Libro ottavo 2 248 2805 2355 576
Messa et salmi 1 177 808 2182 386
sum 19 1568 8545 16899 3289
def summarize_dataset(D):
    all_metadata = D.get_metadata()
    summary = make_summary(all_metadata)
    return make_overview_table(summary.groupby(level=0))


corpus_summary = summarize_dataset(D)
print(corpus_summary.astype(int).to_markdown())
|                      |   pieces |   measures |   length |   notes |   labels |
|:---------------------|---------:|-----------:|---------:|--------:|---------:|
| monteverdi_madrigals |       19 |       1568 |     8545 |   16899 |     3289 |
| sum                  |       19 |       1568 |     8545 |   16899 |     3289 |

Measures#

all_measures = D.get_feature("measures")
print(
    f"{len(all_measures.index)} measures over {len(all_measures.groupby(level=[0,1]))} files."
)
all_measures.head()
1570 measures over 19 files.
mc mn quarterbeats duration_qb keysig timesig act_dur mc_offset numbering_offset dont_count barline breaks repeats next
corpus piece i
monteverdi_madrigals 2-12 0 1 1 0 4.0 -1 4/4 1 0 <NA> <NA> <NA> <NA> firstMeasure (2,)
1 2 2 4 4.0 -1 4/4 1 0 <NA> <NA> <NA> <NA> <NA> (3,)
2 3 3 8 4.0 -1 4/4 1 0 <NA> <NA> <NA> <NA> <NA> (4,)
3 4 4 12 4.0 -1 4/4 1 0 <NA> <NA> <NA> <NA> <NA> (5,)
4 5 5 16 4.0 -1 4/4 1 0 <NA> <NA> <NA> <NA> <NA> (6,)
all_measures.get_default_analysis().plot_grouped()

Harmony labels#

All symbols, independent of the local key (the mode of which changes their semantics).

try:
    all_annotations = D.get_feature("harmonylabels").df
except Exception:
    all_annotations = pd.DataFrame()
n_annotations = len(all_annotations.index)
includes_annotations = n_annotations > 0
if includes_annotations:
    display(all_annotations.head())
    print(f"Concatenated annotation tables contains {all_annotations.shape[0]} rows.")
    no_chord = all_annotations.root.isna()
    if no_chord.sum() > 0:
        print(
            f"{no_chord.sum()} of them are not chords. Their values are:"
            f" {all_annotations.label[no_chord].value_counts(dropna=False).to_dict()}"
        )
    all_chords = all_annotations[~no_chord].copy()
    print(
        f"Dataset contains {all_chords.shape[0]} tokens and {len(all_chords.chord.unique())} types over "
        f"{len(all_chords.groupby(level=[0,1]))} documents."
    )
    all_annotations["corpus_name"] = all_annotations.index.get_level_values(0).map(
        utils.get_corpus_display_name
    )
    all_chords["corpus_name"] = all_chords.index.get_level_values(0).map(
        utils.get_corpus_display_name
    )
else:
    print("Dataset contains no annotations.")
mc mn quarterbeats quarterbeats_all_endings duration_qb mc_onset mn_onset timesig staff voice ... numeral_or_applied_to_numeral intervals_over_bass intervals_over_root scale_degrees scale_degrees_and_mode scale_degrees_major scale_degrees_minor globalkey localkey chord
corpus piece i
monteverdi_madrigals 2-12 0 1 1 0 0 6.0 0 0 4/4 5 1 ... V (M3, P5) (M3, P5) (5, 7, 2) (5, 7, 2), major (5, 7, 2) (5, #7, 2) F I V
1 2 2 6 6 2.0 1/2 1/2 4/4 5 1 ... vi (m3, P5) (m3, P5) (6, 1, 3) (6, 1, 3), major (6, 1, 3) (#6, 1, #3) F I vi
2 3 3 8 8 6.0 0 0 4/4 5 1 ... I (M3, P5) (M3, P5) (1, 3, 5) (1, 3, 5), major (1, 3, 5) (1, #3, 5) F I I
3 4 4 14 14 1.5 1/2 1/2 4/4 5 1 ... ii (m3, P5) (m3, P5) (2, 4, 6) (2, 4, 6), major (2, 4, 6) (2, 4, #6) F I ii
4 4 4 31/2 31/2 0.5 7/8 7/8 4/4 5 1 ... vii (m3, M6) (m3, d5) (2, 4, 7) (2, 4, 7), major (2, 4, 7) (2, 4, #7) F I viio6

5 rows × 51 columns

Concatenated annotation tables contains 3289 rows.
Dataset contains 3289 tokens and 232 types over 19 documents.