Overview

This notebook gives a general overview of the features included in the dataset.

Show imports Hide imports

%load_ext autoreload
%autoreload 2
import os

import dimcat as dc
import pandas as pd
import plotly.express as px
from dimcat import filters, plotting
from IPython.display import display

import utils

RESULTS_PATH = os.path.abspath(os.path.join(utils.OUTPUT_FOLDER, "overview"))
os.makedirs(RESULTS_PATH, exist_ok=True)


def make_output_path(
    filename: str,
    extension=None,
    path=RESULTS_PATH,
) -> str:
    return utils.make_output_path(filename=filename, extension=extension, path=path)


def save_figure_as(
    fig, filename, formats=("png", "pdf"), directory=RESULTS_PATH, **kwargs
):
    if formats is not None:
        for fmt in formats:
            plotting.write_image(fig, filename, directory, format=fmt, **kwargs)
    else:
        plotting.write_image(fig, filename, directory, **kwargs)

Loading data

D = utils.get_dataset("beethoven_piano_sonatas", corpus_release="v2.5")
package = D.inputs.get_package()
package_info = package._package.custom
git_tag = package_info.get("git_tag")
utils.print_heading("Data and software versions")
print("Ludwig van Beethoven – Piano Sonatas version v2.5")
print(f"Datapackage '{package.package_name}' @ {git_tag}")
print(f"dimcat version {dc.__version__}\n")
D

Data and software versions
--------------------------

Ludwig van Beethoven – Piano Sonatas version v2.5
Datapackage 'beethoven_piano_sonatas' @ v2.5
dimcat version 3.4.0

Dataset
=======
{'inputs': {'basepath': None,
            'packages': {'beethoven_piano_sonatas': ["'beethoven_piano_sonatas.measures' "
                                                     '(MuseScoreFacetName.MuseScoreMeasures)',
                                                     "'beethoven_piano_sonatas.notes' "
                                                     '(MuseScoreFacetName.MuseScoreNotes)',
                                                     "'beethoven_piano_sonatas.expanded' "
                                                     '(MuseScoreFacetName.MuseScoreHarmonies)',
                                                     "'beethoven_piano_sonatas.chords' "
                                                     '(MuseScoreFacetName.MuseScoreChords)',
                                                     "'beethoven_piano_sonatas.metadata' (FeatureName.Metadata)"]}},
 'outputs': {'basepath': None, 'packages': {}},
 'pipeline': []}

filtered_D = filters.HasHarmonyLabelsFilter(keep_values=[True]).process(D)
all_metadata = filtered_D.get_metadata()
assert len(all_metadata) > 0, "No pieces selected for analysis."
all_metadata

		TimeSig	KeySig	last_mc	last_mn	length_qb	last_mc_unfolded	last_mn_unfolded	length_qb_unfolded	volta_mcs	all_notes_qb	...	musicbrainz	viaf	wikidata	originalFormat	transcriber	typesetter	staff_1_ambitus	staff_1_instrument	staff_2_ambitus	staff_2_instrument
corpus	piece
beethoven_piano_sonatas	01-1	{1: '2/2'}	{1: -4}	154	152	608.00	308	304	1216.00	()	1476.00	...	https://musicbrainz.org/work/a78520e0-0211-3b5...	https://viaf.org/viaf/179625665	https://www.wikidata.org/wiki/Q145813	xml	<NA>	<NA>	51-89 (Eb3-F6)	piano	32-73 (Ab1-Db5)	piano
	01-2	{1: '3/4'}	{1: -1}	62	61	183.00	124	122	366.00	()	526.17	...	https://musicbrainz.org/work/bea1b893-2732-33a...	https://viaf.org/viaf/179625665	https://www.wikidata.org/wiki/Q145813	xml	<NA>	<NA>	43-89 (G2-F6)	piano	31-77 (G1-F5)	piano
	01-3	{1: '3/4'}	{1: -4, 43: -1}	77	73	219.00	196	186	558.00	()	565.50	...	https://musicbrainz.org/work/2bd7e1ea-c696-3be...	https://viaf.org/viaf/179625665	https://www.wikidata.org/wiki/Q145813	mxl	<NA>	<NA>	53-85 (F3-Db6)	Piano	31-74 (G1-D5)	Piano
	01-4	{1: '2/2'}	{1: -4}	199	196	790.00	392	390	1560.00	(57, 58], [59, 60, 61)	2326.83	...	https://musicbrainz.org/work/b755e900-804a-312...	https://viaf.org/viaf/179625665	https://www.wikidata.org/wiki/Q145813	mxl	<NA>	<NA>	50-89 (D3-F6)	Piano	31-75 (G1-Eb5)	Piano
	02-1	{1: '2/4'}	{1: 3, 127: 0, 230: 3}	342	336	679.50	672	664	1336.00	(115, 116, 117, 118], [119, 120, 121, 122, 123...	1695.75	...	https://musicbrainz.org/work/c001a2eb-9493-327...	https://viaf.org/viaf/179221580	https://www.wikidata.org/wiki/Q145699	xml	<NA>	<NA>	39-89 (D#2-F6)	piano	31-76 (G1-E5)	piano
	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
	31-1	{1: '3/4'}	{1: -4, 69: 4, 78: -4}	116	116	348.00	116	116	348.00	()	1107.25	...	https://musicbrainz.org/work/5102b059-2457-30a...	https://viaf.org/viaf/179823640	https://www.wikidata.org/wiki/Q145699	xml	<NA>	<NA>	36-94 (C2-Bb6)	<NA>	29-80 (F1-Ab5)	<NA>
	31-2	{1: '2/4'}	{1: -4, 43: -5, 98: -4}	162	158	324.00	230	228	460.00	(40, 41], [42]], [[145, 146], [147)	977.00	...	https://musicbrainz.org/work/e83b7804-7b03-37f...	https://viaf.org/viaf/179823640	https://www.wikidata.org/wiki/Q145699	xml	<NA>	<NA>	49-94 (Db3-Bb6)	piano	29-90 (F1-Gb6)	piano
	31-3	{1: '4/4', 9: '12/16', 29: '6/8', 117: '12/16'...	{1: -5, 5: 4, 7: -6, 29: -4, 119: -2, 140: 1, ...	217	212	651.00	217	212	651.00	()	2064.62	...	https://musicbrainz.org/work/989195d3-99b5-35f...	https://viaf.org/viaf/179823640	https://www.wikidata.org/wiki/Q145699	xml	<NA>	<NA>	44-92 (Ab2-Ab6)	<NA>	24-75 (C1-Eb5)	<NA>
	32-1	{1: '4/4'}	{1: -3, 45: -4, 74: -2, 89: -3}	160	157	636.12	210	207	836.12	(70], [71)	1970.58	...	https://musicbrainz.org/work/88fb51a0-8dac-365...	https://viaf.org/viaf/179823640	https://www.wikidata.org/wiki/Q145699	xml	<NA>	<NA>	29-96 (F1-C7)	piano	24-89 (C1-F6)	piano
	32-2	{1: '9/16', 40: '6/16', 59: '12/32', 78: '9/16'}	{1: 0, 130: -3, 144: 0}	191	177	389.25	235	225	469.50	(9], [10]], [[18], [19]], [[27], [28]], [[37, ...	1196.00	...	https://musicbrainz.org/work/607559e1-ac3d-36a...	https://viaf.org/viaf/179823640	https://www.wikidata.org/wiki/Q145699	xml	<NA>	<NA>	31-94 (G1-Bb6)	<NA>	26-89 (D1-F6)	<NA>

64 rows × 57 columns

mean_composition_years = utils.corpus_mean_composition_years(all_metadata)
chronological_order = mean_composition_years.index.to_list()
corpus_colors = dict(zip(chronological_order, utils.CORPUS_COLOR_SCALE))
corpus_names = {
    corp: utils.get_corpus_display_name(corp) for corp in chronological_order
}
chronological_corpus_names = list(corpus_names.values())
corpus_name_colors = {
    corpus_names[corp]: color for corp, color in corpus_colors.items()
}

mean_composition_years

corpus
beethoven_piano_sonatas    1801.96875
Name: mean_composition_year, dtype: float64

Composition dates

This section relies on the dataset’s metadata.

valid_composed_start = pd.to_numeric(all_metadata.composed_start, errors="coerce")
valid_composed_end = pd.to_numeric(all_metadata.composed_end, errors="coerce")
print(
    f"Composition dates range from {int(valid_composed_start.min())} {valid_composed_start.idxmin()} "
    f"to {int(valid_composed_end.max())} {valid_composed_end.idxmax()}."
)

Composition dates range from 1793 ('beethoven_piano_sonatas', '01-1') to 1822 ('beethoven_piano_sonatas', '31-1').

Mean composition years per corpus

def make_summary(metadata_df):
    piece_is_annotated = metadata_df.label_count > 0
    return metadata_df[piece_is_annotated].copy()

Show source Hide source

summary = make_summary(all_metadata)
bar_data = pd.concat(
    [
        mean_composition_years.rename("year"),
        summary.groupby(level="corpus").size().rename("pieces"),
    ],
    axis=1,
).reset_index()

N = len(summary)
fig = px.bar(
    bar_data,
    x="year",
    y="pieces",
    color="corpus",
    color_discrete_map=corpus_colors,
    title=f"Temporal coverage of the {N} annotated pieces in the Distant Listening Corpus",
)
fig.update_traces(width=5)
fig.update_layout(**utils.STD_LAYOUT)
fig.update_traces(width=5)
save_figure_as(fig, "pieces_timeline_bars")
fig.show()

summary

		TimeSig	KeySig	last_mc	last_mn	length_qb	last_mc_unfolded	last_mn_unfolded	length_qb_unfolded	volta_mcs	all_notes_qb	...	musicbrainz	viaf	wikidata	originalFormat	transcriber	typesetter	staff_1_ambitus	staff_1_instrument	staff_2_ambitus	staff_2_instrument
corpus	piece
beethoven_piano_sonatas	01-1	{1: '2/2'}	{1: -4}	154	152	608.00	308	304	1216.00	()	1476.00	...	https://musicbrainz.org/work/a78520e0-0211-3b5...	https://viaf.org/viaf/179625665	https://www.wikidata.org/wiki/Q145813	xml	<NA>	<NA>	51-89 (Eb3-F6)	piano	32-73 (Ab1-Db5)	piano
	01-2	{1: '3/4'}	{1: -1}	62	61	183.00	124	122	366.00	()	526.17	...	https://musicbrainz.org/work/bea1b893-2732-33a...	https://viaf.org/viaf/179625665	https://www.wikidata.org/wiki/Q145813	xml	<NA>	<NA>	43-89 (G2-F6)	piano	31-77 (G1-F5)	piano
	01-3	{1: '3/4'}	{1: -4, 43: -1}	77	73	219.00	196	186	558.00	()	565.50	...	https://musicbrainz.org/work/2bd7e1ea-c696-3be...	https://viaf.org/viaf/179625665	https://www.wikidata.org/wiki/Q145813	mxl	<NA>	<NA>	53-85 (F3-Db6)	Piano	31-74 (G1-D5)	Piano
	01-4	{1: '2/2'}	{1: -4}	199	196	790.00	392	390	1560.00	(57, 58], [59, 60, 61)	2326.83	...	https://musicbrainz.org/work/b755e900-804a-312...	https://viaf.org/viaf/179625665	https://www.wikidata.org/wiki/Q145813	mxl	<NA>	<NA>	50-89 (D3-F6)	Piano	31-75 (G1-Eb5)	Piano
	02-1	{1: '2/4'}	{1: 3, 127: 0, 230: 3}	342	336	679.50	672	664	1336.00	(115, 116, 117, 118], [119, 120, 121, 122, 123...	1695.75	...	https://musicbrainz.org/work/c001a2eb-9493-327...	https://viaf.org/viaf/179221580	https://www.wikidata.org/wiki/Q145699	xml	<NA>	<NA>	39-89 (D#2-F6)	piano	31-76 (G1-E5)	piano
	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
	31-1	{1: '3/4'}	{1: -4, 69: 4, 78: -4}	116	116	348.00	116	116	348.00	()	1107.25	...	https://musicbrainz.org/work/5102b059-2457-30a...	https://viaf.org/viaf/179823640	https://www.wikidata.org/wiki/Q145699	xml	<NA>	<NA>	36-94 (C2-Bb6)	<NA>	29-80 (F1-Ab5)	<NA>
	31-2	{1: '2/4'}	{1: -4, 43: -5, 98: -4}	162	158	324.00	230	228	460.00	(40, 41], [42]], [[145, 146], [147)	977.00	...	https://musicbrainz.org/work/e83b7804-7b03-37f...	https://viaf.org/viaf/179823640	https://www.wikidata.org/wiki/Q145699	xml	<NA>	<NA>	49-94 (Db3-Bb6)	piano	29-90 (F1-Gb6)	piano
	31-3	{1: '4/4', 9: '12/16', 29: '6/8', 117: '12/16'...	{1: -5, 5: 4, 7: -6, 29: -4, 119: -2, 140: 1, ...	217	212	651.00	217	212	651.00	()	2064.62	...	https://musicbrainz.org/work/989195d3-99b5-35f...	https://viaf.org/viaf/179823640	https://www.wikidata.org/wiki/Q145699	xml	<NA>	<NA>	44-92 (Ab2-Ab6)	<NA>	24-75 (C1-Eb5)	<NA>
	32-1	{1: '4/4'}	{1: -3, 45: -4, 74: -2, 89: -3}	160	157	636.12	210	207	836.12	(70], [71)	1970.58	...	https://musicbrainz.org/work/88fb51a0-8dac-365...	https://viaf.org/viaf/179823640	https://www.wikidata.org/wiki/Q145699	xml	<NA>	<NA>	29-96 (F1-C7)	piano	24-89 (C1-F6)	piano
	32-2	{1: '9/16', 40: '6/16', 59: '12/32', 78: '9/16'}	{1: 0, 130: -3, 144: 0}	191	177	389.25	235	225	469.50	(9], [10]], [[18], [19]], [[27], [28]], [[37, ...	1196.00	...	https://musicbrainz.org/work/607559e1-ac3d-36a...	https://viaf.org/viaf/179823640	https://www.wikidata.org/wiki/Q145699	xml	<NA>	<NA>	31-94 (G1-Bb6)	<NA>	26-89 (D1-F6)	<NA>

64 rows × 57 columns

Composition years histogram

Show source Hide source

hist_data = summary.reset_index()
hist_data.corpus = hist_data.corpus.map(corpus_names)
fig = px.histogram(
    hist_data,
    x="composed_end",
    color="corpus",
    labels=dict(
        composed_end="decade",
        count="pieces",
    ),
    color_discrete_map=corpus_name_colors,
    title=f"Temporal coverage of the {N} annotated pieces in the Distant Listening Corpus",
)
fig.update_traces(xbins=dict(size=10))
fig.update_layout(**utils.STD_LAYOUT)
fig.update_legends(font=dict(size=16))
save_figure_as(fig, "pieces_timeline_histogram", height=1250)
fig.show()

Dimensions

Overview

def make_overview_table(groupby, group_name="pieces"):
    n_groups = groupby.size().rename(group_name)
    absolute_numbers = dict(
        measures=groupby.last_mn.sum(),
        length=groupby.length_qb.sum(),
        notes=groupby.n_onsets.sum(),
        labels=groupby.label_count.sum(),
    )
    absolute = pd.DataFrame.from_dict(absolute_numbers)
    absolute = pd.concat([n_groups, absolute], axis=1)
    sum_row = pd.DataFrame(absolute.sum(), columns=["sum"]).T
    absolute = pd.concat([absolute, sum_row])
    return absolute


absolute = make_overview_table(summary.groupby("workTitle"))
# print(absolute.astype(int).to_markdown())
absolute.astype(int)

	pieces	measures	length	notes	labels
Sonata no. 1	4	482	1800	6752	928
Sonata no. 10	3	544	1153	6144	1003
Sonata no. 16	3	719	2304	11421	1291
Sonata no. 17	3	730	1842	8110	1035
Sonata no. 18	4	816	2313	8889	1129
Sonata no. 19	2	274	713	3283	577
Sonata no. 2	4	671	1875	8448	1186
Sonata no. 20	2	242	848	2914	455
Sonata no. 21 'Waldstein'	3	873	2668	13678	1437
Sonata no. 23 ('Appassionata')	3	720	2522	12410	1049
Sonata no. 24	2	288	794	3955	603
Sonata no. 26 'Les Adieux'	3	493	1671	7409	981
Sonata no. 3	4	778	2521	10666	1711
Sonata no. 30	3	479	1422	7135	1228
Sonata no. 31	3	486	1323	7440	1184
Sonata no. 32	2	334	1025	9741	1445
Sonata no. 5	3	518	1564	5609	875
Sonata no. 6	3	522	1217	5721	882
Sonata no. 7	4	630	2347	8381	1105
Sonata no. 8 'Pathetique'	3	593	2233	8329	1011
Sonata no. 9	3	470	1712	5633	848
sum	64	11662	35871	162068	21963

def summarize_dataset(D):
    all_metadata = D.get_metadata()
    summary = make_summary(all_metadata)
    return make_overview_table(summary.groupby(level=0))


corpus_summary = summarize_dataset(D)
print(corpus_summary.astype(int).to_markdown())

|                         |   pieces |   measures |   length |   notes |   labels |
|:------------------------|---------:|-----------:|---------:|--------:|---------:|
| beethoven_piano_sonatas |       64 |      11662 |    35871 |  162068 |    21963 |
| sum                     |       64 |      11662 |    35871 |  162068 |    21963 |

Measures

all_measures = D.get_feature("measures")
print(
    f"{len(all_measures.index)} measures over {len(all_measures.groupby(level=[0,1]))} files."
)
all_measures.head()

16560 measures over 91 files.

			mc	mn	quarterbeats	duration_qb	keysig	timesig	act_dur	mc_offset	numbering_offset	dont_count	barline	breaks	repeats	next	markers	jump_bwd	jump_fwd	play_until	volta
corpus	piece	i
beethoven_piano_sonatas	01-1	0	1	0	0	1.0	-4	2/2	1/4	3/4	<NA>	1	<NA>	<NA>	firstMeasure	(2,)	<NA>	<NA>	<NA>	<NA>	<NA>
		1	2	1	1	4.0	-4	2/2	1	0	<NA>	<NA>	<NA>	<NA>	<NA>	(3,)	<NA>	<NA>	<NA>	<NA>	<NA>
		2	3	2	5	4.0	-4	2/2	1	0	<NA>	<NA>	<NA>	<NA>	<NA>	(4,)	<NA>	<NA>	<NA>	<NA>	<NA>
		3	4	3	9	4.0	-4	2/2	1	0	<NA>	<NA>	<NA>	<NA>	<NA>	(5,)	<NA>	<NA>	<NA>	<NA>	<NA>
		4	5	4	13	4.0	-4	2/2	1	0	<NA>	<NA>	<NA>	<NA>	<NA>	(6,)	<NA>	<NA>	<NA>	<NA>	<NA>

all_measures.get_default_analysis().plot_grouped()

Harmony labels

All symbols, independent of the local key (the mode of which changes their semantics).

try:
    all_annotations = D.get_feature("harmonylabels").df
except Exception:
    all_annotations = pd.DataFrame()
n_annotations = len(all_annotations.index)
includes_annotations = n_annotations > 0
if includes_annotations:
    display(all_annotations.head())
    print(f"Concatenated annotation tables contains {all_annotations.shape[0]} rows.")
    no_chord = all_annotations.root.isna()
    if no_chord.sum() > 0:
        print(
            f"{no_chord.sum()} of them are not chords. Their values are:"
            f" {all_annotations.label[no_chord].value_counts(dropna=False).to_dict()}"
        )
    all_chords = all_annotations[~no_chord].copy()
    print(
        f"Dataset contains {all_chords.shape[0]} tokens and {len(all_chords.chord.unique())} types over "
        f"{len(all_chords.groupby(level=[0,1]))} documents."
    )
    all_annotations["corpus_name"] = all_annotations.index.get_level_values(0).map(
        utils.get_corpus_display_name
    )
    all_chords["corpus_name"] = all_chords.index.get_level_values(0).map(
        utils.get_corpus_display_name
    )
else:
    print("Dataset contains no annotations.")

			mc	mn	quarterbeats	duration_qb	mc_onset	mn_onset	timesig	staff	voice	volta	...	numeral_or_applied_to_numeral	intervals_over_bass	intervals_over_root	scale_degrees	scale_degrees_and_mode	scale_degrees_major	scale_degrees_minor	globalkey	localkey	chord
corpus	piece	i
beethoven_piano_sonatas	01-1	0	1	0	0	9.0	0	3/4	2/2	2	1	<NA>	...	i	(m3, P5)	(m3, P5)	(1, 3, 5)	(1, 3, 5), minor	(1, b3, 5)	(1, 3, 5)	f	i	i
		1	4	3	9	8.0	0	0	2/2	2	1	<NA>	...	V	(m3, d5, m6)	(M3, P5, m7)	(#7, 2, 4, 5)	(#7, 2, 4, 5), minor	(7, 2, 4, 5)	(#7, 2, 4, 5)	f	i	V65
		2	6	5	17	4.0	0	0	2/2	2	1	<NA>	...	i	(m3, P5)	(m3, P5)	(1, 3, 5)	(1, 3, 5), minor	(1, b3, 5)	(1, 3, 5)	f	i	i
		3	7	6	21	4.0	0	0	2/2	2	1	<NA>	...	#vii	(m3, M6)	(m3, d5)	(2, 4, #7)	(2, 4, #7), minor	(2, 4, 7)	(2, 4, #7)	f	i	#viio6
		4	8	7	25	2.0	0	0	2/2	2	1	<NA>	...	i	(M3, M6)	(m3, P5)	(3, 5, 1)	(3, 5, 1), minor	(b3, 5, 1)	(3, 5, 1)	f	i	i6

5 rows × 52 columns

Concatenated annotation tables contains 21509 rows.
Dataset contains 21509 tokens and 1095 types over 64 documents.