Notes#

Notebook settings
-----------------

CORPUS_PATH: '/home/runner/work/workflow_deployment/schubert_winterreise'
ANNOTATED_ONLY: False

Data and software versions
--------------------------

Data repo 'schubert_winterreise' @ 30e2245
dimcat version 0.3.0
ms3 version 2.5.2

dataset = dc.Dataset()
dataset.load(directory=CORPUS_PATH, parse_tsv=False)

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n01'. I'm picking 'labels/v0.1/n01.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n02'. I'm picking 'labels/v0.1/n02.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n03'. I'm picking 'labels/v0.1/n03.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n04'. I'm picking 'labels/v0.1/n04.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n05'. I'm picking 'labels/v0.1/n05.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n06'. I'm picking 'labels/v0.1/n06.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n07'. I'm picking 'labels/v0.1/n07.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n08'. I'm picking 'labels/v0.1/n08.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n09'. I'm picking 'labels/v0.1/n09.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n10'. I'm picking 'labels/v0.1/n10.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n11'. I'm picking 'labels/v0.1/n11.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n12'. I'm picking 'labels/v0.1/n12.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n13'. I'm picking 'labels/v0.1/n13.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n14'. I'm picking 'labels/v0.1/n14.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n15'. I'm picking 'labels/v0.1/n15.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n16'. I'm picking 'labels/v0.1/n16.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n17'. I'm picking 'labels/v0.1/n17.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n18'. I'm picking 'labels/v0.1/n18.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n19'. I'm picking 'labels/v0.1/n19.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n20'. I'm picking 'labels/v0.1/n20.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n21'. I'm picking 'labels/v0.1/n21.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n22'. I'm picking 'labels/v0.1/n22.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n23'. I'm picking 'labels/v0.1/n23.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

WARNING  ms3.utils.functions -- /home/runner/.local/lib/python3.10/site-packages/ms3/utils/functions.py (line 6191) automatically_choose_from_disambiguated_files():
	Unable to automatically choose from the 2 'labels' with piece 'n24'. I'm picking 'labels/v0.1/n24.labels.tsv' because its disambiguation string '/v0.1' is the lexicographically first among ['/v0.1', '/v0.2']

[default|all]
All corpora
-----------
View: This view is called 'default'. It 
	- excludes pieces that are not contained in the metadata,
	- filters out file extensions requiring conversion (such as .xml), and
	- excludes review files and folders.

                          has   active   scores measures           notes          labels        expanded          chords       
                     metadata     view detected detected parsed detected parsed detected parsed detected parsed detected parsed
corpus                                                                                                                         
schubert_winterreise      yes  default       24       24     24       24     24       48     24       24     24       24     24

168/696 files are excluded from this view.

168 files have been excluded based on their subdir.

N = 24 annotated pieces, 120 parsed dataframes.

Metadata#

all_metadata = dataset.data.metadata()
print(f"Concatenated 'metadata.tsv' files cover {len(all_metadata)} of the {dataset.data.count_pieces()} scores.")
all_metadata.reset_index(level=1).groupby(level=0).nth(0).iloc[:,:20]

Concatenated 'metadata.tsv' files cover 24 of the 24 scores.

	piece	TimeSig	KeySig	last_mc	last_mn	length_qb	last_mc_unfolded	last_mn_unfolded	length_qb_unfolded	all_notes_qb	n_onsets	n_onset_positions	guitar_chord_count	form_label_count	label_count	annotated_key	harmony_version	annotators	reviewers	composed_start
corpus
schubert_winterreise	n01	{1: '2/4'}	{1: -1, 71: 2, 99: -1}	105	105	210.0	137	137	274.0	1088.75	2174	505	0	0	214	d	2.1.0	Alexander Faschon	Johannes Hentschel	1827

Compute chronological order

chronological_order = chronological_corpus_order(all_metadata)
corpus_colors = dict(zip(chronological_order, CORPUS_COLOR_SCALE))
chronological_order

['schubert_winterreise']

all_notes = dataset.data.get_all_parsed('notes', force=True, flat=True)
print(f"{len(all_notes.index)} notes over {len(all_notes.groupby(level=[0,1]))} files.")
all_notes.head()

WARNING  ms3.Parse.schubert_winterreise -- /home/runner/.local/lib/python3.10/site-packages/ms3/corpus.py (line 1255) check_number_of_unparsed_scores():
	You have set force=True, which forces me to parse 24 scores iteratively. Next time, call _.parse() on me, so we can speed this up!

26614 notes over 24 files.

			mc	mn	quarterbeats	quarterbeats_all_endings	duration_qb	mc_onset	mn_onset	timesig	staff	voice	...	nominal_duration	scalar	tied	tpc	midi	name	octave	chord_id	gracenote	tremolo
corpus	piece	i
schubert_winterreise	n01	0	1	1	0	0	0.5	0	0	2/4	3	1	...	1/8	1	<NA>	2	50	D3	3	5	<NA>	NaN
		1	1	1	0	0	0.5	0	0	2/4	3	1	...	1/8	1	<NA>	3	57	A3	3	5	<NA>	NaN
		2	1	1	0	0	0.5	0	0	2/4	2	2	...	1/8	1	<NA>	-1	65	F4	4	1	<NA>	NaN
		3	1	1	1/2	1/2	0.5	1/8	1/8	2/4	3	1	...	1/8	1	<NA>	2	50	D3	3	6	<NA>	NaN
		4	1	1	1/2	1/2	0.5	1/8	1/8	2/4	3	1	...	1/8	1	<NA>	3	57	A3	3	6	<NA>	NaN

5 rows × 21 columns

def weight_notes(nl, group_col='midi', precise=True):
    summed_durations = nl.groupby(group_col).duration_qb.sum()
    shortest_duration = summed_durations[summed_durations > 0].min()
    summed_durations /= shortest_duration # normalize such that the shortest duration results in 1 occurrence
    if not precise:
        # This simple trick reduces compute time but also precision:
        # The rationale is to have the smallest value be slightly larger than 0.5 because
        # if it was exactly 0.5 it would be rounded down by repeat_notes_according_to_weights()
        summed_durations /= 1.9999999
    return repeat_notes_according_to_weights(summed_durations)
    
def repeat_notes_according_to_weights(weights):
    try:
        counts = weights.round().astype(int)
    except Exception:
        return pd.Series(dtype=int)
    counts_reflecting_weights = []
    for pitch, count in counts.items():
        counts_reflecting_weights.extend([pitch]*count)
    return pd.Series(counts_reflecting_weights)

Ambitus#

corpus_names = {corp: get_corpus_display_name(corp) for corp in chronological_order}
chronological_corpus_names = list(corpus_names.values())
corpus_name_colors = {corpus_names[corp]: color for corp, color in corpus_colors.items()}
all_notes['corpus_name'] = all_notes.index.get_level_values(0).map(corpus_names)

grouped_notes = all_notes.groupby('corpus_name')
weighted_midi = pd.concat([weight_notes(nl, 'midi', precise=False) for _, nl in grouped_notes], keys=grouped_notes.groups.keys()).reset_index(level=0)
weighted_midi.columns = ['dataset', 'midi']
weighted_midi

	dataset	midi
0	Schubert Winterreise	29
1	Schubert Winterreise	30
2	Schubert Winterreise	30
3	Schubert Winterreise	30
4	Schubert Winterreise	30
...	...	...
11823	Schubert Winterreise	88
11824	Schubert Winterreise	88
11825	Schubert Winterreise	89
11826	Schubert Winterreise	91
11827	Schubert Winterreise	93

11828 rows × 2 columns

yaxis=dict(tickmode= 'array',
           tickvals= [12, 24, 36, 48, 60, 72, 84, 96],
           ticktext = ["C0", "C1", "C2", "C3", "C4", "C5", "C6", "C7"],
           gridcolor='lightgrey',
           )
fig = px.violin(weighted_midi, 
                x='dataset', 
                y='midi', 
                color='dataset', 
                box=True,
                labels=dict(
                    dataset='',
                    midi='distribution of pitches by duration'
                ),
                category_orders=dict(dataset=chronological_corpus_names),
                color_discrete_map=corpus_name_colors,
                width=1000, height=600,
               )
fig.update_traces(spanmode='hard') # do not extend beyond outliers
fig.update_layout(yaxis=yaxis, 
                  **STD_LAYOUT,
                 showlegend=False)
fig.show()

Tonal Pitch Classes (TPC)#

weighted_tpc = pd.concat([weight_notes(nl, 'tpc') for _, nl in grouped_notes], keys=grouped_notes.groups.keys()).reset_index(level=0)
weighted_tpc.columns = ['dataset', 'tpc']
weighted_tpc

	dataset	tpc
0	Schubert Winterreise	-10
1	Schubert Winterreise	-10
2	Schubert Winterreise	-10
3	Schubert Winterreise	-9
4	Schubert Winterreise	-8
...	...	...
15760	Schubert Winterreise	13
15761	Schubert Winterreise	13
15762	Schubert Winterreise	13
15763	Schubert Winterreise	14
15764	Schubert Winterreise	14

15765 rows × 2 columns

As violin plot#

yaxis=dict(
    tickmode= 'array',
    tickvals= [-12, -9, -6, -3, 0, 3, 6, 9, 12, 15, 18],
    ticktext = ["Dbb", "Bbb", "Gb", "Eb", "C", "A", "F#", "D#", "B#", "G##", "E##"],
    gridcolor='lightgrey',
    zerolinecolor='lightgrey',
    zeroline=True
           )
fig = px.violin(weighted_tpc, 
                x='dataset', 
                y='tpc', 
                color='dataset', 
                box=True,
                labels=dict(
                    dataset='',
                    tpc='distribution of tonal pitch classes by duration'
                ),
                category_orders=dict(dataset=chronological_corpus_names),
                color_discrete_map=corpus_name_colors,
                width=1000, 
                height=600,
               )
fig.update_traces(spanmode='hard') # do not extend beyond outliers
fig.update_layout(yaxis=yaxis, 
                  **STD_LAYOUT,
                 showlegend=False)
fig.show()

As bar plots#

bar_data = all_notes.groupby('tpc').duration_qb.sum().reset_index()
x_values = list(range(bar_data.tpc.min(), bar_data.tpc.max()+1))
x_names = ms3.fifths2name(x_values)
fig = px.bar(bar_data, x='tpc', y='duration_qb',
             labels=dict(tpc='Named pitch class',
                             duration_qb='Duration in quarter notes'
                            ),
             color_discrete_sequence=CORPUS_COLOR_SCALE,
             width=1000, height=300,
             )
fig.update_layout(**STD_LAYOUT)
fig.update_yaxes(gridcolor='lightgrey')
fig.update_xaxes(gridcolor='lightgrey', zerolinecolor='grey', tickmode='array', 
                 tickvals=x_values, ticktext = x_names, dtick=1, ticks='outside', tickcolor='black', 
                 minor=dict(dtick=6, gridcolor='grey', showgrid=True),
                )
fig.show()

scatter_data = all_notes.groupby(['corpus_name', 'tpc']).duration_qb.sum().reset_index()
fig = px.bar(scatter_data, x='tpc', y='duration_qb', color='corpus_name', 
                 labels=dict(
                     duration_qb='duration',
                     tpc='named pitch class',
                 ),
                 category_orders=dict(dataset=chronological_corpus_names),
                 color_discrete_map=corpus_name_colors,
                 width=1000, height=500,
                )
fig.update_layout(**STD_LAYOUT)
fig.update_yaxes(gridcolor='lightgrey')
fig.update_xaxes(gridcolor='lightgrey', zerolinecolor='grey', tickmode='array', 
                 tickvals=x_values, ticktext = x_names, dtick=1, ticks='outside', tickcolor='black', 
                 minor=dict(dtick=6, gridcolor='grey', showgrid=True),
                )
fig.show()

As scatter plots#

fig = px.scatter(scatter_data, x='tpc', y='duration_qb', color='corpus_name', 
                 labels=dict(
                     duration_qb='duration',
                     tpc='named pitch class',
                 ),
                 category_orders=dict(dataset=chronological_corpus_names),
                 color_discrete_map=corpus_name_colors,
                 facet_col='corpus_name', facet_col_wrap=3, facet_col_spacing=0.03,
                 width=1000, height=1000,
                )
fig.update_traces(mode='lines+markers')
fig.for_each_annotation(lambda a: a.update(text=a.text.split("=")[-1]))
fig.update_layout(**STD_LAYOUT, showlegend=False)
fig.update_xaxes(gridcolor='lightgrey', zerolinecolor='lightgrey', tickmode='array', tickvals= [-12, -6, 0, 6, 12, 18],
    ticktext = ["Dbb", "Gb", "C", "F#", "B#", "E##"], visible=True, )
fig.update_yaxes(gridcolor='lightgrey', zeroline=False, matches=None, showticklabels=True)
fig.show()

no_accidental = bar_data[bar_data.tpc.between(-1,5)].duration_qb.sum()
with_accidental = bar_data[~bar_data.tpc.between(-1,5)].duration_qb.sum()

entire = no_accidental + with_accidental
f"Fraction of note duration without accidental of the entire durations: {no_accidental} / {entire} = {no_accidental / entire}"

'Fraction of note duration without accidental of the entire durations: 10678.916666666668 / 15766.125000000002 = 0.677332995055327'

Notes and staves#

print("Distribution of notes over staves:")
value_count_df(all_notes.staff)

Distribution of notes over staves:

	counts	%
staff
2	12900	0.484707
3	9576	0.359811
1	4138	0.155482