# matching outside parens (lyrics)
sed '1,/%%/d' 301.gabc | sed 's/) /)∞ /g' | sed 's/([^)]\+)/- /g' - | sed 's/- ∞//g' | tr '\n' ' '  | iconv -f UTF-8 -t ASCII//TRANSLIT | tr -cd 'a-zA-Z- <>/.;:,\!\n' | sed  's/ \+/ /g' | sed -e 's/^[ -]\+//' -e 's/[ -]\+$//' | tr 'j' 'i' | tr 'J' 'I'

# matching inside parens (notes)
sed '1,/%%/d' 301.gabc | grep -oP '\([^)]+\)' -  | tr '\n' ' ' | sed  -e 's/ \+/ /g' -e 's/ \+$//'


# parallel

## lyrics
par "sed '1,/%%/d' {} | sed 's/) /)∞ /g' | sed 's/([^)]\+)/- /g' - | sed 's/- ∞//g' | tr '\n' ' '  | iconv -f UTF-8 -t ASCII//TRANSLIT | tr -cd 'a-zA-Z- <>/.;:,\!\*\n' | sed  's/ \+/ /g' | sed -e 's/^[ -]\+//' -e 's/[ -]\+$//' -e 's/$/\n/' | tr 'j' 'i' | tr 'J' 'I' > Lyrics/{.}.txt" ::: *.gabc

## notes
par "sed '1,/%%/d' {} | grep -oP '\([^)]+\)' -  | tr '\n' ' ' | sed  -e 's/ \+/ /g' -e 's/ \+$//' -e 's/$/\n/' > Notes/{.}.txt" ::: *.gabc




cat Lyrics/* > Lyrics.txt
cat Notes/* > Notes.txt

dd if=/dev/urandom of=rand_shuf.dat bs=1M count=1
shuf --random-source rand_shuf.dat Lyrics.txt > Lyrics_shuffled.txt
shuf --random-source rand_shuf.dat Notes.txt > Notes_shuffled.txt 

mkdir toy-lanotes

# source
head -n -350 Lyrics_shuffled.txt > toy-lanotes/src-train.txt    # first 20745 - 350 = 20395 rows
tail -n 350 Lyrics_shuffled.txt > toy-lanotes/src-val.txt       # last 350 rows
# target
head -n -350 Notes_shuffled.txt > toy-lanotes/tgt-train.txt          # first 289470 - 350 = 20395 rows
tail -n 350 Notes_shuffled.txt > toy-lanotes/tgt-val.txt             # last 350 rows

# check #1
wc -l toy-lanotes/*.txt
#   20395 toy-lanotes/src-train.txt
#     350 toy-lanotes/src-val.txt
#   20395 toy-lanotes/tgt-train.txt
#     350 toy-lanotes/tgt-val.txt

# check #2
parallel echo  {}': ' \&\& ./check_inivisible_chars.py {} ::: toy-lanotes/run/*.{src,tgt}
# reveals that ∃ '\t' chars; this is due to blank lines.
# Find and remove blank lines:
grep -n '^$' toy-lanotes/*-train.txt | cut -d: -f2 | parallel sed -i "{},{}d" toy-lanotes/*-train.txt
grep -n '^$' toy-lanotes/*-val.txt | cut -d: -f2 | parallel sed -i "{},{}d" toy-lanotes/*-val.txt
# (for some reason, these commands have to be run multiple times to remove all blank lines)

# check #3
randomLineFromTwoFiles toy-lanotes/*-train.txt
randomLineFromTwoFiles toy-lanotes/*-val.txt