#!/bin/bash

s=ds
t=no
f=Case


# CONCAT DATA
cat da-ud-train.conllu sv-ud-train.conllu > $s-ud-train.conllu
cat OpenSubtitles2016.da-$t.da OpenSubtitles2016.sv-$t.sv > OpenSubtitles2016.$s-$t.$s
cat OpenSubtitles2016.da-$t.$t OpenSubtitles2016.sv-$t.$t > OpenSubtitles2016.$s-$t.$t

# NORMALIZATION

# Normalize the treebank
./normalize.pl $s < $s-ud-train.conllu > $s-ud-train.norm.conllu


# TRANSLATION

# Train source tagger
udpipe --train --tokenizer=none --parser=none $s.tagger.udpipe $s-ud-train.norm.conllu

# Word-align the treebank
./monogreedy_align.sh $s $t > monogr_$s-$t-tag.aligned

# Create the translation table
./trtable_src2tgt_feats.py < monogr_$s-$t-tag.aligned trtable_$s-$t.pickle

# Translate the treebank
./translate_conll_src2tgt_feats.py < $s-ud-train.norm.conllu trtable_$s-$t.pickle > $s-ud-train.norm.trans-$t.conllu


# EMBEDDINGS

# Tokenize
treex Read::Sentences lines_per_doc=10000 W2A::Tokenize A2W::ConcatenateTokens Write::Sentences < OpenSubtitles2016.$s-$t.$t > OpenSubtitles2016.$s-$t.$t.tok

# Pre-train word2vec
word2vec -train OpenSubtitles2016.$s-$t.$t.tok -output OpenSubtitles2016.$s-$t.$t.vectors -cbow 0 -size 50 -window 10 -negative 5 -hs 0 -sample 1e-1 -threads 4 -binary 0 -iter 15 -min-count 5


# MORPHO FEATS

# Simplify morpho feats
./feats2FEAT.py $f < $s-ud-train.norm.trans-$t.conllu > $s-ud-train.norm.trans-$t.$f.conllu


# PARSER

# Train the parser (and tagger, so that the parser is trained on predicted tags); tgt-xtag UPOS
W=$s.tmp1
cat $s-ud-train.norm.trans-$t.$f.conllu | cut -f5- > $W
udpipe --tag $t.tagger.udpipe $s-ud-train.norm.trans-$t.$f.conllu | \
    cut -f1-4 | paste - $W | sed 's/^\t$//' | \
    udpipe --train --tokenizer=none \
    --tagger='use_lemma=0;provide_lemma=0;use_xpostag=0;provide_xpostag=0' \
    --parser='embedding_form_file=para/OpenSubtitles2016.'$s-$t.$t'.vectors' \
    $s.norm.trans-$t.$f.udpipe
rm $W

# Eval on dev; src-xtag feats
W=$s.tmp2
cat $t-ud-predPoS-dev.conllu | cut -f1-4 > $W
udpipe --tag $s.norm.trans-$t.$f.udpipe $t-ud-predPoS-dev.conllu | \
    cut -f5- | paste $W - | sed 's/^\t$//' | \
    udpipe --accuracy --parse $s.norm.trans-$t.$f.udpipe
rm $W

