Add warnings to eval.jl

Deprecate eval_SC_chucks in favour of eval_SC_chunks (fixing typo) and adding warning regarding homophones/homographs in accuracy_comprehension

Author: MariaHei

src/eval.jl

@@ -25,7 +25,8 @@ function eval_SC_loose end
 """
     accuracy_comprehension(S, Shat, data)
 
-Evaluate comprehension accuracy.
+Evaluate comprehension accuracy for training data.
+NOTE: in case of homophones/homographs in the dataset, the correct/incorrect values for base and inflections may be misleading! See below for more information.
 
 # Obligatory Arguments
 - `S::Matrix`: the (gold standard) S matrix
@@ -47,16 +48,19 @@ accuracy_comprehension(
     base=[:Lexeme],
     inflections=[:Person, :Number, :Tense, :Voice, :Mood]
     )
-
-accuracy_comprehension(
-    S_val,
-    Shat_val,
-    latin_train,
-    target_col=:Words,
-    base=["Lexeme"],
-    inflections=[:Person, :Number, :Tense, :Voice, :Mood]
-    )
 ```
+
+# Note
+In case of homophones/homographs in the dataset, the correct/incorrect values for base and inflections may be misleading!
+Consider the following example: The wordform "Äpfel" in German can be nominative plural, genitive plural and accusative plural.
+Let's assume we have a dataset in which "Äpfel" occurs in all three case/number combinations (i.e. there are homographs).
+If all these wordforms have the same semantic vectors (e.g. because they are derived from word2vec or fasttext which typically
+have a single vector per unique wordform), the predicted semantic vector of the wordform "Äpfel" will be equally correlated
+with all three case/number combinations in the dataset. In such cases, while the algorithm in this function can unambiguously
+conclude that the correct surface form "Äpfel" was comprehended, which of the three possible rows is the correct one will be
+picked somewhat non-deterministically (see https://docs.julialang.org/en/v1/base/collections/#Base.argmax). It is thus possible
+that the algorithm will then use the genitive plural instead of the intended nominative plural as the ground plural, and will
+report that "case" was comprehended incorrectly.
 """
 function accuracy_comprehension(
     S,
@@ -78,10 +82,16 @@ function accuracy_comprehension(
     dfr.r_target = corMat[diagind(corMat)]
     dfr.correct = [dfr.target[i] == dfr.form[i] for i = 1:size(dfr, 1)]
 
+    if length(data[:, target_col]) != length(Set(data[:, target_col]))
+        @warn "This dataset contains homophones/homographs. Note that some of the results on the correctness of comprehended base/inflections may be misleading. See documentation of this function for more information."
+    end
+
     if !isnothing(inflections)
         all_features = vcat(base, inflections)
-    else
+    elseif !isnothing(base)
         all_features = base
+    else
+        all_features = []
     end
 
     for f in all_features
@@ -110,7 +120,9 @@ end
         inflections = nothing,
     )
 
-Evaluate comprehension accuracy.
+Evaluate comprehension accuracy for validation data.
+NOTE: in case of homophones/homographs in the dataset, the correct/incorrect values for base and inflections may be misleading! See below for more information.
+
 
 # Obligatory Arguments
 - `S_val::Matrix`: the (gold standard) S matrix of the validation data
@@ -137,6 +149,18 @@ accuracy_comprehension(
     inflections=[:Person, :Number, :Tense, :Voice, :Mood]
     )
 ```
+
+# Note
+In case of homophones/homographs in the dataset, the correct/incorrect values for base and inflections may be misleading!
+Consider the following example: The wordform "Äpfel" in German can be nominative plural, genitive plural and accusative plural.
+Let's assume we have a dataset in which "Äpfel" occurs in all three case/number combinations (i.e. there are homographs).
+If all these wordforms have the same semantic vectors (e.g. because they are derived from word2vec or fasttext which typically
+have a single vector per unique wordform), the predicted semantic vector of the wordform "Äpfel" will be equally correlated
+with all three case/number combinations in the dataset. In such cases, while the algorithm in this function can unambiguously
+conclude that the correct surface form "Äpfel" was comprehended, which of the three possible rows is the correct one will be
+picked somewhat non-deterministically (see https://docs.julialang.org/en/v1/base/collections/#Base.argmax). It is thus possible
+that the algorithm will then use the genitive plural instead of the intended nominative plural as the ground plural, and will
+report that "case" was comprehended incorrectly.
 """
 function accuracy_comprehension(
     S_val,
@@ -160,6 +184,10 @@ function accuracy_comprehension(
 
     append!(data_combined, data_train, promote=true)
 
+    if length(data_combined[:, target_col]) != length(Set(data_combined[:, target_col]))
+        @warn "This dataset contains homophones/homographs. Note that some of the results on the correctness of comprehended base/inflections may be misleading. See documentation of this function for more information."
+    end
+
     corMat = cor(Shat_val, S, dims = 2)
     top_index = [i[2] for i in argmax(corMat, dims = 2)]
 
@@ -435,7 +463,7 @@ function eval_SC(
 
     # for first parts
     for j = 1:num_chucks-1
-        correct += eval_SC_chucks(
+        correct += eval_SC_chunks(
             SChat_d,
             SC_d,
             (j - 1) * batch_size + 1,
@@ -445,7 +473,7 @@ function eval_SC(
         verbose && ProgressMeter.next!(pb)
     end
     # for last part
-    correct += eval_SC_chucks(
+    correct += eval_SC_chunks(
         SChat_d,
         SC_d,
         (num_chucks - 1) * batch_size + 1,
@@ -504,7 +532,7 @@ function eval_SC(
 
     # for first parts
     for j = 1:num_chucks-1
-        correct += eval_SC_chucks(
+        correct += eval_SC_chunks(
             SChat_d,
             SC_d,
             (j - 1) * batch_size + 1,
@@ -516,7 +544,7 @@ function eval_SC(
         verbose && ProgressMeter.next!(pb)
     end
     # for last part
-    correct += eval_SC_chucks(
+    correct += eval_SC_chunks(
         SChat_d,
         SC_d,
         (num_chucks - 1) * batch_size + 1,
@@ -529,13 +557,18 @@ function eval_SC(
     round(correct / l, digits=digits)
 end
 
-function eval_SC_chucks(SChat, SC, s, e, batch_size)
+function eval_SC_chunks(SChat, SC, s, e, batch_size)
     rSC = cor(SChat[s:e, :], SC, dims = 2)
     v = [(rSC[i[1], i[1]+s-1] == rSC[i]) ? 1 : 0 for i in argmax(rSC, dims = 2)]
     sum(v)
 end
 
-function eval_SC_chucks(SChat, SC, s, e, batch_size, data, target_col)
+function eval_SC_chucks(SChat, SC, s, e, batch_size)
+    @warn "eval_SC_chucks is deprecated and will be removed in version 0.10 in favour of eval_SC_chunks"
+    eval_SC_chunks(SChat, SC, s, e, batch_size)
+end
+
+function eval_SC_chunks(SChat, SC, s, e, batch_size, data, target_col)
     rSC = cor(SChat[s:e, :], SC, dims = 2)
     v = [
         data[i[1]+s-1, target_col] == data[i[2], target_col] ? 1 : 0
@@ -544,13 +577,23 @@ function eval_SC_chucks(SChat, SC, s, e, batch_size, data, target_col)
     sum(v)
 end
 
-function eval_SC_chucks(SChat, SC, s, batch_size)
+function eval_SC_chucks(SChat, SC, s, e, batch_size, data, target_col)
+    @warn "eval_SC_chucks is deprecated and will be removed in version 0.10 in favour of eval_SC_chunks"
+    eval_SC_chunks(SChat, SC, s, e, batch_size, data, target_col)
+end
+
+function eval_SC_chunks(SChat, SC, s, batch_size)
     rSC = cor(SChat[s:end, :], SC, dims = 2)
     v = [(rSC[i[1], i[1]+s-1] == rSC[i]) ? 1 : 0 for i in argmax(rSC, dims = 2)]
     sum(v)
 end
 
-function eval_SC_chucks(SChat, SC, s, batch_size, data, target_col)
+function eval_SC_chucks(SChat, SC, s, batch_size)
+    @warn "eval_SC_chucks is deprecated and will be removed in version 0.10 in favour of eval_SC_chunks"
+    eval_SC_chunks(SChat, SC, s, batch_size)
+end
+
+function eval_SC_chunks(SChat, SC, s, batch_size, data, target_col)
     rSC = cor(SChat[s:end, :], SC, dims = 2)
     v = [
         data[i[1]+s-1, target_col] == data[i[2], target_col] ? 1 : 0
@@ -559,6 +602,11 @@ function eval_SC_chucks(SChat, SC, s, batch_size, data, target_col)
     sum(v)
 end
 
+function eval_SC_chucks(SChat, SC, s, batch_size, data, target_col)
+    @warn "eval_SC_chucks is deprecated and will be removed in version 0.10 in favour of eval_SC_chunks"
+    eval_SC_chunks(SChat, SC, s, batch_size, data, target_col)
+end
+
 """
     eval_SC_loose(SChat, SC, k)