Add warnings about homophones to eval_SC and eval_SC_loose

Author: MariaHei

src/eval.jl

@@ -85,7 +85,7 @@ function accuracy_comprehension(
     dfr.correct = [dfr.target[i] == dfr.form[i] for i = 1:size(dfr, 1)]
 
     if length(data[:, target_col]) != length(Set(data[:, target_col]))
-        @warn "This dataset contains homophones/homographs. Note that some of the results on the correctness of comprehended base/inflections may be misleading. See documentation of this function for more information."
+        @warn "accuracy_comprehension: This dataset contains homophones/homographs. Note that some of the results on the correctness of comprehended base/inflections may be misleading. See documentation of this function for more information."
     end
 
     if !isnothing(inflections)
@@ -189,7 +189,7 @@ function accuracy_comprehension(
     append!(data_combined, data_train, promote=true)
 
     if length(data_combined[:, target_col]) != length(Set(data_combined[:, target_col]))
-        @warn "This dataset contains homophones/homographs. Note that some of the results on the correctness of comprehended base/inflections may be misleading. See documentation of this function for more information."
+        @warn "accuracy_comprehension: This dataset contains homophones/homographs. Note that some of the results on the correctness of comprehended base/inflections may be misleading. See documentation of this function for more information."
     end
 
     corMat = cor(Shat_val, S, dims = 2)
@@ -232,6 +232,9 @@ Assess model accuracy on the basis of the correlations of row vectors of Chat an
 C or Shat and S. Ideally the target words have highest correlations on the diagonal
 of the pertinent correlation matrices.
 
+!!! note
+    If there are homophones/homographs in the dataset, this evaluation method may be misleading: the predicted vector will be equally correlated with the target vector of both words and the one on the diagonal will not necessarily be selected as the most correlated. In such cases, supplying the dataset and `target_col` is recommended which enables taking into account homophones/homographs.
+
 # Obligatory Arguments
 - `SChat::Union{SparseMatrixCSC, Matrix}`: the Chat or Shat matrix
 - `SC::Union{SparseMatrixCSC, Matrix}`: the C or S matrix
@@ -248,6 +251,11 @@ eval_SC(Shat_val, S_val)
 ```
 """
 function eval_SC(SChat::AbstractArray, SC::AbstractArray; digits=4, R=false)
+
+    if size(unique(SC, dims=1), 1) != size(SC, 1)
+        @warn "eval_SC: The C or S matrix contains duplicate vectors (usually because of homophones/homographs). Supplying the dataset and target column is recommended for a realistic evaluation. See the documentation of this function for more information."
+    end
+
     rSC = cor(
         convert(Matrix{Float64}, SChat),
         convert(Matrix{Float64}, SC),
@@ -273,6 +281,9 @@ of the pertinent correlation matrices.
     The order is important. The fist gold standard matrix has to be corresponing
     to the SChat matrix, such as `eval_SC(Shat_train, S_train, S_val)` or `eval_SC(Shat_val, S_val, S_train)`
 
+!!! note
+    If there are homophones/homographs in the dataset, this evaluation method may be misleading: the predicted vector will be equally correlated with the target vector of both words and the one on the diagonal will not necessarily be selected as the most correlated. In such cases, supplying the dataset and target_col is recommended which enables taking into account homophones/homographs.
+
 # Obligatory Arguments
 - `SChat::Union{SparseMatrixCSC, Matrix}`: the Chat or Shat matrix
 - `SC::Union{SparseMatrixCSC, Matrix}`: the training/validation C or S matrix
@@ -427,7 +438,10 @@ end
 Assess model accuracy on the basis of the correlations of row vectors of Chat and
 C or Shat and S. Ideally the target words have highest correlations on the diagonal
 of the pertinent correlation matrices. For large datasets, pass batch_size to
-process evaluation in chucks.
+process evaluation in chunks.
+
+!!! note
+    If there are homophones/homographs in the dataset, this evaluation method may be misleading: the predicted vector will be equally correlated with the target vector of both words and the one on the diagonal will not necessarily be selected as the most correlated. In such cases, supplying the dataset and target_col is recommended which enables taking into account homophones/homographs.
 
 # Obligatory Arguments
 - `SChat`: the Chat or Shat matrix
@@ -455,6 +469,10 @@ function eval_SC(
     verbose = false
     )
 
+    if size(unique(SC, dims=1), 1) != size(SC, 1)
+        @warn "eval_SC: The C or S matrix contains duplicate vectors (usually because of homophones/homographs). Supplying the dataset and target column is recommended for a realistic evaluation. See the documentation of this function for more information."
+    end
+
     l = size(SChat, 1)
     num_chucks = ceil(Int64, l / batch_size)
     verbose && begin
@@ -494,7 +512,7 @@ end
 Assess model accuracy on the basis of the correlations of row vectors of Chat and
 C or Shat and S. Ideally the target words have highest correlations on the diagonal
 of the pertinent correlation matrices. For large datasets, pass batch_size to
-process evaluation in chucks. Support homophones.
+process evaluation in chunks. Support homophones.
 
 # Obligatory Arguments
 - `SChat::AbstractArray`: the Chat or Shat matrix
@@ -617,6 +635,10 @@ end
 Assess model accuracy on the basis of the correlations of row vectors of Chat and
 C or Shat and S. Count it as correct if one of the top k candidates is correct.
 
+!!! note
+    If there are homophones/homographs in the dataset, this evaluation method may be misleading: the predicted vector will be equally correlated with the target vector of both words and it is not guaranteed that the target on the diagonal will be among the k neighbours. In particular, `eval_SC` and `eval_SC_loose` with k=1 are not guaranteed to give the same result. In such cases, supplying the dataset and `target_col` is recommended which enables taking into account homophones/homographs.
+
+
 # Obligatory Arguments
 - `SChat::Union{SparseMatrixCSC, Matrix}`: the Chat or Shat matrix
 - `SC::Union{SparseMatrixCSC, Matrix}`: the C or S matrix
@@ -631,6 +653,14 @@ eval_SC_loose(Shat, S, k)
 ```
 """
 function eval_SC_loose(SChat, SC, k; digits=4)
+
+    if size(unique(SC, dims=1), 1) != size(SC, 1)
+        @warn "eval_SC_loose: The C or S matrix contains duplicate vectors (usually because of homophones/homographs). Supplying the dataset and target column is recommended for a realistic evaluation. See the documentation of this function for more information."
+        if k == 1
+            @warn "eval_SC_loose: You set k=1. Note that if there are duplicate vectors in the S/C matrix, it is not guaranteed that eval_SC_loose with k=1 gives the same result as eval_SC."
+        end
+    end
+
     total = size(SChat, 1)
     correct = 0
     rSC = cor(