eval_SC_loose all

Author: MegamindHenry

docs/src/man/eval.md

@@ -8,10 +8,13 @@ CurrentModule = JudiLing
   Comp_Acc_Struct
   accuracy_comprehension
   eval_SC
+  eval_SC_loose
   eval_SC(Union{SparseMatrixCSC, Matrix}, Union{SparseMatrixCSC, Matrix})
   eval_SC(SChat,SC,data,target_col)
   eval_SC(SChat,SC,batch_size;verbose=false)
   eval_SC(SChat,SC,data,target_col,batch_size;verbose=false)
+  eval_SC_loose(SChat,SC,k)
+  eval_SC_loose(SChat,SC,k,data,target_col)
   eval_acc(::Array, ::Array)
   eval_acc_loose(::Array, ::Array)
   extract_gpi

src/eval.jl

@@ -10,10 +10,18 @@ end
 """
 Assess model accuracy on the basis of the correlations of row vectors of Chat and 
 C or Shat and S. Ideally the target words have highest correlations on the diagonal 
-of the pertinent correlation matrices.
+of the pertinent correlation matrices. Homophones support option is implemented.
 """
 function eval_SC end
 
+
+"""
+Assess model accuracy on the basis of the correlations of row vectors of Chat and 
+C or Shat and S. Count it as correct if one of the top k candidates is correct. 
+Homophones support option is implemented.
+"""
+function eval_SC_loose end
+
 """
     accuracy_comprehension(::Matrix, ::Matrix) -> ::Comp_Acc_Struct
 
@@ -257,6 +265,79 @@ function eval_SC_chucks(SChat,SC,s,batch_size,data,target_col)
   sum(v)
 end
 
+"""
+    eval_SC_loose(SChat, SC, k)
+
+Assess model accuracy on the basis of the correlations of row vectors of Chat and 
+C or Shat and S. Count it as correct if one of the top k candidates is correct. 
+
+...
+# Obligatory Arguments
+- `SChat::Union{SparseMatrixCSC, Matrix}`: the Chat or Shat matrix
+- `SC::Union{SparseMatrixCSC, Matrix}`: the C or S matrix
+- `k`: top k candidates
+
+```julia
+eval_SC_loose(Chat, cue_obj.C, k)
+eval_SC_loose(Shat, S, k)
+```
+...
+"""
+function eval_SC_loose(SChat, SC, k)
+  total = size(SChat, 1)
+  correct = 0
+  rSC = cor(convert(Matrix{Float64}, SChat), convert(Matrix{Float64}, SC), dims=2)
+  
+  for i in 1:total
+    p = sortperm(rSC[i,:],rev=true)
+    p = p[1:k,:]
+    if i in p
+      correct += 1
+    end
+  end
+  return correct/total
+end
+
+"""
+    eval_SC_loose(SChat,SC,k,data,target_col)
+
+Assess model accuracy on the basis of the correlations of row vectors of Chat and 
+C or Shat and S. Count it as correct if one of the top k candidates is correct. 
+Support for homophones.
+
+...
+# Obligatory Arguments
+- `SChat::Union{SparseMatrixCSC, Matrix}`: the Chat or Shat matrix
+- `SC::Union{SparseMatrixCSC, Matrix}`: the C or S matrix
+- `k`: top k candidates
+- `data`: datasets
+- `target_col`: target column name
+
+```julia
+eval_SC_loose(Chat, cue_obj.C, k, latin, :Word)
+eval_SC_loose(Shat, S, k, latin, :Word)
+```
+...
+"""
+function eval_SC_loose(SChat,SC,k,data,target_col)
+  total = size(SChat, 1)
+  correct = 0
+  rSC = cor(convert(Matrix{Float64}, SChat), convert(Matrix{Float64}, SC), dims=2)
+  
+  for i in 1:total
+    p = sortperm(rSC[i,:],rev=true)
+    p = p[1:k]
+    if i in p
+      correct += 1
+    else
+      if data[i,target_col] in data[p,:Word]
+        correct += 1
+      end
+    end
+  end
+  return correct/total
+end
+
 """
     eval_manual(::Array, ::DataFrame, ::Dict) -> ::Nothing
 

test/eval_tests.jl

@@ -90,4 +90,49 @@ end
   catch
     @test false
   end
+end
+
+@testset "eval_SC_loose" begin
+  try
+    latin = DataFrame(
+      Word = ["ABC", "BCD", "CDE", "BCD"],
+      Lexeme = ["A", "B", "C", "B"],
+      Person = ["B", "C", "D", "D"]
+      )
+
+    cue_obj = JudiLing.make_cue_matrix(
+      latin,
+      grams=3,
+      target_col=:Word,
+      tokenized=false,
+      keep_sep=false
+      )
+
+    n_features = size(cue_obj.C, 2)
+    S = JudiLing.make_S_matrix(
+      latin,
+      ["Lexeme"],
+      ["Person"],
+      ncol=n_features)
+
+    G = JudiLing.make_transform_matrix(S, cue_obj.C)
+    Chat = S * G
+    F = JudiLing.make_transform_matrix(cue_obj.C, S)
+    Shat = cue_obj.C * F
+
+    @test JudiLing.eval_SC_loose(Chat, cue_obj.C, 1) == 0.75
+    @test JudiLing.eval_SC_loose(Shat, S, 1) == 0.75
+    @test JudiLing.eval_SC_loose(Chat, cue_obj.C, 1, latin, :Word) == 1
+    @test JudiLing.eval_SC_loose(Shat, S, 1, latin, :Word) == 1
+
+    for k in 2:4
+      @test JudiLing.eval_SC_loose(Chat, cue_obj.C, k) == 1
+      @test JudiLing.eval_SC_loose(Shat, S, k) == 1
+      @test JudiLing.eval_SC_loose(Chat, cue_obj.C, k, latin, :Word) == 1
+      @test JudiLing.eval_SC_loose(Shat, S, k, latin, :Word) == 1
+    end
+
+  catch
+    @test false
+  end
 end