new test_combo

Author: MegamindHenry

docs/src/man/test_combo.md

@@ -5,5 +5,5 @@ CurrentModule = JudiLing
 # Test Combo
 
 ```@docs
-    test_combo
+    test_combo(test_mode;kwargs...)
 ```

src/preprocess.jl

@@ -2,11 +2,6 @@ struct SplitDataException <: Exception
     msg::String
 end
 
-"""
-Split dataset into training and validation datasets.
-"""
-function train_val_split end
-
 """
 Leave p out cross-validation.
 """
@@ -44,30 +39,30 @@ end
 
 function train_val_random_split(
     data_path,
-    test_sample_size,
     output_dir_path,
     data_prefix;
-    max_val = 0,
-    max_val_ratio = 0.0,
+    train_sample_size = 0,
+    val_sample_size = 0,
+    val_ratio = 0.0,
     random_seed = 314,
     verbose = false,
     )
 
     data = DataFrame(CSV.File(data_path))
-    n_data = size(data, 1)
 
-    if test_sample_size != 0
-        data = data[1:test_sample_size,:]
-        n_data = test_sample_size
+    if train_sample_size != 0
+        data = data[1:train_sample_size, :]
     end
 
+    n_data = size(data, 1)
+
     rng = MersenneTwister(random_seed)
     data = data[shuffle(rng, 1:n_data), :]
 
-    max_val = cal_max_val(n_data, max_val, max_val_ratio)
+    val_sample_size = cal_max_val(n_data, val_sample_size, val_ratio)
 
-    data_train = data[max_val+1:end,:]
-    data_val = data[1:max_val,:]
+    data_train = data[val_sample_size+1:end,:]
+    data_val = data[1:val_sample_size,:]
 
     write_split_data(output_dir_path, data_prefix, data_train, data_val)
 
@@ -81,12 +76,12 @@ end
 
 function train_val_carefully_split(
     data_path,
-    test_sample_size,
     output_dir_path,
     n_features_columns;
     data_prefix = "data",
-    max_val = 0,
-    max_val_ratio = 0.0,
+    train_sample_size = 0,
+    val_sample_size = 0,
+    val_ratio = 0.0,
     n_grams_target_col = :PhonWord,
     n_grams_tokenized = false,
     n_grams_sep_token = nothing,
@@ -98,19 +93,19 @@ function train_val_carefully_split(
     )
 
     data = DataFrame(CSV.File(data_path))
-    n_data = size(data, 1)
 
-    if test_sample_size != 0
-        data = data[1:test_sample_size,:]
-        n_data = test_sample_size
+    if train_sample_size != 0
+        data = data[1:train_sample_size, :]
     end
 
+    n_data = size(data, 1)
+
     rng = MersenneTwister(random_seed)
     data = data[shuffle(rng, 1:n_data), :]
 
-    max_val = cal_max_val(n_data, max_val, max_val_ratio)
+    val_sample_size = cal_max_val(n_data, val_sample_size, val_ratio)
 
-    init_num_train = round(Int64, (n_data - max_val) * 0.5)
+    init_num_train = round(Int64, (n_data - val_sample_size) * 0.5)
     data_train = data[1:init_num_train, :]
 
     if n_grams_tokenized && !isnothing(n_grams_sep_token)
@@ -146,7 +141,7 @@ function train_val_carefully_split(
         data_train_features,
         data_train,
         data_val,
-        max_val,
+        val_sample_size,
         grams,
         n_grams_target_col,
         n_grams_tokenized,
@@ -171,175 +166,6 @@ function train_val_carefully_split(
     nothing
 end
 
-function train_val_split(
-    data_path,
-    output_dir_path,
-    n_features_columns;
-    data_prefix = "data",
-    max_test_data = nothing,
-    split_max_ratio = 0.2,
-    n_grams_target_col = :PhonWord,
-    n_grams_tokenized = false,
-    n_grams_sep_token = nothing,
-    grams = 3,
-    n_grams_keep_sep = false,
-    start_end_token = "#",
-    random_seed = 314,
-    verbose = false,
-    )
-
-    # read csv
-    utterances = DataFrame(CSV.File(data_path))
-
-    # shuffle data
-    rng = MersenneTwister(random_seed)
-    utterances = utterances[shuffle(rng, 1:size(utterances, 1)), :]
-
-    if !isnothing(max_test_data)
-        utterances = utterances[1:max_test_data, :]
-    end
-
-    num_utterances = size(utterances, 1)
-
-    max_num_val = round(Int64, num_utterances * split_max_ratio)
-    init_num_train = round(Int64, (num_utterances - max_num_val) * 0.5)
-
-    utterances_train = utterances[1:init_num_train, :]
-
-    if n_grams_tokenized && !isnothing(n_grams_sep_token)
-        tokens =
-            split.(utterances_train[:, n_grams_target_col], n_grams_sep_token)
-    else
-        tokens = split.(utterances_train[:, n_grams_target_col], "")
-    end
-
-    verbose && println("Calculating utterances_train_ngrams ...")
-    utterances_train_ngrams = String[]
-
-    for i = 1:init_num_train
-        push!(
-            utterances_train_ngrams,
-            make_ngrams(
-                tokens[i],
-                grams,
-                n_grams_keep_sep,
-                n_grams_sep_token,
-                start_end_token,
-            )...,
-        )
-    end
-    utterances_train_ngrams = unique(utterances_train_ngrams)
-    utterances_train_features =
-        collect_features(utterances[1:init_num_train, :], n_features_columns)
-    utterances_val = DataFrame()
-
-    perform_split(
-        utterances[init_num_train+1:end, :],
-        utterances_train_ngrams,
-        utterances_train_features,
-        utterances_train,
-        utterances_val,
-        max_num_val,
-        grams,
-        n_grams_target_col,
-        n_grams_tokenized,
-        n_grams_sep_token,
-        n_grams_keep_sep,
-        start_end_token,
-        n_features_columns,
-        verbose = verbose,
-    )
-
-    if size(utterances_train, 1) <= 0 || size(utterances_val, 1) <= 0
-        throw(SplitDataException("Could not split data automaticly"))
-    end
-
-    write_split_data(output_dir_path, data_prefix, data_train, data_val)
-
-    verbose && begin
-        println("Successfully split data into $(size(utterances_train, 1)) training data and $(size(utterances_val, 1)) validation data")
-    end
-
-    nothing
-end
-
-function train_val_split(
-    data_path,
-    output_dir_path;
-    data_prefix = "data",
-    split_max_ratio = 0.2,
-    n_grams_target_col = :Word_n_grams,
-    n_grams_tokenized = false,
-    n_grams_sep_token = nothing,
-    n_features_col_name = :CommunicativeIntention,
-    n_features_tokenized = false,
-    n_features_sep_token = nothing,
-    random_seed = 314,
-    verbose = false,
-)
-
-    # read csv
-    utterances = DataFrame(CSV.File(data_path))
-    num_utterances = size(utterances, 1)
-
-    # shuffle data
-    rng = MersenneTwister(random_seed)
-    utterances = utterances[shuffle(rng, 1:size(utterances, 1)), :]
-
-    init_num_train = round(Int64, num_utterances * 0.4)
-    max_num_val = round(Int64, num_utterances * split_max_ratio)
-    utterances_train = utterances[1:init_num_train, :]
-    utterances_train_ngrams = unique([
-        ngram for i = 1:init_num_train
-        for
-        ngram in split_features(
-            utterances[i, :],
-            n_grams_target_col,
-            n_grams_tokenized,
-            n_grams_sep_token,
-        )
-    ])
-    utterances_train_features = unique([
-        feature for i = 1:init_num_train
-        for
-        feature in split_features(
-            utterances[i, :],
-            n_features_col_name,
-            n_features_tokenized,
-            n_grams_sep_token,
-        )
-    ])
-    utterances_val = DataFrame()
-
-    perform_split(
-        utterances[init_num_train+1:end, :],
-        utterances_train_ngrams,
-        utterances_train_features,
-        utterances_train,
-        utterances_val,
-        max_num_val,
-        n_grams_target_col,
-        n_grams_tokenized,
-        n_grams_sep_token,
-        n_features_col_name,
-        n_features_tokenized,
-        n_features_sep_token,
-        verbose = verbose,
-    )
-
-    if size(utterances_train, 1) <= 0 || size(utterances_val, 1) <= 0
-        throw(SplitDataException("Could not split data automaticly"))
-    end
-
-    write_split_data(output_dir_path, data_prefix, data_train, data_val)
-
-    verbose && begin
-        println("Successfully split data into $(size(utterances_train, 1)) training data and $(size(utterances_val, 1)) validation data")
-        println()
-    end
-    nothing
-end
-
 function split_features(
     datarow,
     col_name,
@@ -563,14 +389,14 @@ end
 function cal_max_val(n_data, max_val, max_val_ratio)
     if max_val == 0
         if max_val_ratio == 0.0
-            throw(ArgumentError("You haven't specify :max_val or " * 
-                " :max_val_ratio yet!"))
+            throw(ArgumentError("You haven't specify :val_sample_size or " * 
+                " :val_ratio yet!"))
         end
         max_val = round(Int64, n_data * max_val_ratio)
     else
         if max_val_ratio != 0.0
-            @warn "You have specified both :max_val and :max_val_ratio. Only" *
-            ":max_val will be used!"
+            @warn "You have specified both :val_sample_size and :val_ratio." *
+            " Only :max_val will be used!"
         end
     end
     return max_val