Merge pull request #2 from skewballfox/test_run

Test run

.gitignore

@@ -10,6 +10,8 @@ __pycache__/
 /logs
 *.egg-info/
 /.pytest_cache/
+/venv/
+/out/
 
 *.pyc
 *.pb

get_base_model_flow.py

@@ -58,33 +58,40 @@ def split_data_and_get_best_model_flow(
 
     print(dialog_handler_instance.render_template("inform-experiment_train_test_split"))
     precise_modeling_operations_instance.run_experimental_training(model_names)
+    # TODO: use model_analytics class to get the best model
     (
-        average_val_acc,
-        standard_deviation_val_acc,
-        average_acc,
-        standard_deviation_acc,
-    ) = precise_modeling_operations_instance.get_models_analytics()
-    # TODO: change to f1 score
+        average_acc_for_min_loss_models,
+        stdev_acc_for_min_loss_models,
+        average_val_acc_for_min_loss_models,
+        stdev_val_acc_for_min_loss_models,
+    ) = precise_modeling_operations_instance.get_optimal_training_model_analytics()
+    # average_val_acc, standard_deviation_val_acc, average_acc, standard_deviation_acc = precise_modeling_operations_instance.get_models_analytics()
     experimental_average_accuracy_dialog = dialog_handler_instance.render_template(
         "inform-accuracy",
-        average_val_acc=average_val_acc,
-        standard_deviation_val_acc=standard_deviation_val_acc,
-        average_acc=average_acc,
-        standard_deviation_acc=standard_deviation_acc,
+        average_val_acc=average_val_acc_for_min_loss_models,
+        standard_deviation_val_acc=stdev_val_acc_for_min_loss_models,
+        average_acc=average_acc_for_min_loss_models,
+        standard_deviation_acc=stdev_acc_for_min_loss_models,
     )
+    # TODO: change to f1 score
+    # experimental_average_accuracy_dialog = dialog_handler_instance.render_template("inform-accuracy", average_val_acc=average_val_acc, standard_deviation_val_acc=standard_deviation_val_acc, average_acc=average_acc, standard_deviation_acc=standard_deviation_acc)
     print(experimental_average_accuracy_dialog)
 
-    (
-        selected_model_name,
-        selected_model_results,
-    ) = precise_modeling_operations_instance.pick_best_model()
-    experimental_best_model = dialog_handler_instance.render_template(
+    best_training_set_accuracy_model = (
+        precise_modeling_operations_instance.pick_best_model()
+    )
+    # selected_model_name, selected_model_results = precise_modeling_operations_instance.pick_best_model()
+    # experimental_best_model = dialog_handler_instance.render_template("inform-best_model", selected_model_name=selected_model_name, selected_model_results=selected_model_results)
+    print(best_training_set_accuracy_model)
+    selected_model_name = list(best_training_set_accuracy_model.keys())[0]
+    selected_model_results = best_training_set_accuracy_model[selected_model_name]
+    experimental_average_accuracy_dialog = dialog_handler_instance.render_template(
         "inform-best_model",
         selected_model_name=selected_model_name,
         selected_model_results=selected_model_results,
     )
-    print(experimental_best_model)
-
+    # return (selected_model_name, selected_model_results, experimental_average_accuracy_dialog)
+    # return(best_training_set_accuracy_model)
     return (
         selected_model_name,
         selected_model_results,
@@ -109,7 +116,8 @@ def train_model_flow(wakeword_model_name, epochs=None):
         )
     )
 
-    precise_modeling_operations_instance.get_last_epoch_model_info(
+    # precise_modeling_operations_instance.get_last_epoch_model_info(wakeword_model_name, training_run)
+    precise_modeling_operations_instance.get_model_info(
         wakeword_model_name, training_run
     )
 
@@ -250,3 +258,41 @@ def get_base_model_flow(
 print(model_info)
 model_info = incremental_training_flow(noise_destination_directory, wakeword_model_name)
 print(model_info)"""
+
+"""import json
+
+with open('data_prep_user_configuration.json', 'r') as file:
+    user_configuration_dictionary = json.load(file)
+source_directory = user_configuration_dictionary['audio_source_directory']
+wakeword_model_name = user_configuration_dictionary['wakeword_model_name']
+pdsounds_directory = user_configuration_dictionary['pdsounds_directory']
+directories_to_process = user_configuration_dictionary['extra_audio_directories_to_process']
+extra_audio_directories_labels = user_configuration_dictionary['extra_audio_directories_labels']
+max_files_from_source_directory = user_configuration_dictionary['max_files_from_source_directory']
+max_files_per_destination_directory = user_configuration_dictionary['max_files_per_destination_directory']
+
+
+with open('data_prep_system_configuration.json', 'r') as file:
+    system_configuration_dictionary = json.load(file)    
+random_split_directories = system_configuration_dictionary['random_split_directories']
+even_odd_split_directories = system_configuration_dictionary['even_odd_split_directories']
+three_four_split_directories = system_configuration_dictionary['three_four_split_directories']
+root_model_name = system_configuration_dictionary['root_model_name']
+source_directories = system_configuration_dictionary['source_directories']
+destination_directories = system_configuration_dictionary['destination_directories']
+directories_to_gauss = system_configuration_dictionary['directories_to_gauss']
+
+best_training_set_accuracy_model = split_data_and_get_best_model_flow(source_directory, random_split_directories, even_odd_split_directories, three_four_split_directories, root_model_name)
+#print(best_training_set_accuracy_model)
+print(best_training_set_accuracy_model)"""
+
+
+"""
+changed experiment_4 to test_wakeword_model_delete_after
+Average accuracies of the base model...
+experiment_4 produces the best results with {'minimum_loss_val_accuracy': {'epoch': 49, 'loss': 0.1925, 'acc': 0.6705, 'val_loss': 0.1503, 'val_acc': 0.6949, 'acc_difference': 0.024399999999999977}}
+Original best model: {'minimum_loss_val_accuracy': {'epoch': 49, 'loss': 0.1925, 'acc': 0.6705, 'val_loss': 0.1503, 'val_acc': 0.6949, 'acc_difference': 0.024399999999999977}}
+Current base model with incremental training on your random audio recordings: {'acc': 0.6878, 'val_acc': 0.6901, 'difference': 0.0023000000000000798}
+Not bad when you consider the average accuracies of the first base model...
+"""
+# TODO: the produces the best results with is wrong, it gives minium_loss_val_acc instead of minimum_loss_acc