diff --git a/README.md b/README.md index 30f5538..8e48e17 100644 --- a/README.md +++ b/README.md @@ -187,7 +187,7 @@ from datasets import generate_data from datasets import metric_3d_to_2d_wrapper from neuraxle_tensorflow.tensorflow_v1 import TensorflowV1ModelStep from neuraxle_tensorflow.tensorflow_v2 import Tensorflow2ModelStep -from plotting import plot_metrics +from train_values import plot_metrics from steps import MeanStdNormalizer, ToNumpy, PlotPredictionsWrapper %matplotlib inline diff --git a/plotting.py b/plotting.py index 99c7d03..c0c1518 100644 --- a/plotting.py +++ b/plotting.py @@ -28,18 +28,15 @@ def plot_predictions(data_inputs, expected_outputs, predicted_outputs, save=Fals plt.show() -def plot_metrics(pipeline, exercice_number): - mse_train = pipeline.get_step_by_name('epoch_metrics').get_metric_train('mse') - print('last mse train: {}'.format(mse_train[-1])) - print('best mse train: {}'.format(min(mse_train))) - - mse_validation = pipeline.get_step_by_name('epoch_metrics').get_metric_validation('mse') - print('last mse validation: {}'.format(mse_validation[-1])) - print('best mse validation: {}'.format(min(mse_validation))) +def plot_metrics(metric_name, train_values, validation_values, exercice_number): + print('last mse train: {}'.format(train_values[-1])) + print('best mse train: {}'.format(min(train_values))) + print('last {} validation: {}'.format(metric_name, validation_values[-1])) + print('best {} validation: {}'.format(metric_name, min(validation_values))) plot_metric( - mse_train, - mse_validation, + train_values, + validation_values, xlabel='epoch', ylabel='mse', title='Exercice {} Model Mean Squared Error'.format(exercice_number) diff --git a/requirements.txt b/requirements.txt index e4f9e33..dbb5aa7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ tensorflow==2.1.0 -neuraxle==0.3.1 -neuraxle_tensorflow==0.1.0 +-e git://github.com/Neuraxio/Neuraxle-Tensorflow.git@save-tensorflow-v2-with-context#egg=neuraxle_tensorflow +-e git://github.com/Neuraxio/Neuraxle.git@master#egg=neuraxle diff --git a/seq2seq.py b/seq2seq.py index 915c51d..e890fdf 100644 --- a/seq2seq.py +++ b/seq2seq.py @@ -4,12 +4,13 @@ import tensorflow as tf from neuraxle.data_container import DataContainer from neuraxle.hyperparams.space import HyperparameterSamples -from neuraxle.metaopt.random import ValidationSplitWrapper -from neuraxle.metrics import MetricsWrapper +from neuraxle.metaopt.auto_ml import Trainer, ValidationSplitter +from neuraxle.metaopt.callbacks import ScoringCallback +from neuraxle.metaopt.trial import Trial from neuraxle.pipeline import Pipeline, MiniBatchSequentialPipeline -from neuraxle.steps.data import EpochRepeater, DataShuffler -from neuraxle.steps.flow import TrainOnlyWrapper from neuraxle.steps.loop import ForEachDataInput +from neuraxle_tensorflow.tensorflow_v1 import TensorflowV1ModelStep +from neuraxle_tensorflow.tensorflow_v2 import Tensorflow2ModelStep from sklearn.metrics import mean_squared_error from tensorflow_core.python.client import device_lib from tensorflow_core.python.keras import Input, Model @@ -18,8 +19,6 @@ from datasets import generate_data from datasets import metric_3d_to_2d_wrapper -from neuraxle_tensorflow.tensorflow_v1 import TensorflowV1ModelStep -from neuraxle_tensorflow.tensorflow_v2 import Tensorflow2ModelStep from plotting import plot_metrics from steps import MeanStdNormalizer, ToNumpy, PlotPredictionsWrapper @@ -178,59 +177,63 @@ def main(chosen_device): 'output_dim': output_dim, 'input_dim': input_dim }) - feature_0_metric = metric_3d_to_2d_wrapper(mean_squared_error) - metrics = {'mse': feature_0_metric} - - signal_prediction_pipeline = Pipeline([ - ForEachDataInput(MeanStdNormalizer()), - ToNumpy(), - PlotPredictionsWrapper(Tensorflow2ModelStep( - # See: https://github.com/Neuraxio/Neuraxle-TensorFlow - create_model=create_model, - create_loss=create_loss, - create_optimizer=create_optimizer, - expected_outputs_dtype=tf.dtypes.float32, - data_inputs_dtype=tf.dtypes.float32, - print_loss=True - ).set_hyperparams(seq2seq_pipeline_hyperparams)) + + pipeline = Pipeline([ + MiniBatchSequentialPipeline([ + ForEachDataInput(MeanStdNormalizer()), + ToNumpy(), + PlotPredictionsWrapper(Tensorflow2ModelStep( + # See: https://github.com/Neuraxio/Neuraxle-TensorFlow + create_model=create_model, + create_loss=create_loss, + create_optimizer=create_optimizer, + expected_outputs_dtype=tf.dtypes.float32, + data_inputs_dtype=tf.dtypes.float32, + device_name=chosen_device, + print_loss=True + ).set_hyperparams(seq2seq_pipeline_hyperparams)) + ], batch_size=batch_size), ]).set_name('SignalPrediction') - pipeline = Pipeline([EpochRepeater( - ValidationSplitWrapper( - MetricsWrapper(Pipeline([ - TrainOnlyWrapper(DataShuffler()), - MiniBatchSequentialPipeline([ - MetricsWrapper( - signal_prediction_pipeline, - metrics=metrics, - name='batch_metrics' - ) - ], batch_size=batch_size) - ]), metrics=metrics, - name='epoch_metrics', - print_metrics=True - ), - test_size=validation_size, - scoring_function=feature_0_metric - ), epochs=epochs)]) - - pipeline, outputs = pipeline.fit_transform(data_inputs, expected_outputs) - - plot_metrics(pipeline=pipeline, exercice_number=exercice_number) - plot_predictions(data_inputs, expected_outputs, pipeline, max_plotted_validation_predictions) - - -def plot_predictions(data_inputs, expected_outputs, pipeline, max_plotted_predictions): - _, _, data_inputs_validation, expected_outputs_validation = \ - pipeline.get_step_by_name('ValidationSplitWrapper').split(data_inputs, expected_outputs) + trainer = Trainer( + epochs=epochs, + validation_splitter=ValidationSplitter(test_size=validation_size), + scoring_callback=ScoringCallback(metric_function=metric_3d_to_2d_wrapper(mean_squared_error), higher_score_is_better=False) + ) + + trial: Trial = trainer.train( + pipeline=pipeline, + data_inputs=data_inputs, + expected_outputs=expected_outputs + ) + + plot_metrics( + metric_name='mse', + train_values=trial.validation_splits[0].metrics_results['main']['train_values'], + validation_values=trial.validation_splits[0].metrics_results['main']['validation_values'], + exercice_number=exercice_number + ) + + # Get trained pipeline + pipeline = trial.get_trained_pipeline(split_number=0) + # Get validation set with trainer.validation_split_function.split function. + _, _, data_inputs_validation, expected_outputs_validation = trainer.validation_split_function.split( + data_inputs=data_inputs, + expected_outputs=expected_outputs + ) + + # Enable the plotting feature inside the PlotPredictionsWrapper wrapper step. pipeline.apply('toggle_plotting') - pipeline.apply('set_max_plotted_predictions', max_plotted_predictions) + pipeline.apply( + method='set_max_plotted_predictions', + max_plotted_predictions=max_plotted_validation_predictions + ) - signal_prediction_pipeline = pipeline.get_step_by_name('SignalPrediction') - signal_prediction_pipeline.transform_data_container(DataContainer( - data_inputs=data_inputs_validation, - expected_outputs=expected_outputs_validation + # Transform the trained pipeline to plot predictions + pipeline.transform_data_container(DataContainer( + data_inputs=data_inputs_validation[0], + expected_outputs=expected_outputs_validation[0] )) diff --git a/steps.py b/steps.py index 2fba801..812a1b6 100644 --- a/steps.py +++ b/steps.py @@ -3,6 +3,7 @@ import numpy as np from neuraxle.base import NonFittableMixin, BaseStep, Identity, ExecutionContext from neuraxle.data_container import DataContainer +from neuraxle.hyperparams.space import RecursiveDict from neuraxle.steps.output_handlers import InputAndOutputTransformerMixin from neuraxle.union import FeatureUnion @@ -80,9 +81,11 @@ def __init__(self, plotting_function: Callable, max_plotted_predictions, enabled def set_max_plotted_predictions(self, max_plotted_predictions): self.max_plotted_predictions = max_plotted_predictions + return RecursiveDict() def toggle_plotting(self): self.enabled = not self.enabled + return RecursiveDict() def transform(self, data_inputs): raise NotImplementedError('must be used inside a pipeline') diff --git a/train_values.py b/train_values.py new file mode 100644 index 0000000..0f94ceb --- /dev/null +++ b/train_values.py @@ -0,0 +1,15 @@ +def plot_metrics(metric_name, train_values, metric_validation, exercice_number): + print('last mse train: {}'.format(train_values[-1])) + print('best mse train: {}'.format(min(mse_train))) + + mse_validation = 0 + print('last mse validation: {}'.format(mse_validation[-1])) + print('best mse validation: {}'.format(min(mse_validation))) + + plot_metric( + mse_train, + mse_validation, + xlabel='epoch', + ylabel='mse', + title='Exercice {} Model Mean Squared Error'.format(exercice_number) + ) \ No newline at end of file