From a2a258dbc44c83e6d6931e6202e9276776c98ce8 Mon Sep 17 00:00:00 2001 From: Nicolay Rusnachenko Date: Sat, 14 Aug 2021 22:26:22 +0300 Subject: [PATCH 1/8] Fixed incorrect path usage, #125 --- tests/contrib/bert/test_output.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/contrib/bert/test_output.py b/tests/contrib/bert/test_output.py index 5d0cc71b..e69a2b0c 100644 --- a/tests/contrib/bert/test_output.py +++ b/tests/contrib/bert/test_output.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from os.path import join, dirname, abspath import sys import unittest @@ -13,9 +14,9 @@ class TestOutputFormatters(unittest.TestCase): - __input_samples_filepath = u"data/test_sample_3l.tsv.gz" - - __google_bert_output_filepath_sample = u"data/test_google_bert_output_3l.tsv" + __current_dir = dirname(__file__) + __input_samples_filepath = join(__current_dir, u"data/test_sample_3l.tsv.gz") + __google_bert_output_filepath_sample = join(__current_dir, u"data/test_google_bert_output_3l.tsv") def test_google_bert_output_formatter(self): row_ids_provider = MultipleIDProvider() From fdbc4fd7296739f387d1860032b03a0803b7fc29 Mon Sep 17 00:00:00 2001 From: Nicolay Rusnachenko Date: Sat, 14 Aug 2021 23:23:48 +0300 Subject: [PATCH 2/8] Using related paths (#156). Refactoring. --- tests/contrib/networks/test_samples_iter.py | 15 ++++++++++----- tests/contrib/networks/test_tf_ctx_compile.py | 6 +++--- tests/contrib/networks/test_tf_mi_feed.py | 7 ++++--- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/tests/contrib/networks/test_samples_iter.py b/tests/contrib/networks/test_samples_iter.py index f38ff303..14263faa 100755 --- a/tests/contrib/networks/test_samples_iter.py +++ b/tests/contrib/networks/test_samples_iter.py @@ -1,3 +1,5 @@ +from os.path import dirname, join + import pandas as pd import gzip import sys @@ -6,22 +8,25 @@ sys.path.append('../../../') +from tests.contrib.networks.labels import TestThreeLabelScaler + from arekit.common.experiment import const from arekit.contrib.networks.core.input.rows_parser import ParsedSampleRow from arekit.contrib.networks.context.configurations.base.base import DefaultNetworkConfig from arekit.contrib.networks.sample import InputSample -from tests.contrib.networks.labels import TestThreeLabelScaler - class TestSamplesIteration(unittest.TestCase): __show_examples = False __show_shifted_examples = False + def __get_local_dir(self, local_filepath): + return join(dirname(__file__), local_filepath) + def test_check_all_samples(self): - vocab_filepath = u"test_data/vocab.txt.gz" - samples_filepath = u"test_data/sample-train.tsv.gz" + vocab_filepath = self.__get_local_dir(u"test_data/vocab.txt.gz") + samples_filepath = self.__get_local_dir(u"test_data/sample-train.tsv.gz") words_vocab = self.__read_vocab(vocab_filepath) config = DefaultNetworkConfig() config.modify_terms_per_context(50) @@ -82,7 +87,7 @@ def __test_core(self, words_vocab, config, samples_filepath): for i, row in enumerate(self.__iter_tsv_gzip(input_file=samples_filepath)): # Perform row parsing process. - row = ParsedSampleRow(row, labels_scaler=labels_scaler) + row = ParsedSampleRow(row) subj_ind = row.SubjectIndex obj_ind = row.ObjectIndex diff --git a/tests/contrib/networks/test_tf_ctx_compile.py b/tests/contrib/networks/test_tf_ctx_compile.py index 6f2ac527..6f8031ad 100755 --- a/tests/contrib/networks/test_tf_ctx_compile.py +++ b/tests/contrib/networks/test_tf_ctx_compile.py @@ -4,12 +4,12 @@ sys.path.append('../../../') -from arekit.common.languages.ru.pos_service import PartOfSpeechTypesService -from arekit.contrib.networks.context.configurations.base.base import DefaultNetworkConfig - from tests.contrib.networks.tf_networks.supported import get_supported from tests.contrib.networks.tf_networks.utils import init_config +from arekit.common.languages.ru.pos_service import PartOfSpeechTypesService +from arekit.contrib.networks.context.configurations.base.base import DefaultNetworkConfig + class TestContextNetworkCompilation(unittest.TestCase): diff --git a/tests/contrib/networks/test_tf_mi_feed.py b/tests/contrib/networks/test_tf_mi_feed.py index 3206f805..36123aa3 100755 --- a/tests/contrib/networks/test_tf_mi_feed.py +++ b/tests/contrib/networks/test_tf_mi_feed.py @@ -5,6 +5,10 @@ sys.path.append('../../../') +from tests.contrib.networks.labels import TestNeutralLabel +from tests.contrib.networks.test_tf_ctx_feed import TestContextNetworkFeeding +from tests.contrib.networks.tf_networks.supported import get_supported + from arekit.common.labels.scaler import BaseLabelScaler from arekit.contrib.networks.core.feeding.bags.bag import Bag @@ -15,9 +19,6 @@ from arekit.contrib.networks.sample import InputSample from arekit.contrib.networks.multi.architectures.max_pooling import MaxPoolingOverSentences -from tests.contrib.networks.labels import TestNeutralLabel -from tests.contrib.networks.test_tf_ctx_feed import TestContextNetworkFeeding -from tests.contrib.networks.tf_networks.supported import get_supported class TestMultiInstanceFeed(unittest.TestCase): From fcde39057e82940ee5a0dd5fc915513eeec756c3 Mon Sep 17 00:00:00 2001 From: Nicolay Rusnachenko Date: Sun, 15 Aug 2021 08:00:59 +0300 Subject: [PATCH 3/8] Fix related to #158 and #156. --- tests/contrib/networks/test_samples_iter.py | 3 --- tests/contrib/networks/test_tf_ctx_feed.py | 15 +++++++++++---- tests/contrib/networks/test_tf_mi_feed.py | 9 +++++---- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/tests/contrib/networks/test_samples_iter.py b/tests/contrib/networks/test_samples_iter.py index 14263faa..80551838 100755 --- a/tests/contrib/networks/test_samples_iter.py +++ b/tests/contrib/networks/test_samples_iter.py @@ -8,8 +8,6 @@ sys.path.append('../../../') -from tests.contrib.networks.labels import TestThreeLabelScaler - from arekit.common.experiment import const from arekit.contrib.networks.core.input.rows_parser import ParsedSampleRow from arekit.contrib.networks.context.configurations.base.base import DefaultNetworkConfig @@ -83,7 +81,6 @@ def __test_core(self, words_vocab, config, samples_filepath): assert(isinstance(samples_filepath, unicode)) samples = [] - labels_scaler = TestThreeLabelScaler() for i, row in enumerate(self.__iter_tsv_gzip(input_file=samples_filepath)): # Perform row parsing process. diff --git a/tests/contrib/networks/test_tf_ctx_feed.py b/tests/contrib/networks/test_tf_ctx_feed.py index 884861d5..c9fd36d2 100755 --- a/tests/contrib/networks/test_tf_ctx_feed.py +++ b/tests/contrib/networks/test_tf_ctx_feed.py @@ -7,7 +7,12 @@ sys.path.append('../../../') +from tests.contrib.networks.labels import TestThreeLabelScaler +from tests.contrib.networks.tf_networks.supported import get_supported +from tests.contrib.networks.tf_networks.utils import init_config + from arekit.common.experiment.data_type import DataType +from arekit.common.labels.scaler import BaseLabelScaler from arekit.contrib.networks.context.configurations.base.base import DefaultNetworkConfig from arekit.contrib.networks.sample import InputSample @@ -15,9 +20,6 @@ from arekit.contrib.networks.core.feeding.batch.base import MiniBatch from arekit.contrib.networks.core.nn import NeuralNetwork -from tests.contrib.networks.tf_networks.supported import get_supported -from tests.contrib.networks.tf_networks.utils import init_config - class TestContextNetworkFeeding(unittest.TestCase): @@ -48,16 +50,18 @@ def create_minibatch(config, labels_count): @staticmethod def run_feeding(network, network_config, create_minibatch_func, logger, + labels_scaler, display_hidden_values=True, display_idp_values=True): assert(isinstance(network, NeuralNetwork)) assert(isinstance(network_config, DefaultNetworkConfig)) + assert(isinstance(labels_scaler, BaseLabelScaler)) assert(callable(create_minibatch_func)) init_config(network_config) # Init network. network.compile(config=network_config, reset_graph=True, graph_seed=42) - minibatch = create_minibatch_func(config=network_config, labels_count=3) + minibatch = create_minibatch_func(config=network_config, labels_scaler=labels_scaler) network_optimiser = network_config.Optimiser.minimize(network.Cost) @@ -107,11 +111,14 @@ def test(self): logger = logging.getLogger(__name__) logging.basicConfig(level=logging.DEBUG) + labels_scaler = TestThreeLabelScaler() + for cfg, network in get_supported(): logger.debug("Feed to the network: {}".format(type(network))) self.run_feeding(network=network, network_config=cfg, create_minibatch_func=self.create_minibatch, + labels_scaler=labels_scaler, logger=logger) diff --git a/tests/contrib/networks/test_tf_mi_feed.py b/tests/contrib/networks/test_tf_mi_feed.py index 36123aa3..eb6cbd45 100755 --- a/tests/contrib/networks/test_tf_mi_feed.py +++ b/tests/contrib/networks/test_tf_mi_feed.py @@ -5,20 +5,18 @@ sys.path.append('../../../') -from tests.contrib.networks.labels import TestNeutralLabel +from tests.contrib.networks.labels import TestNeutralLabel, TestThreeLabelScaler from tests.contrib.networks.test_tf_ctx_feed import TestContextNetworkFeeding from tests.contrib.networks.tf_networks.supported import get_supported -from arekit.common.labels.scaler import BaseLabelScaler - from arekit.contrib.networks.core.feeding.bags.bag import Bag from arekit.contrib.networks.core.feeding.batch.multi import MultiInstanceMiniBatch - from arekit.contrib.networks.multi.configurations.max_pooling import MaxPoolingOverSentencesConfig from arekit.contrib.networks.context.configurations.base.base import DefaultNetworkConfig from arekit.contrib.networks.sample import InputSample from arekit.contrib.networks.multi.architectures.max_pooling import MaxPoolingOverSentences +from arekit.common.labels.scaler import BaseLabelScaler class TestMultiInstanceFeed(unittest.TestCase): @@ -51,6 +49,8 @@ def test(self): logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) + labels_scaler = TestThreeLabelScaler() + for ctx_config, ctx_network in get_supported(): for config, network in self.multiinstances_supported(ctx_config, ctx_network): logger.info(type(network)) @@ -59,6 +59,7 @@ def test(self): network_config=config, create_minibatch_func=self.__create_minibatch, logger=logger, + labels_scaler=labels_scaler, display_idp_values=False) From 8434e3c9892b4d426b539d494538feefa0fcb76e Mon Sep 17 00:00:00 2001 From: Nicolay Rusnachenko Date: Sun, 15 Aug 2021 09:31:03 +0300 Subject: [PATCH 4/8] Fix related to #158 and #156. --- tests/contrib/networks/test_tf_ctx_feed.py | 13 +++++-------- tests/contrib/networks/test_tf_mi_feed.py | 4 ++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/tests/contrib/networks/test_tf_ctx_feed.py b/tests/contrib/networks/test_tf_ctx_feed.py index c9fd36d2..9690dec5 100755 --- a/tests/contrib/networks/test_tf_ctx_feed.py +++ b/tests/contrib/networks/test_tf_ctx_feed.py @@ -31,16 +31,13 @@ def init_session(): return sess @staticmethod - def create_minibatch(config, labels_count): + def __create_minibatch(config, labels_scaler): assert(isinstance(config, DefaultNetworkConfig)) - assert(isinstance(labels_count, int)) - - l_uint_min = 0 - l_uint_max = labels_count - 1 + assert(isinstance(labels_scaler, BaseLabelScaler)) bags = [] for i in range(config.BagsPerMinibatch): - uint_label = random.randint(l_uint_min, l_uint_max) + uint_label = random.randint(0, labels_scaler.LabelsCount) bag = Bag(uint_label=uint_label) for j in range(config.BagSize): bag.add_sample(InputSample._generate_test(config)) @@ -61,7 +58,7 @@ def run_feeding(network, network_config, create_minibatch_func, logger, init_config(network_config) # Init network. network.compile(config=network_config, reset_graph=True, graph_seed=42) - minibatch = create_minibatch_func(config=network_config, labels_scaler=labels_scaler) + minibatch = create_minibatch_func(config=network_config, labels_count=labels_scaler) network_optimiser = network_config.Optimiser.minimize(network.Cost) @@ -117,7 +114,7 @@ def test(self): logger.debug("Feed to the network: {}".format(type(network))) self.run_feeding(network=network, network_config=cfg, - create_minibatch_func=self.create_minibatch, + create_minibatch_func=self.__create_minibatch, labels_scaler=labels_scaler, logger=logger) diff --git a/tests/contrib/networks/test_tf_mi_feed.py b/tests/contrib/networks/test_tf_mi_feed.py index eb6cbd45..159818bc 100755 --- a/tests/contrib/networks/test_tf_mi_feed.py +++ b/tests/contrib/networks/test_tf_mi_feed.py @@ -26,12 +26,12 @@ def __create_minibatch(config, labels_scaler): assert(isinstance(config, DefaultNetworkConfig)) assert(isinstance(labels_scaler, BaseLabelScaler)) bags = [] - label = TestNeutralLabel() + no_label = labels_scaler.get_no_label_instance() empty_sample = InputSample.create_empty(terms_per_context=config.TermsPerContext, frames_per_context=config.FramesPerContext, synonyms_per_context=config.SynonymsPerContext) for i in range(config.BagsPerMinibatch): - bag = Bag(label) + bag = Bag(labels_scaler.label_to_uint(no_label)) for j in range(config.BagSize): bag.add_sample(empty_sample) bags.append(bag) From 2b12ce7f5383d0048d02e8be4e3e6e7947f0d024 Mon Sep 17 00:00:00 2001 From: Nicolay Rusnachenko Date: Sun, 15 Aug 2021 09:32:52 +0300 Subject: [PATCH 5/8] Uncomment other tests --- tests/contrib/networks/test_tf_mi_feed.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/contrib/networks/test_tf_mi_feed.py b/tests/contrib/networks/test_tf_mi_feed.py index 159818bc..42f7da7b 100755 --- a/tests/contrib/networks/test_tf_mi_feed.py +++ b/tests/contrib/networks/test_tf_mi_feed.py @@ -5,10 +5,12 @@ sys.path.append('../../../') -from tests.contrib.networks.labels import TestNeutralLabel, TestThreeLabelScaler +from tests.contrib.networks.labels import TestThreeLabelScaler from tests.contrib.networks.test_tf_ctx_feed import TestContextNetworkFeeding from tests.contrib.networks.tf_networks.supported import get_supported +from arekit.contrib.networks.multi.configurations.att_self import AttSelfOverSentencesConfig +from arekit.contrib.networks.multi.architectures.att_self import AttSelfOverSentences from arekit.contrib.networks.core.feeding.bags.bag import Bag from arekit.contrib.networks.core.feeding.batch.multi import MultiInstanceMiniBatch from arekit.contrib.networks.multi.configurations.max_pooling import MaxPoolingOverSentencesConfig @@ -42,7 +44,7 @@ def __create_minibatch(config, labels_scaler): def multiinstances_supported(ctx_config, ctx_network): return [ (MaxPoolingOverSentencesConfig(ctx_config), MaxPoolingOverSentences(ctx_network)), - # (AttSelfOverSentencesConfig(ctx_config), AttSelfOverSentences(ctx_network)) + (AttSelfOverSentencesConfig(ctx_config), AttSelfOverSentences(ctx_network)) ] def test(self): From 43e5745fd02859e68498b92eaae5bb92a61d8cca Mon Sep 17 00:00:00 2001 From: Nicolay Rusnachenko Date: Sun, 15 Aug 2021 09:37:20 +0300 Subject: [PATCH 6/8] fixed --- tests/contrib/networks/test_tf_ctx_feed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/contrib/networks/test_tf_ctx_feed.py b/tests/contrib/networks/test_tf_ctx_feed.py index 9690dec5..e8ee789b 100755 --- a/tests/contrib/networks/test_tf_ctx_feed.py +++ b/tests/contrib/networks/test_tf_ctx_feed.py @@ -58,7 +58,7 @@ def run_feeding(network, network_config, create_minibatch_func, logger, init_config(network_config) # Init network. network.compile(config=network_config, reset_graph=True, graph_seed=42) - minibatch = create_minibatch_func(config=network_config, labels_count=labels_scaler) + minibatch = create_minibatch_func(config=network_config, labels_scaler=labels_scaler) network_optimiser = network_config.Optimiser.minimize(network.Cost) From 51208a7cef8905d9eed99c5de436c11e39486f31 Mon Sep 17 00:00:00 2001 From: Nicolay Rusnachenko Date: Sun, 15 Aug 2021 13:20:15 +0300 Subject: [PATCH 7/8] added release notes generator --- tests/run_gen_release_notes.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 tests/run_gen_release_notes.sh diff --git a/tests/run_gen_release_notes.sh b/tests/run_gen_release_notes.sh new file mode 100755 index 00000000..3a3654ff --- /dev/null +++ b/tests/run_gen_release_notes.sh @@ -0,0 +1,6 @@ +#!/bin/bash +docker run -it --rm -v "$(pwd)":/usr/local/src/your-app/ githubchangeloggenerator/github-changelog-generator \ + -u nicolay-r \ + -p AREkit \ + --token \ + --since-tag v0.20.5-rc From 14ea80197090d7e0bb425c185926b820db4397b2 Mon Sep 17 00:00:00 2001 From: Nicolay Rusnachenko Date: Sun, 15 Aug 2021 13:20:44 +0300 Subject: [PATCH 8/8] removing sudo --- tests/run_test_all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/run_test_all.sh b/tests/run_test_all.sh index 42218ab0..045f1e26 100755 --- a/tests/run_test_all.sh +++ b/tests/run_test_all.sh @@ -2,7 +2,7 @@ # Performing a quick library installation. # https://stackoverflow.com/questions/19048732/python-setup-py-develop-vs-install -sudo pip install -e ../ --no-deps +pip install -e ../ --no-deps # Run all unit tests. python2.7 -m unittest discover .