Skip to content

Commit

Permalink
Merge pull request #1413 from lscheinkman/NUP-2506
Browse files Browse the repository at this point in the history 
NUP-2514: Fix traversal limit
  • Loading branch information
@rhyolight
rhyolight authored Apr 4, 2018
2 parents e10b2cb + 5f1ddcb commit 6d483ff
Show file tree
Hide file tree
Showing 13 changed files with 112 additions and 96 deletions.
23 changes: 16 additions & 7 deletions src/nupic/bindings/algorithms.i
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ else:
from nupic.proto.SvmProto_capnp import (SvmDenseProto, Svm01Proto)
from nupic.proto.TemporalMemoryProto_capnp import TemporalMemoryProto

# Capnp reader traveral limit (see capnp::ReaderOptions)
_TRAVERSAL_LIMIT_IN_WORDS = 1 << 63

_ALGORITHMS = _algorithms
%}
Expand Down Expand Up @@ -338,7 +340,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
:param: Destination SvmDenseProto message builder
"""
reader = SvmDenseProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
reader = SvmDenseProto.from_bytes(self._writeAsCapnpPyBytes(),
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
pyBuilder.from_dict(reader.to_dict()) # copy

@classmethod
Expand Down Expand Up @@ -450,7 +453,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
:param: Destination Svm01Proto message builder
"""
reader = Svm01Proto.from_bytes(self._writeAsCapnpPyBytes()) # copy
reader = Svm01Proto.from_bytes(self._writeAsCapnpPyBytes(),
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
pyBuilder.from_dict(reader.to_dict()) # copy

@classmethod
Expand Down Expand Up @@ -899,7 +903,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
:param: Destination Cells4Proto message builder
"""
reader = Cells4Proto.from_bytes(self._writeAsCapnpPyBytes()) # copy
reader = Cells4Proto.from_bytes(self._writeAsCapnpPyBytes(),
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
pyBuilder.from_dict(reader.to_dict()) # copy

%}
Expand Down Expand Up @@ -1148,7 +1153,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
:param: Destination SpatialPoolerProto message builder
"""
reader = SpatialPoolerProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
reader = SpatialPoolerProto.from_bytes(self._writeAsCapnpPyBytes(),
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
pyBuilder.from_dict(reader.to_dict()) # copy


Expand Down Expand Up @@ -1465,7 +1471,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
:param: Destination SdrClassifierProto message builder
"""
reader = SdrClassifierProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
reader = SdrClassifierProto.from_bytes(self._writeAsCapnpPyBytes(),
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
pyBuilder.from_dict(reader.to_dict()) # copy

@classmethod
Expand Down Expand Up @@ -1581,7 +1588,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
:param: Destination ConnectionsProto message builder
"""
reader = ConnectionsProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
reader = ConnectionsProto.from_bytes(self._writeAsCapnpPyBytes(),
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
pyBuilder.from_dict(reader.to_dict()) # copy


Expand Down Expand Up @@ -1782,7 +1790,8 @@ void forceRetentionOfImageSensorLiteLibrary(void) {
:param: Destination TemporalMemoryProto message builder
"""
reader = TemporalMemoryProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
reader = TemporalMemoryProto.from_bytes(self._writeAsCapnpPyBytes(),
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
pyBuilder.from_dict(reader.to_dict()) # copy


Expand Down
9 changes: 7 additions & 2 deletions src/nupic/bindings/engine_internal.i
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,9 @@ except ImportError:
else:
from nupic.proto.NetworkProto_capnp import NetworkProto
from nupic.proto.PyRegionProto_capnp import PyRegionProto

# Capnp reader traveral limit (see capnp::ReaderOptions)
_TRAVERSAL_LIMIT_IN_WORDS = 1 << 63
%}

%{
Expand Down Expand Up @@ -287,7 +290,8 @@ class IterablePair(object):
:param: Destination NetworkProto message builder
"""
reader = NetworkProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
reader = NetworkProto.from_bytes(self._writeAsCapnpPyBytes(),
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
pyBuilder.from_dict(reader.to_dict()) # copy


Expand Down Expand Up @@ -364,7 +368,8 @@ class _PyCapnpHelper(object):
:returns: The deserialized python region instance.
"""
pyRegionProto = PyRegionProto.from_bytes(pyRegionProtoBytes)
pyRegionProto = PyRegionProto.from_bytes(pyRegionProtoBytes,
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)

return getattr(regionCls, methodName)(pyRegionProto)

Expand Down
6 changes: 5 additions & 1 deletion src/nupic/bindings/math.i
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,9 @@ except ImportError:
else:
from nupic.proto.RandomProto_capnp import RandomProto

# Capnp reader traveral limit (see capnp::ReaderOptions)
_TRAVERSAL_LIMIT_IN_WORDS = 1 << 63

_MATH = _math
%}

Expand Down Expand Up @@ -195,7 +198,8 @@ def write(self, pyBuilder):
:param: Destination RandomProto message builder
"""
reader = RandomProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
reader = RandomProto.from_bytes(self._writeAsCapnpPyBytes(),
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
pyBuilder.from_dict(reader.to_dict()) # copy


Expand Down
12 changes: 9 additions & 3 deletions src/nupic/bindings/sparse_matrix.i
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@ except ImportError:
else:
from nupic.proto.SparseMatrixProto_capnp import SparseMatrixProto
from nupic.proto.SparseBinaryMatrixProto_capnp import SparseBinaryMatrixProto

# Capnp reader traveral limit (see capnp::ReaderOptions)
_TRAVERSAL_LIMIT_IN_WORDS = 1 << 63
%}


Expand Down Expand Up @@ -423,7 +426,8 @@ def write(self, pyBuilder):
:param: Destination SparseMatrixProto message builder
"""
reader = SparseMatrixProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
reader = SparseMatrixProto.from_bytes(self._writeAsCapnpPyBytes(),
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
pyBuilder.from_dict(reader.to_dict()) # copy

@classmethod
Expand Down Expand Up @@ -2973,7 +2977,8 @@ def write(self, pyBuilder):
:param: Destination SparseBinaryMatrixProto message builder
"""
reader = SparseBinaryMatrixProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
reader = SparseBinaryMatrixProto.from_bytes(self._writeAsCapnpPyBytes(),
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
pyBuilder.from_dict(reader.to_dict()) # copy
def read(self, proto):
Expand Down Expand Up @@ -3484,7 +3489,8 @@ def write(self, pyBuilder):
:param: Destination SparseBinaryMatrixProto message builder
"""
reader = SparseBinaryMatrixProto.from_bytes(self._writeAsCapnpPyBytes()) # copy
reader = SparseBinaryMatrixProto.from_bytes(self._writeAsCapnpPyBytes(),
traversal_limit_in_words=_TRAVERSAL_LIMIT_IN_WORDS)
pyBuilder.from_dict(reader.to_dict()) # copy

def read(self, proto):
Expand Down
75 changes: 5 additions & 70 deletions src/nupic/engine/Region.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -436,79 +436,14 @@ const Timer &Region::getExecuteTimer() const { return executeTimer_; }

bool Region::operator==(const Region &o) const {

if (name_ != o.name_ || type_ != o.type_ || dims_ != o.dims_ ||
phases_ != o.phases_ || dimensionInfo_ != o.dimensionInfo_ ||
initialized_ != o.initialized_ || outputs_.size() != o.outputs_.size() ||
if (initialized_ != o.initialized_ || outputs_.size() != o.outputs_.size() ||
inputs_.size() != o.inputs_.size()) {
return false;
}
if (spec_ != nullptr && o.spec_ != nullptr) {
// Compare specs
if (spec_->singleNodeOnly != o.spec_->singleNodeOnly ||
spec_->description != o.spec_->description) {
return false;
}

// Parameters
for (size_t i = 0; i < spec_->parameters.getCount(); ++i) {
const std::pair<std::string, ParameterSpec> &p1 =
spec_->parameters.getByIndex(i);
const std::pair<std::string, ParameterSpec> &p2 =
o.spec_->parameters.getByIndex(i);
if (p1.first != p2.first || p1.second.count != p2.second.count ||
p1.second.description != p2.second.description ||
p1.second.constraints != p2.second.constraints ||
p1.second.defaultValue != p2.second.defaultValue ||
p1.second.dataType != p2.second.dataType ||
p1.second.accessMode != p2.second.accessMode) {
return false;
}
}
// Outputs
for (size_t i = 0; i < spec_->outputs.getCount(); ++i) {
const std::pair<std::string, OutputSpec> &p1 =
spec_->outputs.getByIndex(i);
const std::pair<std::string, OutputSpec> &p2 =
o.spec_->outputs.getByIndex(i);
if (p1.first != p2.first || p1.second.count != p2.second.count ||
p1.second.regionLevel != p2.second.regionLevel ||
p1.second.isDefaultOutput != p2.second.isDefaultOutput ||
p1.second.sparse != p2.second.sparse ||
p1.second.description != p2.second.description ||
p1.second.dataType != p2.second.dataType) {
return false;
}
}

// Outputs
for (size_t i = 0; i < spec_->inputs.getCount(); ++i) {
const std::pair<std::string, InputSpec> &p1 = spec_->inputs.getByIndex(i);
const std::pair<std::string, InputSpec> &p2 =
o.spec_->inputs.getByIndex(i);
if (p1.first != p2.first || p1.second.count != p2.second.count ||
p1.second.regionLevel != p2.second.regionLevel ||
p1.second.isDefaultInput != p2.second.isDefaultInput ||
p1.second.sparse != p2.second.sparse ||
p1.second.requireSplitterMap != p2.second.requireSplitterMap ||
p1.second.required != p2.second.required ||
p1.second.description != p2.second.description ||
p1.second.dataType != p2.second.dataType) {
return false;
}
}
// Commands
for (size_t i = 0; i < spec_->commands.getCount(); ++i) {
const std::pair<std::string, CommandSpec> &p1 =
spec_->commands.getByIndex(i);
const std::pair<std::string, CommandSpec> &p2 =
o.spec_->commands.getByIndex(i);
if (p1.first != p2.first ||
p1.second.description != p2.second.description) {
return false;
}
}
} else if (spec_ != o.spec_) {
// One of them is not null
if (name_ != o.name_ || type_ != o.type_ || dims_ != o.dims_ ||
spec_ != o.spec_ || phases_ != o.phases_ ||
dimensionInfo_ != o.dimensionInfo_) {
return false;
}

Expand Down Expand Up @@ -552,6 +487,6 @@ bool Region::operator==(const Region &o) const {
}

return true;
}
} // namespace nupic

} // namespace nupic
29 changes: 27 additions & 2 deletions src/nupic/engine/Spec.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,14 @@ Implementation of Spec API
namespace nupic {

Spec::Spec() : singleNodeOnly(false), description("") {}

bool Spec::operator==(const Spec &o) const {
if (singleNodeOnly != o.singleNodeOnly || description != o.description ||
parameters != o.parameters || outputs != o.outputs ||
inputs != o.inputs || commands != o.commands) {
return false;
}
return true;
}
std::string Spec::getDefaultInputName() const {
if (inputs.getCount() == 0)
return "";
Expand Down Expand Up @@ -87,16 +94,29 @@ InputSpec::InputSpec(std::string description, NTA_BasicType dataType,
required(required), regionLevel(regionLevel),
isDefaultInput(isDefaultInput), requireSplitterMap(requireSplitterMap),
sparse(sparse) {}

bool InputSpec::operator==(const InputSpec &o) const {
return required == o.required && regionLevel == o.regionLevel &&
isDefaultInput == o.isDefaultInput && sparse == o.sparse &&
requireSplitterMap == o.requireSplitterMap && dataType == o.dataType &&
count == o.count && description == o.description;
}
OutputSpec::OutputSpec(std::string description, NTA_BasicType dataType,
size_t count, bool regionLevel, bool isDefaultOutput,
bool sparse)
: description(std::move(description)), dataType(dataType), count(count),
regionLevel(regionLevel), isDefaultOutput(isDefaultOutput),
sparse(sparse) {}
bool OutputSpec::operator==(const OutputSpec &o) const {
return regionLevel == o.regionLevel && isDefaultOutput == o.isDefaultOutput &&
sparse == o.sparse && dataType == o.dataType && count == o.count &&
description == o.description;
}

CommandSpec::CommandSpec(std::string description)
: description(std::move(description)) {}
bool CommandSpec::operator==(const CommandSpec &o) const {
return description == o.description;
}

ParameterSpec::ParameterSpec(std::string description, NTA_BasicType dataType,
size_t count, std::string constraints,
Expand All @@ -109,6 +129,11 @@ ParameterSpec::ParameterSpec(std::string description, NTA_BasicType dataType,
if (dataType == NTA_BasicType_Byte && count > 0)
NTA_THROW << "Parameters of type 'byte' are not supported";
}
bool ParameterSpec::operator==(const ParameterSpec &o) const {
return dataType == o.dataType && count == o.count &&
description == o.description && constraints == o.constraints &&
defaultValue == o.defaultValue && accessMode == o.accessMode;
}

std::string Spec::toString() const {
// TODO -- minimal information here; fill out with the rest of
Expand Down
23 changes: 18 additions & 5 deletions src/nupic/engine/Spec.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,10 @@ class InputSpec {
InputSpec(std::string description, NTA_BasicType dataType, UInt32 count,
bool required, bool regionLevel, bool isDefaultInput,
bool requireSplitterMap = true, bool sparse = false);

bool operator==(const InputSpec &other) const;
inline bool operator!=(const InputSpec &other) const {
return !operator==(other);
}
std::string description;
NTA_BasicType dataType;
// TBD: Omit? isn't it always of unknown size?
Expand All @@ -59,7 +62,10 @@ class OutputSpec {
OutputSpec(std::string description, const NTA_BasicType dataType,
size_t count, bool regionLevel, bool isDefaultOutput,
bool sparse = false);

bool operator==(const OutputSpec &other) const;
inline bool operator!=(const OutputSpec &other) const {
return !operator==(other);
}
std::string description;
NTA_BasicType dataType;
// Size, in number of elements. If size is fixed, specify it here.
Expand All @@ -74,7 +80,10 @@ class CommandSpec {
public:
CommandSpec() {}
CommandSpec(std::string description);

bool operator==(const CommandSpec &other) const;
inline bool operator!=(const CommandSpec &other) const {
return !operator==(other);
}
std::string description;
};

Expand All @@ -89,7 +98,10 @@ class ParameterSpec {
ParameterSpec(std::string description, NTA_BasicType dataType, size_t count,
std::string constraints, std::string defaultValue,
AccessMode accessMode);

bool operator==(const ParameterSpec &other) const;
inline bool operator!=(const ParameterSpec &other) const {
return !operator==(other);
}
std::string description;

// [open: current basic types are bytes/{u}int16/32/64, real32/64, BytePtr. Is
Expand All @@ -109,7 +121,8 @@ struct Spec {
// TODO: should this be in the base API or layered? In the API right
// now since we do not build layered libraries.
std::string toString() const;

bool operator==(const Spec &other) const;
inline bool operator!=(const Spec &other) const { return !operator==(other); }
// Some RegionImpls support only a single node in a region.
// Such regions always have dimension [1]
bool singleNodeOnly;
Expand Down
9 changes: 8 additions & 1 deletion src/nupic/ntypes/Collection.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,14 @@ namespace nupic {
template <typename T> Collection<T>::Collection() {}

template <typename T> Collection<T>::~Collection() {}

template <typename T>
bool Collection<T>::operator==(const Collection<T> &o) const {
const static auto compare = [](std::pair<std::string, T> a,
std::pair<std::string, T> b) {
return a.first == b.first && a.second == b.second;
};
return std::equal(vec_.begin(), vec_.end(), o.vec_.begin(), compare);
}
template <typename T> size_t Collection<T>::getCount() const {
return vec_.size();
}
Expand Down
Loading

0 comments on commit 6d483ff

Please sign in to comment.