diff --git a/doc/usage/Crashcourse.md b/doc/usage/Crashcourse.md index b8a87ca03..9b2deff0b 100644 --- a/doc/usage/Crashcourse.md +++ b/doc/usage/Crashcourse.md @@ -111,7 +111,7 @@ Try queries like: "Fahrradwerkstatt München", "Berlin Alexanderplatz", "San Fra #### Build your own index -Try python/scripts/compile_json.py and compile your own JSON. The format should be: +Try python/examples/lookup/compile_json.py and compile your own JSON. The format should be: key {"city": "Munich", "state": "Germany"} diff --git a/keyvi/include/keyvi/dictionary/fsa/comparable_state_traverser.h b/keyvi/include/keyvi/dictionary/fsa/comparable_state_traverser.h index a9ac8b6fa..3449ce07e 100644 --- a/keyvi/include/keyvi/dictionary/fsa/comparable_state_traverser.h +++ b/keyvi/include/keyvi/dictionary/fsa/comparable_state_traverser.h @@ -31,6 +31,8 @@ #include #include "keyvi/dictionary/fsa/automata.h" +#include "keyvi/dictionary/fsa/traversal/traversal_base.h" +#include "keyvi/dictionary/fsa/traversal/weighted_traversal.h" #include "keyvi/dictionary/fsa/traverser_types.h" // #define ENABLE_TRACING @@ -65,7 +67,7 @@ class ComparableStateTraverser final { using label_t = typename innerTraverserType::label_t; using transition_t = typename innerTraverserType::transition_t; - explicit ComparableStateTraverser(const innerTraverserType &&traverser, const bool advance = true, + explicit ComparableStateTraverser(const innerTraverserType&& traverser, const bool advance = true, const size_t order = 0) : state_traverser_(std::move(traverser)), order_(order) { if (advance) { @@ -85,7 +87,7 @@ class ComparableStateTraverser final { : ComparableStateTraverser(f, f->GetStartState(), advance, order) {} explicit ComparableStateTraverser(const automata_t f, const uint64_t start_state, - traversal::TraversalPayload &&payload, const bool advance = true, + traversal::TraversalPayload&& payload, const bool advance = true, const size_t order = 0) : state_traverser_(f, start_state, std::move(payload), false), order_(order) { if (advance) { @@ -94,13 +96,13 @@ class ComparableStateTraverser final { } ComparableStateTraverser() = delete; - ComparableStateTraverser &operator=(ComparableStateTraverser const &) = delete; - ComparableStateTraverser(const ComparableStateTraverser &that) = delete; + ComparableStateTraverser& operator=(ComparableStateTraverser const&) = delete; + ComparableStateTraverser(const ComparableStateTraverser& that) = delete; /** * Comparison of the state traverser for the purpose of ordering them */ - bool operator<(const ComparableStateTraverser &rhs) const { + bool operator<(const ComparableStateTraverser& rhs) const { int compare = std::memcmp(label_stack_.data(), rhs.label_stack_.data(), std::min(label_stack_.size(), rhs.label_stack_.size()) * sizeof(label_t)); if (compare != 0) { @@ -114,16 +116,16 @@ class ComparableStateTraverser final { return order_ > rhs.order_; } - bool operator>(const ComparableStateTraverser &rhs) const { return rhs.operator<(*this); } + bool operator>(const ComparableStateTraverser& rhs) const { return rhs.operator<(*this); } - bool operator<=(const ComparableStateTraverser &rhs) const { return !operator>(rhs); } + bool operator<=(const ComparableStateTraverser& rhs) const { return !operator>(rhs); } - bool operator>=(const ComparableStateTraverser &rhs) const { return !operator<(rhs); } + bool operator>=(const ComparableStateTraverser& rhs) const { return !operator<(rhs); } /** * Compare traverser with another one, _ignoring_ the order value */ - bool operator==(const ComparableStateTraverser &rhs) const { + bool operator==(const ComparableStateTraverser& rhs) const { if (label_stack_.size() != rhs.label_stack_.size()) { return false; } @@ -131,7 +133,7 @@ class ComparableStateTraverser final { return std::memcmp(label_stack_.data(), rhs.label_stack_.data(), label_stack_.size() * sizeof(label_t)) == 0; } - bool operator!=(const ComparableStateTraverser &rhs) const { return !operator==(rhs); } + bool operator!=(const ComparableStateTraverser& rhs) const { return !operator==(rhs); } operator bool() const { return state_traverser_; } @@ -162,7 +164,7 @@ class ComparableStateTraverser final { label_t GetStateLabel() const { return state_traverser_.GetStateLabel(); } - const std::vector &GetStateLabels() const { return label_stack_; } + [[nodiscard]] const std::vector& GetStateLabels() const { return label_stack_; } size_t GetOrder() const { return order_; } @@ -186,22 +188,22 @@ class ComparableStateTraverser final { template friend class matching::NearMatching; - traversal::TraversalState &GetStates() { return state_traverser_.GetStates(); } + traversal::TraversalState& GetStates() { return state_traverser_.GetStates(); } - traversal::TraversalPayload &GetTraversalPayload() { return state_traverser_.GetTraversalPayload(); } + traversal::TraversalPayload& GetTraversalPayload() { return state_traverser_.GetTraversalPayload(); } - const traversal::TraversalPayload &GetTraversalPayload() const { + [[nodiscard]] const traversal::TraversalPayload& GetTraversalPayload() const { return state_traverser_.GetTraversalPayload(); } }; -inline bool CompareWeights(const traversal::TraversalState &i, - const traversal::TraversalState &j) { - return i.GetNextInnerWeight() == j.GetNextInnerWeight(); +inline bool CompareWeights(const traversal::TraversalState& lhs, + const traversal::TraversalState& rhs) { + return lhs.GetNextInnerWeight() == rhs.GetNextInnerWeight(); } template <> -inline bool ComparableStateTraverser::operator<(const ComparableStateTraverser &rhs) const { +inline bool ComparableStateTraverser::operator<(const ComparableStateTraverser& rhs) const { TRACE("operator< (weighted state specialization)"); TRACE("depth %ld %ld", state_traverser_.GetDepth(), rhs.state_traverser_.GetDepth()); @@ -231,7 +233,7 @@ inline bool ComparableStateTraverser::operator<(const Co } template <> -inline bool ComparableStateTraverser::operator==(const ComparableStateTraverser &rhs) const { +inline bool ComparableStateTraverser::operator==(const ComparableStateTraverser& rhs) const { if (label_stack_.size() != rhs.label_stack_.size()) { return false; } @@ -244,7 +246,7 @@ inline bool ComparableStateTraverser::operator==(const Compa } template <> -inline bool ComparableStateTraverser::operator<(const ComparableStateTraverser &rhs) const { +inline bool ComparableStateTraverser::operator<(const ComparableStateTraverser& rhs) const { TRACE("operator< (near state specialization)"); if (GetTraversalPayload().exact != rhs.GetTraversalPayload().exact) { diff --git a/keyvi/include/keyvi/dictionary/fsa/internal/lru_generation_cache.h b/keyvi/include/keyvi/dictionary/fsa/internal/lru_generation_cache.h index 613051f3f..e62d31912 100644 --- a/keyvi/include/keyvi/dictionary/fsa/internal/lru_generation_cache.h +++ b/keyvi/include/keyvi/dictionary/fsa/internal/lru_generation_cache.h @@ -66,21 +66,21 @@ class LeastRecentlyUsedGenerationsCache final { ~LeastRecentlyUsedGenerationsCache() { delete current_generation_; - for (MinimizationHash *generation : generations_) { + for (MinimizationHash* generation : generations_) { delete generation; } } LeastRecentlyUsedGenerationsCache() = delete; - LeastRecentlyUsedGenerationsCache &operator=(LeastRecentlyUsedGenerationsCache const &) = delete; - LeastRecentlyUsedGenerationsCache(const LeastRecentlyUsedGenerationsCache &that) = delete; + LeastRecentlyUsedGenerationsCache& operator=(LeastRecentlyUsedGenerationsCache const&) = delete; + LeastRecentlyUsedGenerationsCache(const LeastRecentlyUsedGenerationsCache& that) = delete; /** Add this object. * @param key The key to add */ void Add(EntryT key) { if (current_generation_->Size() >= size_per_generation_) { - MinimizationHash *newGeneration = nullptr; + MinimizationHash* newGeneration = nullptr; if (generations_.size() + 1 == max_number_of_generations_) { // remove(free) the first generation newGeneration = generations_[0]; @@ -101,7 +101,7 @@ class LeastRecentlyUsedGenerationsCache final { } template - const EntryT Get(EqualityType &key) { // NOLINT + const EntryT Get(EqualityType& key) { // NOLINT EntryT state = current_generation_->Get(key); if (!state.IsEmpty()) { @@ -126,7 +126,7 @@ class LeastRecentlyUsedGenerationsCache final { */ void Clear() { current_generation_->Clear(); - for (MinimizationHash *generation : generations_) { + for (MinimizationHash* generation : generations_) { delete generation; } generations_.clear(); @@ -148,8 +148,8 @@ class LeastRecentlyUsedGenerationsCache final { private: size_t size_per_generation_; size_t max_number_of_generations_; - MinimizationHash *current_generation_; - std::vector *> generations_; + MinimizationHash* current_generation_; + std::vector*> generations_; }; } /* namespace internal */ diff --git a/keyvi/include/keyvi/dictionary/fsa/internal/minimization_hash.h b/keyvi/include/keyvi/dictionary/fsa/internal/minimization_hash.h index 26253a490..18886aa2a 100644 --- a/keyvi/include/keyvi/dictionary/fsa/internal/minimization_hash.h +++ b/keyvi/include/keyvi/dictionary/fsa/internal/minimization_hash.h @@ -180,7 +180,7 @@ class MinimizationHash final { * @return the equal state or an empty value */ template - inline const T Get(EqualityType &key) const { // NOLINT + inline const T Get(EqualityType& key) const { // NOLINT size_t hash = key.GetHashcode() & 0x7fffffff; size_t bucket = hash % hash_size_; @@ -209,7 +209,7 @@ class MinimizationHash final { * @return the equal state or an empty value */ template - inline const T GetAndMove(EqualityType &key, MinimizationHash *other) { // NOLINT + inline const T GetAndMove(EqualityType& key, MinimizationHash* other) { // NOLINT size_t hash = key.GetHashcode() & 0x7fffffff; size_t bucket = hash % hash_size_; T entry = entries_[bucket]; @@ -325,10 +325,10 @@ class MinimizationHash final { size_t rehash_limit_ = 0; /// the actual data storage - T *entries_ = 0; + T* entries_ = 0; /// overflow data storage for colliding entries - T *overflow_entries_ = 0; + T* overflow_entries_ = 0; /// number of items in the data size_t count_ = 0; @@ -399,10 +399,10 @@ class MinimizationHash final { hash_size_ = hash_size_step_table_[hash_size_step_]; rehash_limit_ = static_cast(hash_size_ * load_factor_); - T *old_entries = entries_; + T* old_entries = entries_; entries_ = new T[hash_size_]; - T *old_overflow_entries = overflow_entries_; + T* old_overflow_entries = overflow_entries_; overflow_entries_size_ = std::min(hash_size_ >> 2, max_cookie_size_); overflow_entries_ = new T[overflow_entries_size_]; diff --git a/keyvi/include/keyvi/dictionary/fsa/state_traverser.h b/keyvi/include/keyvi/dictionary/fsa/state_traverser.h index 667f3ca82..46c9495e2 100644 --- a/keyvi/include/keyvi/dictionary/fsa/state_traverser.h +++ b/keyvi/include/keyvi/dictionary/fsa/state_traverser.h @@ -25,6 +25,7 @@ #ifndef KEYVI_DICTIONARY_FSA_STATE_TRAVERSER_H_ #define KEYVI_DICTIONARY_FSA_STATE_TRAVERSER_H_ +#include #include #include "keyvi/dictionary/fsa/automata.h" @@ -47,33 +48,33 @@ class StateTraverser final { using label_t = unsigned char; using transition_t = TransitionT; - explicit StateTraverser(automata_t f) - : fsa_(f), current_state_(f->GetStartState()), current_weight_(0), current_label_(0), stack_() { + explicit StateTraverser(const automata_t& fsa) + : fsa_(fsa), current_state_(fsa->GetStartState()), current_weight_(0), current_label_(0), stack_() { TRACE("StateTraverser starting with Start state %d", current_state_); - f->GetOutGoingTransitions(current_state_, &stack_.GetStates(), &stack_.traversal_stack_payload, 0); + fsa->GetOutGoingTransitions(current_state_, &stack_.GetStates(), &stack_.traversal_stack_payload, 0); this->operator++(0); } - StateTraverser(automata_t f, const uint64_t start_state, traversal::TraversalPayload &&payload, + StateTraverser(const automata_t& fsa, const uint64_t start_state, traversal::TraversalPayload&& payload, const bool advance = true) - : fsa_(f), current_weight_(0), current_label_(0), stack_(std::move(payload)) { + : fsa_(fsa), current_weight_(0), current_label_(0), stack_(std::move(payload)) { current_state_ = start_state; TRACE("StateTraverser starting with Start state %d", current_state_); - f->GetOutGoingTransitions(start_state, &stack_.GetStates(), &stack_.traversal_stack_payload, - f->GetInnerWeight(start_state)); + fsa->GetOutGoingTransitions(start_state, &stack_.GetStates(), &stack_.traversal_stack_payload, + fsa->GetInnerWeight(start_state)); if (advance) { this->operator++(0); } } - StateTraverser(automata_t f, const uint64_t start_state, const bool advance = true) - : fsa_(f), current_state_(start_state), current_weight_(0), current_label_(0), stack_() { + StateTraverser(const automata_t& fsa, const uint64_t start_state, const bool advance = true) + : fsa_(fsa), current_state_(start_state), current_weight_(0), current_label_(0), stack_() { TRACE("StateTraverser starting with Start state %d", current_state_); - f->GetOutGoingTransitions(start_state, &stack_.GetStates(), &stack_.traversal_stack_payload, - f->GetInnerWeight(start_state)); + fsa->GetOutGoingTransitions(start_state, &stack_.GetStates(), &stack_.traversal_stack_payload, + fsa->GetInnerWeight(start_state)); if (advance) { this->operator++(0); @@ -81,10 +82,10 @@ class StateTraverser final { } StateTraverser() = delete; - StateTraverser &operator=(StateTraverser const &) = delete; - StateTraverser(const StateTraverser &that) = delete; + StateTraverser& operator=(StateTraverser const&) = delete; + StateTraverser(const StateTraverser& that) = delete; - StateTraverser(StateTraverser &&other) + StateTraverser(StateTraverser&& other) noexcept : fsa_(other.fsa_), current_state_(other.current_state_), current_weight_(other.current_weight_), @@ -182,13 +183,15 @@ class StateTraverser final { template friend class ComparableStateTraverser; - const traversal::TraversalStack &GetStack() const { return stack_; } + [[nodiscard]] const traversal::TraversalStack& GetStack() const { return stack_; } - traversal::TraversalState &GetStates() { return stack_.GetStates(); } + traversal::TraversalState& GetStates() { return stack_.GetStates(); } - traversal::TraversalPayload &GetTraversalPayload() { return stack_.traversal_stack_payload; } + traversal::TraversalPayload& GetTraversalPayload() { return stack_.traversal_stack_payload; } - const traversal::TraversalPayload &GetTraversalPayload() const { return stack_.traversal_stack_payload; } + [[nodiscard]] const traversal::TraversalPayload& GetTraversalPayload() const { + return stack_.traversal_stack_payload; + } }; /** diff --git a/keyvi/include/keyvi/dictionary/fsa/zip_state_traverser.h b/keyvi/include/keyvi/dictionary/fsa/zip_state_traverser.h index f787e4023..c26047117 100644 --- a/keyvi/include/keyvi/dictionary/fsa/zip_state_traverser.h +++ b/keyvi/include/keyvi/dictionary/fsa/zip_state_traverser.h @@ -26,6 +26,7 @@ #define KEYVI_DICTIONARY_FSA_ZIP_STATE_TRAVERSER_H_ #include +#include #include #include #include @@ -38,6 +39,7 @@ #include "keyvi/dictionary/fsa/automata.h" #include "keyvi/dictionary/fsa/comparable_state_traverser.h" +#include "keyvi/dictionary/fsa/traversal/traversal_base.h" #include "keyvi/dictionary/fsa/traverser_types.h" // #define ENABLE_TRACING @@ -66,7 +68,7 @@ class ZipStateTraverser final { using traverser_t = std::shared_ptr>; struct TraverserCompare { - bool operator()(const traverser_t &t1, const traverser_t &t2) const { return *t1 > *t2; } + bool operator()(const traverser_t& lhs, const traverser_t& rhs) const { return *lhs > *rhs; } }; public: @@ -74,10 +76,10 @@ class ZipStateTraverser final { using transition_t = typename innerTraverserType::transition_t; using heap_t = boost::heap::skew_heap, boost::heap::mutable_>; - explicit ZipStateTraverser(const std::vector &fsas, const bool advance = true) { + explicit ZipStateTraverser(const std::vector& fsas, const bool advance = true) { size_t order = 0; - for (const automata_t &f : fsas) { - traverser_t traverser = std::make_shared>(f, advance, order++); + for (const automata_t& fsa : fsas) { + traverser_t traverser = std::make_shared>(fsa, advance, order++); // the traverser could be exhausted after it has been advanced if (*traverser) { traverser_queue_.push(traverser); @@ -88,8 +90,8 @@ class ZipStateTraverser final { explicit ZipStateTraverser(const std::initializer_list fsas, const bool advance = true) { size_t order = 0; - for (auto f : fsas) { - traverser_t traverser = std::make_shared>(f, advance, order++); + for (auto fsa : fsas) { + traverser_t traverser = std::make_shared>(fsa, advance, order++); // the traverser could be exhausted after it has been advanced if (*traverser) { traverser_queue_.push(traverser); @@ -98,13 +100,13 @@ class ZipStateTraverser final { FillInValues(); } - explicit ZipStateTraverser(const std::vector> &fsa_start_state_pairs, + explicit ZipStateTraverser(const std::vector>& fsa_start_state_pairs, const bool advance = true) { size_t order = 0; - for (auto f : fsa_start_state_pairs) { - if (f.second > 0) { + for (auto pair : fsa_start_state_pairs) { + if (pair.second > 0) { traverser_t traverser = - std::make_shared>(f.first, f.second, advance, order++); + std::make_shared>(pair.first, pair.second, advance, order++); // the traverser could be exhausted after it has been advanced if (*traverser) { traverser_queue_.push(traverser); @@ -114,15 +116,15 @@ class ZipStateTraverser final { FillInValues(); } - explicit ZipStateTraverser(std::vector>> - &&fsa_start_state_payloads, + explicit ZipStateTraverser(std::vector>>&& + fsa_start_state_payloads, const bool advance = true) { size_t order = 0; - for (auto &f : fsa_start_state_payloads) { - if (std::get<1>(f) > 0) { + for (auto& entry : fsa_start_state_payloads) { + if (std::get<1>(entry) > 0) { traverser_t traverser = std::make_shared>( - std::get<0>(f), std::get<1>(f), std::move(std::get<2>(f)), advance, order++); + std::get<0>(entry), std::get<1>(entry), std::move(std::get<2>(entry)), advance, order++); // the traverser could be exhausted after it has been advanced if (*traverser) { @@ -134,10 +136,10 @@ class ZipStateTraverser final { } ZipStateTraverser() = delete; - ZipStateTraverser &operator=(ZipStateTraverser const &) = delete; - ZipStateTraverser(const ZipStateTraverser &that) = delete; + ZipStateTraverser& operator=(ZipStateTraverser const&) = delete; + ZipStateTraverser(const ZipStateTraverser& that) = delete; - ZipStateTraverser(ZipStateTraverser &&other) + ZipStateTraverser(ZipStateTraverser&& other) : traverser_queue_(std::move(other.traverser_queue_)), final_(other.final_), depth_(other.depth_), @@ -212,7 +214,7 @@ class ZipStateTraverser final { label_t GetStateLabel() const { return state_label_; } - const std::vector &GetStateLabels() const { return traverser_queue_.top()->GetStateLabels(); } + [[nodiscard]] const std::vector& GetStateLabels() const { return traverser_queue_.top()->GetStateLabels(); } /** * Set the minimum weight states must be greater or equal to. @@ -231,7 +233,7 @@ class ZipStateTraverser final { uint32_t inner_weight_ = 0; uint64_t state_id_ = 0; label_t state_label_ = 0; - size_t order_; + size_t order_{}; automata_t fsa_; size_t equal_states_ = 1; bool pruned = false; @@ -245,17 +247,17 @@ class ZipStateTraverser final { pruned = false; if (!traverser_queue_.empty()) { - const traverser_t &t = traverser_queue_.top(); - TRACE("take values from traverser %lu", t->GetOrder()); - - final_ = t->IsFinalState(); - depth_ = t->GetDepth(); - state_value_ = t->GetStateValue(); - inner_weight_ = t->GetInnerWeight(); - state_id_ = t->GetStateId(); - state_label_ = t->GetStateLabel(); - fsa_ = t->GetFsa(); - order_ = t->GetOrder(); + const traverser_t& top = traverser_queue_.top(); + TRACE("take values from traverser %lu", top->GetOrder()); + + final_ = top->IsFinalState(); + depth_ = top->GetDepth(); + state_value_ = top->GetStateValue(); + inner_weight_ = top->GetInnerWeight(); + state_id_ = top->GetStateId(); + state_label_ = top->GetStateLabel(); + fsa_ = top->GetFsa(); + order_ = top->GetOrder(); // memorize how many inner traverser are at a equal inner state equal_states_ = 1; @@ -265,7 +267,7 @@ class ZipStateTraverser final { TRACE("label: %c", state_label_); - while (traverser_queue_.size() > equal_states_ && *t == *(*it)) { + while (traverser_queue_.size() > equal_states_ && *top == *(*it)) { TRACE("dedup"); equal_states_++; // if not final yet check if other states are final @@ -295,7 +297,7 @@ class ZipStateTraverser final { template friend class matching::NearMatching; - const traversal::TraversalPayload &GetTraversalPayload() const { + [[nodiscard]] const traversal::TraversalPayload& GetTraversalPayload() const { return traverser_queue_.top()->GetTraversalPayload(); } }; @@ -322,7 +324,7 @@ inline void ZipStateTraverser::PreIncrement() { size_t steps = equal_states_; while (steps > 0) { - for (const transition_t &transition : (*it)->GetStates().traversal_state_payload.transitions) { + for (const transition_t& transition : (*it)->GetStates().traversal_state_payload.transitions) { if (global_weights.count(transition.label) == 0 || global_weights.at(transition.label) < transition.weight) { global_weights[transition.label] = transition.weight; } @@ -335,7 +337,7 @@ inline void ZipStateTraverser::PreIncrement() { it = traverser_queue_.ordered_begin(); steps = equal_states_; while (steps > 0) { - for (transition_t &transition : (*it)->GetStates().traversal_state_payload.transitions) { + for (transition_t& transition : (*it)->GetStates().traversal_state_payload.transitions) { transition.weight = global_weights.at(transition.label); } // re-sort transitions @@ -353,8 +355,9 @@ inline ZipStateTraverser::ZipStateTraverser(const std::i size_t order = 0; if (fsas.size() < 2) { - for (auto f : fsas) { - traverser_t traverser = std::make_shared>(f, advance, order++); + for (const auto& fsa : fsas) { + traverser_t const traverser = + std::make_shared>(fsa, advance, order++); // the traverser could be exhausted after it has been advanced if (*traverser) { traverser_queue_.push(traverser); @@ -364,34 +367,35 @@ inline ZipStateTraverser::ZipStateTraverser(const std::i std::map global_weights; std::vector traversers; - for (auto f : fsas) { - traverser_t traverser = std::make_shared>(f, false, order++); + for (const auto& fsa : fsas) { + traverser_t const traverser = + std::make_shared>(fsa, false, order++); traversers.push_back(traverser); } // 1st pass collect all weights per label - for (const auto &t : traversers) { - for (const transition_t &transition : t->GetStates().traversal_state_payload.transitions) { + for (const auto& trav : traversers) { + for (const transition_t& transition : trav->GetStates().traversal_state_payload.transitions) { if (global_weights.count(transition.label) == 0 || global_weights.at(transition.label) < transition.weight) { global_weights[transition.label] = transition.weight; } } } // 2nd pass apply global weights - for (const auto &t : traversers) { - for (transition_t &transition : t->GetStates().traversal_state_payload.transitions) { + for (const auto& trav : traversers) { + for (transition_t& transition : trav->GetStates().traversal_state_payload.transitions) { transition.weight = global_weights.at(transition.label); } // re-sort transitions - t->GetStates().PostProcess(&t->GetTraversalPayload()); + trav->GetStates().PostProcess(&trav->GetTraversalPayload()); // now advance? if (advance) { - t->operator++(0); + trav->operator++(0); } // the traverser could be exhausted after it has been advanced - if (*t) { - traverser_queue_.push(t); + if (*trav) { + traverser_queue_.push(trav); } } } @@ -400,14 +404,15 @@ inline ZipStateTraverser::ZipStateTraverser(const std::i } template <> -inline ZipStateTraverser::ZipStateTraverser(const std::vector &fsas, +inline ZipStateTraverser::ZipStateTraverser(const std::vector& fsas, const bool advance) { TRACE("construct (weighted state specialization)"); size_t order = 0; if (fsas.size() < 2) { - for (auto f : fsas) { - traverser_t traverser = std::make_shared>(f, advance, order++); + for (const auto& fsa : fsas) { + traverser_t const traverser = + std::make_shared>(fsa, advance, order++); // the traverser could be exhausted after it has been advanced if (*traverser) { traverser_queue_.push(traverser); @@ -417,34 +422,35 @@ inline ZipStateTraverser::ZipStateTraverser(const std::v std::map global_weights; std::vector traversers; - for (auto f : fsas) { - traverser_t traverser = std::make_shared>(f, false, order++); + for (const auto& fsa : fsas) { + const traverser_t traverser = + std::make_shared>(fsa, false, order++); traversers.push_back(traverser); } // 1st pass collect all weights per label - for (const auto &t : traversers) { - for (const transition_t &transition : t->GetStates().traversal_state_payload.transitions) { + for (const auto& trav : traversers) { + for (const transition_t& transition : trav->GetStates().traversal_state_payload.transitions) { if (global_weights.count(transition.label) == 0 || global_weights.at(transition.label) < transition.weight) { global_weights[transition.label] = transition.weight; } } } // 2nd pass apply global weights - for (const auto &t : traversers) { - for (transition_t &transition : t->GetStates().traversal_state_payload.transitions) { + for (const auto& trav : traversers) { + for (transition_t& transition : trav->GetStates().traversal_state_payload.transitions) { transition.weight = global_weights.at(transition.label); } // re-sort transitions - t->GetStates().PostProcess(&t->GetTraversalPayload()); + trav->GetStates().PostProcess(&trav->GetTraversalPayload()); // now advance? if (advance) { - t->operator++(0); + trav->operator++(0); } // the traverser could be exhausted after it has been advanced - if (*t) { - traverser_queue_.push(t); + if (*trav) { + traverser_queue_.push(trav); } } } @@ -454,14 +460,14 @@ inline ZipStateTraverser::ZipStateTraverser(const std::v template <> inline ZipStateTraverser::ZipStateTraverser( - const std::vector> &fsa_start_state_pairs, const bool advance) { + const std::vector>& fsa_start_state_pairs, const bool advance) { size_t order = 0; if (fsa_start_state_pairs.size() < 2) { - for (auto f : fsa_start_state_pairs) { - if (f.second > 0) { - traverser_t traverser = - std::make_shared>(f.first, f.second, advance, order++); + for (const auto& pair : fsa_start_state_pairs) { + if (pair.second > 0) { + const traverser_t traverser = std::make_shared>( + pair.first, pair.second, advance, order++); // the traverser could be exhausted after it has been advanced if (*traverser) { traverser_queue_.push(traverser); @@ -472,39 +478,39 @@ inline ZipStateTraverser::ZipStateTraverser( // there is more than 1 inner traverser std::map global_weights; std::vector traversers; - for (auto f : fsa_start_state_pairs) { - if (f.second > 0) { + for (const auto& pair : fsa_start_state_pairs) { + if (pair.second > 0) { traverser_t traverser = - std::make_shared>(f.first, f.second, false, order++); + std::make_shared>(pair.first, pair.second, false, order++); traversers.push_back(traverser); } } // 1st pass collect all weights per label - for (const auto &t : traversers) { - for (const transition_t &transition : t->GetStates().traversal_state_payload.transitions) { + for (const auto& trav : traversers) { + for (const transition_t& transition : trav->GetStates().traversal_state_payload.transitions) { if (global_weights.count(transition.label) == 0 || global_weights.at(transition.label) < transition.weight) { global_weights[transition.label] = transition.weight; } } } // 2nd pass apply global weights - for (const auto &t : traversers) { - for (transition_t &transition : t->GetStates().traversal_state_payload.transitions) { + for (const auto& trav : traversers) { + for (transition_t& transition : trav->GetStates().traversal_state_payload.transitions) { transition.weight = global_weights.at(transition.label); } - TRACE("resort %ld", t->GetOrder()); + TRACE("resort %ld", trav->GetOrder()); // re-sort transitions - t->GetStates().PostProcess(&t->GetTraversalPayload()); + trav->GetStates().PostProcess(&trav->GetTraversalPayload()); // now advance? if (advance) { - t->operator++(0); + trav->operator++(0); } // the traverser could be exhausted after it has been advanced - if (*t) { - traverser_queue_.push(t); + if (*trav) { + traverser_queue_.push(trav); } } } @@ -518,8 +524,8 @@ inline ZipStateTraverser::ZipStateTraverser( */ template <> inline void ZipStateTraverser::SetMinWeight(uint32_t weight) { - for (auto t : traverser_queue_) { - t->SetMinWeight(weight); + for (const auto& trav : traverser_queue_) { + trav->SetMinWeight(weight); } } diff --git a/keyvi/tests/keyvi/dictionary/completion/prefix_completion_test.cpp b/keyvi/tests/keyvi/dictionary/completion/prefix_completion_test.cpp index de104eee9..4d8cf0de3 100644 --- a/keyvi/tests/keyvi/dictionary/completion/prefix_completion_test.cpp +++ b/keyvi/tests/keyvi/dictionary/completion/prefix_completion_test.cpp @@ -23,6 +23,9 @@ * Author: hendrik */ +#include +#include + #include #include diff --git a/keyvi/tests/keyvi/dictionary/dictionary_index_compiler_test.cpp b/keyvi/tests/keyvi/dictionary/dictionary_index_compiler_test.cpp index b089f08da..f6c489f86 100644 --- a/keyvi/tests/keyvi/dictionary/dictionary_index_compiler_test.cpp +++ b/keyvi/tests/keyvi/dictionary/dictionary_index_compiler_test.cpp @@ -23,6 +23,9 @@ * Author: hendrik */ +#include +#include + #include #include "keyvi/dictionary/dictionary.h" diff --git a/keyvi/tests/keyvi/dictionary/dictionary_properties_test.cpp b/keyvi/tests/keyvi/dictionary/dictionary_properties_test.cpp index b78a67638..fb58e9a47 100644 --- a/keyvi/tests/keyvi/dictionary/dictionary_properties_test.cpp +++ b/keyvi/tests/keyvi/dictionary/dictionary_properties_test.cpp @@ -17,6 +17,7 @@ // #include +#include #include #include diff --git a/keyvi/tests/keyvi/dictionary/fsa/internal/memory_map_manager_test.cpp b/keyvi/tests/keyvi/dictionary/fsa/internal/memory_map_manager_test.cpp index e3d0bf140..f3f2ef179 100644 --- a/keyvi/tests/keyvi/dictionary/fsa/internal/memory_map_manager_test.cpp +++ b/keyvi/tests/keyvi/dictionary/fsa/internal/memory_map_manager_test.cpp @@ -24,6 +24,7 @@ */ #include +#include #include diff --git a/keyvi/tests/keyvi/dictionary/fsa/internal/minimization_hash_test.cpp b/keyvi/tests/keyvi/dictionary/fsa/internal/minimization_hash_test.cpp index 24024de9c..abb831308 100644 --- a/keyvi/tests/keyvi/dictionary/fsa/internal/minimization_hash_test.cpp +++ b/keyvi/tests/keyvi/dictionary/fsa/internal/minimization_hash_test.cpp @@ -35,7 +35,7 @@ namespace internal { BOOST_AUTO_TEST_SUITE(MinimizationHashTests) BOOST_AUTO_TEST_CASE(insert) { - MinimizationHash> *hash = new MinimizationHash>(); + auto* hash = new MinimizationHash>(); PackedState<> p1 = {10, 25, 2}; hash->Add(p1); PackedState<> p2 = {12, 25, 3}; @@ -54,7 +54,7 @@ BOOST_AUTO_TEST_CASE(insert) { } BOOST_AUTO_TEST_CASE(reset) { - MinimizationHash> *hash = new MinimizationHash>(); + auto* hash = new MinimizationHash>(); PackedState<> p1 = {10, 25, 2}; hash->Add(p1); PackedState<> p2 = {12, 25, 3}; diff --git a/keyvi/tests/keyvi/dictionary/fsa/traversal/near_traversal_test.cpp b/keyvi/tests/keyvi/dictionary/fsa/traversal/near_traversal_test.cpp index e02d6b6cb..3161f4fcd 100644 --- a/keyvi/tests/keyvi/dictionary/fsa/traversal/near_traversal_test.cpp +++ b/keyvi/tests/keyvi/dictionary/fsa/traversal/near_traversal_test.cpp @@ -23,7 +23,10 @@ * Author: hendrik */ +#include +#include #include +#include #include diff --git a/keyvi/tests/keyvi/dictionary/fsa/zip_state_traverser_test.cpp b/keyvi/tests/keyvi/dictionary/fsa/zip_state_traverser_test.cpp index 3602adca0..152b06c09 100644 --- a/keyvi/tests/keyvi/dictionary/fsa/zip_state_traverser_test.cpp +++ b/keyvi/tests/keyvi/dictionary/fsa/zip_state_traverser_test.cpp @@ -24,6 +24,9 @@ */ #include +#include +#include +#include #include @@ -510,7 +513,9 @@ BOOST_AUTO_TEST_CASE(basic) { BOOST_CHECK(!t); } -std::vector GetAllKeys(ZipStateTraverser> *zip_traverser) { +namespace { + +std::vector GetAllKeys(ZipStateTraverser>* zip_traverser) { std::vector label_stack; std::vector keys; @@ -527,6 +532,8 @@ std::vector GetAllKeys(ZipStateTraverser> *zip_tra return keys; } +} // namespace + BOOST_AUTO_TEST_CASE(append) { std::vector test_data1 = {"aa", "bb", "cc", "dd", "ee"}; testing::TempDictionary dictionary1(&test_data1); diff --git a/keyvi/tests/keyvi/dictionary/matching/multiword_completion_matching_test.cpp b/keyvi/tests/keyvi/dictionary/matching/multiword_completion_matching_test.cpp index d7f78d043..dffcf6a5a 100644 --- a/keyvi/tests/keyvi/dictionary/matching/multiword_completion_matching_test.cpp +++ b/keyvi/tests/keyvi/dictionary/matching/multiword_completion_matching_test.cpp @@ -18,6 +18,9 @@ #include "keyvi/dictionary/matching/multiword_completion_matching.h" #include +#include +#include + #include #include "keyvi/dictionary/dictionary.h" diff --git a/keyvi/tests/keyvi/dictionary/matching/prefix_completion_matching_test.cpp b/keyvi/tests/keyvi/dictionary/matching/prefix_completion_matching_test.cpp index c5acdfda3..3abe76775 100644 --- a/keyvi/tests/keyvi/dictionary/matching/prefix_completion_matching_test.cpp +++ b/keyvi/tests/keyvi/dictionary/matching/prefix_completion_matching_test.cpp @@ -18,6 +18,9 @@ #include "keyvi/dictionary/matching/prefix_completion_matching.h" #include +#include +#include + #include #include "keyvi/dictionary/dictionary.h" diff --git a/keyvi/tests/keyvi/index/index_limits_test.cpp b/keyvi/tests/keyvi/index/index_limits_test.cpp index 141e82cb9..34e6acc15 100644 --- a/keyvi/tests/keyvi/index/index_limits_test.cpp +++ b/keyvi/tests/keyvi/index/index_limits_test.cpp @@ -46,9 +46,7 @@ inline std::string get_keyvimerger_bin() { } inline size_t limit_filedescriptors(size_t file_descriptor_limit) { - struct rlimit limit { - 0 - }; + struct rlimit limit{0}; getrlimit(RLIMIT_NOFILE, &limit); const size_t old_limit = limit.rlim_cur; diff --git a/keyvi/tests/keyvi/index/internal/index_settings_test.cpp b/keyvi/tests/keyvi/index/internal/index_settings_test.cpp index 9b2dc02a1..f6ea6ce7f 100644 --- a/keyvi/tests/keyvi/index/internal/index_settings_test.cpp +++ b/keyvi/tests/keyvi/index/internal/index_settings_test.cpp @@ -23,6 +23,8 @@ * Author: hendrik */ +#include + #include #include "keyvi/index/internal/index_settings.h" diff --git a/keyvi/tests/keyvi/stringdistance/needleman_wunsch_test.cpp b/keyvi/tests/keyvi/stringdistance/needleman_wunsch_test.cpp index 94c48880d..f795176b8 100644 --- a/keyvi/tests/keyvi/stringdistance/needleman_wunsch_test.cpp +++ b/keyvi/tests/keyvi/stringdistance/needleman_wunsch_test.cpp @@ -24,6 +24,8 @@ */ #include +#include +#include #include diff --git a/keyvi/tests/keyvi/util/configuration_test.cpp b/keyvi/tests/keyvi/util/configuration_test.cpp index 424feb555..b6282c220 100644 --- a/keyvi/tests/keyvi/util/configuration_test.cpp +++ b/keyvi/tests/keyvi/util/configuration_test.cpp @@ -24,6 +24,7 @@ */ #include +#include #include diff --git a/keyvi/tests/keyvi/vector/basic_test.cpp b/keyvi/tests/keyvi/vector/basic_test.cpp index 1787584ab..935f7cad4 100644 --- a/keyvi/tests/keyvi/vector/basic_test.cpp +++ b/keyvi/tests/keyvi/vector/basic_test.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include diff --git a/python/scripts/compile_json.py b/python/scripts/compile_json.py deleted file mode 100755 index c7b35b2b0..000000000 --- a/python/scripts/compile_json.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import sys -import Queue -import threading -import keyvi -import argparse -import os -import gzip -import unicodedata, re - -control_chars = ''.join(map(unichr, range(0,32))) - -control_char_re = re.compile('[%s]' % re.escape(control_chars)) - -def remove_control_chars(s): - return control_char_re.sub('', s) - -def print_progress(a,b): - print "Progress {}/{}".format(a,b) - -def compile_worker(): - while True: - compiler, output = compile_queue.get() - compiler.Compile(print_progress) - compiler.WriteToFile(output) - compile_queue.task_done() - -compile_queue = Queue.Queue() - -def compile_file(input, output, jobs, shards): - skipped_keys = 0 - - compilers = {} - for i in range (0, shards): - compilers[i] = keyvi.JsonDictionaryCompiler() - - if os.path.isdir(input): - input_files = [os.path.join(input,d) for d in os.listdir(input)] - else: - input_files = [input] - - for input_file in input_files: - if input_file.endswith(".gz"): - input_fd = gzip.open(input_file) - else: - input_fd = open(input_file) - - for line in input_fd: - try: - parts = line.split("\t") - key = parts[0] - - if key != remove_control_chars(key): - print "skip key: " + ":".join("{:02x}".format(ord(c)) for c in key) + " due to containing control characters" - skipped_keys +=1 - - value = parts[1] - - shard = keyvi.JumpConsistentHashString(key, shards) - compilers[shard].Add(key, value) - except: - print "failed to add: " + line - print "Skipped keys " + str(skipped_keys) - - for i in range(jobs): - t = threading.Thread(target=compile_worker) - t.daemon = True - t.start() - - for i in range (0, shards): - compile_queue.put((compilers[i], output + "-" + str(i))) - - compile_queue.join() - - -ARGV = [ - ('-i', '--input', str, None, 'input file'), - ('-o', '--output', str, None, 'output'), - ('-b', '--bucket', str, None, 's3 bucket to read from'), - ('-k', '--s3key', str, None, 's3 key/folder to read from'), - ('-j', '--jobs', int, 1, 'number of parallel jobs'), - ('-s', '--shards', int, 1, 'number of shards'), -] - - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Compile json keyvi dictionary') - for arg in ARGV: - parser.add_argument(*arg[0:2], type=arg[2], default=arg[3], help=arg[4]) - args = parser.parse_args() - if args.input: - compile_file(args.input, args.output, args.jobs, args.shards)