Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion doc/usage/Crashcourse.md
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ Try queries like: "Fahrradwerkstatt München", "Berlin Alexanderplatz", "San Fra

#### Build your own index

Try python/scripts/compile_json.py and compile your own JSON. The format should be:
Try python/examples/lookup/compile_json.py and compile your own JSON. The format should be:

key {"city": "Munich", "state": "Germany"}

Expand Down
42 changes: 22 additions & 20 deletions keyvi/include/keyvi/dictionary/fsa/comparable_state_traverser.h
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
#include <vector>

#include "keyvi/dictionary/fsa/automata.h"
#include "keyvi/dictionary/fsa/traversal/traversal_base.h"
#include "keyvi/dictionary/fsa/traversal/weighted_traversal.h"
#include "keyvi/dictionary/fsa/traverser_types.h"

// #define ENABLE_TRACING
Expand Down Expand Up @@ -65,7 +67,7 @@ class ComparableStateTraverser final {
using label_t = typename innerTraverserType::label_t;
using transition_t = typename innerTraverserType::transition_t;

explicit ComparableStateTraverser(const innerTraverserType &&traverser, const bool advance = true,
explicit ComparableStateTraverser(const innerTraverserType&& traverser, const bool advance = true,
const size_t order = 0)
: state_traverser_(std::move(traverser)), order_(order) {
if (advance) {
Expand All @@ -85,7 +87,7 @@ class ComparableStateTraverser final {
: ComparableStateTraverser(f, f->GetStartState(), advance, order) {}

explicit ComparableStateTraverser(const automata_t f, const uint64_t start_state,
traversal::TraversalPayload<transition_t> &&payload, const bool advance = true,
traversal::TraversalPayload<transition_t>&& payload, const bool advance = true,
Comment thread
hendrikmuhs marked this conversation as resolved.
const size_t order = 0)
: state_traverser_(f, start_state, std::move(payload), false), order_(order) {
if (advance) {
Expand All @@ -94,13 +96,13 @@ class ComparableStateTraverser final {
}

ComparableStateTraverser() = delete;
ComparableStateTraverser &operator=(ComparableStateTraverser const &) = delete;
ComparableStateTraverser(const ComparableStateTraverser &that) = delete;
ComparableStateTraverser& operator=(ComparableStateTraverser const&) = delete;
ComparableStateTraverser(const ComparableStateTraverser& that) = delete;

/**
* Comparison of the state traverser for the purpose of ordering them
*/
bool operator<(const ComparableStateTraverser &rhs) const {
bool operator<(const ComparableStateTraverser& rhs) const {
int compare = std::memcmp(label_stack_.data(), rhs.label_stack_.data(),
std::min(label_stack_.size(), rhs.label_stack_.size()) * sizeof(label_t));
if (compare != 0) {
Expand All @@ -114,24 +116,24 @@ class ComparableStateTraverser final {
return order_ > rhs.order_;
}

bool operator>(const ComparableStateTraverser &rhs) const { return rhs.operator<(*this); }
bool operator>(const ComparableStateTraverser& rhs) const { return rhs.operator<(*this); }

bool operator<=(const ComparableStateTraverser &rhs) const { return !operator>(rhs); }
bool operator<=(const ComparableStateTraverser& rhs) const { return !operator>(rhs); }

bool operator>=(const ComparableStateTraverser &rhs) const { return !operator<(rhs); }
bool operator>=(const ComparableStateTraverser& rhs) const { return !operator<(rhs); }

/**
* Compare traverser with another one, _ignoring_ the order value
*/
bool operator==(const ComparableStateTraverser &rhs) const {
bool operator==(const ComparableStateTraverser& rhs) const {
if (label_stack_.size() != rhs.label_stack_.size()) {
return false;
}

return std::memcmp(label_stack_.data(), rhs.label_stack_.data(), label_stack_.size() * sizeof(label_t)) == 0;
}

bool operator!=(const ComparableStateTraverser &rhs) const { return !operator==(rhs); }
bool operator!=(const ComparableStateTraverser& rhs) const { return !operator==(rhs); }

operator bool() const { return state_traverser_; }

Expand Down Expand Up @@ -162,7 +164,7 @@ class ComparableStateTraverser final {

label_t GetStateLabel() const { return state_traverser_.GetStateLabel(); }

const std::vector<label_t> &GetStateLabels() const { return label_stack_; }
[[nodiscard]] const std::vector<label_t>& GetStateLabels() const { return label_stack_; }

size_t GetOrder() const { return order_; }

Expand All @@ -186,22 +188,22 @@ class ComparableStateTraverser final {
template <class nearInnerTraverserType>
friend class matching::NearMatching;

traversal::TraversalState<transition_t> &GetStates() { return state_traverser_.GetStates(); }
traversal::TraversalState<transition_t>& GetStates() { return state_traverser_.GetStates(); }
Comment thread
hendrikmuhs marked this conversation as resolved.

traversal::TraversalPayload<transition_t> &GetTraversalPayload() { return state_traverser_.GetTraversalPayload(); }
traversal::TraversalPayload<transition_t>& GetTraversalPayload() { return state_traverser_.GetTraversalPayload(); }

const traversal::TraversalPayload<transition_t> &GetTraversalPayload() const {
[[nodiscard]] const traversal::TraversalPayload<transition_t>& GetTraversalPayload() const {
return state_traverser_.GetTraversalPayload();
}
};

inline bool CompareWeights(const traversal::TraversalState<traversal::WeightedTransition> &i,
const traversal::TraversalState<traversal::WeightedTransition> &j) {
return i.GetNextInnerWeight() == j.GetNextInnerWeight();
inline bool CompareWeights(const traversal::TraversalState<traversal::WeightedTransition>& lhs,
const traversal::TraversalState<traversal::WeightedTransition>& rhs) {
return lhs.GetNextInnerWeight() == rhs.GetNextInnerWeight();
}

template <>
inline bool ComparableStateTraverser<WeightedStateTraverser>::operator<(const ComparableStateTraverser &rhs) const {
inline bool ComparableStateTraverser<WeightedStateTraverser>::operator<(const ComparableStateTraverser& rhs) const {
TRACE("operator< (weighted state specialization)");

TRACE("depth %ld %ld", state_traverser_.GetDepth(), rhs.state_traverser_.GetDepth());
Expand Down Expand Up @@ -231,7 +233,7 @@ inline bool ComparableStateTraverser<WeightedStateTraverser>::operator<(const Co
}

template <>
inline bool ComparableStateTraverser<NearStateTraverser>::operator==(const ComparableStateTraverser &rhs) const {
inline bool ComparableStateTraverser<NearStateTraverser>::operator==(const ComparableStateTraverser& rhs) const {
if (label_stack_.size() != rhs.label_stack_.size()) {
return false;
}
Expand All @@ -244,7 +246,7 @@ inline bool ComparableStateTraverser<NearStateTraverser>::operator==(const Compa
}

template <>
inline bool ComparableStateTraverser<NearStateTraverser>::operator<(const ComparableStateTraverser &rhs) const {
inline bool ComparableStateTraverser<NearStateTraverser>::operator<(const ComparableStateTraverser& rhs) const {
TRACE("operator< (near state specialization)");

if (GetTraversalPayload().exact != rhs.GetTraversalPayload().exact) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,21 +66,21 @@ class LeastRecentlyUsedGenerationsCache final {

~LeastRecentlyUsedGenerationsCache() {
delete current_generation_;
for (MinimizationHash<EntryT> *generation : generations_) {
for (MinimizationHash<EntryT>* generation : generations_) {
delete generation;
Comment thread
hendrikmuhs marked this conversation as resolved.
}
}

LeastRecentlyUsedGenerationsCache() = delete;
LeastRecentlyUsedGenerationsCache &operator=(LeastRecentlyUsedGenerationsCache const &) = delete;
LeastRecentlyUsedGenerationsCache(const LeastRecentlyUsedGenerationsCache &that) = delete;
LeastRecentlyUsedGenerationsCache& operator=(LeastRecentlyUsedGenerationsCache const&) = delete;
LeastRecentlyUsedGenerationsCache(const LeastRecentlyUsedGenerationsCache& that) = delete;

/** Add this object.
* @param key The key to add
*/
void Add(EntryT key) {
if (current_generation_->Size() >= size_per_generation_) {
MinimizationHash<EntryT> *newGeneration = nullptr;
MinimizationHash<EntryT>* newGeneration = nullptr;
if (generations_.size() + 1 == max_number_of_generations_) {
// remove(free) the first generation
newGeneration = generations_[0];
Expand All @@ -101,7 +101,7 @@ class LeastRecentlyUsedGenerationsCache final {
}

template <typename EqualityType>
const EntryT Get(EqualityType &key) { // NOLINT
const EntryT Get(EqualityType& key) { // NOLINT
EntryT state = current_generation_->Get(key);

if (!state.IsEmpty()) {
Expand All @@ -126,7 +126,7 @@ class LeastRecentlyUsedGenerationsCache final {
*/
void Clear() {
current_generation_->Clear();
for (MinimizationHash<EntryT> *generation : generations_) {
for (MinimizationHash<EntryT>* generation : generations_) {
delete generation;
Comment thread
hendrikmuhs marked this conversation as resolved.
}
generations_.clear();
Expand All @@ -148,8 +148,8 @@ class LeastRecentlyUsedGenerationsCache final {
private:
size_t size_per_generation_;
size_t max_number_of_generations_;
MinimizationHash<EntryT> *current_generation_;
std::vector<MinimizationHash<EntryT> *> generations_;
MinimizationHash<EntryT>* current_generation_;
std::vector<MinimizationHash<EntryT>*> generations_;
};

} /* namespace internal */
Expand Down
12 changes: 6 additions & 6 deletions keyvi/include/keyvi/dictionary/fsa/internal/minimization_hash.h
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ class MinimizationHash final {
* @return the equal state or an empty value
*/
template <typename EqualityType>
inline const T Get(EqualityType &key) const { // NOLINT
inline const T Get(EqualityType& key) const { // NOLINT
size_t hash = key.GetHashcode() & 0x7fffffff;
size_t bucket = hash % hash_size_;

Expand Down Expand Up @@ -209,7 +209,7 @@ class MinimizationHash final {
* @return the equal state or an empty value
*/
template <typename EqualityType>
inline const T GetAndMove(EqualityType &key, MinimizationHash<T> *other) { // NOLINT
inline const T GetAndMove(EqualityType& key, MinimizationHash<T>* other) { // NOLINT
size_t hash = key.GetHashcode() & 0x7fffffff;
size_t bucket = hash % hash_size_;
T entry = entries_[bucket];
Expand Down Expand Up @@ -325,10 +325,10 @@ class MinimizationHash final {
size_t rehash_limit_ = 0;

/// the actual data storage
T *entries_ = 0;
T* entries_ = 0;

/// overflow data storage for colliding entries
T *overflow_entries_ = 0;
T* overflow_entries_ = 0;

/// number of items in the data
size_t count_ = 0;
Expand Down Expand Up @@ -399,10 +399,10 @@ class MinimizationHash final {
hash_size_ = hash_size_step_table_[hash_size_step_];
rehash_limit_ = static_cast<int>(hash_size_ * load_factor_);

T *old_entries = entries_;
T* old_entries = entries_;
entries_ = new T[hash_size_];

T *old_overflow_entries = overflow_entries_;
T* old_overflow_entries = overflow_entries_;
overflow_entries_size_ = std::min(hash_size_ >> 2, max_cookie_size_);
overflow_entries_ = new T[overflow_entries_size_];

Expand Down
39 changes: 21 additions & 18 deletions keyvi/include/keyvi/dictionary/fsa/state_traverser.h
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#ifndef KEYVI_DICTIONARY_FSA_STATE_TRAVERSER_H_
#define KEYVI_DICTIONARY_FSA_STATE_TRAVERSER_H_

#include <cstdint>
#include <utility>

#include "keyvi/dictionary/fsa/automata.h"
Expand All @@ -47,44 +48,44 @@ class StateTraverser final {
using label_t = unsigned char;
using transition_t = TransitionT;

explicit StateTraverser(automata_t f)
: fsa_(f), current_state_(f->GetStartState()), current_weight_(0), current_label_(0), stack_() {
explicit StateTraverser(const automata_t& fsa)
: fsa_(fsa), current_state_(fsa->GetStartState()), current_weight_(0), current_label_(0), stack_() {
TRACE("StateTraverser starting with Start state %d", current_state_);
f->GetOutGoingTransitions(current_state_, &stack_.GetStates(), &stack_.traversal_stack_payload, 0);
fsa->GetOutGoingTransitions(current_state_, &stack_.GetStates(), &stack_.traversal_stack_payload, 0);

this->operator++(0);
}

StateTraverser(automata_t f, const uint64_t start_state, traversal::TraversalPayload<TransitionT> &&payload,
StateTraverser(const automata_t& fsa, const uint64_t start_state, traversal::TraversalPayload<TransitionT>&& payload,
const bool advance = true)
: fsa_(f), current_weight_(0), current_label_(0), stack_(std::move(payload)) {
: fsa_(fsa), current_weight_(0), current_label_(0), stack_(std::move(payload)) {
current_state_ = start_state;

TRACE("StateTraverser starting with Start state %d", current_state_);
f->GetOutGoingTransitions(start_state, &stack_.GetStates(), &stack_.traversal_stack_payload,
f->GetInnerWeight(start_state));
fsa->GetOutGoingTransitions(start_state, &stack_.GetStates(), &stack_.traversal_stack_payload,
fsa->GetInnerWeight(start_state));

if (advance) {
this->operator++(0);
}
}

StateTraverser(automata_t f, const uint64_t start_state, const bool advance = true)
: fsa_(f), current_state_(start_state), current_weight_(0), current_label_(0), stack_() {
StateTraverser(const automata_t& fsa, const uint64_t start_state, const bool advance = true)
: fsa_(fsa), current_state_(start_state), current_weight_(0), current_label_(0), stack_() {
TRACE("StateTraverser starting with Start state %d", current_state_);
f->GetOutGoingTransitions(start_state, &stack_.GetStates(), &stack_.traversal_stack_payload,
f->GetInnerWeight(start_state));
fsa->GetOutGoingTransitions(start_state, &stack_.GetStates(), &stack_.traversal_stack_payload,
fsa->GetInnerWeight(start_state));

if (advance) {
this->operator++(0);
}
}

StateTraverser() = delete;
StateTraverser &operator=(StateTraverser const &) = delete;
StateTraverser(const StateTraverser &that) = delete;
StateTraverser& operator=(StateTraverser const&) = delete;
StateTraverser(const StateTraverser& that) = delete;

StateTraverser(StateTraverser &&other)
StateTraverser(StateTraverser&& other) noexcept
: fsa_(other.fsa_),
current_state_(other.current_state_),
current_weight_(other.current_weight_),
Expand Down Expand Up @@ -182,13 +183,15 @@ class StateTraverser final {

template <class innerTraverserType>
friend class ComparableStateTraverser;
const traversal::TraversalStack<TransitionT> &GetStack() const { return stack_; }
[[nodiscard]] const traversal::TraversalStack<TransitionT>& GetStack() const { return stack_; }

traversal::TraversalState<transition_t> &GetStates() { return stack_.GetStates(); }
traversal::TraversalState<transition_t>& GetStates() { return stack_.GetStates(); }

traversal::TraversalPayload<TransitionT> &GetTraversalPayload() { return stack_.traversal_stack_payload; }
traversal::TraversalPayload<TransitionT>& GetTraversalPayload() { return stack_.traversal_stack_payload; }

const traversal::TraversalPayload<TransitionT> &GetTraversalPayload() const { return stack_.traversal_stack_payload; }
[[nodiscard]] const traversal::TraversalPayload<TransitionT>& GetTraversalPayload() const {
return stack_.traversal_stack_payload;
}
};

/**
Expand Down
Loading
Loading