SentenceSearch/preprocessing.py at master · leonard-git-acc/SentenceSearch · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
import word2vec
import numpy as np

def doc_padding(doc, docSize, sentenceSize, vectorSize):
    """
    Padds/Truncates sentences of a document and words of its sentences.
    Parameters:
    -----------
    doc : 3D numpy array (sentences, words, vector_size)
        Takes in a array to be transformed.
        This array represents a document, containing sentences,
        which contain words embedded as vectors.
    docSize : integer
        Target size of the first dimension of 'doc'.
        Represents amount of sentences in a document.
    sentenceSize : integer
        PTarget size for the third dimension.
        Represents amount of words in a sentence.
    vectorSize : integer
        Size of the embedded word vectors.
    Returns:
    --------
    out : 3D numpy array
        Returns 'doc' in the correct size.
    """
    target = np.zeros((docSize, sentenceSize, vectorSize))
    for sent in range(min(len(doc), docSize)):
        sentence = sentence_padding(doc[sent], sentenceSize, vectorSize)
        for word in range(sentenceSize):
            target[sent][word] = sentence[word]

    return target


def sentence_padding(sentence, sentenceSize, vectorSize):
    """
    Padds/Truncates words of a sentence.
    Parameters:
    -----------
    sentence : 2D numpy array (words, vector_size)
        Takes in a array to be transformed.
        This array represents a sentence,
        with words embedded as vectors.
    sentenceSize : integer
        Target size of the first dim of the array.
        Represents amount of words in a sentence.
    vectorSize : integer
        Size of the embedded word vectors
    Returns:
    --------
    out : 2D numpy array
        Returns 'sentence' in the correct size.
    """
    target = np.zeros((sentenceSize, vectorSize))

    for i in range(min(len(sentence), sentenceSize)):
        target[i] = sentence[i]

    return target


def vectorize_sentences(sentences, vectors):
    """
    Vectorizes sentences with word2vec.
    Parameters:
    -----------
    sentences : list of strings
    vectors : WordVectors
    Returns:
    -------
    out : 2D numpy array
        Contains sentences with word embeddings as vectors
    """
    vec = []
    for sen in sentences:
        res = vectors.vectorize_string(sen)
        vec.append(res)
    vec = np.array(vec)

    return vec

def get_sentence_index(sentences, charIndex):
    """Finds the index of a sentence by a character index from the previous text"""
    count = 0
    for i, se in enumerate(sentences):
        count = count + len(se)
        if count > charIndex:
            return i
        elif i >= len(sentences) - 1:
            return i


def get_sample_shape(docSize, sentenceSize, vectorSize):
    """Calculates the length of one sample, containing question, possible answer and entire document"""
    return (docSize * sentenceSize + 2 * sentenceSize, vectorSize)


def shuffle_in_unison(a, b):
    """Shuffles two arrays in unison"""
    rng_state = np.random.get_state()
    np.random.shuffle(a)
    np.random.set_state(rng_state)
    np.random.shuffle(b)


def checksum(data):
    sum = 0
    for doc in data:
        docsum = 0
        for sample in doc:
            docsum = docsum + np.sum(sample)
        sum = sum + docsum

    return sum