get_vectors.py

import sys

from sentence_transformers import SentenceTransformer
import numpy as np

filename = sys.argv[1]
number_of_overlays = int(sys.argv[2]) + 1 # +1 because we want to include the original sentence

def process_file(filename):
    model_path = "buddhist-nlp/bod-eng-similarity"
    model = SentenceTransformer(model_path)

    model.max_seq_length = 500
    file = open(filename,'r')

    sentences = [line.rstrip('\n').strip() for line in file]
    sentences_overlay = []

    for x in range(len(sentences)):
        val = number_of_overlays
        if (len(sentences) - x) < val:
            val = (len(sentences) - x) + 1
        for i in range(1,val):
            sentences_overlay.append(' '.join(sentences[x:x+i]))
    overlay_string = "\n".join(sentences_overlay)
    vectors = np.array(model.encode(sentences_overlay,show_progress_bar=False))
    print("LEN SENTENCES",len(sentences_overlay))
    print("LEN VECTORS",len(vectors))
    with open(sys.argv[1] + "_overlay", "w") as text_file:
        text_file.write(overlay_string)

    np.save(sys.argv[1] + "_vectors",vectors)

process_file(filename)