Merge pull request #106 from topoteretes/fix/setting-alternative-vect…

…or-db

fix: allow alternative vector db engine to be used

.github/workflows/test_common.yml

@@ -1,4 +1,4 @@
-name: common | common
+name: test | environments
 
 on:
   pull_request:
@@ -83,16 +83,7 @@ jobs:
       - name: Run tests
         run: poetry run pytest tests/
 
-
-
-      # - name: Download NLTK Punkt Tokenizer Models
-      #   run: |
-      #     python -m nltk.downloader punkt
-      #     python -m nltk.downloader stopwords
-      #     python -m nltk.downloader wordnet
-      #     python -m nltk.downloader averaged_perceptron_tagger
-
-      - name: Run test script
+      - name: Run default basic pipeline
         env:
           OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
           LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}

.github/workflows/test_neo4j.yml

@@ -0,0 +1,63 @@
+name: test neo4j
+
+on:
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUNTIME__LOG_LEVEL: ERROR
+
+jobs:
+  get_docs_changes:
+    name: docs changes
+    uses: ./.github/workflows/get_docs_changes.yml
+
+  run_common:
+    name: test
+    needs: get_docs_changes
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
+    runs-on: macos-latest
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1.3.2
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Install dependencies
+        run: poetry install --no-interaction
+
+      - name: Create .cognee_system directory and print path
+        run: |
+          mkdir .cognee_system
+          echo $(pwd)/.cognee_system
+
+      - name: Run default Neo4j
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          GRAPH_DATABASE_URL: ${{ secrets.NEO4J_API_URL }}
+          GRAPH_DATABASE_PASSWORD: ${{ secrets.NEO4J_API_KEY }}
+          GRAPH_DATABASE_USERNAME: "neo4j"
+          ENV: 'dev'
+        run: poetry run python ./cognee/tests/test_neo4j.py

.github/workflows/test_qdrant.yml

@@ -0,0 +1,62 @@
+name: test | qdrant
+
+on:
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUNTIME__LOG_LEVEL: ERROR
+
+jobs:
+  get_docs_changes:
+    name: docs changes
+    uses: ./.github/workflows/get_docs_changes.yml
+
+  run_qdrant_integration_test:
+    name: test
+    needs: get_docs_changes
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
+    runs-on: macos-latest
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1.3.2
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Install dependencies
+        run: poetry install --no-interaction
+
+      - name: Create .cognee_system directory and print path
+        run: |
+          mkdir .cognee_system
+          echo $(pwd)/.cognee_system
+
+      - name: Run default Qdrant
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          VECTOR_DB_URL: ${{ secrets.QDRANT_API_URL }}
+          VECTOR_DB_KEY: ${{ secrets.QDRANT_API_KEY }}
+          ENV: 'dev'
+        run: poetry run python ./cognee/tests/test_qdrant.py

.github/workflows/test_weaviate.yml

@@ -0,0 +1,62 @@
+name: test | weaviate
+
+on:
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  RUNTIME__LOG_LEVEL: ERROR
+
+jobs:
+  get_docs_changes:
+    name: docs changes
+    uses: ./.github/workflows/get_docs_changes.yml
+
+  run_weaviate_integration_test:
+    name: test
+    needs: get_docs_changes
+    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
+    runs-on: macos-latest
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - name: Check out
+        uses: actions/checkout@v2
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.11
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1.3.2
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Install dependencies
+        run: poetry install --no-interaction
+
+      - name: Create .cognee_system directory and print path
+        run: |
+          mkdir .cognee_system
+          echo $(pwd)/.cognee_system
+
+      - name: Run default Weaviate
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          LLM_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          VECTOR_DB_URL: ${{ secrets.WEAVIATE_API_URL }}
+          VECTOR_DB_KEY: ${{ secrets.WEAVIATE_API_KEY }}
+          ENV: 'dev'
+        run: poetry run python ./cognee/tests/test_weaviate.py

.gitignore

@@ -10,6 +10,10 @@ __pycache__/
 *.py[cod]
 *$py.class
 
+notebooks/
+full_run.ipynb
+evals/
+
 # C extensions
 *.so
 

README.md

@@ -74,6 +74,12 @@ or
 import cognee
 cognee.config.llm_api_key = "YOUR_OPENAI_API_KEY"
 ```
+If you are using Networkx, create an account on Graphistry to vizualize results:
+```
+   
+   cognee.config.set_graphistry_username = "YOUR_USERNAME"
+   cognee.config.set_graphistry_password = "YOUR_PASSWORD"
+```
 
 To run the UI, run:
 ```
@@ -95,10 +101,9 @@ cognee.add([text], "example_dataset") # Add a new piece of information
 
 cognee.cognify() # Use LLMs and cognee to create knowledge
 
-search_results = cognee.search("SIMILARITY", "computer science") # Query cognee for the knowledge
+search_results = cognee.search("SIMILARITY", {'query': 'Tell me about NLP'}) # Query cognee for the knowledge
 
-for result_text in search_results[0]:
-    print(result_text)
+print(search_results)
 
 ```
 Add alternative data types:

cognee-frontend/src/app/globals.css

@@ -6,6 +6,7 @@
     "Fira Mono", "Droid Sans Mono", "Courier New", monospace;
 
   --global-background-default: #0D051C;
+  --textarea-default-color: #0D051C !important;
 }
 
 * {

cognee-frontend/src/app/page.tsx

@@ -112,7 +112,7 @@ export default function Home() {
             expireIn={notification.expireIn}
             onClose={notification.delete}
           >
-            <Text>{notification.message}</Text>
+            <Text nowrap>{notification.message}</Text>
           </Notification>
         ))}
       </NotificationContainer>

cognee-frontend/src/app/wizard/CognifyStep/CognifyStep.tsx

@@ -1,8 +1,7 @@
-import { useCallback, useEffect } from 'react';
-import { CTAButton, Spacer, Stack, Text, useBoolean } from 'ohmy-ui';
+import { useEffect, useRef } from 'react';
+import { CTAButton, Stack, Text, useBoolean } from 'ohmy-ui';
 import { Divider } from '@/ui/Layout';
-import { CognifyLoadingIndicator, LoadingIndicator } from '@/ui/App';
-import { getExplorationGraphUrl } from '@/modules/exploration';
+import { CognifyLoadingIndicator } from '@/ui/App';
 import { WizardHeading } from '@/ui/Partials/Wizard';
 import cognifyDataset from '@/modules/datasets/cognifyDataset';
 
@@ -16,9 +15,14 @@ export default function CognifyStep({ onNext, dataset }: ConfigStepProps) {
     value: isCognifyRunning,
     setFalse: stopCognifyIndicator,
   } = useBoolean(true);
+  const cognifyPromise = useRef<Promise<void>>()
 
   useEffect(() => {
-    cognifyDataset(dataset)
+    if (cognifyPromise.current) {
+      return;
+    }
+
+    cognifyPromise.current = cognifyDataset(dataset)
       .then(() => {
         stopCognifyIndicator();        
       });

cognee-frontend/src/modules/exploration/getExplorationGraphUrl.ts

@@ -1,5 +1,11 @@
 export default function getExplorationGraphUrl(dataset: { id: string }) {
   return fetch(`http://0.0.0.0:8000/datasets/${dataset.id}/graph`)
+      .then(async (response) => {
+        if (response.status !== 200) {
+          throw new Error((await response.text()).replaceAll("\"", ""));
+        }
+        return response;
+      })
       .then((response) => response.text())
       .then((text) => text.replace('"', ''));
 }

cognee-frontend/src/ui/Partials/Explorer/Explorer.tsx

@@ -1,6 +1,6 @@
 import { useCallback, useEffect, useState } from 'react';
 import classNames from 'classnames';
-import { Spacer, Stack } from 'ohmy-ui';
+import { Spacer, Stack, Text } from 'ohmy-ui';
 import { getExplorationGraphUrl } from '@/modules/exploration';
 import { IFrameView, SearchView } from '@/ui/Partials';
 import { LoadingIndicator } from '@/ui/App';
@@ -13,12 +13,17 @@ interface ExplorerProps {
 }
 
 export default function Explorer({ dataset, className, style }: ExplorerProps) {
+  const [error, setError] = useState<Error | null>(null);
   const [graphUrl, setGraphUrl] = useState<string | null>(null);
 
   const exploreData = useCallback(() => {
     getExplorationGraphUrl(dataset)
       .then((graphUrl) => {
+        setError(null);
         setGraphUrl(graphUrl);
+      })
+      .catch((error) => {
+        setError(error);
       });
   }, [dataset]);
 
@@ -34,12 +39,18 @@ export default function Explorer({ dataset, className, style }: ExplorerProps) {
       className={classNames(styles.explorerContent, className)}
     >
       <div className={styles.graphExplorer}>
-        {!graphUrl ? (
-          <Spacer horizontal="2" wrap>
-            <LoadingIndicator />
-          </Spacer>
+        {error ? (
+          <Text color="red">{error.message}</Text>
         ) : (
-          <IFrameView src={graphUrl} />
+          <>
+            {!graphUrl ? (
+              <Spacer horizontal="2" wrap>
+                <LoadingIndicator />
+              </Spacer>
+            ) : (
+              <IFrameView src={graphUrl} />
+            )}
+          </>
         )}
       </div>
       <div className={styles.chat}>