diff --git a/Cargo.toml b/Cargo.toml index d40d127..1a535c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ authors = ["Tony Solomonik @ tony.solomonik@gmail.com"] description = "A search engine on object storage." repository = "https://github.com/tontinton/toshokan" homepage = "https://github.com/tontinton/toshokan" +readme = "./README.md" license = "Apache-2.0 OR MIT" [profile.release] diff --git a/README.md b/README.md index 980e86c..345d69a 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,21 @@ +## Introduction + +`toshokan` is a search engine (think Elasticsearch, Splunk), but storing the data on object storage, most similar to Quickwit. + +It uses: +* tantivy - for building and searching the inverted index data structure. +* Apache OpenDAL - for an abstraction over object storages. + +I've also posted a blog post explaining the benefits and drawbacks of using an object storage for data intensive applications. + +## Architecture + + + + + +## How to use + ```sh toshokan create example_config.yaml @@ -22,5 +40,8 @@ toshokan search test "tenant_id:[60 TO 65} AND severity_text:INFO" --limit 1 | j # "timestamp": "2016-04-13T06:46:54Z" # } +# Merge index files for faster searching. +toshokan merge test + toshokan drop test ``` diff --git a/architecture.svg b/architecture.svg new file mode 100644 index 0000000..3eb7a73 --- /dev/null +++ b/architecture.svg @@ -0,0 +1 @@ +Index ClusterPodJSONdocumentsObject Storage(Inverted index filesPostgres(Metadata)1. PUT splitWrite path2. UPDATERead path1. SELECT2. GET splitsSearchqueryPodPodSearch ClusterPodPodPod \ No newline at end of file