diff --git a/build-drivers.gradle b/build-drivers.gradle index d01aa60f..f57e221d 100644 --- a/build-drivers.gradle +++ b/build-drivers.gradle @@ -50,6 +50,7 @@ dependencies { api project(':drivers:linkedbq') api project(':drivers:atomicq') api project(':drivers:syncq') + api project(':drivers:elasticsearch') /* api project(':drivers:sbktemplate') */ /* above line is a signature */ } diff --git a/checkstyle/import-control.xml b/checkstyle/import-control.xml index ba0826bf..649eb976 100644 --- a/checkstyle/import-control.xml +++ b/checkstyle/import-control.xml @@ -76,4 +76,6 @@ + + \ No newline at end of file diff --git a/drivers/elasticsearch/README.md b/drivers/elasticsearch/README.md new file mode 100644 index 00000000..602905d8 --- /dev/null +++ b/drivers/elasticsearch/README.md @@ -0,0 +1,202 @@ + +# Elasticsearch Benchmarking with SBK +The SBK supports an analytically distributed Elasticsearch engine with multiple writers and readers. + +you can access the Elasticsearch console via http://localhost:9200 or else choose chrome extention like elastic-head for GUI interface. + +An Example SBK command for end to end benchmarking single single reader is as follows + +``` +./build/distributions/sbk-5.4/bin/sbk -class elasticsearch -size 10 -readers 1 -seconds 60 +``` +In the above example, the data size is 10 bytes. + +example output: + +``` +~/Desktop/SBK$ ./build/distributions/sbk-5.4/bin/sbk -class elasticsearch -size 10 -readers 1 -seconds 60 +2024-10-04 23:12:35 INFO + _____ ____ _ __ + / ____| | _ \ | | / / + | (___ | |_) | | |/ / + \___ \ | _ < | < + ____) | | |_) | | |\ \ + |_____/ |____/ |_| \_\ + +2024-10-04 23:12:35 INFO Storage Benchmark Kit +2024-10-04 23:12:35 INFO SBK Version: 5.4 +2024-10-04 23:12:35 INFO SBK Website: https://github.com/kmgowda/SBK +2024-10-04 23:12:35 INFO Arguments List: [-class, elasticsearch, -size, 10, -readers, 1, -seconds, 60] +2024-10-04 23:12:35 INFO Java Runtime Version: 17.0.12+7-Ubuntu-1ubuntu224.04 +2024-10-04 23:12:35 INFO SBP Version Major: 3, Minor: 0 +2024-10-04 23:12:35 INFO Storage Drivers Package: io.sbk.driver +2024-10-04 23:12:35 INFO Logger Package: io.sbk.logger +2024-10-04 23:12:35 INFO sbk.applicationName: sbk +2024-10-04 23:12:35 INFO sbk.appHome: /home/rak/Desktop/SBK/build/distributions/sbk-5.4 +2024-10-04 23:12:35 INFO sbk.className: +2024-10-04 23:12:35 INFO '-class': elasticsearch +2024-10-04 23:12:35 INFO '-out': +2024-10-04 23:12:35 INFO Storage Classes in package 'io.sbk.driver': 50 [Activemq, +Artemis, AsyncFile, Atomicq, BookKeeper, Cassandra, CephS3, ConcurrentQ, Conqueue, +Couchbase, CouchDB, CSV, Db2, Derby, Dynamodb, Elasticsearch, Exasol, FdbRecord, +File, FileStream, FoundationDB, H2, HDFS, Hive, Jdbc, Kafka, LevelDB, Linkedbq, MariaDB, +Memcached, MinIO, MongoDB, MsSql, MySQL, Nats, NatsStream, Nsq, Null, OpenIO, PostgreSQL, +Pravega, Pulsar, RabbitMQ, Redis, RedPanda, RocketMQ, RocksDB, SeaweedS3, SQLite, +Syncq] +2024-10-04 23:12:35 INFO Logger Classes in package 'io.sbk.logger': 5 [CSVLogger, +GrpcLogger, PrometheusLogger, Sl4jLogger, SystemLogger] +2024-10-04 23:12:35 WARN The option '-out' is not supplied; default logger: SystemLogger +2024-10-04 23:12:35 INFO Arguments to Driver 'Elasticsearch' : [-size, 10, -readers, 1, -seconds, 60] +2024-10-04 23:12:35 INFO Time Unit: MILLISECONDS +2024-10-04 23:12:35 INFO Minimum Latency: 0 ms +2024-10-04 23:12:35 INFO Maximum Latency: 180000 ms +2024-10-04 23:12:35 INFO Window Latency Store: Array, Size: 1 MB +2024-10-04 23:12:35 INFO Total Window Latency Store: HashMap, Size: 256 MB +2024-10-04 23:12:35 INFO Total Window Extension: None, Size: 0 MB +2024-10-04 23:12:35 INFO SBK Benchmark Started +2024-10-04 23:12:35 INFO Attempting to connect to Elasticsearch... +2024-10-04 23:12:36 INFO ElasticSearch Client Connected..... +2024-10-04 23:12:36 INFO CQueuePerl Start +2024-10-04 23:12:36 INFO PerformanceRecorderIdleBusyWait Started : 1000000 nanoseconds idle busy wait +2024-10-04 23:12:36 INFO SBK Benchmark initiated Readers +2024-10-04 23:12:36 INFO Reader 0 started , run seconds: 60 +Elasticsearch Reading 0 writers, 1 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 935 records, 187.0 records/sec, 0.00 MB/sec, 5.3 ms avg latency, 2 ms min latency, 112 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 1, SLC-2: 6; Latency Percentiles: 3 ms 5th, 3 ms 10th, 4 ms 20th, 4 ms 25th, 4 ms 30th, 4 ms 40th, 5 ms 50th, 5 ms 60th, 6 ms 70th, 6 ms 75th, 7 ms 80th, 7 ms 90th, 7 ms 92.5th, 8 ms 95th, 8 ms 97.5th, 11 ms 99th, 11 ms 99.25th, 11 ms 99.5th, 22 ms 99.75th, 112 ms 99.9th, 112 ms 99.95th, 112 ms 99.99th + +Elasticsearch Reading 0 writers, 1 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 1607 records, 321.3 records/sec, 0.00 MB/sec, 3.1 ms avg latency, 1 ms min latency, 10 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 1, SLC-2: 2; Latency Percentiles: 2 ms 5th, 2 ms 10th, 2 ms 20th, 2 ms 25th, 2 ms 30th, 3 ms 40th, 3 ms 50th, 3 ms 60th, 3 ms 70th, 4 ms 75th, 4 ms 80th, 5 ms 90th, 5 ms 92.5th, 6 ms 95th, 6 ms 97.5th, 7 ms 99th, 7 ms 99.25th, 7 ms 99.5th, 8 ms 99.75th, 10 ms 99.9th, 10 ms 99.95th, 10 ms 99.99th + +Elasticsearch Reading 0 writers, 1 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 2237 records, 447.3 records/sec, 0.00 MB/sec, 2.2 ms avg latency, 1 ms min latency, 36 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 1, SLC-2: 5; Latency Percentiles: 1 ms 5th, 1 ms 10th, 2 ms 20th, 2 ms 25th, 2 ms 30th, 2 ms 40th, 2 ms 50th, 2 ms 60th, 2 ms 70th, 2 ms 75th, 3 ms 80th, 3 ms 90th, 3 ms 92.5th, 3 ms 95th, 4 ms 97.5th, 5 ms 99th, 6 ms 99.25th, 8 ms 99.5th, 18 ms 99.75th, 24 ms 99.9th, 28 ms 99.95th, 36 ms 99.99th + +Elasticsearch Reading 0 writers, 1 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 2515 records, 502.4 records/sec, 0.00 MB/sec, 2.0 ms avg latency, 1 ms min latency, 99 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 1, SLC-2: 12; Latency Percentiles: 1 ms 5th, 1 ms 10th, 1 ms 20th, 1 ms 25th, 1 ms 30th, 2 ms 40th, 2 ms 50th, 2 ms 60th, 2 ms 70th, 2 ms 75th, 2 ms 80th, 3 ms 90th, 3 ms 92.5th, 3 ms 95th, 5 ms 97.5th, 7 ms 99th, 11 ms 99.25th, 15 ms 99.5th, 25 ms 99.75th, 45 ms 99.9th, 53 ms 99.95th, 99 ms 99.99th + +Elasticsearch Reading 0 writers, 1 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 2182 records, 436.2 records/sec, 0.00 MB/sec, 2.3 ms avg latency, 0 ms min latency, 149 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 1, SLC-2: 19; Latency Percentiles: 1 ms 5th, 1 ms 10th, 1 ms 20th, 1 ms 25th, 1 ms 30th, 1 ms 40th, 2 ms 50th, 2 ms 60th, 2 ms 70th, 2 ms 75th, 3 ms 80th, 3 ms 90th, 4 ms 92.5th, 5 ms 95th, 6 ms 97.5th, 17 ms 99th, 23 ms 99.25th, 31 ms 99.5th, 38 ms 99.75th, 91 ms 99.9th, 100 ms 99.95th, 149 ms 99.99th + +Elasticsearch Reading 0 writers, 1 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 5178 records, 1035.4 records/sec, 0.00 MB/sec, 1.0 ms avg latency, 0 ms min latency, 64 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 6; Latency Percentiles: 0 ms 5th, 0 ms 10th, 1 ms 20th, 1 ms 25th, 1 ms 30th, 1 ms 40th, 1 ms 50th, 1 ms 60th, 1 ms 70th, 1 ms 75th, 1 ms 80th, 1 ms 90th, 1 ms 92.5th, 2 ms 95th, 2 ms 97.5th, 3 ms 99th, 3 ms 99.25th, 3 ms 99.5th, 5 ms 99.75th, 15 ms 99.9th, 29 ms 99.95th, 64 ms 99.99th + +Elasticsearch Reading 0 writers, 1 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 8013 records, 1602.3 records/sec, 0.00 MB/sec, 0.6 ms avg latency, 0 ms min latency, 37 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 3; Latency Percentiles: 0 ms 5th, 0 ms 10th, 0 ms 20th, 0 ms 25th, 0 ms 30th, 0 ms 40th, 1 ms 50th, 1 ms 60th, 1 ms 70th, 1 ms 75th, 1 ms 80th, 1 ms 90th, 1 ms 92.5th, 1 ms 95th, 1 ms 97.5th, 2 ms 99th, 2 ms 99.25th, 3 ms 99.5th, 4 ms 99.75th, 10 ms 99.9th, 15 ms 99.95th, 37 ms 99.99th + +Elasticsearch Reading 0 writers, 1 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 6274 records, 1254.5 records/sec, 0.00 MB/sec, 0.8 ms avg latency, 0 ms min latency, 73 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 9; Latency Percentiles: 0 ms 5th, 0 ms 10th, 0 ms 20th, 0 ms 25th, 0 ms 30th, 0 ms 40th, 1 ms 50th, 1 ms 60th, 1 ms 70th, 1 ms 75th, 1 ms 80th, 1 ms 90th, 1 ms 92.5th, 2 ms 95th, 3 ms 97.5th, 5 ms 99th, 6 ms 99.25th, 8 ms 99.5th, 14 ms 99.75th, 34 ms 99.9th, 54 ms 99.95th, 73 ms 99.99th + +Elasticsearch Reading 0 writers, 1 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 6714 records, 1342.5 records/sec, 0.00 MB/sec, 0.7 ms avg latency, 0 ms min latency, 67 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 5; Latency Percentiles: 0 ms 5th, 0 ms 10th, 0 ms 20th, 0 ms 25th, 0 ms 30th, 1 ms 40th, 1 ms 50th, 1 ms 60th, 1 ms 70th, 1 ms 75th, 1 ms 80th, 1 ms 90th, 2 ms 92.5th, 2 ms 95th, 2 ms 97.5th, 3 ms 99th, 4 ms 99.25th, 4 ms 99.5th, 6 ms 99.75th, 10 ms 99.9th, 19 ms 99.95th, 67 ms 99.99th + +Elasticsearch Reading 0 writers, 1 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 9389 records, 1877.4 records/sec, 0.00 MB/sec, 0.5 ms avg latency, 0 ms min latency, 39 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 0; Latency Percentiles: 0 ms 5th, 0 ms 10th, 0 ms 20th, 0 ms 25th, 0 ms 30th, 0 ms 40th, 0 ms 50th, 1 ms 60th, 1 ms 70th, 1 ms 75th, 1 ms 80th, 1 ms 90th, 1 ms 92.5th, 1 ms 95th, 2 ms 97.5th, 2 ms 99th, 2 ms 99.25th, 2 ms 99.5th, 3 ms 99.75th, 3 ms 99.9th, 7 ms 99.95th, 39 ms 99.99th + +Elasticsearch Reading 0 writers, 1 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 13628 records, 2725.1 records/sec, 0.01 MB/sec, 0.4 ms avg latency, 0 ms min latency, 18 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 0; Latency Percentiles: 0 ms 5th, 0 ms 10th, 0 ms 20th, 0 ms 25th, 0 ms 30th, 0 ms 40th, 0 ms 50th, 0 ms 60th, 1 ms 70th, 1 ms 75th, 1 ms 80th, 1 ms 90th, 1 ms 92.5th, 1 ms 95th, 1 ms 97.5th, 1 ms 99th, 1 ms 99.25th, 1 ms 99.5th, 1 ms 99.75th, 2 ms 99.9th, 3 ms 99.95th, 13 ms 99.99th + +Elasticsearch Reading 0 writers, 1 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 4 seconds, 0.0 MB, 14695 records, 2949.0 records/sec, 0.01 MB/sec, 0.3 ms avg latency, 0 ms min latency, 17 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 0; Latency Percentiles: 0 ms 5th, 0 ms 10th, 0 ms 20th, 0 ms 25th, 0 ms 30th, 0 ms 40th, 0 ms 50th, 0 ms 60th, 1 ms 70th, 1 ms 75th, 1 ms 80th, 1 ms 90th, 1 ms 92.5th, 1 ms 95th, 1 ms 97.5th, 1 ms 99th, 1 ms 99.25th, 2 ms 99.5th, 2 ms 99.75th, 2 ms 99.9th, 2 ms 99.95th, 3 ms 99.99th + +2024-10-04 23:13:36 INFO Reader 0 exited +Total Elasticsearch Reading 0 writers, 0 readers, 0 max writers, 1 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 60 seconds, 0.1 MB, 73367 records, 1222.8 records/sec, 0.00 MB/sec, 0.8 ms avg latency, 0 ms min latency, 149 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 7; Latency Percentiles: 0 ms 5th, 0 ms 10th, 0 ms 20th, 0 ms 25th, 0 ms 30th, 0 ms 40th, 1 ms 50th, 1 ms 60th, 1 ms 70th, 1 ms 75th, 1 ms 80th, 2 ms 90th, 2 ms 92.5th, 2 ms 95th, 4 ms 97.5th, 5 ms 99th, 6 ms 99.25th, 7 ms 99.5th, 8 ms 99.75th, 16 ms 99.9th, 29 ms 99.95th, 64 ms 99.99th + +2024-10-04 23:13:36 INFO PerformanceRecorderIdleBusyWait Exited +2024-10-04 23:13:36 INFO CQueuePerl Shutdown +2024-10-04 23:13:38 INFO Elasticsearch shut down successfully. +2024-10-04 23:13:38 INFO Elasticsearch shut down successfully. +2024-10-04 23:13:39 INFO SBK Benchmark Shutdown + +``` + +To write the driver for elasticsearch following examples are referred: +1. https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/index.html. +2. https://www.elastic.co/guide/en/elasticsearch/client/java-api-client/current/getting-started-java.html. +3. https://www.baeldung.com/elasticsearch-java. + + +An Example SBK command for end to end benchmarking single single reader is as follows + +``` +./build/distributions/sbk-5.4/bin/sbk -class elasticsearch -size 10 -writers 1 -seconds 60 +``` +example output: + +``` +-D:~/Desktop/SBK$ ./build/distributions/sbk-5.4/bin/sbk -class elasticsearch -size 10 -writers 1 -seconds 60 +2024-10-04 23:32:20 INFO Reflections took 122 ms to scan 52 urls, producing 26 keys and 159 values +2024-10-04 23:32:20 INFO Reflections took 18 ms to scan 1 urls, producing 20 keys and 22 values +2024-10-04 23:32:20 INFO + _____ ____ _ __ + / ____| | _ \ | | / / + | (___ | |_) | | |/ / + \___ \ | _ < | < + ____) | | |_) | | |\ \ + |_____/ |____/ |_| \_\ + +2024-10-04 23:32:20 INFO Storage Benchmark Kit +2024-10-04 23:32:20 INFO SBK Version: 5.4 +2024-10-04 23:32:20 INFO SBK Website: https://github.com/kmgowda/SBK +2024-10-04 23:32:20 INFO Arguments List: [-class, elasticsearch, -size, 10, -writers, 1, -seconds, 60] +2024-10-04 23:32:20 INFO Java Runtime Version: 17.0.12+7-Ubuntu-1ubuntu224.04 +2024-10-04 23:32:20 INFO SBP Version Major: 3, Minor: 0 +2024-10-04 23:32:20 INFO Storage Drivers Package: io.sbk.driver +2024-10-04 23:32:20 INFO Logger Package: io.sbk.logger +2024-10-04 23:32:20 INFO sbk.applicationName: sbk +2024-10-04 23:32:20 INFO sbk.appHome: /home/rak/Desktop/SBK/build/distributions/sbk-5.4 +2024-10-04 23:32:20 INFO sbk.className: +2024-10-04 23:32:20 INFO '-class': elasticsearch +2024-10-04 23:32:20 INFO '-out': +2024-10-04 23:32:20 INFO Storage Classes in package 'io.sbk.driver': 50 [Activemq, +Artemis, AsyncFile, Atomicq, BookKeeper, Cassandra, CephS3, ConcurrentQ, Conqueue, +Couchbase, CouchDB, CSV, Db2, Derby, Dynamodb, Elasticsearch, Exasol, FdbRecord, +File, FileStream, FoundationDB, H2, HDFS, Hive, Jdbc, Kafka, LevelDB, Linkedbq, MariaDB, +Memcached, MinIO, MongoDB, MsSql, MySQL, Nats, NatsStream, Nsq, Null, OpenIO, PostgreSQL, +Pravega, Pulsar, RabbitMQ, Redis, RedPanda, RocketMQ, RocksDB, SeaweedS3, SQLite, +Syncq] +2024-10-04 23:32:20 INFO Logger Classes in package 'io.sbk.logger': 5 [CSVLogger, +GrpcLogger, PrometheusLogger, Sl4jLogger, SystemLogger] +2024-10-04 23:32:20 WARN The option '-out' is not supplied; default logger: SystemLogger +2024-10-04 23:32:20 INFO Arguments to Driver 'Elasticsearch' : [-size, 10, -writers, 1, -seconds, 60] +2024-10-04 23:32:21 INFO Time Unit: MILLISECONDS +2024-10-04 23:32:21 INFO Minimum Latency: 0 ms +2024-10-04 23:32:21 INFO Maximum Latency: 180000 ms +2024-10-04 23:32:21 INFO Window Latency Store: Array, Size: 1 MB +2024-10-04 23:32:21 INFO Total Window Latency Store: HashMap, Size: 256 MB +2024-10-04 23:32:21 INFO Total Window Extension: None, Size: 0 MB +2024-10-04 23:32:21 INFO SBK Benchmark Started +2024-10-04 23:32:21 INFO Attempting to connect to Elasticsearch... +2024-10-04 23:32:21 INFO ElasticSearch Client Connected..... +2024-10-04 23:32:22 INFO sbkjCreated Successfully +2024-10-04 23:32:22 INFO CQueuePerl Start +2024-10-04 23:32:22 INFO PerformanceRecorderIdleBusyWait Started : 1000000 nanoseconds idle busy wait +2024-10-04 23:32:22 INFO SBK Benchmark initiated Writers +2024-10-04 23:32:22 INFO Writer 0 started , run seconds: 60 +Elasticsearch Writing 1 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 351 records, 70.2 records/sec, 0.00 MB/sec, 14.2 ms avg latency, 8 ms min latency, 235 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 1, SLC-2: 6; Latency Percentiles: 9 ms 5th, 9 ms 10th, 10 ms 20th, 10 ms 25th, 11 ms 30th, 11 ms 40th, 12 ms 50th, 13 ms 60th, 14 ms 70th, 15 ms 75th, 17 ms 80th, 20 ms 90th, 20 ms 92.5th, 22 ms 95th, 25 ms 97.5th, 33 ms 99th, 42 ms 99.25th, 91 ms 99.5th, 235 ms 99.75th, 235 ms 99.9th, 235 ms 99.95th, 235 ms 99.99th + +Elasticsearch Writing 1 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 423 records, 84.6 records/sec, 0.00 MB/sec, 11.8 ms avg latency, 7 ms min latency, 79 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 3; Latency Percentiles: 8 ms 5th, 8 ms 10th, 9 ms 20th, 9 ms 25th, 9 ms 30th, 10 ms 40th, 10 ms 50th, 12 ms 60th, 13 ms 70th, 14 ms 75th, 14 ms 80th, 16 ms 90th, 17 ms 92.5th, 19 ms 95th, 22 ms 97.5th, 24 ms 99th, 30 ms 99.25th, 33 ms 99.5th, 36 ms 99.75th, 79 ms 99.9th, 79 ms 99.95th, 79 ms 99.99th + +Elasticsearch Writing 1 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 478 records, 95.3 records/sec, 0.00 MB/sec, 10.5 ms avg latency, 6 ms min latency, 54 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 1, SLC-2: 2; Latency Percentiles: 7 ms 5th, 8 ms 10th, 8 ms 20th, 8 ms 25th, 8 ms 30th, 9 ms 40th, 9 ms 50th, 10 ms 60th, 11 ms 70th, 11 ms 75th, 12 ms 80th, 15 ms 90th, 16 ms 92.5th, 18 ms 95th, 21 ms 97.5th, 29 ms 99th, 29 ms 99.25th, 32 ms 99.5th, 36 ms 99.75th, 54 ms 99.9th, 54 ms 99.95th, 54 ms 99.99th + +Elasticsearch Writing 1 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 279 records, 55.7 records/sec, 0.00 MB/sec, 17.9 ms avg latency, 9 ms min latency, 332 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 1, SLC-2: 10; Latency Percentiles: 11 ms 5th, 11 ms 10th, 12 ms 20th, 12 ms 25th, 12 ms 30th, 13 ms 40th, 14 ms 50th, 14 ms 60th, 15 ms 70th, 16 ms 75th, 16 ms 80th, 20 ms 90th, 22 ms 92.5th, 27 ms 95th, 42 ms 97.5th, 164 ms 99th, 164 ms 99.25th, 307 ms 99.5th, 332 ms 99.75th, 332 ms 99.9th, 332 ms 99.95th, 332 ms 99.99th + +Elasticsearch Writing 1 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 294 records, 58.8 records/sec, 0.00 MB/sec, 17.0 ms avg latency, 9 ms min latency, 208 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 6; Latency Percentiles: 10 ms 5th, 10 ms 10th, 11 ms 20th, 11 ms 25th, 12 ms 30th, 12 ms 40th, 13 ms 50th, 14 ms 60th, 15 ms 70th, 16 ms 75th, 18 ms 80th, 23 ms 90th, 26 ms 92.5th, 30 ms 95th, 64 ms 97.5th, 88 ms 99th, 88 ms 99.25th, 103 ms 99.5th, 208 ms 99.75th, 208 ms 99.9th, 208 ms 99.95th, 208 ms 99.99th + +Elasticsearch Writing 1 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 328 records, 65.6 records/sec, 0.00 MB/sec, 15.2 ms avg latency, 8 ms min latency, 83 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 2; Latency Percentiles: 12 ms 5th, 12 ms 10th, 12 ms 20th, 13 ms 25th, 13 ms 30th, 14 ms 40th, 14 ms 50th, 15 ms 60th, 15 ms 70th, 15 ms 75th, 16 ms 80th, 18 ms 90th, 19 ms 92.5th, 22 ms 95th, 29 ms 97.5th, 36 ms 99th, 38 ms 99.25th, 43 ms 99.5th, 83 ms 99.75th, 83 ms 99.9th, 83 ms 99.95th, 83 ms 99.99th + +Elasticsearch Writing 1 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 302 records, 60.3 records/sec, 0.00 MB/sec, 16.6 ms avg latency, 11 ms min latency, 60 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 2; Latency Percentiles: 13 ms 5th, 13 ms 10th, 14 ms 20th, 14 ms 25th, 14 ms 30th, 15 ms 40th, 16 ms 50th, 16 ms 60th, 17 ms 70th, 18 ms 75th, 18 ms 80th, 20 ms 90th, 22 ms 92.5th, 25 ms 95th, 29 ms 97.5th, 38 ms 99th, 39 ms 99.25th, 39 ms 99.5th, 60 ms 99.75th, 60 ms 99.9th, 60 ms 99.95th, 60 ms 99.99th + +Elasticsearch Writing 1 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 277 records, 55.2 records/sec, 0.00 MB/sec, 18.1 ms avg latency, 10 ms min latency, 95 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 1, SLC-2: 3; Latency Percentiles: 12 ms 5th, 13 ms 10th, 14 ms 20th, 15 ms 25th, 15 ms 30th, 16 ms 40th, 16 ms 50th, 17 ms 60th, 18 ms 70th, 18 ms 75th, 19 ms 80th, 24 ms 90th, 25 ms 92.5th, 29 ms 95th, 41 ms 97.5th, 83 ms 99th, 83 ms 99.25th, 87 ms 99.5th, 95 ms 99.75th, 95 ms 99.9th, 95 ms 99.95th, 95 ms 99.99th + +Elasticsearch Writing 1 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 320 records, 63.9 records/sec, 0.00 MB/sec, 15.6 ms avg latency, 10 ms min latency, 69 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 2; Latency Percentiles: 12 ms 5th, 12 ms 10th, 12 ms 20th, 13 ms 25th, 13 ms 30th, 14 ms 40th, 15 ms 50th, 15 ms 60th, 16 ms 70th, 17 ms 75th, 18 ms 80th, 20 ms 90th, 21 ms 92.5th, 23 ms 95th, 27 ms 97.5th, 31 ms 99th, 33 ms 99.25th, 41 ms 99.5th, 69 ms 99.75th, 69 ms 99.9th, 69 ms 99.95th, 69 ms 99.99th + +Elasticsearch Writing 1 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 288 records, 57.6 records/sec, 0.00 MB/sec, 17.3 ms avg latency, 11 ms min latency, 79 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 2; Latency Percentiles: 12 ms 5th, 12 ms 10th, 13 ms 20th, 14 ms 25th, 14 ms 30th, 15 ms 40th, 15 ms 50th, 16 ms 60th, 18 ms 70th, 19 ms 75th, 19 ms 80th, 22 ms 90th, 26 ms 92.5th, 29 ms 95th, 36 ms 97.5th, 56 ms 99th, 56 ms 99.25th, 71 ms 99.5th, 79 ms 99.75th, 79 ms 99.9th, 79 ms 99.95th, 79 ms 99.99th + +Elasticsearch Writing 1 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 5 seconds, 0.0 MB, 321 records, 64.1 records/sec, 0.00 MB/sec, 15.6 ms avg latency, 10 ms min latency, 58 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 1, SLC-2: 2; Latency Percentiles: 11 ms 5th, 12 ms 10th, 12 ms 20th, 13 ms 25th, 13 ms 30th, 14 ms 40th, 14 ms 50th, 15 ms 60th, 16 ms 70th, 16 ms 75th, 18 ms 80th, 20 ms 90th, 21 ms 92.5th, 22 ms 95th, 28 ms 97.5th, 47 ms 99th, 53 ms 99.25th, 54 ms 99.5th, 58 ms 99.75th, 58 ms 99.9th, 58 ms 99.95th, 58 ms 99.99th + +Elasticsearch Writing 1 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 4 seconds, 0.0 MB, 347 records, 70.3 records/sec, 0.00 MB/sec, 14.2 ms avg latency, 9 ms min latency, 49 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 0, SLC-2: 2; Latency Percentiles: 11 ms 5th, 12 ms 10th, 12 ms 20th, 12 ms 25th, 13 ms 30th, 13 ms 40th, 13 ms 50th, 14 ms 60th, 15 ms 70th, 15 ms 75th, 16 ms 80th, 18 ms 90th, 18 ms 92.5th, 19 ms 95th, 21 ms 97.5th, 29 ms 99th, 32 ms 99.25th, 48 ms 99.5th, 49 ms 99.75th, 49 ms 99.9th, 49 ms 99.95th, 49 ms 99.99th + +2024-10-04 23:33:22 INFO Writer 0 exited +^CTotal Elasticsearch Writing 0 writers, 0 readers, 1 max writers, 0 max readers, 0.0 write request MB, 0 write request records, 0.0 write request records/sec, 0.00 write request MB/sec, 0.0 read request MB, 0 read request records, 0.0 read request records/sec, 0.00 read request MB/sec, 0.00 write response pending MB, 0 write response pending records, 0.00 read response pending MB, 0 read response pending records, 0.00 write read request pending MB, 0 write read request pending records, 0 write timeout events, 0.00 write timeout events/sec, 0 read timeout events, 0.00 read timeout events/sec, 60 seconds, 0.0 MB, 4008 records, 66.8 records/sec, 0.00 MB/sec, 15.0 ms avg latency, 6 ms min latency, 332 ms max latency; 0 invalid latencies; Discarded Latencies: 0 lower, 0 higher; SLC-1: 1, SLC-2: 5; Latency Percentiles: 8 ms 5th, 9 ms 10th, 11 ms 20th, 12 ms 25th, 12 ms 30th, 13 ms 40th, 14 ms 50th, 14 ms 60th, 15 ms 70th, 16 ms 75th, 17 ms 80th, 20 ms 90th, 21 ms 92.5th, 23 ms 95th, 29 ms 97.5th, 43 ms 99th, 53 ms 99.25th, 69 ms 99.5th, 88 ms 99.75th, 164 ms 99.9th, 235 ms 99.95th, 332 ms 99.99th + +2024-10-04 23:33:22 INFO PerformanceRecorderIdleBusyWait Exited +2024-10-04 23:33:22 INFO CQueuePerl Shutdown +Elasticsearch shut down successfully. +2024-10-04 23:33:23 INFO Elasticsearch shut down successfully. +2024-10-04 23:33:24 INFO SBK Benchmark Shutdown + +``` diff --git a/drivers/elasticsearch/build.gradle b/drivers/elasticsearch/build.gradle new file mode 100644 index 00000000..5f48a610 --- /dev/null +++ b/drivers/elasticsearch/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'java' +} + +repositories { + mavenCentral() +} + +dependencies { + api project(":sbk-api") + + // Add your storage driver specific dependencies here + api 'co.elastic.clients:elasticsearch-java:8.15.0' +} diff --git a/drivers/elasticsearch/src/main/java/io/sbk/driver/Elasticsearch/Elasticsearch.java b/drivers/elasticsearch/src/main/java/io/sbk/driver/Elasticsearch/Elasticsearch.java new file mode 100644 index 00000000..17b074cd --- /dev/null +++ b/drivers/elasticsearch/src/main/java/io/sbk/driver/Elasticsearch/Elasticsearch.java @@ -0,0 +1,171 @@ +/** + * Copyright (c) KMG. All Rights Reserved.. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package io.sbk.driver.Elasticsearch; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch.indices.CreateIndexRequest; +import co.elastic.clients.elasticsearch.indices.CreateIndexResponse; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.ElasticsearchTransport; +import co.elastic.clients.transport.endpoints.BooleanResponse; +import co.elastic.clients.transport.rest_client.RestClientTransport; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.javaprop.JavaPropsFactory; +import io.sbk.api.DataReader; +import io.sbk.api.DataWriter; +import io.sbk.data.impl.SbkString; +import io.sbk.params.ParameterOptions; +import io.sbk.api.Storage; +import io.sbk.data.DataType; +import io.sbk.params.InputOptions; +import io.sbk.system.Printer; +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestClientBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * Class for Elasticsearch storage driver. + * + * Incase if your data type in other than byte[] (Byte Array) + * then change the datatype and getDataType. + */ +public class Elasticsearch implements Storage { + private final static String CONFIGFILE = "Elasticsearch.properties"; + private ElasticsearchConfig config; + private ElasticsearchClient elasticsearchClient; + + public static long generateStartKey(int id) { + return (long) id * (long) Integer.MAX_VALUE; + } + + @Override + public void addArgs(final InputOptions params) throws IllegalArgumentException { + final ObjectMapper mapper = new ObjectMapper(new JavaPropsFactory()) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + try { + config = mapper.readValue( + Objects.requireNonNull(Elasticsearch.class.getClassLoader().getResourceAsStream(CONFIGFILE)), + ElasticsearchConfig.class); + } catch (Exception ex) { + ex.printStackTrace(); + throw new IllegalArgumentException(ex); + } + + // change and uncomment the below code as per your driver specific parameters + // params.addOption("param", true, "Elasticsearch parameter, default param: " + config.param); + params.addOption("user", true, "ElasticSearch user : " + config.user); + params.addOption("password", true, "ElasticSearch Password " + config.password); + params.addOption("url", true, "ElasticSearch URL:" + config.url); + params.addOption("index", true, "ElasticSearch Index: " + config.index); + } + + @Override + public void parseArgs(final ParameterOptions params) throws IllegalArgumentException { + // change and uncommnet the below code as per your driver specific parameters + // config.param = params.getOptionValue("param", config.param); + config.user = params.getOptionValue("user", config.user); + config.password = params.getOptionValue("password", config.password); + config.url = params.getOptionValue("url", config.url); + config.index = params.getOptionValue("index", config.index); + } + + @Override + public void openStorage(final ParameterOptions params) throws IOException { + try { + elasticsearchClient = connect(); + Printer.log.info("ElasticSearch Client Connected....."); + String index1 = config.index.trim(); + if (!indexExists(index1)) { + createIndex(elasticsearchClient, index1); + } + } catch (ElasticsearchException e ) { + Printer.log.error(e.getMessage()); + throw new RuntimeException(e); + } + } + + private ElasticsearchClient connect() { + Printer.log.info("Attempting to connect to Elasticsearch..."); + try { + final BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(config.user, config.password)); + + RestClientBuilder builder = RestClient.builder(HttpHost.create(config.url)) + .setHttpClientConfigCallback(httpClientBuilder -> httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider)); + + RestClient restClient = builder.build(); + ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper()); + return new ElasticsearchClient(transport); + } catch (ElasticsearchException e) { + Printer.log.error("Error connecting to Elasticsearch: " + e.getMessage()); + throw new RuntimeException(e); + } + } + + private boolean indexExists(String indexName) { + try { + BooleanResponse response = elasticsearchClient. + indices().exists(e -> e.index(indexName)); + return response.value(); + } catch (IOException e) { + e.printStackTrace(); + return false; + } + } + + private void createIndex(ElasticsearchClient client, String indexName) { + try { + CreateIndexRequest createIndexRequest = CreateIndexRequest.of(c -> c.index(indexName)); + CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest); + + if (createIndexResponse.acknowledged()) { + Printer.log.info(indexName + "Created Successfully"); + } else { + Printer.log.info("Index creation was not acknowledged."); + } + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public void closeStorage(final ParameterOptions params) throws IOException { + try { + elasticsearchClient._transport().close(); + } catch (ElasticsearchException e) { + e.printStackTrace(); + Printer.log.error("Failed to close the connection"); + } + + } + + @Override + public DataWriter createWriter(final int id, final ParameterOptions params) { + return new ElasticsearchWriter(id, params, config, elasticsearchClient); + } + + @Override + public DataReader createReader(final int id, final ParameterOptions params) { + return new ElasticsearchReader(id, params, config, elasticsearchClient); + } + + @Override + public DataType getDataType() { + return new SbkString(); + } +} \ No newline at end of file diff --git a/drivers/elasticsearch/src/main/java/io/sbk/driver/Elasticsearch/ElasticsearchConfig.java b/drivers/elasticsearch/src/main/java/io/sbk/driver/Elasticsearch/ElasticsearchConfig.java new file mode 100644 index 00000000..9fd14b47 --- /dev/null +++ b/drivers/elasticsearch/src/main/java/io/sbk/driver/Elasticsearch/ElasticsearchConfig.java @@ -0,0 +1,22 @@ +/** + * Copyright (c) KMG. All Rights Reserved.. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ + +package io.sbk.driver.Elasticsearch; + +/** + * Class for Elasticsearch storage configuration. + */ +public class ElasticsearchConfig { + // Add Elasticsearch Storage driver configuration parameters + public String user; + public String password; + public String url; + public String index; +} \ No newline at end of file diff --git a/drivers/elasticsearch/src/main/java/io/sbk/driver/Elasticsearch/ElasticsearchReader.java b/drivers/elasticsearch/src/main/java/io/sbk/driver/Elasticsearch/ElasticsearchReader.java new file mode 100644 index 00000000..a1aad68f --- /dev/null +++ b/drivers/elasticsearch/src/main/java/io/sbk/driver/Elasticsearch/ElasticsearchReader.java @@ -0,0 +1,55 @@ +/** + * Copyright (c) KMG. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package io.sbk.driver.Elasticsearch; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch.core.GetRequest; +import co.elastic.clients.elasticsearch.core.GetResponse; +import io.sbk.params.ParameterOptions; +import io.sbk.api.Reader; +import io.sbk.system.Printer; + +import java.io.IOException; +import java.util.Map; + +/** + * Class for Elasticsearch Reader. + */ +public class ElasticsearchReader implements Reader { + private final ElasticsearchConfig config; + private final ElasticsearchClient client; + private long id; + + public ElasticsearchReader(int readerId, ParameterOptions params, ElasticsearchConfig config, ElasticsearchClient client) { + this.id = Elasticsearch.generateStartKey(readerId); + this.config = config; + this.client = client; + } + + @Override + public String read() throws IOException { + try { + GetRequest request = GetRequest.of(g -> g + .index(config.index.trim()) + .id(String.valueOf(id++)) + ); + GetResponse response = client.get(request, Map.class); + return response.fields().toString(); + } catch (ElasticsearchException e) { + Printer.log.error("Elastic Search: recordRead failed !"); + throw new IOException(e); + } + } + + @Override + public void close() throws IOException { + } +} \ No newline at end of file diff --git a/drivers/elasticsearch/src/main/java/io/sbk/driver/Elasticsearch/ElasticsearchWriter.java b/drivers/elasticsearch/src/main/java/io/sbk/driver/Elasticsearch/ElasticsearchWriter.java new file mode 100644 index 00000000..425af5e2 --- /dev/null +++ b/drivers/elasticsearch/src/main/java/io/sbk/driver/Elasticsearch/ElasticsearchWriter.java @@ -0,0 +1,74 @@ +/** + * Copyright (c) KMG. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + */ +package io.sbk.driver.Elasticsearch; + +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch.core.IndexRequest; +import io.sbk.params.ParameterOptions; +import io.sbk.api.Writer; +import io.sbk.system.Printer; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + + +/** + * Class for Elasticsearch Writer. + */ +public class ElasticsearchWriter implements Writer { + private final ElasticsearchConfig config; + private final ElasticsearchClient client; + private long id; + + public ElasticsearchWriter(int writerID, ParameterOptions params, ElasticsearchConfig config, ElasticsearchClient client) { + this.id = Elasticsearch.generateStartKey(writerID); + this.config = config; + this.client = client; + } + + @Override + public CompletableFuture writeAsync(String data) throws IOException { + try { + writeData(data); + } catch (ElasticsearchException ex ) { + Printer.log.error("Elastic Search: recordWrite failed !"); + throw new IOException(ex); + } + return null; + } + + @Override + public void sync() throws IOException { + } + + @Override + public void close() throws IOException { + } + + + private void writeData(String data) { + Map document = new HashMap<>(); + document.put("data", data); + try { + IndexRequest> request = IndexRequest.of(i -> i + .index(config.index.trim()) + .id(String.valueOf(id++)) + .document(document) + ); + client.index(request); + } catch (ElasticsearchException | IOException ex ) { + Printer.log.error("Elastic Search: recordWrite failed !"); + throw new RuntimeException(ex); + } + } +} \ No newline at end of file diff --git a/drivers/elasticsearch/src/main/resources/Elasticsearch.properties b/drivers/elasticsearch/src/main/resources/Elasticsearch.properties new file mode 100644 index 00000000..914a428c --- /dev/null +++ b/drivers/elasticsearch/src/main/resources/Elasticsearch.properties @@ -0,0 +1,13 @@ +#Copyright (c) KMG. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +# Elasticsearch storage driver default Properties/parameters +user= rakesh +password= jain@88 +url= http://localhost:9200 +index=sbkj diff --git a/settings-drivers.gradle b/settings-drivers.gradle index 8989152d..c8462910 100644 --- a/settings-drivers.gradle +++ b/settings-drivers.gradle @@ -60,5 +60,6 @@ include 'drivers:conqueue' include 'drivers:linkedbq' include 'drivers:atomicq' include 'drivers:syncq' +include 'drivers:elasticsearch' /* include 'drivers:sbktemplate' */ /* above line is a signature */