diff --git a/Makefile b/Makefile index 9154927123..9d067c58ec 100644 --- a/Makefile +++ b/Makefile @@ -227,6 +227,7 @@ docker-integration-test: @./scripts/docker-integration-tests/prometheus/test.sh @./scripts/docker-integration-tests/carbon/test.sh @./scripts/docker-integration-tests/aggregator/test.sh + @./scripts/docker-integration-tests/query_fanout/test.sh .PHONY: site-build site-build: diff --git a/scripts/docker-integration-tests/query_fanout/test.sh b/scripts/docker-integration-tests/query_fanout/test.sh index 7c33ed0812..f607b3b893 100755 --- a/scripts/docker-integration-tests/query_fanout/test.sh +++ b/scripts/docker-integration-tests/query_fanout/test.sh @@ -74,7 +74,6 @@ curl -vvvsS -X POST 0.0.0.0:19003/writetagged -d '{ } }' - function read { RESPONSE=$(curl "http://0.0.0.0:7201/api/v1/query?query=test_metric") ACTUAL=$(echo $RESPONSE | jq .data.result[].metric.cluster) @@ -90,3 +89,67 @@ function read_sum { } ATTEMPTS=5 TIMEOUT=1 retry_with_backoff read_sum + +echo "Write local tagged data to cluster a" +curl -vvvsS -X POST 0.0.0.0:19003/writetagged -d '{ + "namespace": "unagg", + "id": "{__name__=\"test_metric\",cluster=\"cluster-b\",endpoint=\"/request\",local-only=\"local\"}", + "tags": [ + { + "name": "__name__", + "value": "test_metric" + }, + { + "name": "cluster", + "value": "cluster-b" + }, + { + "name": "endpoint", + "value": "/request" + }, + { + "name": "local-only", + "value": "local" + } + ], + "datapoint": { + "timestamp":'"$(date +"%s")"', + "value": 42.123456789 + } +}' + +echo "Write remote tagged data to cluster b" +curl -vvvsS -X POST 0.0.0.0:19003/writetagged -d '{ + "namespace": "unagg", + "id": "{__name__=\"test_metric\",cluster=\"cluster-b\",endpoint=\"/request\",remote-only=\"remote\"}", + "tags": [ + { + "name": "__name__", + "value": "test_metric" + }, + { + "name": "cluster", + "value": "cluster-b" + }, + { + "name": "endpoint", + "value": "/request" + }, + { + "name": "remote-only", + "value": "remote" + } + ], + "datapoint": { + "timestamp":'"$(date +"%s")"', + "value": 42.123456789 + } +}' + +function complete_tags { + RESPONSE=$(curl "http://0.0.0.0:7201/api/v1/labels") + ACTUAL=$(echo $RESPONSE | jq .data[]) + test "$(echo $ACTUAL)" = '"__name__" "cluster" "endpoint" "local-only" "remote-only"' +} + +ATTEMPTS=5 TIMEOUT=1 retry_with_backoff complete_tags diff --git a/src/query/generated/proto/rpcpb/query.pb.go b/src/query/generated/proto/rpcpb/query.pb.go index cfffca612a..9982a2a198 100644 --- a/src/query/generated/proto/rpcpb/query.pb.go +++ b/src/query/generated/proto/rpcpb/query.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. // source: github.com/m3db/m3/src/query/generated/proto/rpcpb/query.proto -// Copyright (c) 2018 Uber Technologies, Inc. +// Copyright (c) 2019 Uber Technologies, Inc. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -945,6 +945,8 @@ func _SearchResponse_OneofSizer(msg proto.Message) (n int) { type CompleteTagsRequestOptions struct { Type CompleteTagsType `protobuf:"varint,1,opt,name=type,proto3,enum=rpc.CompleteTagsType" json:"type,omitempty"` FilterNameTags [][]byte `protobuf:"bytes,2,rep,name=filterNameTags" json:"filterNameTags,omitempty"` + Start int64 `protobuf:"varint,3,opt,name=start,proto3" json:"start,omitempty"` + End int64 `protobuf:"varint,4,opt,name=end,proto3" json:"end,omitempty"` } func (m *CompleteTagsRequestOptions) Reset() { *m = CompleteTagsRequestOptions{} } @@ -966,6 +968,20 @@ func (m *CompleteTagsRequestOptions) GetFilterNameTags() [][]byte { return nil } +func (m *CompleteTagsRequestOptions) GetStart() int64 { + if m != nil { + return m.Start + } + return 0 +} + +func (m *CompleteTagsRequestOptions) GetEnd() int64 { + if m != nil { + return m.End + } + return 0 +} + type CompleteTagsRequest struct { // Types that are valid to be assigned to Matchers: // *CompleteTagsRequest_TagMatchers @@ -2222,6 +2238,16 @@ func (m *CompleteTagsRequestOptions) MarshalTo(dAtA []byte) (int, error) { i += copy(dAtA[i:], b) } } + if m.Start != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintQuery(dAtA, i, uint64(m.Start)) + } + if m.End != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintQuery(dAtA, i, uint64(m.End)) + } return i, nil } @@ -2753,6 +2779,12 @@ func (m *CompleteTagsRequestOptions) Size() (n int) { n += 1 + l + sovQuery(uint64(l)) } } + if m.Start != 0 { + n += 1 + sovQuery(uint64(m.Start)) + } + if m.End != 0 { + n += 1 + sovQuery(uint64(m.End)) + } return n } @@ -4958,6 +4990,44 @@ func (m *CompleteTagsRequestOptions) Unmarshal(dAtA []byte) error { m.FilterNameTags = append(m.FilterNameTags, make([]byte, postIndex-iNdEx)) copy(m.FilterNameTags[len(m.FilterNameTags)-1], dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Start", wireType) + } + m.Start = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Start |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field End", wireType) + } + m.End = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowQuery + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.End |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipQuery(dAtA[iNdEx:]) @@ -5588,73 +5658,73 @@ func init() { } var fileDescriptorQuery = []byte{ - // 1075 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0x6d, 0x6f, 0x1b, 0x45, - 0x10, 0xf6, 0xf9, 0x6c, 0xc7, 0x1e, 0xbf, 0xd4, 0x4c, 0x02, 0x75, 0xa3, 0x28, 0x44, 0x07, 0x94, - 0x10, 0x20, 0x0e, 0x76, 0x04, 0x14, 0x89, 0x97, 0x84, 0xba, 0x09, 0x52, 0x93, 0x34, 0xeb, 0x2b, - 0x8a, 0x50, 0x3f, 0xb0, 0xbe, 0xdb, 0x3a, 0xa7, 0xfa, 0x7c, 0xd7, 0xbb, 0x35, 0xaa, 0xfb, 0x17, - 0xf8, 0x82, 0xf8, 0x2d, 0xf0, 0x1f, 0xf8, 0xc8, 0x4f, 0x40, 0xe1, 0x8f, 0xa0, 0xdd, 0xdb, 0x7b, - 0xb3, 0x5d, 0xde, 0xbe, 0xdd, 0xcc, 0x3c, 0x33, 0x3b, 0x33, 0xfb, 0xf8, 0xf1, 0xc2, 0x17, 0x63, - 0x87, 0x5f, 0xcf, 0x46, 0xfb, 0x96, 0xe7, 0x76, 0xdd, 0xbe, 0x3d, 0xea, 0xba, 0xfd, 0x6e, 0x18, - 0x58, 0xdd, 0xe7, 0x33, 0x16, 0xcc, 0xbb, 0x63, 0x36, 0x65, 0x01, 0xe5, 0xcc, 0xee, 0xfa, 0x81, - 0xc7, 0xbd, 0x6e, 0xe0, 0x5b, 0xfe, 0x28, 0x8a, 0xed, 0x4b, 0x0f, 0xea, 0x81, 0x6f, 0x19, 0x2f, - 0xa0, 0xf1, 0x80, 0x71, 0xeb, 0x9a, 0xb0, 0xe7, 0x33, 0x16, 0x72, 0xdc, 0x80, 0x72, 0xc8, 0x69, - 0xc0, 0x3b, 0xda, 0x8e, 0xb6, 0xab, 0x93, 0xc8, 0xc0, 0x36, 0xe8, 0x6c, 0x6a, 0x77, 0x8a, 0xd2, - 0x27, 0x3e, 0xf1, 0x10, 0xea, 0x9c, 0x8e, 0xcf, 0x28, 0xb7, 0xae, 0x59, 0x10, 0x76, 0xf4, 0x1d, - 0x6d, 0xb7, 0xde, 0x6b, 0xef, 0x07, 0xbe, 0xb5, 0x6f, 0xa6, 0xfe, 0xd3, 0x02, 0xc9, 0xc2, 0x8e, - 0x01, 0xaa, 0xae, 0xfa, 0x36, 0xbe, 0x82, 0x7a, 0x06, 0x89, 0x1f, 0xe5, 0x0b, 0x6a, 0x3b, 0xfa, - 0x6e, 0xbd, 0x77, 0x6b, 0xa1, 0x60, 0xae, 0x9a, 0xf1, 0x04, 0x20, 0x0d, 0x21, 0x42, 0x69, 0x4a, - 0x5d, 0x26, 0x1b, 0x6f, 0x10, 0xf9, 0x2d, 0xa6, 0xf9, 0x81, 0x4e, 0x66, 0x4c, 0x76, 0xde, 0x20, - 0x91, 0x81, 0x6f, 0x43, 0x89, 0xcf, 0x7d, 0x26, 0x9b, 0x6e, 0xa9, 0xa6, 0x55, 0x15, 0x73, 0xee, - 0x33, 0x22, 0xa3, 0xc6, 0x21, 0x34, 0xd5, 0x66, 0x42, 0xdf, 0x9b, 0x86, 0x0c, 0xdf, 0x82, 0x4a, - 0xc8, 0x02, 0x87, 0xc5, 0xcd, 0xd5, 0x65, 0xe2, 0x50, 0xba, 0x88, 0x0a, 0x19, 0xbf, 0x68, 0x50, - 0x89, 0x5c, 0xf8, 0x2e, 0x94, 0x5c, 0xc6, 0xa9, 0x6c, 0xa8, 0xde, 0x5b, 0xcf, 0xa0, 0xcf, 0x18, - 0xa7, 0x36, 0xe5, 0x94, 0x48, 0x00, 0x7e, 0x0e, 0x0d, 0x9b, 0x59, 0x9e, 0xeb, 0x07, 0x2c, 0x0c, - 0x59, 0xb4, 0xe6, 0x7a, 0xef, 0xb6, 0x4c, 0xb8, 0x9f, 0x09, 0x44, 0xc9, 0xa7, 0x05, 0x92, 0x83, - 0xe3, 0x3d, 0x80, 0x4c, 0xb2, 0x9e, 0x49, 0x3e, 0xeb, 0x7f, 0xbd, 0x9c, 0x9c, 0x01, 0x1f, 0xaf, - 0xa9, 0xfd, 0x18, 0x57, 0xd0, 0xca, 0xb7, 0x86, 0x2d, 0x28, 0x3a, 0xb6, 0x5a, 0x66, 0xd1, 0xb1, - 0x71, 0x0b, 0x6a, 0x92, 0x0b, 0xa6, 0xe3, 0x32, 0x45, 0x84, 0xd4, 0x81, 0x1d, 0x58, 0x63, 0x53, - 0x5b, 0xc6, 0x74, 0x19, 0x8b, 0x4d, 0x63, 0x04, 0xb8, 0x3c, 0x03, 0xee, 0x03, 0x88, 0x53, 0x7c, - 0xcf, 0x99, 0xf2, 0x78, 0x9f, 0xad, 0x68, 0xe0, 0xd8, 0x4d, 0x32, 0x08, 0xdc, 0x82, 0x12, 0xa7, - 0xe3, 0xb0, 0x53, 0x94, 0xc8, 0x6a, 0x4c, 0x0b, 0x22, 0xbd, 0xc6, 0x97, 0x50, 0x4b, 0xd2, 0x44, - 0xa3, 0xdc, 0x71, 0x59, 0xc8, 0xa9, 0xeb, 0x2b, 0x16, 0xa7, 0x8e, 0x3c, 0x23, 0x34, 0xc5, 0x08, - 0xa3, 0x0b, 0xba, 0x49, 0xc7, 0xff, 0x9e, 0x42, 0xc6, 0x0b, 0xc0, 0xe5, 0xe5, 0xe2, 0x5d, 0x68, - 0xa5, 0x93, 0x9a, 0xa2, 0xdf, 0xa8, 0xd2, 0x82, 0x17, 0x3f, 0x83, 0x6a, 0xc0, 0xfc, 0x89, 0x63, - 0xd1, 0x78, 0xa2, 0xed, 0xa5, 0xfb, 0xfa, 0x56, 0x9c, 0x13, 0x92, 0x08, 0x46, 0x12, 0xbc, 0x71, - 0x0a, 0x77, 0x5e, 0x09, 0xc3, 0xf7, 0xa1, 0x1a, 0xb2, 0xb1, 0xcb, 0xd2, 0xa5, 0xde, 0x52, 0x85, - 0x87, 0xca, 0x4d, 0x12, 0x80, 0xf1, 0x3d, 0x40, 0xea, 0xc7, 0xbb, 0x50, 0x71, 0x59, 0x30, 0x66, - 0xb6, 0xe2, 0x6b, 0x2b, 0x9f, 0x48, 0x54, 0x14, 0xf7, 0xa0, 0x3a, 0x9b, 0x2a, 0x64, 0x31, 0x73, - 0x6f, 0x29, 0x32, 0x89, 0x1b, 0x1e, 0xd4, 0x12, 0xb7, 0x58, 0xee, 0x35, 0xa3, 0x31, 0xa5, 0xe4, - 0xb7, 0xf0, 0x71, 0xea, 0x4c, 0xd4, 0x6e, 0xe5, 0x77, 0x9e, 0x68, 0xfa, 0x22, 0xd1, 0xb6, 0xa0, - 0x36, 0x9a, 0x78, 0xd6, 0xb3, 0xa1, 0xf3, 0x92, 0x75, 0x4a, 0x51, 0x34, 0x71, 0x18, 0x97, 0xd0, - 0x1c, 0x32, 0x1a, 0xa4, 0x72, 0x76, 0xb8, 0xa8, 0x2a, 0xff, 0x59, 0xa6, 0x4e, 0xa0, 0x79, 0xd6, - 0x37, 0xe9, 0xf8, 0x51, 0xe0, 0xf9, 0x2c, 0xe0, 0xf3, 0xa5, 0x1f, 0xc6, 0xf2, 0xa5, 0x17, 0x57, - 0x5d, 0xba, 0x31, 0x80, 0x5b, 0xd9, 0x42, 0x82, 0x2f, 0x3d, 0x00, 0x3f, 0xb1, 0xd4, 0x85, 0xa1, - 0xda, 0x66, 0xe6, 0x48, 0x92, 0x41, 0x19, 0x9f, 0x48, 0xd9, 0x4c, 0xba, 0x69, 0x83, 0xfe, 0x8c, - 0xcd, 0x55, 0x3b, 0xe2, 0x13, 0xdf, 0x80, 0x8a, 0xe4, 0x68, 0xdc, 0x87, 0xb2, 0x8c, 0x23, 0x68, - 0xe6, 0x4f, 0x3f, 0x58, 0x71, 0x7a, 0xb2, 0x9a, 0x95, 0x67, 0xff, 0xa8, 0x09, 0x99, 0x88, 0xf6, - 0xab, 0x44, 0xf1, 0xd3, 0x05, 0xed, 0x8a, 0x36, 0x8c, 0x0b, 0x65, 0x56, 0xc9, 0xd6, 0xc7, 0x39, - 0xd9, 0x8a, 0x34, 0x6f, 0x63, 0x69, 0xf8, 0xbf, 0xd1, 0x2c, 0x0f, 0x36, 0xc5, 0xef, 0x60, 0xc2, - 0x38, 0x13, 0x0b, 0x56, 0x57, 0x7e, 0xe1, 0x73, 0xc7, 0x9b, 0x86, 0xf8, 0x9e, 0x12, 0x79, 0x4d, - 0x8a, 0xfc, 0xeb, 0xb2, 0x70, 0x16, 0x9e, 0x2a, 0xbd, 0xb8, 0xc1, 0xa7, 0xce, 0x84, 0xb3, 0xe0, - 0x9c, 0xba, 0xcc, 0x8c, 0x65, 0xa6, 0x41, 0x16, 0xbc, 0xc6, 0xcf, 0x1a, 0xac, 0xaf, 0x38, 0xf1, - 0xff, 0x91, 0x0c, 0xef, 0xc1, 0x9a, 0x17, 0xf5, 0xaa, 0x86, 0x7f, 0x73, 0xa9, 0xc7, 0xfc, 0x48, - 0x24, 0xc6, 0xe7, 0xf8, 0xb9, 0x03, 0x55, 0x93, 0x8e, 0x45, 0x8f, 0xa1, 0xd0, 0x2a, 0xa1, 0x59, - 0xd1, 0x65, 0x36, 0x48, 0x64, 0x18, 0x87, 0x12, 0x21, 0x85, 0xe2, 0x1f, 0xe8, 0xa2, 0x67, 0xe8, - 0xd2, 0x83, 0x5a, 0x9c, 0x15, 0xe2, 0x3b, 0x09, 0x28, 0xa2, 0x49, 0x33, 0x1e, 0x4e, 0xc6, 0x93, - 0x9c, 0x97, 0xb0, 0x91, 0x6f, 0x5f, 0x91, 0x64, 0x0f, 0xd6, 0x6c, 0xf6, 0x94, 0xce, 0x26, 0x3c, - 0x27, 0x2e, 0x49, 0xfd, 0xd3, 0x02, 0x89, 0x01, 0xf8, 0x21, 0xd4, 0x64, 0xdb, 0x17, 0xd3, 0xc9, - 0x5c, 0x2d, 0x26, 0x39, 0x4d, 0x4e, 0x79, 0x5a, 0x20, 0x29, 0x22, 0x61, 0xc3, 0xde, 0x13, 0xa8, - 0x67, 0xfe, 0xc3, 0xb1, 0x06, 0xe5, 0xc1, 0xe5, 0xe3, 0xa3, 0x87, 0xed, 0x02, 0x36, 0xa0, 0x7a, - 0x7e, 0x61, 0x46, 0x96, 0x86, 0x00, 0x15, 0x32, 0x38, 0x19, 0x5c, 0x3d, 0x6a, 0x17, 0xb1, 0x09, - 0xb5, 0xf3, 0x0b, 0x53, 0x99, 0xba, 0x08, 0x0d, 0xae, 0xbe, 0x19, 0x9a, 0xc3, 0x76, 0x49, 0x85, - 0x94, 0x59, 0xde, 0xfb, 0x00, 0xda, 0x8b, 0xe4, 0xc1, 0x3a, 0xac, 0xdd, 0x1f, 0x3c, 0x38, 0x7a, - 0xfc, 0xd0, 0x6c, 0x17, 0x84, 0x61, 0x1e, 0x9d, 0x9c, 0x1f, 0x9d, 0x0d, 0xda, 0x5a, 0xef, 0x57, - 0x0d, 0xca, 0x97, 0xe2, 0xa5, 0x85, 0x07, 0x50, 0x96, 0x8f, 0x08, 0x7c, 0x4d, 0xce, 0x90, 0x7d, - 0x6a, 0x6d, 0x62, 0xd6, 0x15, 0x6d, 0xea, 0x40, 0xc3, 0xbe, 0x78, 0x3f, 0x88, 0x9f, 0x18, 0xa2, - 0x7a, 0x31, 0x64, 0xf4, 0x6c, 0x73, 0x3d, 0xe7, 0x4b, 0x92, 0x06, 0xd0, 0xc8, 0xb6, 0x87, 0x9d, - 0x57, 0x51, 0x69, 0xf3, 0xce, 0x8a, 0x48, 0x5c, 0xe6, 0xf8, 0xf6, 0x6f, 0x37, 0xdb, 0xda, 0xef, - 0x37, 0xdb, 0xda, 0x1f, 0x37, 0xdb, 0xda, 0x4f, 0x7f, 0x6e, 0x17, 0xbe, 0x2b, 0xcb, 0x67, 0xe3, - 0xa8, 0x22, 0x5f, 0x8c, 0xfd, 0xbf, 0x02, 0x00, 0x00, 0xff, 0xff, 0xe6, 0x41, 0x85, 0x94, 0x73, - 0x0a, 0x00, 0x00, + // 1087 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xdd, 0x6e, 0x1b, 0x45, + 0x14, 0xf6, 0x7a, 0x6d, 0xc7, 0x3e, 0xfe, 0x89, 0x39, 0x09, 0xd4, 0x8d, 0xa2, 0x10, 0x2d, 0x50, + 0x42, 0x80, 0x38, 0xd8, 0x11, 0x50, 0x24, 0x7e, 0x12, 0xea, 0x26, 0x48, 0x75, 0xd2, 0x8c, 0xb7, + 0x28, 0x42, 0xbd, 0x60, 0xec, 0x9d, 0x3a, 0xab, 0x7a, 0xbd, 0xdb, 0xdd, 0x31, 0xaa, 0xfb, 0x0a, + 0xdc, 0x20, 0x6e, 0x78, 0x11, 0x78, 0x07, 0x2e, 0x79, 0x04, 0x14, 0x5e, 0x04, 0xcd, 0xec, 0xec, + 0x9f, 0xed, 0xf2, 0xd3, 0xbb, 0x99, 0x73, 0xbe, 0x73, 0xe6, 0x9c, 0x33, 0xdf, 0x7e, 0x3b, 0xf0, + 0xc5, 0xd8, 0xe6, 0xd7, 0xb3, 0xe1, 0xc1, 0xc8, 0x75, 0xda, 0x4e, 0xd7, 0x1a, 0xb6, 0x9d, 0x6e, + 0x3b, 0xf0, 0x47, 0xed, 0x67, 0x33, 0xe6, 0xcf, 0xdb, 0x63, 0x36, 0x65, 0x3e, 0xe5, 0xcc, 0x6a, + 0x7b, 0xbe, 0xcb, 0xdd, 0xb6, 0xef, 0x8d, 0xbc, 0x61, 0xe8, 0x3b, 0x90, 0x16, 0xd4, 0x7d, 0x6f, + 0x64, 0x3c, 0x87, 0xda, 0x7d, 0xc6, 0x47, 0xd7, 0x84, 0x3d, 0x9b, 0xb1, 0x80, 0xe3, 0x26, 0x14, + 0x03, 0x4e, 0x7d, 0xde, 0xd2, 0x76, 0xb5, 0x3d, 0x9d, 0x84, 0x1b, 0x6c, 0x82, 0xce, 0xa6, 0x56, + 0x2b, 0x2f, 0x6d, 0x62, 0x89, 0x47, 0x50, 0xe5, 0x74, 0xdc, 0xa7, 0x7c, 0x74, 0xcd, 0xfc, 0xa0, + 0xa5, 0xef, 0x6a, 0x7b, 0xd5, 0x4e, 0xf3, 0xc0, 0xf7, 0x46, 0x07, 0x66, 0x62, 0x3f, 0xcb, 0x91, + 0x34, 0xec, 0x04, 0xa0, 0xec, 0xa8, 0xb5, 0xf1, 0x15, 0x54, 0x53, 0x48, 0xfc, 0x28, 0x9b, 0x50, + 0xdb, 0xd5, 0xf7, 0xaa, 0x9d, 0xf5, 0x85, 0x84, 0x99, 0x6c, 0xc6, 0x63, 0x80, 0xc4, 0x85, 0x08, + 0x85, 0x29, 0x75, 0x98, 0x2c, 0xbc, 0x46, 0xe4, 0x5a, 0x74, 0xf3, 0x03, 0x9d, 0xcc, 0x98, 0xac, + 0xbc, 0x46, 0xc2, 0x0d, 0xbe, 0x0d, 0x05, 0x3e, 0xf7, 0x98, 0x2c, 0xba, 0xa1, 0x8a, 0x56, 0x59, + 0xcc, 0xb9, 0xc7, 0x88, 0xf4, 0x1a, 0x47, 0x50, 0x57, 0x93, 0x09, 0x3c, 0x77, 0x1a, 0x30, 0x7c, + 0x0b, 0x4a, 0x01, 0xf3, 0x6d, 0x16, 0x15, 0x57, 0x95, 0x81, 0x03, 0x69, 0x22, 0xca, 0x65, 0xfc, + 0xaa, 0x41, 0x29, 0x34, 0xe1, 0xbb, 0x50, 0x70, 0x18, 0xa7, 0xb2, 0xa0, 0x6a, 0x67, 0x23, 0x85, + 0xee, 0x33, 0x4e, 0x2d, 0xca, 0x29, 0x91, 0x00, 0xfc, 0x1c, 0x6a, 0x16, 0x1b, 0xb9, 0x8e, 0xe7, + 0xb3, 0x20, 0x60, 0xe1, 0x98, 0xab, 0x9d, 0x5b, 0x32, 0xe0, 0x5e, 0xca, 0x11, 0x06, 0x9f, 0xe5, + 0x48, 0x06, 0x8e, 0x77, 0x01, 0x52, 0xc1, 0x7a, 0x2a, 0xb8, 0xdf, 0xfd, 0x7a, 0x39, 0x38, 0x05, + 0x3e, 0x59, 0x53, 0xf3, 0x31, 0xae, 0xa0, 0x91, 0x2d, 0x0d, 0x1b, 0x90, 0xb7, 0x2d, 0x35, 0xcc, + 0xbc, 0x6d, 0xe1, 0x36, 0x54, 0x24, 0x17, 0x4c, 0xdb, 0x61, 0x8a, 0x08, 0x89, 0x01, 0x5b, 0xb0, + 0xc6, 0xa6, 0x96, 0xf4, 0xe9, 0xd2, 0x17, 0x6d, 0x8d, 0x21, 0xe0, 0x72, 0x0f, 0x78, 0x00, 0x20, + 0x4e, 0xf1, 0x5c, 0x7b, 0xca, 0xa3, 0x79, 0x36, 0xc2, 0x86, 0x23, 0x33, 0x49, 0x21, 0x70, 0x1b, + 0x0a, 0x9c, 0x8e, 0x83, 0x56, 0x5e, 0x22, 0xcb, 0x11, 0x2d, 0x88, 0xb4, 0x1a, 0x5f, 0x42, 0x25, + 0x0e, 0x13, 0x85, 0x72, 0xdb, 0x61, 0x01, 0xa7, 0x8e, 0xa7, 0x58, 0x9c, 0x18, 0xb2, 0x8c, 0xd0, + 0x14, 0x23, 0x8c, 0x36, 0xe8, 0x26, 0x1d, 0xff, 0x77, 0x0a, 0x19, 0xcf, 0x01, 0x97, 0x87, 0x8b, + 0x77, 0xa0, 0x91, 0x74, 0x6a, 0x8a, 0x7a, 0xc3, 0x4c, 0x0b, 0x56, 0xfc, 0x0c, 0xca, 0x3e, 0xf3, + 0x26, 0xf6, 0x88, 0x46, 0x1d, 0xed, 0x2c, 0xdd, 0xd7, 0xb7, 0xe2, 0x9c, 0x80, 0x84, 0x30, 0x12, + 0xe3, 0x8d, 0x33, 0xb8, 0xfd, 0x52, 0x18, 0xbe, 0x0f, 0xe5, 0x80, 0x8d, 0x1d, 0x96, 0x0c, 0x75, + 0x5d, 0x25, 0x1e, 0x28, 0x33, 0x89, 0x01, 0xc6, 0xf7, 0x00, 0x89, 0x1d, 0xef, 0x40, 0xc9, 0x61, + 0xfe, 0x98, 0x59, 0x8a, 0xaf, 0x8d, 0x6c, 0x20, 0x51, 0x5e, 0xdc, 0x87, 0xf2, 0x6c, 0xaa, 0x90, + 0xf9, 0xd4, 0xbd, 0x25, 0xc8, 0xd8, 0x6f, 0xb8, 0x50, 0x89, 0xcd, 0x62, 0xb8, 0xd7, 0x8c, 0x46, + 0x94, 0x92, 0x6b, 0x61, 0xe3, 0xd4, 0x9e, 0xa8, 0xd9, 0xca, 0x75, 0x96, 0x68, 0xfa, 0x22, 0xd1, + 0xb6, 0xa1, 0x32, 0x9c, 0xb8, 0xa3, 0xa7, 0x03, 0xfb, 0x05, 0x6b, 0x15, 0x42, 0x6f, 0x6c, 0x30, + 0x2e, 0xa1, 0x3e, 0x60, 0xd4, 0x4f, 0xe4, 0xec, 0x68, 0x51, 0x55, 0xfe, 0xb7, 0x4c, 0x9d, 0x42, + 0xbd, 0xdf, 0x35, 0xe9, 0xf8, 0xa1, 0xef, 0x7a, 0xcc, 0xe7, 0xf3, 0xa5, 0x0f, 0x63, 0xf9, 0xd2, + 0xf3, 0xab, 0x2e, 0xdd, 0xe8, 0xc1, 0x7a, 0x3a, 0x91, 0xe0, 0x4b, 0x07, 0xc0, 0x8b, 0x77, 0xea, + 0xc2, 0x50, 0x4d, 0x33, 0x75, 0x24, 0x49, 0xa1, 0x8c, 0x4f, 0xa4, 0x6c, 0xc6, 0xd5, 0x34, 0x41, + 0x7f, 0xca, 0xe6, 0xaa, 0x1c, 0xb1, 0xc4, 0x37, 0xa0, 0x24, 0x39, 0x1a, 0xd5, 0xa1, 0x76, 0xc6, + 0x31, 0xd4, 0xb3, 0xa7, 0x1f, 0xae, 0x38, 0x3d, 0x1e, 0xcd, 0xca, 0xb3, 0x7f, 0xd4, 0x84, 0x4c, + 0x84, 0xf3, 0x55, 0xa2, 0xf8, 0xe9, 0x82, 0x76, 0x85, 0x13, 0xc6, 0x85, 0x34, 0xab, 0x64, 0xeb, + 0xe3, 0x8c, 0x6c, 0x85, 0x9a, 0xb7, 0xb9, 0xd4, 0xfc, 0x3f, 0x68, 0xd6, 0x2f, 0x1a, 0x6c, 0x89, + 0x0f, 0x61, 0xc2, 0x38, 0x13, 0x13, 0x56, 0x77, 0x7e, 0xe1, 0x71, 0xdb, 0x9d, 0x06, 0xf8, 0x9e, + 0x52, 0x79, 0x4d, 0xaa, 0xfc, 0xeb, 0x32, 0x73, 0x1a, 0x9e, 0x48, 0xbd, 0xb8, 0xc2, 0x27, 0xf6, + 0x84, 0x33, 0xff, 0x9c, 0x3a, 0xcc, 0x8c, 0x74, 0xa6, 0x46, 0x16, 0xac, 0xc9, 0xcf, 0x51, 0x5f, + 0xf1, 0x73, 0x2c, 0xc4, 0x3f, 0x47, 0xe3, 0x67, 0x0d, 0x36, 0x56, 0x54, 0xf6, 0x6a, 0x6c, 0xc4, + 0xbb, 0xb0, 0xe6, 0x86, 0x3d, 0xa9, 0x29, 0xbd, 0xb9, 0xd4, 0x4b, 0xb6, 0x75, 0x12, 0xe1, 0x33, + 0x44, 0xde, 0x85, 0xb2, 0x49, 0xc7, 0xa2, 0x17, 0xd9, 0x88, 0x10, 0xb7, 0xf0, 0xd6, 0x6b, 0x24, + 0xdc, 0x18, 0x47, 0x12, 0x21, 0x15, 0xe5, 0x5f, 0x78, 0xa5, 0xa7, 0x78, 0xd5, 0x81, 0x4a, 0x14, + 0x15, 0xe0, 0x3b, 0x31, 0x28, 0xe4, 0x53, 0x3d, 0x6a, 0x4e, 0xfa, 0xe3, 0x98, 0x17, 0xb0, 0x99, + 0x2d, 0x5f, 0xb1, 0x69, 0x1f, 0xd6, 0x2c, 0xf6, 0x84, 0xce, 0x26, 0x3c, 0xa3, 0x42, 0x71, 0xfe, + 0xb3, 0x1c, 0x89, 0x00, 0xf8, 0x21, 0x54, 0x64, 0xd9, 0x17, 0xd3, 0xc9, 0x5c, 0x0d, 0x26, 0x3e, + 0x4d, 0x76, 0x79, 0x96, 0x23, 0x09, 0x22, 0xa6, 0xcd, 0xfe, 0x63, 0xa8, 0xa6, 0x7e, 0xf6, 0x58, + 0x81, 0x62, 0xef, 0xf2, 0xd1, 0xf1, 0x83, 0x66, 0x0e, 0x6b, 0x50, 0x3e, 0xbf, 0x30, 0xc3, 0x9d, + 0x86, 0x00, 0x25, 0xd2, 0x3b, 0xed, 0x5d, 0x3d, 0x6c, 0xe6, 0xb1, 0x0e, 0x95, 0xf3, 0x0b, 0x53, + 0x6d, 0x75, 0xe1, 0xea, 0x5d, 0x7d, 0x33, 0x30, 0x07, 0xcd, 0x82, 0x72, 0xa9, 0x6d, 0x71, 0xff, + 0x03, 0x68, 0x2e, 0x92, 0x0c, 0xab, 0xb0, 0x76, 0xaf, 0x77, 0xff, 0xf8, 0xd1, 0x03, 0xb3, 0x99, + 0x13, 0x1b, 0xf3, 0xf8, 0xf4, 0xfc, 0xb8, 0xdf, 0x6b, 0x6a, 0x9d, 0xdf, 0x34, 0x28, 0x5e, 0x8a, + 0x27, 0x19, 0x1e, 0x42, 0x51, 0xbe, 0x36, 0xf0, 0x35, 0xd9, 0x43, 0xfa, 0x4d, 0xb6, 0x85, 0x69, + 0x53, 0x38, 0xa9, 0x43, 0x0d, 0xbb, 0xe2, 0xa1, 0x21, 0xbe, 0x45, 0x44, 0xf5, 0xb4, 0x48, 0x09, + 0xdf, 0xd6, 0x46, 0xc6, 0x16, 0x07, 0xf5, 0xa0, 0x96, 0x2e, 0x0f, 0x5b, 0x2f, 0xa3, 0xd2, 0xd6, + 0xed, 0x15, 0x9e, 0x28, 0xcd, 0xc9, 0xad, 0xdf, 0x6f, 0x76, 0xb4, 0x3f, 0x6e, 0x76, 0xb4, 0x3f, + 0x6f, 0x76, 0xb4, 0x9f, 0xfe, 0xda, 0xc9, 0x7d, 0x57, 0x94, 0xef, 0xcb, 0x61, 0x49, 0x3e, 0x2d, + 0xbb, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0x8d, 0x75, 0x78, 0x03, 0x9c, 0x0a, 0x00, 0x00, } diff --git a/src/query/generated/proto/rpcpb/query.proto b/src/query/generated/proto/rpcpb/query.proto index 60b2514ffc..bc07d2dc89 100644 --- a/src/query/generated/proto/rpcpb/query.proto +++ b/src/query/generated/proto/rpcpb/query.proto @@ -135,6 +135,8 @@ enum CompleteTagsType { message CompleteTagsRequestOptions { CompleteTagsType type = 1; repeated bytes filterNameTags = 2; + int64 start = 3; + int64 end = 4; } message CompleteTagsRequest { diff --git a/src/query/server/server.go b/src/query/server/server.go index a5e51a48dd..01a14423e7 100644 --- a/src/query/server/server.go +++ b/src/query/server/server.go @@ -116,6 +116,10 @@ type RunOptions struct { // InterruptCh is a programmatic interrupt channel to supply to // interrupt and shutdown the server. InterruptCh <-chan error + + // ListenerCh is a programmatic channel to receive the server listener + // on once it has opened. + ListenerCh chan<- net.Listener } // Run runs the server programmatically given a filename for the configuration file. @@ -280,11 +284,21 @@ func Run(runOpts RunOptions) { } }() + listener, err := net.Listen("tcp", listenAddress) + if err != nil { + logger.Fatal("unable to listen on listen address", + zap.String("address", listenAddress), + zap.Error(err)) + } + if runOpts.ListenerCh != nil { + runOpts.ListenerCh <- listener + } go func() { logger.Info("starting API server", zap.String("address", listenAddress)) - if err := srv.ListenAndServe(); err != nil && err != http.ErrServerClosed { - logger.Fatal("server error while listening", - zap.String("address", listenAddress), zap.Error(err)) + if err := srv.Serve(listener); err != nil && err != http.ErrServerClosed { + logger.Fatal("server serve error", + zap.String("address", listenAddress), + zap.Error(err)) } }() diff --git a/src/query/server/server_test.go b/src/query/server/server_test.go index 2b1db221cb..4cd28c3541 100644 --- a/src/query/server/server_test.go +++ b/src/query/server/server_test.go @@ -48,19 +48,17 @@ import ( "google.golang.org/grpc" ) -var queryPort = 25123 - var configYAML = ` listenAddress: type: "config" - value: "127.0.0.1:25123" + value: "127.0.0.1:0" metrics: scope: prefix: "coordinator" prometheus: handlerPath: /metrics - listenAddress: "127.0.0.1:18202" + listenAddress: "127.0.0.1:0" sanitization: prometheus samplingRate: 1.0 @@ -88,7 +86,6 @@ writeWorkerPoolPolicy: ` -//TODO: Use randomly assigned port here func TestRun(t *testing.T) { ctrl := gomock.NewController(xtest.Reporter{T: t}) defer ctrl.Finish() @@ -136,24 +133,30 @@ func TestRun(t *testing.T) { return dbClient, nil }) - interruptCh := make(chan error) - doneCh := make(chan struct{}) + interruptCh := make(chan error, 1) + doneCh := make(chan struct{}, 1) + listenerCh := make(chan net.Listener, 1) go func() { Run(RunOptions{ Config: cfg, InterruptCh: interruptCh, + ListenerCh: listenerCh, }) doneCh <- struct{}{} }() + // Wait for listener + listener := <-listenerCh + addr := listener.Addr().String() + // Wait for server to come up - waitForServerHealthy(t, queryPort) + waitForServerHealthy(t, addr) // Send Prometheus write request promReq := test.GeneratePromWriteRequest() promReqBody := test.GeneratePromWriteRequestBody(t, promReq) req, err := http.NewRequest(http.MethodPost, - fmt.Sprintf("http://127.0.0.1:%d", queryPort)+remote.PromWriteURL, promReqBody) + fmt.Sprintf("http://%s%s", addr, remote.PromWriteURL), promReqBody) require.NoError(t, err) res, err := http.DefaultClient.Do(req) @@ -179,11 +182,11 @@ func newTestFile(t *testing.T, fileName, contents string) (*os.File, closeFn) { } } -func waitForServerHealthy(t *testing.T, port int) { +func waitForServerHealthy(t *testing.T, addr string) { maxWait := 10 * time.Second startAt := time.Now() for time.Since(startAt) < maxWait { - req, err := http.NewRequest("GET", fmt.Sprintf("http://127.0.0.1:%d/health", port), nil) + req, err := http.NewRequest("GET", fmt.Sprintf("http://%s/health", addr), nil) require.NoError(t, err) res, err := http.DefaultClient.Do(req) if err != nil || res.StatusCode != http.StatusOK { @@ -232,24 +235,26 @@ func (s *queryServer) CompleteTags( } func TestGRPCBackend(t *testing.T) { - var grpcConfigYAML = ` + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + grpcAddr := lis.Addr().String() + var grpcConfigYAML = fmt.Sprintf(` listenAddress: type: "config" - value: "127.0.0.1:17221" + value: "127.0.0.1:0" metrics: scope: prefix: "coordinator" prometheus: handlerPath: /metrics - listenAddress: "127.0.0.1:17223" + listenAddress: "127.0.0.1:0" onError: stderr sanitization: prometheus samplingRate: 1.0 rpc: - remoteListenAddresses: - - "127.0.0.1:17222" + remoteListenAddresses: ["%s"] backend: grpc @@ -268,17 +273,11 @@ writeWorkerPoolPolicy: size: 100 shards: 1000 killProbability: 0.3 -` - - // TODO(arnikola): REVERT - t.SkipNow() +`, grpcAddr) ctrl := gomock.NewController(xtest.Reporter{T: t}) defer ctrl.Finish() - port := "127.0.0.1:17222" - lis, err := net.Listen("tcp", port) - require.NoError(t, err) s := grpc.NewServer() defer s.GracefulStop() qs := &queryServer{} @@ -300,22 +299,28 @@ writeWorkerPoolPolicy: interruptCh := make(chan error) doneCh := make(chan struct{}) + listenerCh := make(chan net.Listener, 1) go func() { Run(RunOptions{ Config: cfg, InterruptCh: interruptCh, + ListenerCh: listenerCh, }) doneCh <- struct{}{} }() + // Wait for listener + listener := <-listenerCh + addr := listener.Addr().String() + // Wait for server to come up - waitForServerHealthy(t, 17221) + waitForServerHealthy(t, addr) // Send Prometheus read request promReq := test.GeneratePromReadRequest() promReqBody := test.GeneratePromReadRequestBody(t, promReq) req, err := http.NewRequest(http.MethodPost, - "http://127.0.0.1:17221"+remote.PromReadURL, promReqBody) + fmt.Sprintf("http://%s%s", addr, remote.PromReadURL), promReqBody) require.NoError(t, err) _, err = http.DefaultClient.Do(req) diff --git a/src/query/storage/fanout/storage.go b/src/query/storage/fanout/storage.go index 2cc3d2ea95..6e233eb06b 100644 --- a/src/query/storage/fanout/storage.go +++ b/src/query/storage/fanout/storage.go @@ -175,8 +175,13 @@ func (s *fanoutStorage) CompleteTags( query *storage.CompleteTagsQuery, options *storage.FetchOptions, ) (*storage.CompleteTagsResult, error) { - accumulatedTags := storage.NewCompleteTagsResultBuilder(query.CompleteNameOnly) stores := filterCompleteTagsStores(s.stores, s.completeTagsFilter, *query) + // short circuit complete tags + if len(stores) == 1 { + return stores[0].CompleteTags(ctx, query, options) + } + + accumulatedTags := storage.NewCompleteTagsResultBuilder(query.CompleteNameOnly) for _, store := range stores { result, err := store.CompleteTags(ctx, query, options) if err != nil { diff --git a/src/query/tsdb/remote/codecs_complete_tags.go b/src/query/tsdb/remote/codecs_complete_tags.go index 872e520002..62b083e913 100644 --- a/src/query/tsdb/remote/codecs_complete_tags.go +++ b/src/query/tsdb/remote/codecs_complete_tags.go @@ -93,6 +93,80 @@ func encodeCompleteTagsRequest( Options: &rpc.CompleteTagsRequestOptions{ Type: completionType, FilterNameTags: query.FilterNameTags, + Start: fromTime(query.Start), + End: fromTime(query.End), }, }, nil } + +func decodeCompleteTagsRequest( + request *rpc.CompleteTagsRequest, +) (*storage.CompleteTagsQuery, error) { + var ( + opts = request.GetOptions() + matchers = request.GetTagMatchers() + ) + + completeNameOnly := opts.GetType() == rpc.CompleteTagsType_TAGNAME + tagMatchers, err := decodeTagMatchers(matchers) + if err != nil { + return nil, err + } + + return &storage.CompleteTagsQuery{ + CompleteNameOnly: completeNameOnly, + FilterNameTags: opts.GetFilterNameTags(), + TagMatchers: tagMatchers, + Start: toTime(opts.GetStart()), + End: toTime(opts.GetEnd()), + }, nil +} + +func encodeToCompressedCompleteTagsDefaultResult( + results *storage.CompleteTagsResult, +) (*rpc.CompleteTagsResponse, error) { + tags := results.CompletedTags + values := make([]*rpc.TagValue, 0, len(tags)) + for _, tag := range tags { + values = append(values, &rpc.TagValue{ + Key: tag.Name, + Values: tag.Values, + }) + } + + return &rpc.CompleteTagsResponse{ + Value: &rpc.CompleteTagsResponse_Default{ + Default: &rpc.TagValues{ + Values: values, + }, + }, + }, nil +} + +func encodeToCompressedCompleteTagsNameOnlyResult( + results *storage.CompleteTagsResult, +) (*rpc.CompleteTagsResponse, error) { + tags := results.CompletedTags + names := make([][]byte, 0, len(tags)) + for _, tag := range tags { + names = append(names, tag.Name) + } + + return &rpc.CompleteTagsResponse{ + Value: &rpc.CompleteTagsResponse_NamesOnly{ + NamesOnly: &rpc.TagNames{ + Names: names, + }, + }, + }, nil +} + +func encodeToCompressedCompleteTagsResult( + results *storage.CompleteTagsResult, +) (*rpc.CompleteTagsResponse, error) { + if results.CompleteNameOnly { + return encodeToCompressedCompleteTagsNameOnlyResult(results) + } + + return encodeToCompressedCompleteTagsDefaultResult(results) +} diff --git a/src/query/tsdb/remote/server.go b/src/query/tsdb/remote/server.go index 5c4ce34b2b..2a48c6ed52 100644 --- a/src/query/tsdb/remote/server.go +++ b/src/query/tsdb/remote/server.go @@ -25,7 +25,6 @@ import ( "time" "github.com/m3db/m3/src/dbnode/encoding" - "github.com/m3db/m3/src/query/errors" rpc "github.com/m3db/m3/src/query/generated/proto/rpcpb" "github.com/m3db/m3/src/query/models" "github.com/m3db/m3/src/query/pools" @@ -168,5 +167,36 @@ func (s *grpcServer) CompleteTags( message *rpc.CompleteTagsRequest, stream rpc.Query_CompleteTagsServer, ) error { - return errors.ErrNotImplemented + var err error + + ctx := retrieveMetadata(stream.Context()) + logger := logging.WithContext(ctx) + completeTagsQuery, err := decodeCompleteTagsRequest(message) + if err != nil { + logger.Error("unable to decode complete tags query", zap.Error(err)) + return err + } + + // TODO(r): Allow propagation of limit from RPC request + fetchOpts := storage.NewFetchOptions() + fetchOpts.Limit = s.queryContextOpts.LimitMaxTimeseries + + results, err := s.storage.CompleteTags(ctx, completeTagsQuery, fetchOpts) + if err != nil { + logger.Error("unable to complete tags", zap.Error(err)) + return err + } + + response, err := encodeToCompressedCompleteTagsResult(results) + if err != nil { + logger.Error("unable to encode complete tags result", zap.Error(err)) + return err + } + + err = stream.SendMsg(response) + if err != nil { + logger.Error("unable to send complete tags result", zap.Error(err)) + } + + return err }