From d7e827bb438a017386355473446fa19f05fac5f4 Mon Sep 17 00:00:00 2001 From: wonderf00l Date: Tue, 5 Dec 2023 23:31:10 +0300 Subject: [PATCH 01/10] TP-c01_ci-cd: add Dockerfiles for nginx and microservices, add service deps,healthchecks --- deployments/Dockerfile.auth | 20 +++++++ deployments/Dockerfile.main | 21 +++++++ deployments/Dockerfile.messenger | 20 +++++++ deployments/Dockerfile.realtime | 20 +++++++ deployments/docker-compose.yml | 95 +++++++++++++++++++++++++++++++- 5 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 deployments/Dockerfile.auth create mode 100644 deployments/Dockerfile.main create mode 100644 deployments/Dockerfile.messenger create mode 100644 deployments/Dockerfile.realtime diff --git a/deployments/Dockerfile.auth b/deployments/Dockerfile.auth new file mode 100644 index 0000000..88735ef --- /dev/null +++ b/deployments/Dockerfile.auth @@ -0,0 +1,20 @@ +FROM golang:1.19.13-alpine AS build + +RUN apk --no-cache add make + +WORKDIR /pinspire_auth + +COPY go.mod go.sum /pinspire_auth/ +RUN go mod download + +COPY . . + +RUN make build_auth + +FROM alpine:latest + +WORKDIR / + +COPY --from=build /pinspire_auth/bin/auth . + +ENTRYPOINT [ "./auth" ] diff --git a/deployments/Dockerfile.main b/deployments/Dockerfile.main new file mode 100644 index 0000000..c595010 --- /dev/null +++ b/deployments/Dockerfile.main @@ -0,0 +1,21 @@ +FROM golang:1.19.13-alpine AS build + +RUN apk --no-cache add make + +WORKDIR /pinspire_main + +COPY go.mod go.sum /pinspire_main/ +RUN go mod download + +COPY . . + +RUN make build + +FROM alpine:latest + +WORKDIR / + +COPY --from=build /pinspire_main/bin/app . +COPY --from=build /pinspire_main/configs configs + +ENTRYPOINT [ "./app" ] diff --git a/deployments/Dockerfile.messenger b/deployments/Dockerfile.messenger new file mode 100644 index 0000000..6fa332f --- /dev/null +++ b/deployments/Dockerfile.messenger @@ -0,0 +1,20 @@ +FROM golang:1.19.13-alpine AS build + +RUN apk --no-cache add make + +WORKDIR /pinspire_messenger + +COPY go.mod go.sum /pinspire_messenger/ +RUN go mod download + +COPY . . + +RUN make build_messenger + +FROM alpine:latest + +WORKDIR / + +COPY --from=build /pinspire_messenger/bin/messenger . + +ENTRYPOINT [ "./messenger" ] diff --git a/deployments/Dockerfile.realtime b/deployments/Dockerfile.realtime new file mode 100644 index 0000000..7c94295 --- /dev/null +++ b/deployments/Dockerfile.realtime @@ -0,0 +1,20 @@ +FROM golang:1.19.13-alpine AS build + +RUN apk --no-cache add make + +WORKDIR /pinspire_realtime + +COPY go.mod go.sum /pinspire_realtime/ +RUN go mod download + +COPY . . + +RUN make build_realtime + +FROM alpine:latest + +WORKDIR / + +COPY --from=build /pinspire_realtime/bin/realtime . + +ENTRYPOINT [ "./realtime" ] diff --git a/deployments/docker-compose.yml b/deployments/docker-compose.yml index 8ee69d6..094bbab 100644 --- a/deployments/docker-compose.yml +++ b/deployments/docker-compose.yml @@ -1,5 +1,6 @@ version: '3.8' + services: postgres: image: postgres:latest @@ -10,6 +11,8 @@ services: - ../db/migrations:/docker-entrypoint-initdb.d ports: - 5432:5432 + healthcheck: + test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}"] redis: image: redis:latest @@ -19,6 +22,82 @@ services: command: redis-server /usr/local/etc/redis/redis.conf ports: - 6379:6379 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + + main_service: + build: + context: ./.. + dockerfile: deployments/Dockerfile.main + container_name: pinspireMainService + env_file: + - ../.env + environment: + - POSTGRES_HOST=postgres + - AUTH_SERVICE_HOST=auth_service + - MESSENGER_SERVICE_HOST=messenger_service + - REALTIME_SERVICE_HOST=realtime_service + depends_on: + postgres: + condition: 'service_healthy' + auth_service: + condition: 'service_started' + messenger_service: + condition: 'service_started' + realtime_service: + condition: 'service_started' + ports: + - 8100:8080 + + auth_service: + build: + context: ./.. + dockerfile: deployments/Dockerfile.auth + container_name: pinspireAuthService + env_file: + - ../.env + environment: + - POSTGRES_HOST=postgres + - REDIS_HOST=redis + depends_on: + postgres: + condition: 'service_healthy' + redis: + condition: 'service_healthy' + ports: + - 8086:8086 + # - 8101:8085 + + messenger_service: + build: + context: ./.. + dockerfile: deployments/Dockerfile.messenger + container_name: pinspireMessengerService + env_file: + - ../.env + environment: + - POSTGRES_HOST=postgres + depends_on: + postgres: + condition: 'service_healthy' + ports: + - 8096:8096 + # - 8102:8095 + + realtime_service: + build: + context: ./.. + dockerfile: deployments/Dockerfile.realtime + container_name: pinspireRealtimeService + env_file: + - ../.env + environment: + - KAFKA_BROKER_ADDRESS=kafka + depends_on: + - kafka + ports: + - 8091:8091 + # - 8103:8090 zookeeper: image: bitnami/zookeeper:latest @@ -41,7 +120,7 @@ services: - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092 - - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 + - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092 depends_on: - zookeeper @@ -56,6 +135,8 @@ services: grafana: image: grafana/grafana:latest container_name: pinspireGrafana + env_file: + - ../.env ports: - 3000:3000 volumes: @@ -75,6 +156,18 @@ services: - /:/rootfs:ro ports: - "9100:9100" + + nginx: + image: nginx:latest + container_name: pinspireNginx + volumes: + - '/etc/nginx/sites-available/pinspire.conf:/etc/nginx/conf.d/pinspire.conf:ro' + network_mode: 'host' + depends_on: + main_service: + condition: 'service_started' + realtime_service: + condition: 'service_started' volumes: From f8ec2a3eef23016170bfe649c7613b64a121173f Mon Sep 17 00:00:00 2001 From: wonderf00l Date: Tue, 5 Dec 2023 23:33:22 +0300 Subject: [PATCH 02/10] TP-c01_ci-cd: changed prometheus targets, servers addresses from localhost: to :, replaced string servers params with env variables --- cmd/app/config.go | 8 ++++++-- cmd/auth/config.go | 2 +- cmd/realtime/main.go | 2 +- configs/config.yml | 2 +- configs/prometheus.yml | 10 +++++----- internal/app/app.go | 4 +++- internal/app/auth/auth.go | 13 +++++++++---- internal/app/config.go | 8 ++++---- internal/app/messenger/messenger.go | 2 +- internal/app/redis_conn.go | 2 +- internal/microservices/realtime/node.go | 4 +++- internal/pkg/delivery/websocket/websocket.go | 4 +++- 12 files changed, 38 insertions(+), 23 deletions(-) diff --git a/cmd/app/config.go b/cmd/app/config.go index 0cb9a84..42f8ed3 100644 --- a/cmd/app/config.go +++ b/cmd/app/config.go @@ -1,8 +1,12 @@ package main -import "github.com/go-park-mail-ru/2023_2_OND_team/internal/app" +import ( + "os" + + "github.com/go-park-mail-ru/2023_2_OND_team/internal/app" +) var configFiles = app.ConfigFiles{ ServerConfigFile: "configs/config.yml", - AddrAuthServer: "localhost:8085", + AddrAuthServer: os.Getenv("AUTH_SERVICE_HOST") + ":" + os.Getenv("AUTH_SERVICE_PORT"), // "localhost:8085", } diff --git a/cmd/auth/config.go b/cmd/auth/config.go index bf1142d..2ffd53a 100644 --- a/cmd/auth/config.go +++ b/cmd/auth/config.go @@ -3,6 +3,6 @@ package main import "github.com/go-park-mail-ru/2023_2_OND_team/internal/app/auth" var configAuth = auth.Config{ - Addr: "localhost:8085", + Addr: "0.0.0.0:8085", RedisFileConfig: "redis.conf", } diff --git a/cmd/realtime/main.go b/cmd/realtime/main.go index a68532f..1022c65 100644 --- a/cmd/realtime/main.go +++ b/cmd/realtime/main.go @@ -13,7 +13,7 @@ import ( "github.com/go-park-mail-ru/2023_2_OND_team/pkg/logger" ) -const _address = "localhost:8090" +const _address = "0.0.0.0:8090" func main() { log, err := logger.New() diff --git a/configs/config.yml b/configs/config.yml index 88ea0c1..4d1fda6 100644 --- a/configs/config.yml +++ b/configs/config.yml @@ -1,6 +1,6 @@ app: server: - host: 127.0.0.1 + host: 0.0.0.0 port: 8080 https: false certFile: /home/ond_team/cert/fullchain.pem diff --git a/configs/prometheus.yml b/configs/prometheus.yml index 255697d..57406b3 100644 --- a/configs/prometheus.yml +++ b/configs/prometheus.yml @@ -5,23 +5,23 @@ global: scrape_configs: - job_name: 'api' static_configs: - - targets: ['host.docker.internal:8080'] + - targets: ['main_service:8080'] - job_name: 'auth' static_configs: - - targets: ['pinspire.online:8086'] + - targets: ['auth_service:8086'] - job_name: 'messenger' static_configs: - - targets: ['pinspire.online:8096'] + - targets: ['messenger_service:8096'] - job_name: 'realtime' static_configs: - - targets: ['pinspire.online:8091'] + - targets: ['realtime_service:8091'] - job_name: 'node_exporter' static_configs: - - targets: ['pinspire.online:9100'] + - targets: ['node_exporter:9100'] - job_name: 'pinspire' scheme: https diff --git a/internal/app/app.go b/internal/app/app.go index 0437156..5d3d18f 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -2,6 +2,7 @@ package app import ( "context" + "os" "time" "github.com/joho/godotenv" @@ -57,7 +58,8 @@ func Run(ctx context.Context, log *log.Logger, cfg ConfigFiles) { } defer pool.Close() - connMessMS, err := grpc.Dial("localhost:8095", grpc.WithTransportCredentials(insecure.NewCredentials())) + // connMessMS, err := grpc.Dial("localhost:8095", grpc.WithTransportCredentials(insecure.NewCredentials())) + connMessMS, err := grpc.Dial(os.Getenv("MESSENGER_SERVICE_HOST")+":"+os.Getenv("MESSENGER_SERVICE_PORT"), grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Error(err.Error()) return diff --git a/internal/app/auth/auth.go b/internal/app/auth/auth.go index cfcfbd6..ddfc24e 100644 --- a/internal/app/auth/auth.go +++ b/internal/app/auth/auth.go @@ -3,6 +3,7 @@ package auth import ( "context" "net" + "os" "time" "github.com/joho/godotenv" @@ -54,11 +55,15 @@ func Run(ctx context.Context, log *logger.Logger, cfg Config) { ctxRedis, cancelCtxRedis := context.WithTimeout(ctx, _timeoutForConnRedis) defer cancelCtxRedis() - redisCfg, err := app.NewConfig(cfg.RedisFileConfig) - if err != nil { - log.Error(err.Error()) - return + // redisCfg, err := app.NewConfig(cfg.RedisFileConfig) + redisCfg := app.RedisConfig{ + Addr: os.Getenv("REDIS_HOST") + ":" + os.Getenv("REDIS_PORT"), + Password: os.Getenv("REDIS_PASSWORD"), } + // if err != nil { + // log.Error(err.Error()) + // return + // } redisCl, err := app.NewRedisClient(ctxRedis, redisCfg) if err != nil { diff --git a/internal/app/config.go b/internal/app/config.go index 7fa6c87..4fb3bb2 100644 --- a/internal/app/config.go +++ b/internal/app/config.go @@ -11,18 +11,18 @@ type ConfigFiles struct { AddrAuthServer string } -type redisConfig struct { +type RedisConfig struct { Password string Addr string } -func NewConfig(filename string) (redisConfig, error) { +func NewConfig(filename string) (RedisConfig, error) { cfg, err := config.ParseConfig(filename) if err != nil { - return redisConfig{}, fmt.Errorf("new redis config: %w", err) + return RedisConfig{}, fmt.Errorf("new redis config: %w", err) } - return redisConfig{ + return RedisConfig{ Password: cfg.Get("requirepass"), Addr: cfg.Get("host") + ":" + cfg.Get("port"), }, nil diff --git a/internal/app/messenger/messenger.go b/internal/app/messenger/messenger.go index f57c930..27ca153 100644 --- a/internal/app/messenger/messenger.go +++ b/internal/app/messenger/messenger.go @@ -48,7 +48,7 @@ func Run(ctx context.Context, log *logger.Logger) { )) messenger.RegisterMessengerServer(server, messMS.New(log, messageCase)) - l, err := net.Listen("tcp", "localhost:8095") + l, err := net.Listen("tcp", "0.0.0.0:8095") if err != nil { log.Error(err.Error()) return diff --git a/internal/app/redis_conn.go b/internal/app/redis_conn.go index 8a7527c..8ee910e 100644 --- a/internal/app/redis_conn.go +++ b/internal/app/redis_conn.go @@ -7,7 +7,7 @@ import ( redis "github.com/redis/go-redis/v9" ) -func NewRedisClient(ctx context.Context, cfg redisConfig) (*redis.Client, error) { +func NewRedisClient(ctx context.Context, cfg RedisConfig) (*redis.Client, error) { redisCl := redis.NewClient(&redis.Options{ Addr: cfg.Addr, Password: cfg.Password, diff --git a/internal/microservices/realtime/node.go b/internal/microservices/realtime/node.go index db17bf6..f6f39c7 100644 --- a/internal/microservices/realtime/node.go +++ b/internal/microservices/realtime/node.go @@ -2,6 +2,7 @@ package realtime import ( "fmt" + "os" "sync" "google.golang.org/protobuf/proto" @@ -20,7 +21,8 @@ func NewNode() (*Node, error) { node := &Node{} broker, err := NewKafkaBroker(node, KafkaConfig{ - Addres: []string{"localhost:9092"}, + // Addres: []string{"localhost:9092"}, + Addres: []string{os.Getenv("KAFKA_BROKER_ADDRESS") + ":" + os.Getenv("KAFKA_BROKER_PORT")}, PartitionsOnTopic: _numWorkers, MaxNumTopic: 10, }) diff --git a/internal/pkg/delivery/websocket/websocket.go b/internal/pkg/delivery/websocket/websocket.go index 6074730..c0042de 100644 --- a/internal/pkg/delivery/websocket/websocket.go +++ b/internal/pkg/delivery/websocket/websocket.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/http" + "os" "time" "google.golang.org/grpc" @@ -36,7 +37,8 @@ func SetOriginPatterns(patterns []string) Option { } func New(log *log.Logger, mesCase usecase.Usecase, opts ...Option) *HandlerWebSocket { - gRPCConn, err := grpc.Dial("localhost:8090", grpc.WithTransportCredentials(insecure.NewCredentials())) + // gRPCConn, err := grpc.Dial("localhost:8090", grpc.WithTransportCredentials(insecure.NewCredentials())) + gRPCConn, err := grpc.Dial(os.Getenv("REALTIME_SERVICE_HOST"+":"+os.Getenv("REALTIME_SERVICE_PORT")), grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Error(fmt.Errorf("grpc dial: %w", err).Error()) } From f862979bc2f69d009ffdde11aee7077ca306c7ee Mon Sep 17 00:00:00 2001 From: wonderf00l Date: Thu, 7 Dec 2023 21:03:48 +0300 Subject: [PATCH 03/10] TP-c01_ci-cd: add linter configuration --- Makefile | 18 +++- configs/.golangci.yml | 206 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 1 deletion(-) create mode 100644 configs/.golangci.yml diff --git a/Makefile b/Makefile index 7b0652c..ee03e17 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ .PHONY: build run test test_with_coverage cleantest retest doc generate cover_all currcover -.PHONY: build_auth build_realtime build_messenger +.PHONY: build_auth build_realtime build_messenger build_all +.PHONY: .install-linter lint lint-fast ENTRYPOINT=cmd/app/main.go DOC_DIR=./docs @@ -7,6 +8,11 @@ COV_OUT=coverage.out COV_HTML=coverage.html CURRCOVER=github.com/go-park-mail-ru/2023_2_OND_team/internal/pkg/delivery/http/v1 +PROJECT_DIR = $(shell pwd) +PROJECT_BIN = $(PROJECT_DIR)/bin +$(shell [ -f bin ] || mkdir -p $(PROJECT_BIN)) +GOLANGCI_LINT = $(PROJECT_BIN)/golangci-lint + build: go build -o bin/app cmd/app/*.go @@ -19,6 +25,8 @@ build_realtime: build_messenger: go build -o bin/messenger cmd/messenger/*.go +build_all: build build_auth build_realtime build_messenger + run: build ./bin/app @@ -51,3 +59,11 @@ currcover: go test -cover -v -coverprofile=cover.out ${CURRCOVER} go tool cover -html=cover.out -o cover.html +.install-linter: + [ -f $(PROJECT_BIN)/golangci-lint ] || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(PROJECT_BIN) v1.55.2 + +lint: .install-linter + $(GOLANGCI_LINT) run ./... --config=configs/.golangci.yml + +lint-fast: .install-linter + $(GOLANGCI_LINT) run ./... --fast --config=configs/.golangci.yml diff --git a/configs/.golangci.yml b/configs/.golangci.yml new file mode 100644 index 0000000..cac2d04 --- /dev/null +++ b/configs/.golangci.yml @@ -0,0 +1,206 @@ +# This code is licensed under the terms of the MIT license https://opensource.org/license/mit +# Copyright (c) 2021 Marat Reymers + +## Golden config for golangci-lint v1.55.2 +# +# This is the best config for golangci-lint based on my experience and opinion. +# It is very strict, but not extremely strict. +# Feel free to adapt and change it for your needs. + +run: + # Timeout for analysis, e.g. 30s, 5m. + # Default: 1m + timeout: 3m + skip-dirs: + - .. + + + +# This file contains only configs which differ from defaults. +# All possible options can be found here https://github.com/golangci/golangci-lint/blob/master/.golangci.reference.yml +linters-settings: + cyclop: + # The maximal code complexity to report. + # Default: 10 + max-complexity: 30 + # The maximal average package complexity. + # If it's higher than 0.0 (float) the check is enabled + # Default: 0.0 + package-average: 10.0 + + errcheck: + # Report about not checking of errors in type assertions: `a := b.(MyStruct)`. + # Such cases aren't reported by default. + # Default: false + check-type-assertions: true + + exhaustive: + # Program elements to check for exhaustiveness. + # Default: [ switch ] + check: + - switch + - map + + exhaustruct: + # List of regular expressions to exclude struct packages and names from check. + # Default: [] + exclude: + # std libs + - "^net/http.Client$" + - "^net/http.Cookie$" + - "^net/http.Request$" + - "^net/http.Response$" + - "^net/http.Server$" + - "^net/http.Transport$" + - "^net/url.URL$" + - "^os/exec.Cmd$" + - "^reflect.StructField$" + # public libs + - "^github.com/Shopify/sarama.Config$" + - "^github.com/Shopify/sarama.ProducerMessage$" + - "^github.com/mitchellh/mapstructure.DecoderConfig$" + - "^github.com/prometheus/client_golang/.+Opts$" + - "^github.com/spf13/cobra.Command$" + - "^github.com/spf13/cobra.CompletionOptions$" + - "^github.com/stretchr/testify/mock.Mock$" + - "^github.com/testcontainers/testcontainers-go.+Request$" + - "^github.com/testcontainers/testcontainers-go.FromDockerfile$" + - "^golang.org/x/tools/go/analysis.Analyzer$" + - "^google.golang.org/protobuf/.+Options$" + - "^gopkg.in/yaml.v3.Node$" + + funlen: + # Checks the number of lines in a function. + # If lower than 0, disable the check. + # Default: 60 + lines: 100 + # Checks the number of statements in a function. + # If lower than 0, disable the check. + # Default: 40 + statements: 50 + # Ignore comments when counting lines. + # Default false + ignore-comments: true + + gocognit: + # Minimal code complexity to report. + # Default: 30 (but we recommend 10-20) + min-complexity: 20 + + gocritic: + # Settings passed to gocritic. + # The settings key is the name of a supported gocritic checker. + # The list of supported checkers can be find in https://go-critic.github.io/overview. + settings: + captLocal: + # Whether to restrict checker to params only. + # Default: true + paramsOnly: false + underef: + # Whether to skip (*x).method() calls where x is a pointer receiver. + # Default: true + skipRecvDeref: false + + gomnd: + # List of function patterns to exclude from analysis. + # Values always ignored: `time.Date`, + # `strconv.FormatInt`, `strconv.FormatUint`, `strconv.FormatFloat`, + # `strconv.ParseInt`, `strconv.ParseUint`, `strconv.ParseFloat`. + # Default: [] + ignored-functions: + - flag.Arg + - flag.Duration.* + - flag.Float.* + - flag.Int.* + - flag.Uint.* + - os.Chmod + - os.Mkdir.* + - os.OpenFile + - os.WriteFile + - prometheus.ExponentialBuckets.* + - prometheus.LinearBuckets + + gomodguard: + blocked: + # List of blocked modules. + # Default: [] + modules: + - github.com/golang/protobuf: + recommendations: + - google.golang.org/protobuf + reason: "see https://developers.google.com/protocol-buffers/docs/reference/go/faq#modules" + - github.com/satori/go.uuid: + recommendations: + - github.com/google/uuid + reason: "satori's package is not maintained" + - github.com/gofrs/uuid: + recommendations: + - github.com/google/uuid + reason: "gofrs' package is not go module" + + govet: + # Enable all analyzers. + # Default: false + enable-all: true + # Disable analyzers by name. + # Run `go tool vet help` to see all analyzers. + # Default: [] + disable: + - fieldalignment # too strict + # Settings per analyzer. + settings: + shadow: + # Whether to be strict about shadowing; can be noisy. + # Default: false + strict: true + + nakedret: + # Make an issue if func has more lines of code than this setting, and it has naked returns. + # Default: 30 + max-func-lines: 0 + + nolintlint: + # Exclude following linters from requiring an explanation. + # Default: [] + allow-no-explanation: [ funlen, gocognit, lll ] + # Enable to require an explanation of nonzero length after each nolint directive. + # Default: false + require-explanation: true + # Enable to require nolint directives to mention the specific linter being suppressed. + # Default: false + require-specific: true + + rowserrcheck: + # database/sql is always checked + # Default: [] + packages: + - github.com/jmoiron/sqlx + + tenv: + # The option `all` will run against whole test files (`_test.go`) regardless of method/function signatures. + # Otherwise, only methods that take `*testing.T`, `*testing.B`, and `testing.TB` as arguments are checked. + # Default: false + all: true + + +issues: + # Maximum count of issues with the same text. + # Set to 0 to disable. + # Default: 3 + exclude-use-default: true + max-same-issues: 50 + + exclude-rules: + - source: "(noinspection|TODO)" + linters: [ godot ] + - source: "//noinspection" + linters: [ gocritic ] + - path: "_test\\.go" + linters: + - bodyclose + - dupl + - funlen + - goconst + - gosec + - noctx + - wrapcheck \ No newline at end of file From ea75fda9e0de36917c696dc38bb50a1425387fad Mon Sep 17 00:00:00 2001 From: wonderf00l Date: Thu, 7 Dec 2023 21:22:28 +0300 Subject: [PATCH 04/10] TP-c01_ci-cd: add named volumes for postgres and redis, add compose.prod.yml for pulling images while deploying --- deployments/Dockerfile.auth | 6 +++--- deployments/Dockerfile.main | 8 ++++---- deployments/Dockerfile.messenger | 6 +++--- deployments/Dockerfile.realtime | 6 +++--- deployments/compose.prod.yml | 14 ++++++++++++++ deployments/docker-compose.yml | 12 ++++++++---- 6 files changed, 35 insertions(+), 17 deletions(-) create mode 100644 deployments/compose.prod.yml diff --git a/deployments/Dockerfile.auth b/deployments/Dockerfile.auth index 88735ef..74b9443 100644 --- a/deployments/Dockerfile.auth +++ b/deployments/Dockerfile.auth @@ -2,9 +2,9 @@ FROM golang:1.19.13-alpine AS build RUN apk --no-cache add make -WORKDIR /pinspire_auth +WORKDIR /pinspire -COPY go.mod go.sum /pinspire_auth/ +COPY go.mod go.sum /pinspire/ RUN go mod download COPY . . @@ -15,6 +15,6 @@ FROM alpine:latest WORKDIR / -COPY --from=build /pinspire_auth/bin/auth . +COPY --from=build /pinspire/bin/auth . ENTRYPOINT [ "./auth" ] diff --git a/deployments/Dockerfile.main b/deployments/Dockerfile.main index c595010..4d500c8 100644 --- a/deployments/Dockerfile.main +++ b/deployments/Dockerfile.main @@ -2,9 +2,9 @@ FROM golang:1.19.13-alpine AS build RUN apk --no-cache add make -WORKDIR /pinspire_main +WORKDIR /pinspire -COPY go.mod go.sum /pinspire_main/ +COPY go.mod go.sum /pinspire/ RUN go mod download COPY . . @@ -15,7 +15,7 @@ FROM alpine:latest WORKDIR / -COPY --from=build /pinspire_main/bin/app . -COPY --from=build /pinspire_main/configs configs +COPY --from=build /pinspire/bin/app . +COPY --from=build /pinspire/configs configs ENTRYPOINT [ "./app" ] diff --git a/deployments/Dockerfile.messenger b/deployments/Dockerfile.messenger index 6fa332f..cdaec41 100644 --- a/deployments/Dockerfile.messenger +++ b/deployments/Dockerfile.messenger @@ -2,9 +2,9 @@ FROM golang:1.19.13-alpine AS build RUN apk --no-cache add make -WORKDIR /pinspire_messenger +WORKDIR /pinspire -COPY go.mod go.sum /pinspire_messenger/ +COPY go.mod go.sum /pinspire/ RUN go mod download COPY . . @@ -15,6 +15,6 @@ FROM alpine:latest WORKDIR / -COPY --from=build /pinspire_messenger/bin/messenger . +COPY --from=build /pinspire/bin/messenger . ENTRYPOINT [ "./messenger" ] diff --git a/deployments/Dockerfile.realtime b/deployments/Dockerfile.realtime index 7c94295..da5610f 100644 --- a/deployments/Dockerfile.realtime +++ b/deployments/Dockerfile.realtime @@ -2,9 +2,9 @@ FROM golang:1.19.13-alpine AS build RUN apk --no-cache add make -WORKDIR /pinspire_realtime +WORKDIR /pinspire -COPY go.mod go.sum /pinspire_realtime/ +COPY go.mod go.sum /pinspire/ RUN go mod download COPY . . @@ -15,6 +15,6 @@ FROM alpine:latest WORKDIR / -COPY --from=build /pinspire_realtime/bin/realtime . +COPY --from=build /pinspire/bin/realtime . ENTRYPOINT [ "./realtime" ] diff --git a/deployments/compose.prod.yml b/deployments/compose.prod.yml new file mode 100644 index 0000000..5156d5d --- /dev/null +++ b/deployments/compose.prod.yml @@ -0,0 +1,14 @@ +version: '3.8' + +services: + main_service: + image: pinspireapp/main:latest + + auth_service: + image: pinspireapp/auth:latest + + messenger_service: + image: pinspireapp/messenger:latest + + realtime_service: + image: pinspireapp/realtime:latest diff --git a/deployments/docker-compose.yml b/deployments/docker-compose.yml index 094bbab..d36f370 100644 --- a/deployments/docker-compose.yml +++ b/deployments/docker-compose.yml @@ -1,6 +1,5 @@ version: '3.8' - services: postgres: image: postgres:latest @@ -9,6 +8,7 @@ services: - ../.env volumes: - ../db/migrations:/docker-entrypoint-initdb.d + - 'postgres_storage:/var/lib/postgresql/data' ports: - 5432:5432 healthcheck: @@ -19,6 +19,7 @@ services: container_name: pinspireRedis volumes: - ../redis.conf:/usr/local/etc/redis/redis.conf + - 'redis_storage:/data' command: redis-server /usr/local/etc/redis/redis.conf ports: - 6379:6379 @@ -65,7 +66,7 @@ services: redis: condition: 'service_healthy' ports: - - 8086:8086 + - 8186:8086 # - 8101:8085 messenger_service: @@ -81,7 +82,7 @@ services: postgres: condition: 'service_healthy' ports: - - 8096:8096 + - 8196:8096 # - 8102:8095 realtime_service: @@ -96,7 +97,7 @@ services: depends_on: - kafka ports: - - 8091:8091 + - 8191:8091 # - 8103:8090 zookeeper: @@ -171,8 +172,11 @@ services: volumes: + postgres_storage: {} + redis_storage: {} zookeeper_data: driver: local kafka_data: driver: local grafana_storage: {} + From 6c8a215a984a03c080ecdf37e7489420d35afc6d Mon Sep 17 00:00:00 2001 From: wonderf00l Date: Thu, 7 Dec 2023 21:39:15 +0300 Subject: [PATCH 05/10] TP-c01_ci-cd: add workflows description, add configuration file for ansible, add inventory in .gitignore --- .github/workflows/ci.yml | 27 ++++++++++++++++ .github/workflows/deployment.yml | 55 ++++++++++++++++++++++++++++++++ .gitignore | 2 ++ configs/playbook.yml | 16 ++++++++++ 4 files changed, 100 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/deployment.yml create mode 100644 configs/playbook.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..66b52c8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,27 @@ +name: Start pinspire CI + +on: + workflow_dispatch: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Get repository code + uses: actions/checkout@v4 + - name: Test application + run: go test ./... + lint: + runs-on: ubuntu-latest + steps: + - name: Get repository code + uses: actions/checkout@v4 + - name: Lint application + run: make lint + build: + runs-o: ubuntu-latest + steps: + - name: Get repository code + uses: actions/checkout@v4 + - name: Build application + run: make build_all diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml new file mode 100644 index 0000000..43876cf --- /dev/null +++ b/.github/workflows/deployment.yml @@ -0,0 +1,55 @@ +name: Start Pinspire deployment + +on: + workflow_dispatch: + +jobs: + build_images: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + ref: dev3 + - name: Login to DockerHub Registry + run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin + - name: Build docker images of services + run: | + docker build -t pinspireapp/main:latest -f deployments/Dockerfile.main . & + docker build -t pinspireapp/auth:latest -f deployments/Dockerfile.auth . & + docker build -t pinspireapp/realtime:latest -f deployments/Dockerfile.realtime . & + docker build -t pinspireapp/messenger:latest -f deployments/Dockerfile.messenger . & + for p in $(jobs -p); do wait "$p" || { echo "job $p failed" >&2; exit; }; done + - name: Push docker images + run: | + docker push pinspireapp/main:latest & + docker push pinspireapp/auth:latest & + docker push pinspireapp/realtime:latest & + docker push pinspireapp/messenger:latest & + for p in $(jobs -p); do wait "$p" || { echo "job $p failed" >&2; exit; }; done + + deploy: + runs-on: ubuntu-latest + needs: build_images + steps: + - name: fetch changes + uses: appleboy/ssh-action@master + with: + host: pinspire.online + username: ${{ secrets.REMOTE_USERNAME }} + key: ${{ secrets.PRIVATE_KEY }} + script: | + cd ${{ secrets.PINSPIRE_BACKEND_PATH }} + sudo git switch dev3 + sudo git pull + - name: deploy application + uses: appleboy/ssh-action@master + with: + host: pinspire.online + username: ${{ secrets.REMOTE_USERNAME }} + key: ${{ secrets.PRIVATE_KEY }} + script: | + cd ${{ secrets.PINSPIRE_BACKEND_PATH }}/deployments + sudo docker compose down main_service auth_service realtime_service messenger_service + sudo docker rmi pinspireapp/main:latest pinspireapp/auth:latest pinspireapp/realtime:latest pinspireapp/messenger:latest + docker compose --env-file=../.env -f docker-compose.yml -f compose.prod.yml up -d + \ No newline at end of file diff --git a/.gitignore b/.gitignore index 1d94894..e33fae4 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ testdata/ cert/ .env redis.conf +inventory +script* \ No newline at end of file diff --git a/configs/playbook.yml b/configs/playbook.yml new file mode 100644 index 0000000..c3d8bf0 --- /dev/null +++ b/configs/playbook.yml @@ -0,0 +1,16 @@ +- name: "Provide configuration files" + become: yes + hosts: pinspire + tasks: + - name: "Provide .env file" + copy: + src: ../.env + dest: /home/ond_team/go/src/github.com/go-park-mail-ru/ci-cd/.env + - name: "Provide redis config" + copy: + src: ../redis.conf + dest: /home/ond_team/go/src/github.com/go-park-mail-ru/ci-cd/redis.conf + - name: "Provide nginx config" + copy: + src: /etc/nginx/sites-available/pinspire.conf + dest: /etc/nginx/sites-available/pinspire.conf \ No newline at end of file From 7318fe6d77898364d2574bbf0e39d572d19193a0 Mon Sep 17 00:00:00 2001 From: wonderf00l Date: Fri, 8 Dec 2023 13:55:34 +0300 Subject: [PATCH 06/10] TP-c01_ci-cd: add kafka healthcheck --- .github/workflows/deployment.yml | 2 +- deployments/docker-compose.yml | 24 ++++++++++++++++++++---- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 43876cf..1928b3a 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -51,5 +51,5 @@ jobs: cd ${{ secrets.PINSPIRE_BACKEND_PATH }}/deployments sudo docker compose down main_service auth_service realtime_service messenger_service sudo docker rmi pinspireapp/main:latest pinspireapp/auth:latest pinspireapp/realtime:latest pinspireapp/messenger:latest - docker compose --env-file=../.env -f docker-compose.yml -f compose.prod.yml up -d + docker compose -f docker-compose.yml -f compose.prod.yml up -d \ No newline at end of file diff --git a/deployments/docker-compose.yml b/deployments/docker-compose.yml index d36f370..1aa64f2 100644 --- a/deployments/docker-compose.yml +++ b/deployments/docker-compose.yml @@ -12,8 +12,12 @@ services: ports: - 5432:5432 healthcheck: - test: ["CMD", "pg_isready", "-U", "${POSTGRES_USER}"] - + test: ["CMD", "pg_isready"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 15s + redis: image: redis:latest container_name: pinspireRedis @@ -25,6 +29,10 @@ services: - 6379:6379 healthcheck: test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 15s main_service: build: @@ -95,7 +103,8 @@ services: environment: - KAFKA_BROKER_ADDRESS=kafka depends_on: - - kafka + kafka: + condition: 'service_healthy' ports: - 8191:8091 # - 8103:8090 @@ -122,6 +131,14 @@ services: - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092 - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092 + healthcheck: + test: | + curl localhost:9092 + [ $(echo $?) = '52' ] && exit 0 || exit -1 + interval: 5s + timeout: 5s + retries: 10 + start_period: 15s depends_on: - zookeeper @@ -179,4 +196,3 @@ volumes: kafka_data: driver: local grafana_storage: {} - From eaa7ece7ea3861d95c9fbd5328b75be74b0a7e8c Mon Sep 17 00:00:00 2001 From: wonderf00l Date: Fri, 8 Dec 2023 14:15:45 +0300 Subject: [PATCH 07/10] TP-c01_ci-cd: deleted nginx config from compose.yml, changed triggers, changed ports --- .github/workflows/ci.yml | 15 ++++++++++++--- .github/workflows/deployment.yml | 13 ++++++++++--- cmd/realtime/main.go | 2 ++ configs/config.yml | 2 +- configs/playbook.yml | 4 ---- configs/prometheus.yml | 2 +- deployments/docker-compose.yml | 20 +++++--------------- internal/microservices/realtime/node.go | 1 - internal/pkg/delivery/websocket/websocket.go | 2 +- 9 files changed, 32 insertions(+), 29 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66b52c8..4a5ad53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,16 @@ name: Start pinspire CI on: - workflow_dispatch: - + workflow_dispatch: {} + push: + branches: + - TP-c01_ci-cd + - dev3 + - dev4 + pull_request: + types: [opened, edited, reopened] + branches: [main, dev4] + jobs: test: runs-on: ubuntu-latest @@ -10,6 +18,7 @@ jobs: - name: Get repository code uses: actions/checkout@v4 - name: Test application + continue-on-error: true run: go test ./... lint: runs-on: ubuntu-latest @@ -19,7 +28,7 @@ jobs: - name: Lint application run: make lint build: - runs-o: ubuntu-latest + runs-on: ubuntu-latest steps: - name: Get repository code uses: actions/checkout@v4 diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index 1928b3a..bb54dc3 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -1,7 +1,14 @@ name: Start Pinspire deployment on: - workflow_dispatch: + workflow_dispatch: {} + push: + branches: + - TP-c01_ci-cd + - dev4 + pull_request: + types: [opened, edited, reopened] + branches: [main, dev4] jobs: build_images: @@ -39,7 +46,7 @@ jobs: key: ${{ secrets.PRIVATE_KEY }} script: | cd ${{ secrets.PINSPIRE_BACKEND_PATH }} - sudo git switch dev3 + sudo git switch TP-c01_ci-cd sudo git pull - name: deploy application uses: appleboy/ssh-action@master @@ -51,5 +58,5 @@ jobs: cd ${{ secrets.PINSPIRE_BACKEND_PATH }}/deployments sudo docker compose down main_service auth_service realtime_service messenger_service sudo docker rmi pinspireapp/main:latest pinspireapp/auth:latest pinspireapp/realtime:latest pinspireapp/messenger:latest - docker compose -f docker-compose.yml -f compose.prod.yml up -d + sudo docker compose -f docker-compose.yml -f compose.prod.yml up -d \ No newline at end of file diff --git a/cmd/realtime/main.go b/cmd/realtime/main.go index 1022c65..a01f79f 100644 --- a/cmd/realtime/main.go +++ b/cmd/realtime/main.go @@ -11,11 +11,13 @@ import ( grpcMetrics "github.com/go-park-mail-ru/2023_2_OND_team/internal/pkg/metrics/grpc" "github.com/go-park-mail-ru/2023_2_OND_team/internal/pkg/middleware/grpc/interceptor" "github.com/go-park-mail-ru/2023_2_OND_team/pkg/logger" + "github.com/joho/godotenv" ) const _address = "0.0.0.0:8090" func main() { + godotenv.Load() log, err := logger.New() if err != nil { fmt.Println(err) diff --git a/configs/config.yml b/configs/config.yml index 4d1fda6..87b2acd 100644 --- a/configs/config.yml +++ b/configs/config.yml @@ -2,6 +2,6 @@ app: server: host: 0.0.0.0 port: 8080 - https: false + https: true certFile: /home/ond_team/cert/fullchain.pem keyFile: /home/ond_team/cert/privkey.pem diff --git a/configs/playbook.yml b/configs/playbook.yml index c3d8bf0..2daa6b2 100644 --- a/configs/playbook.yml +++ b/configs/playbook.yml @@ -10,7 +10,3 @@ copy: src: ../redis.conf dest: /home/ond_team/go/src/github.com/go-park-mail-ru/ci-cd/redis.conf - - name: "Provide nginx config" - copy: - src: /etc/nginx/sites-available/pinspire.conf - dest: /etc/nginx/sites-available/pinspire.conf \ No newline at end of file diff --git a/configs/prometheus.yml b/configs/prometheus.yml index 57406b3..a65d7b8 100644 --- a/configs/prometheus.yml +++ b/configs/prometheus.yml @@ -5,7 +5,7 @@ global: scrape_configs: - job_name: 'api' static_configs: - - targets: ['main_service:8080'] + - targets: ['main_service:8079'] - job_name: 'auth' static_configs: diff --git a/deployments/docker-compose.yml b/deployments/docker-compose.yml index 1aa64f2..44d21c6 100644 --- a/deployments/docker-compose.yml +++ b/deployments/docker-compose.yml @@ -46,6 +46,9 @@ services: - AUTH_SERVICE_HOST=auth_service - MESSENGER_SERVICE_HOST=messenger_service - REALTIME_SERVICE_HOST=realtime_service + volumes: + - '/home/ond_team/cert/fullchain.pem:/home/ond_team/cert/fullchain.pem:ro' + - '/home/ond_team/cert/privkey.pem:/home/ond_team/cert/privkey.pem:ro' depends_on: postgres: condition: 'service_healthy' @@ -56,7 +59,7 @@ services: realtime_service: condition: 'service_started' ports: - - 8100:8080 + - 8079:8080 auth_service: build: @@ -130,7 +133,7 @@ services: - ALLOW_PLAINTEXT_LISTENER=yes - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181 - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092 - - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://:9092 + - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092 healthcheck: test: | curl localhost:9092 @@ -175,19 +178,6 @@ services: ports: - "9100:9100" - nginx: - image: nginx:latest - container_name: pinspireNginx - volumes: - - '/etc/nginx/sites-available/pinspire.conf:/etc/nginx/conf.d/pinspire.conf:ro' - network_mode: 'host' - depends_on: - main_service: - condition: 'service_started' - realtime_service: - condition: 'service_started' - - volumes: postgres_storage: {} redis_storage: {} diff --git a/internal/microservices/realtime/node.go b/internal/microservices/realtime/node.go index f6f39c7..53d64cc 100644 --- a/internal/microservices/realtime/node.go +++ b/internal/microservices/realtime/node.go @@ -19,7 +19,6 @@ type Node struct { func NewNode() (*Node, error) { node := &Node{} - broker, err := NewKafkaBroker(node, KafkaConfig{ // Addres: []string{"localhost:9092"}, Addres: []string{os.Getenv("KAFKA_BROKER_ADDRESS") + ":" + os.Getenv("KAFKA_BROKER_PORT")}, diff --git a/internal/pkg/delivery/websocket/websocket.go b/internal/pkg/delivery/websocket/websocket.go index c0042de..07635de 100644 --- a/internal/pkg/delivery/websocket/websocket.go +++ b/internal/pkg/delivery/websocket/websocket.go @@ -38,7 +38,7 @@ func SetOriginPatterns(patterns []string) Option { func New(log *log.Logger, mesCase usecase.Usecase, opts ...Option) *HandlerWebSocket { // gRPCConn, err := grpc.Dial("localhost:8090", grpc.WithTransportCredentials(insecure.NewCredentials())) - gRPCConn, err := grpc.Dial(os.Getenv("REALTIME_SERVICE_HOST"+":"+os.Getenv("REALTIME_SERVICE_PORT")), grpc.WithTransportCredentials(insecure.NewCredentials())) + gRPCConn, err := grpc.Dial((os.Getenv("REALTIME_SERVICE_HOST") + ":" + os.Getenv("REALTIME_SERVICE_PORT")), grpc.WithTransportCredentials(insecure.NewCredentials())) if err != nil { log.Error(fmt.Errorf("grpc dial: %w", err).Error()) } From 8d07119e3b71dec010e1f951a51d7c61f0aefff6 Mon Sep 17 00:00:00 2001 From: wonderf00l Date: Sat, 9 Dec 2023 19:55:59 +0300 Subject: [PATCH 08/10] TP-c01_ci-cd: deleted dev3 from triggers --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a5ad53..69188a3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,6 @@ on: push: branches: - TP-c01_ci-cd - - dev3 - dev4 pull_request: types: [opened, edited, reopened] From df49eed9f32451eb0ef66afcfd88e0bdb8232a5f Mon Sep 17 00:00:00 2001 From: wonderf00l Date: Sat, 9 Dec 2023 20:06:28 +0300 Subject: [PATCH 09/10] TP-c01_ci-cd: changed triggers, add step --- .github/workflows/ci.yml | 5 +---- .github/workflows/deployment.yml | 5 ++--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 69188a3..e0f2a3f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,10 +2,7 @@ name: Start pinspire CI on: workflow_dispatch: {} - push: - branches: - - TP-c01_ci-cd - - dev4 + push: {} pull_request: types: [opened, edited, reopened] branches: [main, dev4] diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml index bb54dc3..9036f2b 100644 --- a/.github/workflows/deployment.yml +++ b/.github/workflows/deployment.yml @@ -14,9 +14,8 @@ jobs: build_images: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - ref: dev3 + - name: get repository code + uses: actions/checkout@v4 - name: Login to DockerHub Registry run: echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin - name: Build docker images of services From d63df1b8277f245d81fdd6ea4d7cc4b16e8b5c4b Mon Sep 17 00:00:00 2001 From: Gvidow <96253031+Gvidow@users.noreply.github.com> Date: Tue, 19 Dec 2023 11:41:40 +0300 Subject: [PATCH 10/10] Update websocket.go: delete unused package "os" --- internal/pkg/delivery/websocket/websocket.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/pkg/delivery/websocket/websocket.go b/internal/pkg/delivery/websocket/websocket.go index 5d55376..bd7db73 100644 --- a/internal/pkg/delivery/websocket/websocket.go +++ b/internal/pkg/delivery/websocket/websocket.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "net/http" - "os" "time" ws "nhooyr.io/websocket"