From f30a7680a585d1c085f3466e20e161fcce6cca6b Mon Sep 17 00:00:00 2001 From: Jon Date: Thu, 1 Aug 2024 05:57:58 -0500 Subject: [PATCH 1/4] Do not set repo1-*-region if repo.Storage = "gcs" --- pkg/cluster/k8sres.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 3af900a8..833c01dc 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2961,7 +2961,11 @@ func (c *Cluster) generatePgbackrestConfigmap() (*v1.ConfigMap, error) { } else { config += fmt.Sprintf("\n%s-%s-bucket = %s", repo.Name, repo.Storage, repo.Resource) config += fmt.Sprintf("\n%s-%s-endpoint = %s", repo.Name, repo.Storage, repo.Endpoint) - config += fmt.Sprintf("\n%s-%s-region = %s", repo.Name, repo.Storage, repo.Region) + // https://github.com/cybertec-postgresql/CYBERTEC-pg-container/issues/50 + // P00 WARN: configuration file contains invalid option 'repo1-gcs-region' + if repo.Storage != "gcs" { + config += fmt.Sprintf("\n%s-%s-region = %s", repo.Name, repo.Storage, repo.Region) + } config += fmt.Sprintf("\n%s-type = %s", repo.Name, repo.Storage) } } From cc83f3450f7f6fc482cefc4d58edf7d8a0a20496 Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 1 Aug 2024 18:37:50 +0200 Subject: [PATCH 2/4] modified pgbackrest config and crd for more options regarding gcs and azure --- .../cpo.opensource.cybertec.at/v1/crds.go | 12 ++++++++ .../v1/postgresql_type.go | 18 +++++++----- pkg/cluster/k8sres.go | 29 ++++++++++++++----- 3 files changed, 45 insertions(+), 14 deletions(-) diff --git a/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go b/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go index 73132338..8b315d73 100644 --- a/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go +++ b/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go @@ -1186,6 +1186,18 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "region": { Type: "string", }, + "projectId": { + Type: "string", + }, + "account": { + Type: "string", + }, + "key": { + Type: "string", + }, + "keyType": { + Type: "string", + }, "schedule": { Type: "object", Properties: map[string]apiextv1.JSONSchemaProps{ diff --git a/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go b/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go index 98b9e3ce..4701281a 100644 --- a/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go +++ b/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go @@ -283,13 +283,17 @@ type Pgbackrest struct { } type Repo struct { - Name string `json:"name"` - Storage string `json:"storage"` - Resource string `json:"resource"` - Endpoint string `json:"endpoint"` - Region string `json:"region"` - Schedule map[string]string `json:"schedule"` - Volume `json:"volume,omitempty"` + Name string `json:"name"` + Storage string `json:"storage"` + Resource string `json:"resource"` + Endpoint string `json:"endpoint"` + Region string `json:"region"` + ProjectId string `json:"projectId"` + Account string `json:"account"` + Key string `json:"key"` + KeyType string `json:"keyType"` + Schedule map[string]string `json:"schedule"` + Volume `json:"volume,omitempty"` } type Restore struct { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 833c01dc..0a2f37f7 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2950,7 +2950,8 @@ func (c *Cluster) generatePgbackrestConfigmap() (*v1.ConfigMap, error) { if len(repos) >= 1 { for i, repo := range repos { - if repo.Storage == "pvc" { + switch repo.Storage { + case "pvc": c.logger.Debugf("DEBUG_OUTPUT %s %s", c.clusterName().Name, c.Namespace) config += "\nrepo" + fmt.Sprintf("%d", i+1) + "-host = " + c.clusterName().Name + "-pgbackrest-repo-host-0." + c.serviceName(ClusterPods) + "." + c.Namespace + ".svc." + c.OpConfig.ClusterDomain config += "\nrepo" + fmt.Sprintf("%d", i+1) + "-host-ca-file = /etc/pgbackrest/certs/pgbackrest.ca-roots" @@ -2958,16 +2959,30 @@ func (c *Cluster) generatePgbackrestConfigmap() (*v1.ConfigMap, error) { config += "\nrepo" + fmt.Sprintf("%d", i+1) + "-host-key-file = /etc/pgbackrest/certs/pgbackrest-client.key" config += "\nrepo" + fmt.Sprintf("%d", i+1) + "-host-type = tls" config += "\nrepo" + fmt.Sprintf("%d", i+1) + "-host-user = postgres" - } else { + + case "s3": config += fmt.Sprintf("\n%s-%s-bucket = %s", repo.Name, repo.Storage, repo.Resource) config += fmt.Sprintf("\n%s-%s-endpoint = %s", repo.Name, repo.Storage, repo.Endpoint) - // https://github.com/cybertec-postgresql/CYBERTEC-pg-container/issues/50 - // P00 WARN: configuration file contains invalid option 'repo1-gcs-region' - if repo.Storage != "gcs" { - config += fmt.Sprintf("\n%s-%s-region = %s", repo.Name, repo.Storage, repo.Region) - } + config += fmt.Sprintf("\n%s-%s-region = %s", repo.Name, repo.Storage, repo.Region) + config += fmt.Sprintf("\n%s-type = %s", repo.Name, repo.Storage) + + case "gcs": + config += fmt.Sprintf("\n%s-%s-bucket = %s", repo.Name, repo.Storage, repo.Resource) + config += fmt.Sprintf("\n%s-%s-project-id = %s", repo.Name, repo.Storage, repo.ProjectId) + config += fmt.Sprintf("\n%s-%s-key = %s", repo.Name, repo.Storage, repo.Key) + config += fmt.Sprintf("\n%s-%s-key-typ = %s", repo.Name, repo.Storage, repo.KeyType) config += fmt.Sprintf("\n%s-type = %s", repo.Name, repo.Storage) + + case "azure": + config += fmt.Sprintf("\n%s-%s-container = %s", repo.Name, repo.Storage, repo.Resource) + config += fmt.Sprintf("\n%s-%s-endpoint = %s", repo.Name, repo.Storage, repo.Endpoint) + config += fmt.Sprintf("\n%s-%s-key = %s", repo.Name, repo.Storage, repo.Key) + config += fmt.Sprintf("\n%s-%s-account = %s", repo.Name, repo.Storage, repo.Account) + + config += fmt.Sprintf("\n%s-type = %s", repo.Name, repo.Storage) + default: } + } } } From ad1c83776691efcf327ac42d8cb6d442717961fc Mon Sep 17 00:00:00 2001 From: matthias Date: Thu, 1 Aug 2024 20:58:30 +0200 Subject: [PATCH 3/4] corrected parameters for gcs --- .../cpo.opensource.cybertec.at/v1/crds.go | 3 --- .../v1/postgresql_type.go | 21 +++++++++---------- pkg/cluster/k8sres.go | 14 ++++++------- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go b/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go index 8b315d73..4b07713c 100644 --- a/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go +++ b/pkg/apis/cpo.opensource.cybertec.at/v1/crds.go @@ -1186,9 +1186,6 @@ var PostgresCRDResourceValidation = apiextv1.CustomResourceValidation{ "region": { Type: "string", }, - "projectId": { - Type: "string", - }, "account": { Type: "string", }, diff --git a/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go b/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go index 4701281a..4582d203 100644 --- a/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go +++ b/pkg/apis/cpo.opensource.cybertec.at/v1/postgresql_type.go @@ -283,17 +283,16 @@ type Pgbackrest struct { } type Repo struct { - Name string `json:"name"` - Storage string `json:"storage"` - Resource string `json:"resource"` - Endpoint string `json:"endpoint"` - Region string `json:"region"` - ProjectId string `json:"projectId"` - Account string `json:"account"` - Key string `json:"key"` - KeyType string `json:"keyType"` - Schedule map[string]string `json:"schedule"` - Volume `json:"volume,omitempty"` + Name string `json:"name"` + Storage string `json:"storage"` + Resource string `json:"resource"` + Endpoint string `json:"endpoint"` + Region string `json:"region"` + Account string `json:"account"` + Key string `json:"key"` + KeyType string `json:"keyType"` + Schedule map[string]string `json:"schedule"` + Volume `json:"volume,omitempty"` } type Restore struct { diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 0a2f37f7..e0a6b436 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2935,11 +2935,7 @@ func ensurePath(file string, defaultDir string, defaultFile string) string { func (c *Cluster) generatePgbackrestConfigmap() (*v1.ConfigMap, error) { config := "[db]\npg1-path = /home/postgres/pgdata/pgroot/data\npg1-port = 5432\npg1-socket-path = /var/run/postgresql/\n" config += "\n[global]\nlog-path = /home/postgres/pgdata/pgbackrest/log\nspool-path = /home/postgres/pgdata/pgbackrest/spool-path" - config += "\ntls-server-address=*" - config += "\ntls-server-ca-file = /etc/pgbackrest/certs/pgbackrest.ca-roots" - config += "\ntls-server-cert-file = /etc/pgbackrest/certs/pgbackrest-client.crt" - config += "\ntls-server-key-file = /etc/pgbackrest/certs/pgbackrest-client.key" - config += "\ntls-server-auth = " + c.clientCommonName() + "=*" + if c.Postgresql.Spec.Backup != nil && c.Postgresql.Spec.Backup.Pgbackrest != nil { if global := c.Postgresql.Spec.Backup.Pgbackrest.Global; global != nil { for k, v := range global { @@ -2953,6 +2949,11 @@ func (c *Cluster) generatePgbackrestConfigmap() (*v1.ConfigMap, error) { switch repo.Storage { case "pvc": c.logger.Debugf("DEBUG_OUTPUT %s %s", c.clusterName().Name, c.Namespace) + config += "\ntls-server-address=*" + config += "\ntls-server-ca-file = /etc/pgbackrest/certs/pgbackrest.ca-roots" + config += "\ntls-server-cert-file = /etc/pgbackrest/certs/pgbackrest-client.crt" + config += "\ntls-server-key-file = /etc/pgbackrest/certs/pgbackrest-client.key" + config += "\ntls-server-auth = " + c.clientCommonName() + "=*" config += "\nrepo" + fmt.Sprintf("%d", i+1) + "-host = " + c.clusterName().Name + "-pgbackrest-repo-host-0." + c.serviceName(ClusterPods) + "." + c.Namespace + ".svc." + c.OpConfig.ClusterDomain config += "\nrepo" + fmt.Sprintf("%d", i+1) + "-host-ca-file = /etc/pgbackrest/certs/pgbackrest.ca-roots" config += "\nrepo" + fmt.Sprintf("%d", i+1) + "-host-cert-file = /etc/pgbackrest/certs/pgbackrest-client.crt" @@ -2968,9 +2969,8 @@ func (c *Cluster) generatePgbackrestConfigmap() (*v1.ConfigMap, error) { case "gcs": config += fmt.Sprintf("\n%s-%s-bucket = %s", repo.Name, repo.Storage, repo.Resource) - config += fmt.Sprintf("\n%s-%s-project-id = %s", repo.Name, repo.Storage, repo.ProjectId) config += fmt.Sprintf("\n%s-%s-key = %s", repo.Name, repo.Storage, repo.Key) - config += fmt.Sprintf("\n%s-%s-key-typ = %s", repo.Name, repo.Storage, repo.KeyType) + config += fmt.Sprintf("\n%s-%s-key-type = %s", repo.Name, repo.Storage, repo.KeyType) config += fmt.Sprintf("\n%s-type = %s", repo.Name, repo.Storage) case "azure": From ee20c5cc058aac803642c543f8ad9e30d6948066 Mon Sep 17 00:00:00 2001 From: matthias Date: Fri, 2 Aug 2024 20:19:18 +0200 Subject: [PATCH 4/4] update gcs key --- pkg/cluster/k8sres.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index e0a6b436..dd8c4373 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2969,7 +2969,7 @@ func (c *Cluster) generatePgbackrestConfigmap() (*v1.ConfigMap, error) { case "gcs": config += fmt.Sprintf("\n%s-%s-bucket = %s", repo.Name, repo.Storage, repo.Resource) - config += fmt.Sprintf("\n%s-%s-key = %s", repo.Name, repo.Storage, repo.Key) + config += fmt.Sprintf("\n%s-%s-key = /etc/pgbackrest/conf.d/%s", repo.Name, repo.Storage, repo.Key) config += fmt.Sprintf("\n%s-%s-key-type = %s", repo.Name, repo.Storage, repo.KeyType) config += fmt.Sprintf("\n%s-type = %s", repo.Name, repo.Storage)