diff --git a/.gitignore b/.gitignore index bb269abd8..a7b708488 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ .DS_Store +.envrc .gobuild bin logs diff --git a/CHANGELOG.md b/CHANGELOG.md index 69f10c8f0..9f32826ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,7 @@ # Change Log -## [Unreleased](https://github.com/arangodb/kube-arangodb/tree/HEAD) - -[Full Changelog](https://github.com/arangodb/kube-arangodb/compare/0.0.1...HEAD) +## [0.1.0](https://github.com/arangodb/kube-arangodb/tree/0.1.0) (2018-04-06) +[Full Changelog](https://github.com/arangodb/kube-arangodb/compare/0.0.1...0.1.0) **Closed issues:** @@ -12,9 +11,38 @@ **Merged pull requests:** +- Fixed down/upgrading resilient single deployments. [\#123](https://github.com/arangodb/kube-arangodb/pull/123) +- Various docs improvements & fixes [\#122](https://github.com/arangodb/kube-arangodb/pull/122) +- Added tests for query cursors on various deployments. [\#121](https://github.com/arangodb/kube-arangodb/pull/121) +- Remove upgrade resilient single 3.2 -\> 3.3 test. [\#120](https://github.com/arangodb/kube-arangodb/pull/120) +- Various renamings in tests such that common names are used. [\#119](https://github.com/arangodb/kube-arangodb/pull/119) +- Added envvar \(CLEANUPDEPLOYMENTS\) to cleanup failed tests. [\#118](https://github.com/arangodb/kube-arangodb/pull/118) +- Added test that removes PV, PVC & Pod or dbserver. \[ci VERBOSE=1\] \[ci LONG=1\] \[ci TESTOPTIONS="-test.run ^TestResiliencePVDBServer$"\] [\#117](https://github.com/arangodb/kube-arangodb/pull/117) +- Fixed expected value for ENGINE file in init container of dbserver. [\#116](https://github.com/arangodb/kube-arangodb/pull/116) +- Improved liveness detection [\#115](https://github.com/arangodb/kube-arangodb/pull/115) +- Run chaos-monkey in go-routine to avoid blocking the operator [\#114](https://github.com/arangodb/kube-arangodb/pull/114) +- Added examples for exposing metrics to Prometheus [\#113](https://github.com/arangodb/kube-arangodb/pull/113) +- Replace HTTP server with HTTPS server [\#112](https://github.com/arangodb/kube-arangodb/pull/112) +- Disabled colorizing logs [\#111](https://github.com/arangodb/kube-arangodb/pull/111) +- Safe resource watcher [\#110](https://github.com/arangodb/kube-arangodb/pull/110) +- Archive log files [\#109](https://github.com/arangodb/kube-arangodb/pull/109) +- Doc - Follow file name conventions of main docs, move to Tutorials [\#108](https://github.com/arangodb/kube-arangodb/pull/108) +- Quickly fail when deployment no longer exists [\#107](https://github.com/arangodb/kube-arangodb/pull/107) +- BREAKING CHANGE: Renamed all enum values to title case [\#104](https://github.com/arangodb/kube-arangodb/pull/104) +- Changed TLSSpec.TTL to new string based `Duration` type [\#103](https://github.com/arangodb/kube-arangodb/pull/103) +- Added automatic renewal of TLS server certificates [\#102](https://github.com/arangodb/kube-arangodb/pull/102) +- Adding GettingStarted page and structuring docs for website [\#101](https://github.com/arangodb/kube-arangodb/pull/101) +- Added LivenessProbe & Readiness probe [\#100](https://github.com/arangodb/kube-arangodb/pull/100) +- Patch latest version number in README [\#99](https://github.com/arangodb/kube-arangodb/pull/99) +- Adding CHANGELOG.md generation [\#98](https://github.com/arangodb/kube-arangodb/pull/98) +- Adding chaos-monkey for deployments [\#96](https://github.com/arangodb/kube-arangodb/pull/96) +- Check contents of persisted volume when dbserver is restarting [\#95](https://github.com/arangodb/kube-arangodb/pull/95) - Added helper to prepull arangodb \(enterprise\) image. This allows the normal tests to have decent timeouts while prevent a timeout caused by a long during image pull. [\#94](https://github.com/arangodb/kube-arangodb/pull/94) +- Fixing PV cleanup [\#93](https://github.com/arangodb/kube-arangodb/pull/93) +- Check member failure [\#92](https://github.com/arangodb/kube-arangodb/pull/92) - Tracking recent pod terminations [\#91](https://github.com/arangodb/kube-arangodb/pull/91) - Enable LONG on kube-arangodb-long test [\#90](https://github.com/arangodb/kube-arangodb/pull/90) +- Tests/multi deployment [\#89](https://github.com/arangodb/kube-arangodb/pull/89) - Tests/modes [\#88](https://github.com/arangodb/kube-arangodb/pull/88) - increase timeout for long running tests [\#87](https://github.com/arangodb/kube-arangodb/pull/87) - fix rocksdb\_encryption\_test [\#86](https://github.com/arangodb/kube-arangodb/pull/86) diff --git a/Jenkinsfile.groovy b/Jenkinsfile.groovy index 4407bbde3..22d58beb4 100644 --- a/Jenkinsfile.groovy +++ b/Jenkinsfile.groovy @@ -58,16 +58,19 @@ def kubeConfigRoot = "/home/jenkins/.kube" def buildBuildSteps(Map myParams) { return { timestamps { - withEnv([ - "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", - "IMAGETAG=jenkins-test", - "LONG=${myParams.LONG ? 1 : 0}", - "TESTOPTIONS=${myParams.TESTOPTIONS}", - ]) { - sh "make" - sh "make run-unit-tests" - sh "make docker-test" + timeout(time: 15) { + withEnv([ + "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", + "IMAGETAG=jenkins-test", + "LONG=${myParams.LONG ? 1 : 0}", + "TESTOPTIONS=${myParams.TESTOPTIONS}", + ]) { + sh "make clean" + sh "make" + sh "make run-unit-tests" + sh "make docker-test" + } } } } @@ -76,18 +79,20 @@ def buildBuildSteps(Map myParams) { def buildTestSteps(Map myParams, String kubeConfigRoot, String kubeconfig) { return { timestamps { - withCredentials([string(credentialsId: 'ENTERPRISEIMAGE', variable: 'DEFAULTENTERPRISEIMAGE')]) { - withEnv([ - "CLEANDEPLOYMENTS=1", - "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", - "ENTERPRISEIMAGE=${myParams.ENTERPRISEIMAGE}", - "IMAGETAG=jenkins-test", - "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", - "LONG=${myParams.LONG ? 1 : 0}", - "TESTOPTIONS=${myParams.TESTOPTIONS}", - ]) { - sh "make run-tests" + timeout(time: myParams.LONG ? 180 : 30) { + withCredentials([string(credentialsId: 'ENTERPRISEIMAGE', variable: 'DEFAULTENTERPRISEIMAGE')]) { + withEnv([ + "CLEANDEPLOYMENTS=1", + "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", + "ENTERPRISEIMAGE=${myParams.ENTERPRISEIMAGE}", + "IMAGETAG=jenkins-test", + "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", + "LONG=${myParams.LONG ? 1 : 0}", + "TESTOPTIONS=${myParams.TESTOPTIONS}", + ]) { + sh "make run-tests" + } } } } @@ -97,14 +102,16 @@ def buildTestSteps(Map myParams, String kubeConfigRoot, String kubeconfig) { def buildCleanupSteps(Map myParams, String kubeConfigRoot, String kubeconfig) { return { timestamps { - withEnv([ - "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", - "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", - "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", - ]) { - sh "./scripts/collect_logs.sh ${env.DEPLOYMENTNAMESPACE} ${kubeconfig}" - archive includes: 'logs/*' - sh "make cleanup-tests" + timeout(time: 15) { + withEnv([ + "DEPLOYMENTNAMESPACE=${myParams.TESTNAMESPACE}-${env.GIT_COMMIT}", + "DOCKERNAMESPACE=${myParams.DOCKERNAMESPACE}", + "KUBECONFIG=${kubeConfigRoot}/${kubeconfig}", + ]) { + sh "./scripts/collect_logs.sh ${env.DEPLOYMENTNAMESPACE} ${kubeconfig}" + archive includes: 'logs/*' + sh "make cleanup-tests" + } } } } diff --git a/Makefile b/Makefile index 345f19cdd..dbcb53c82 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,7 @@ DOCKERCLI := $(shell which docker) GOBUILDDIR := $(SCRIPTDIR)/.gobuild SRCDIR := $(SCRIPTDIR) +CACHEVOL := $(PROJECT)-gocache BINDIR := $(ROOTDIR)/bin VENDORDIR := $(ROOTDIR)/deps @@ -26,6 +27,7 @@ PULSAR := $(GOBUILDDIR)/bin/pulsar$(shell go env GOEXE) DOCKERFILE := Dockerfile DOCKERTESTFILE := Dockerfile.test +DOCKERDURATIONTESTFILE := tests/duration/Dockerfile ifndef LOCALONLY PUSHIMAGES := 1 @@ -49,6 +51,7 @@ ifndef MANIFESTSUFFIX endif endif MANIFESTPATHDEPLOYMENT := manifests/arango-deployment$(MANIFESTSUFFIX).yaml +MANIFESTPATHDEPLOYMENTREPLICATION := manifests/arango-deployment-replication$(MANIFESTSUFFIX).yaml MANIFESTPATHSTORAGE := manifests/arango-storage$(MANIFESTSUFFIX).yaml MANIFESTPATHTEST := manifests/arango-test$(MANIFESTSUFFIX).yaml ifndef DEPLOYMENTNAMESPACE @@ -61,6 +64,9 @@ endif ifndef TESTIMAGE TESTIMAGE := $(DOCKERNAMESPACE)/kube-arangodb-test$(IMAGESUFFIX) endif +ifndef DURATIONTESTIMAGE + DURATIONTESTIMAGE := $(DOCKERNAMESPACE)/kube-arangodb-durationtest$(IMAGESUFFIX) +endif ifndef ENTERPRISEIMAGE ENTERPRISEIMAGE := $(DEFAULTENTERPRISEIMAGE) endif @@ -73,6 +79,8 @@ BINNAME := $(PROJECT) BIN := $(BINDIR)/$(BINNAME) TESTBINNAME := $(PROJECT)_test TESTBIN := $(BINDIR)/$(TESTBINNAME) +DURATIONTESTBINNAME := $(PROJECT)_duration_test +DURATIONTESTBIN := $(BINDIR)/$(DURATIONTESTBINNAME) RELEASE := $(GOBUILDDIR)/bin/release GHRELEASE := $(GOBUILDDIR)/bin/github-release @@ -125,6 +133,9 @@ $(GOBUILDDIR): @rm -f $(REPODIR) && ln -sf ../../../.. $(REPODIR) GOPATH=$(GOBUILDDIR) $(PULSAR) go flatten -V $(VENDORDIR) +$(CACHEVOL): + @docker volume create $(CACHEVOL) + .PHONY: update-vendor update-vendor: @mkdir -p $(GOBUILDDIR) @@ -168,7 +179,7 @@ update-generated: $(GOBUILDDIR) "all" \ "github.com/arangodb/kube-arangodb/pkg/generated" \ "github.com/arangodb/kube-arangodb/pkg/apis" \ - "deployment:v1alpha storage:v1alpha" \ + "deployment:v1alpha replication:v1alpha storage:v1alpha" \ --go-header-file "./tools/codegen/boilerplate.go.txt" \ $(VERIFYARGS) @@ -176,11 +187,13 @@ update-generated: $(GOBUILDDIR) verify-generated: @${MAKE} -B -s VERIFYARGS=--verify-only update-generated -$(BIN): $(GOBUILDDIR) $(SOURCES) +$(BIN): $(GOBUILDDIR) $(CACHEVOL) $(SOURCES) @mkdir -p $(BINDIR) docker run \ --rm \ -v $(SRCDIR):/usr/code \ + -v $(CACHEVOL):/usr/gocache \ + -e GOCACHE=/usr/gocache \ -e GOPATH=/usr/code/.gobuild \ -e GOOS=linux \ -e GOARCH=amd64 \ @@ -214,6 +227,8 @@ run-unit-tests: $(GOBUILDDIR) $(SOURCES) docker run \ --rm \ -v $(SRCDIR):/usr/code \ + -v $(CACHEVOL):/usr/gocache \ + -e GOCACHE=/usr/gocache \ -e GOPATH=/usr/code/.gobuild \ -e GOOS=linux \ -e GOARCH=amd64 \ @@ -222,9 +237,11 @@ run-unit-tests: $(GOBUILDDIR) $(SOURCES) golang:$(GOVERSION) \ go test $(TESTVERBOSEOPTIONS) \ $(REPOPATH)/pkg/apis/deployment/v1alpha \ + $(REPOPATH)/pkg/apis/replication/v1alpha \ $(REPOPATH)/pkg/apis/storage/v1alpha \ $(REPOPATH)/pkg/deployment/reconcile \ $(REPOPATH)/pkg/deployment/resources \ + $(REPOPATH)/pkg/storage \ $(REPOPATH)/pkg/util/k8sutil \ $(REPOPATH)/pkg/util/k8sutil/test \ $(REPOPATH)/pkg/util/probe \ @@ -235,6 +252,8 @@ $(TESTBIN): $(GOBUILDDIR) $(SOURCES) docker run \ --rm \ -v $(SRCDIR):/usr/code \ + -v $(CACHEVOL):/usr/gocache \ + -e GOCACHE=/usr/gocache \ -e GOPATH=/usr/code/.gobuild \ -e GOOS=linux \ -e GOARCH=amd64 \ @@ -260,10 +279,33 @@ endif kubectl apply -f manifests/crd.yaml kubectl apply -f $(MANIFESTPATHSTORAGE) kubectl apply -f $(MANIFESTPATHDEPLOYMENT) + kubectl apply -f $(MANIFESTPATHDEPLOYMENTREPLICATION) kubectl apply -f $(MANIFESTPATHTEST) $(ROOTDIR)/scripts/kube_create_storage.sh $(DEPLOYMENTNAMESPACE) $(ROOTDIR)/scripts/kube_run_tests.sh $(DEPLOYMENTNAMESPACE) $(TESTIMAGE) "$(ENTERPRISEIMAGE)" $(TESTTIMEOUT) $(TESTLENGTHOPTIONS) +$(DURATIONTESTBIN): $(GOBUILDDIR) $(SOURCES) + @mkdir -p $(BINDIR) + docker run \ + --rm \ + -v $(SRCDIR):/usr/code \ + -v $(CACHEVOL):/usr/gocache \ + -e GOCACHE=/usr/gocache \ + -e GOPATH=/usr/code/.gobuild \ + -e GOOS=linux \ + -e GOARCH=amd64 \ + -e CGO_ENABLED=0 \ + -w /usr/code/ \ + golang:$(GOVERSION) \ + go build -installsuffix cgo -ldflags "-X main.projectVersion=$(VERSION) -X main.projectBuild=$(COMMIT)" -o /usr/code/bin/$(DURATIONTESTBINNAME) $(REPOPATH)/tests/duration + +.PHONY: docker-duration-test +docker-duration-test: $(DURATIONTESTBIN) + docker build --quiet -f $(DOCKERDURATIONTESTFILE) -t $(DURATIONTESTIMAGE) . +ifdef PUSHIMAGES + docker push $(DURATIONTESTIMAGE) +endif + .PHONY: cleanup-tests cleanup-tests: ifneq ($(DEPLOYMENTNAMESPACE), default) @@ -335,6 +377,7 @@ minikube-start: delete-operator: kubectl delete -f $(MANIFESTPATHTEST) --ignore-not-found kubectl delete -f $(MANIFESTPATHDEPLOYMENT) --ignore-not-found + kubectl delete -f $(MANIFESTPATHDEPLOYMENTREPLICATION) --ignore-not-found kubectl delete -f $(MANIFESTPATHSTORAGE) --ignore-not-found .PHONY: redeploy-operator @@ -342,5 +385,6 @@ redeploy-operator: delete-operator manifests kubectl apply -f manifests/crd.yaml kubectl apply -f $(MANIFESTPATHSTORAGE) kubectl apply -f $(MANIFESTPATHDEPLOYMENT) + kubectl apply -f $(MANIFESTPATHDEPLOYMENTREPLICATION) kubectl apply -f $(MANIFESTPATHTEST) kubectl get pods diff --git a/README.md b/README.md index 9b7bdb08c..86e920227 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ read the [tutorial](./docs/Manual/Tutorials/Kubernetes/README.md). The ArangoDB Kubernetes Operator is still in **heavy development**. -Running ArangoDB deployments (single, resilient-single or cluster) +Running ArangoDB deployments (single, active-failover or cluster) is becoming reasonably stable, but you should **not yet use it for production environments**. @@ -23,10 +23,10 @@ support. That is still completely missing. ## Installation of latest release ```bash -kubectl apply -f https://raw.githubusercontent.com/arangodb/kube-arangodb/0.1.0/manifests/crd.yaml -kubectl apply -f https://raw.githubusercontent.com/arangodb/kube-arangodb/0.1.0/manifests/arango-deployment.yaml +kubectl apply -f https://raw.githubusercontent.com/arangodb/kube-arangodb/0.2.0/manifests/crd.yaml +kubectl apply -f https://raw.githubusercontent.com/arangodb/kube-arangodb/0.2.0/manifests/arango-deployment.yaml # To use `ArangoLocalStorage`, also run -kubectl apply -f https://raw.githubusercontent.com/arangodb/kube-arangodb/0.1.0/manifests/arango-storage.yaml +kubectl apply -f https://raw.githubusercontent.com/arangodb/kube-arangodb/0.2.0/manifests/arango-storage.yaml ``` ## Building diff --git a/VERSION b/VERSION index 6c6aa7cb0..341cf11fa 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.0 \ No newline at end of file +0.2.0 \ No newline at end of file diff --git a/deps/github.com/arangodb-helper/go-certificates/cli/certificates.go b/deps/github.com/arangodb-helper/go-certificates/cli/certificates.go index f749808e8..9bcd52e0e 100644 --- a/deps/github.com/arangodb-helper/go-certificates/cli/certificates.go +++ b/deps/github.com/arangodb-helper/go-certificates/cli/certificates.go @@ -43,6 +43,9 @@ const ( // Client authentication valid for defaults defaultClientAuthValidFor = time.Hour * 24 * 365 * 1 // 1 years defaultClientAuthCAValidFor = time.Hour * 24 * 365 * 15 // 15 years + // TLS curve defaults + defaultTLSCurve = "P256" + defaultClientAuthCurve = "P521" ) var ( @@ -148,11 +151,11 @@ type createCAOptions struct { ecdsaCurve string } -func (o *createCAOptions) ConfigureFlags(f *pflag.FlagSet, defaultFName string, defaultValidFor time.Duration) { +func (o *createCAOptions) ConfigureFlags(f *pflag.FlagSet, defaultFName string, defaultValidFor time.Duration, defaultCurve string) { f.StringVar(&o.certFile, "cert", defaultFName+".crt", "Filename of the generated CA certificate") f.StringVar(&o.keyFile, "key", defaultFName+".key", "Filename of the generated CA private key") f.DurationVar(&o.validFor, "validfor", defaultValidFor, "Lifetime of the certificate until expiration") - f.StringVar(&o.ecdsaCurve, "curve", "P521", "ECDSA curve used for private key") + f.StringVar(&o.ecdsaCurve, "curve", defaultCurve, "ECDSA curve used for private key") } func (o *createCAOptions) CreateCA() { @@ -184,13 +187,13 @@ type createCertificateBaseOptions struct { ecdsaCurve string } -func (o *createCertificateBaseOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration) { +func (o *createCertificateBaseOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration, defaultCurve string) { f.StringVar(&o.caCertFile, "cacert", defaultCAFName+".crt", "File containing TLS CA certificate") f.StringVar(&o.caKeyFile, "cakey", defaultCAFName+".key", "File containing TLS CA private key") f.StringSliceVar(&o.hosts, "host", nil, "Host name to include in the certificate") f.StringSliceVar(&o.emailAddresses, "email", nil, "Email address to include in the certificate") f.DurationVar(&o.validFor, "validfor", defaultValidFor, "Lifetime of the certificate until expiration") - f.StringVar(&o.ecdsaCurve, "curve", "P521", "ECDSA curve used for private key") + f.StringVar(&o.ecdsaCurve, "curve", defaultCurve, "ECDSA curve used for private key") } // Create a certificate from given options. @@ -206,8 +209,8 @@ func (o *createCertificateBaseOptions) CreateCertificate(isClientAuth bool) (str // Create certificate options := certificates.CreateCertificateOptions{ - Hosts: o.hosts, - EmailAddresses: o.emailAddresses, + Hosts: removeEmptyStrings(o.hosts), + EmailAddresses: removeEmptyStrings(o.emailAddresses), ValidFor: o.validFor, ECDSACurve: o.ecdsaCurve, IsClientAuth: isClientAuth, @@ -225,8 +228,8 @@ type createKeyFileOptions struct { keyFile string } -func (o *createKeyFileOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration) { - o.createCertificateBaseOptions.ConfigureFlags(f, defaultCAFName, defaultFName, defaultValidFor) +func (o *createKeyFileOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration, defaultCurve string) { + o.createCertificateBaseOptions.ConfigureFlags(f, defaultCAFName, defaultFName, defaultValidFor, defaultCurve) f.StringVar(&o.keyFile, "keyfile", defaultFName+".keyfile", "Filename of keyfile to generate") } @@ -247,8 +250,8 @@ type createCertificateOptions struct { keyFile string } -func (o *createCertificateOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration) { - o.createCertificateBaseOptions.ConfigureFlags(f, defaultCAFName, defaultFName, defaultValidFor) +func (o *createCertificateOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration, defaultCurve string) { + o.createCertificateBaseOptions.ConfigureFlags(f, defaultCAFName, defaultFName, defaultValidFor, defaultCurve) f.StringVar(&o.certFile, "cert", defaultFName+".crt", "Filename of the generated certificate") f.StringVar(&o.keyFile, "key", defaultFName+".key", "Filename of the generated private key") } @@ -272,8 +275,8 @@ type createKeystoreOptions struct { alias string } -func (o *createKeystoreOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration) { - o.createCertificateBaseOptions.ConfigureFlags(f, defaultCAFName, defaultFName, defaultValidFor) +func (o *createKeystoreOptions) ConfigureFlags(f *pflag.FlagSet, defaultCAFName, defaultFName string, defaultValidFor time.Duration, defaultCurve string) { + o.createCertificateBaseOptions.ConfigureFlags(f, defaultCAFName, defaultFName, defaultValidFor, defaultCurve) f.StringVar(&o.keystoreFile, "keystore", defaultFName+".jks", "Filename of the generated keystore") f.StringVar(&o.keystorePassword, "keystore-password", "", "Password of the generated keystore") f.StringVar(&o.alias, "alias", "", "Aliases use to store the certificate under in the keystore") @@ -317,12 +320,12 @@ func AddCommands(cmd *cobra.Command, logFatalFunc func(error, string), showUsage cmdCreateClientAuth.AddCommand(cmdCreateClientAuthKeyFile) createOptions.jwtsecret.ConfigureFlags(cmdCreateJWTSecret.Flags()) - createOptions.tls.ca.ConfigureFlags(cmdCreateTLSCA.Flags(), "tls-ca", defaultTLSCAValidFor) - createOptions.tls.keyFile.ConfigureFlags(cmdCreateTLSKeyFile.Flags(), "tls-ca", "tls", defaultTLSValidFor) - createOptions.tls.certificate.ConfigureFlags(cmdCreateTLSCertificate.Flags(), "tls-ca", "tls", defaultTLSValidFor) - createOptions.tls.keystore.ConfigureFlags(cmdCreateTLSKeystore.Flags(), "tls-ca", "tls", defaultTLSValidFor) - createOptions.clientAuth.ca.ConfigureFlags(cmdCreateClientAuthCA.Flags(), "client-auth-ca", defaultClientAuthCAValidFor) - createOptions.clientAuth.keyFile.ConfigureFlags(cmdCreateClientAuthKeyFile.Flags(), "client-auth-ca", "client-auth", defaultClientAuthValidFor) + createOptions.tls.ca.ConfigureFlags(cmdCreateTLSCA.Flags(), "tls-ca", defaultTLSCAValidFor, defaultTLSCurve) + createOptions.tls.keyFile.ConfigureFlags(cmdCreateTLSKeyFile.Flags(), "tls-ca", "tls", defaultTLSValidFor, defaultTLSCurve) + createOptions.tls.certificate.ConfigureFlags(cmdCreateTLSCertificate.Flags(), "tls-ca", "tls", defaultTLSValidFor, defaultTLSCurve) + createOptions.tls.keystore.ConfigureFlags(cmdCreateTLSKeystore.Flags(), "tls-ca", "tls", defaultTLSValidFor, defaultTLSCurve) + createOptions.clientAuth.ca.ConfigureFlags(cmdCreateClientAuthCA.Flags(), "client-auth-ca", defaultClientAuthCAValidFor, defaultClientAuthCurve) + createOptions.clientAuth.keyFile.ConfigureFlags(cmdCreateClientAuthKeyFile.Flags(), "client-auth-ca", "client-auth", defaultClientAuthValidFor, defaultClientAuthCurve) } // Cobra run function using the usage of the given command @@ -401,3 +404,14 @@ func mustReadFile(filename string, flagName string) string { } return string(content) } + +// removeEmptyStrings returns the given slice without all empty entries removed. +func removeEmptyStrings(slice []string) []string { + result := make([]string, 0, len(slice)) + for _, x := range slice { + if x != "" { + result = append(result, x) + } + } + return result +} diff --git a/deps/github.com/arangodb-helper/go-certificates/create.go b/deps/github.com/arangodb-helper/go-certificates/create.go index 0cd489bee..d8bdf82a8 100644 --- a/deps/github.com/arangodb-helper/go-certificates/create.go +++ b/deps/github.com/arangodb-helper/go-certificates/create.go @@ -44,6 +44,7 @@ const ( ) type CreateCertificateOptions struct { + Subject *pkix.Name // If set, this name is used for the subject of the certificate and CommonName is ignored. CommonName string // Common name set in the certificate. If not specified, defaults to first email address, then first host and if all not set 'ArangoDB'. Hosts []string // Comma-separated hostnames and IPs to generate a certificate for EmailAddresses []string // List of email address to include in the certificate as alternative name @@ -101,14 +102,18 @@ func CreateCertificate(options CreateCertificateOptions, ca *CA) (string, string } else if len(options.Hosts) > 0 { commonName = options.Hosts[0] } + var subject pkix.Name + if options.Subject != nil { + subject = *options.Subject + } else { + subject.CommonName = commonName + subject.Organization = []string{"ArangoDB"} + } template := x509.Certificate{ SerialNumber: serialNumber, - Subject: pkix.Name{ - CommonName: commonName, - Organization: []string{"ArangoDB"}, - }, - NotBefore: notBefore, - NotAfter: notAfter, + Subject: subject, + NotBefore: notBefore, + NotAfter: notAfter, KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageAny}, diff --git a/deps/github.com/arangodb-helper/go-certificates/keyfile.go b/deps/github.com/arangodb-helper/go-certificates/keyfile.go index 1f57e2c91..376d080db 100644 --- a/deps/github.com/arangodb-helper/go-certificates/keyfile.go +++ b/deps/github.com/arangodb-helper/go-certificates/keyfile.go @@ -38,14 +38,13 @@ import ( "strings" ) -// LoadKeyFile loads a SSL keyfile formatted for the arangod server. -func LoadKeyFile(keyFile string) (tls.Certificate, error) { - raw, err := ioutil.ReadFile(keyFile) - if err != nil { - return tls.Certificate{}, maskAny(err) - } +// Keyfile contains 1 or more certificates and a private key. +type Keyfile tls.Certificate - result := tls.Certificate{} +// NewKeyfile creates a keyfile from given content. +func NewKeyfile(content string) (Keyfile, error) { + raw := []byte(content) + result := Keyfile{} for { var derBlock *pem.Block derBlock, raw = pem.Decode(raw) @@ -56,22 +55,74 @@ func LoadKeyFile(keyFile string) (tls.Certificate, error) { result.Certificate = append(result.Certificate, derBlock.Bytes) } else if derBlock.Type == "PRIVATE KEY" || strings.HasSuffix(derBlock.Type, " PRIVATE KEY") { if result.PrivateKey == nil { + var err error result.PrivateKey, err = parsePrivateKey(derBlock.Bytes) if err != nil { - return tls.Certificate{}, maskAny(err) + return Keyfile{}, maskAny(err) } } } } + return result, nil +} - if len(result.Certificate) == 0 { - return tls.Certificate{}, maskAny(fmt.Errorf("No certificates found in %s", keyFile)) +// Validate the contents of the keyfile +func (kf Keyfile) Validate() error { + if len(kf.Certificate) == 0 { + return maskAny(fmt.Errorf("No certificates found in keyfile")) } - if result.PrivateKey == nil { - return tls.Certificate{}, maskAny(fmt.Errorf("No private key found in %s", keyFile)) + if kf.PrivateKey == nil { + return maskAny(fmt.Errorf("No private key found in keyfile")) } - return result, nil + return nil +} + +// EncodeCACertificates extracts the CA certificate(s) from the given keyfile (if any). +func (kf Keyfile) EncodeCACertificates() (string, error) { + buf := &bytes.Buffer{} + for _, derBytes := range kf.Certificate { + c, err := x509.ParseCertificate(derBytes) + if err != nil { + return "", maskAny(err) + } + if c.IsCA { + pem.Encode(buf, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + } + } + + return buf.String(), nil +} + +// EncodeCertificates extracts all certificates from the given keyfile and encodes them as PEM blocks. +func (kf Keyfile) EncodeCertificates() string { + buf := &bytes.Buffer{} + for _, derBytes := range kf.Certificate { + pem.Encode(buf, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + } + + return buf.String() +} + +// EncodePrivateKey extract the private key from the given keyfile and encodes is as PEM block. +func (kf Keyfile) EncodePrivateKey() string { + buf := &bytes.Buffer{} + pem.Encode(buf, pemBlockForKey(kf.PrivateKey)) + return buf.String() +} + +// LoadKeyFile loads a SSL keyfile formatted for the arangod server. +func LoadKeyFile(keyFile string) (tls.Certificate, error) { + raw, err := ioutil.ReadFile(keyFile) + if err != nil { + return tls.Certificate{}, maskAny(err) + } + + kf, err := NewKeyfile(string(raw)) + if err != nil { + return tls.Certificate{}, maskAny(err) + } + return tls.Certificate(kf), nil } // ExtractCACertificateFromKeyFile loads a SSL keyfile formatted for the arangod server and diff --git a/deps/github.com/arangodb/arangosync/client/api.go b/deps/github.com/arangodb/arangosync/client/api.go new file mode 100644 index 000000000..a00a17ab0 --- /dev/null +++ b/deps/github.com/arangodb/arangosync/client/api.go @@ -0,0 +1,333 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package client + +import ( + "context" + "time" + + "github.com/arangodb/arangosync/tasks" + "github.com/pkg/errors" +) + +// API of a sync master/worker +type API interface { + // Close this client + Close() error + // Get the version of the sync master/worker + Version(ctx context.Context) (VersionInfo, error) + // Get the role of the sync master/worker + Role(ctx context.Context) (Role, error) + // Health performs a quick health check. + // Returns an error when anything is wrong. If so, check Status. + Health(ctx context.Context) error + // Returns the master API (only valid when Role returns master) + Master() MasterAPI + // Returns the worker API (only valid when Role returns worker) + Worker() WorkerAPI + + // Set the ID of the client that is making requests. + SetClientID(id string) + // SetShared marks the client as shared. + // Closing a shared client will not close all idle connections. + SetShared() + // SynchronizeMasterEndpoints ensures that the client is using all known master + // endpoints. + // Do not use for connections to workers. + // Returns true when endpoints have changed. + SynchronizeMasterEndpoints(ctx context.Context) (bool, error) + // Endpoint returns the currently used endpoint for this client. + Endpoint() Endpoint +} + +const ( + // ClientIDHeaderKey is the name of a request header containing the ID that is + // making the request. + ClientIDHeaderKey = "X-ArangoSync-Client-ID" +) + +// MasterAPI contains API of sync master +type MasterAPI interface { + // Gets the current status of synchronization towards the local cluster. + Status(ctx context.Context) (SyncInfo, error) + // Configure the master to synchronize the local cluster from a given remote cluster. + Synchronize(ctx context.Context, input SynchronizationRequest) error + // Configure the master to stop & completely cancel the current synchronization of the + // local cluster from a remote cluster. + // Errors: + // - RequestTimeoutError when input.WaitTimeout is non-zero and the inactive stage is not reached in time. + CancelSynchronization(ctx context.Context, input CancelSynchronizationRequest) (CancelSynchronizationResponse, error) + // Reset a failed shard synchronization. + ResetShardSynchronization(ctx context.Context, dbName, colName string, shardIndex int) error + // Update the maximum allowed time between messages in a task channel. + SetMessageTimeout(ctx context.Context, timeout time.Duration) error + // Return a list of all known master endpoints of this datacenter. + // The resulting endpoints are usable from inside and outside the datacenter. + GetEndpoints(ctx context.Context) (Endpoint, error) + // Return a list of master endpoints of the leader (syncmaster) of this datacenter. + // Length of returned list will be 1 or the call will fail because no master is available. + // In the very rare occasion that the leadership is changing during this call, a list + // of length 0 can be returned. + // The resulting endpoint is usable only within the same datacenter. + GetLeaderEndpoint(ctx context.Context) (Endpoint, error) + // Return a list of known masters in this datacenter. + Masters(ctx context.Context) ([]MasterInfo, error) + + InternalMasterAPI +} + +// WorkerAPI contains API of sync worker +type WorkerAPI interface { + InternalWorkerAPI +} + +type VersionInfo struct { + Version string `json:"version"` + Build string `json:"build"` +} + +// MasterInfo contains information about a single master. +type MasterInfo struct { + // Unique identifier of the master + ID string `json:"id"` + // Internal endpoint of the master + Endpoint string `json:"endpoint"` + // Is this master the current leader + Leader bool `json:"leader"` +} + +type RoleInfo struct { + Role Role `json:"role"` +} + +type Role string + +const ( + RoleMaster Role = "master" + RoleWorker Role = "worker" +) + +func (r Role) IsMaster() bool { return r == RoleMaster } +func (r Role) IsWorker() bool { return r == RoleWorker } + +type ChannelPrefixInfo struct { + Prefix string `json:"prefix"` +} + +// SyncInfo holds the JSON info returned from `GET /_api/sync` +type SyncInfo struct { + Source Endpoint `json:"source"` // Endpoint of sync master on remote cluster + Status SyncStatus `json:"status"` // Overall status of (incoming) synchronization + Shards []ShardSyncInfo `json:"shards,omitempty"` // Status of incoming synchronization per shard + Outgoing []OutgoingSyncInfo `json:"outgoing,omitempty"` // Status of outgoing synchronization + MessageTimeout time.Duration `json:"messageTimeout,omitempty"` // Maximum time between messages in a task channel +} + +// OutgoingSyncInfo holds JSON info returned as part of `GET /_api/sync` +// regarding a specific target for outgoing synchronization data. +type OutgoingSyncInfo struct { + ID string `json:"id"` // ID of sync master to which data is being send + Endpoint Endpoint `json:"endpoint"` // Endpoint of sync masters to which data is being send + Status SyncStatus `json:"status"` // Overall status for this outgoing target + Shards []ShardSyncInfo `json:"shards,omitempty"` // Status of outgoing synchronization per shard for this target +} + +// ShardSyncInfo holds JSON info returned as part of `GET /_api/sync` +// regarding a specific shard. +type ShardSyncInfo struct { + Database string `json:"database"` // Database containing the collection - shard + Collection string `json:"collection"` // Collection containing the shard + ShardIndex int `json:"shardIndex"` // Index of the shard (0..) + Status SyncStatus `json:"status"` // Status of this shard + StatusMessage string `json:"status_message,omitempty"` // Human readable message about the status of this shard + Delay time.Duration `json:"delay,omitempty"` // Delay between other datacenter and us. + LastMessage time.Time `json:"last_message"` // Time of last message received by the task handling this shard + LastDataChange time.Time `json:"last_data_change"` // Time of last message that resulted in a data change, received by the task handling this shard + LastShardMasterChange time.Time `json:"last_shard_master_change"` // Time of when we last had a change in the status of the shard master + ShardMasterKnown bool `json:"shard_master_known"` // Is the shard master known? +} + +type SyncStatus string + +const ( + // SyncStatusInactive indicates that no synchronization is taking place + SyncStatusInactive SyncStatus = "inactive" + // SyncStatusInitializing indicates that synchronization tasks are being setup + SyncStatusInitializing SyncStatus = "initializing" + // SyncStatusInitialSync indicates that initial synchronization of collections is ongoing + SyncStatusInitialSync SyncStatus = "initial-sync" + // SyncStatusRunning indicates that all collections have been initially synchronized + // and normal transaction synchronization is active. + SyncStatusRunning SyncStatus = "running" + // SyncStatusCancelling indicates that the synchronization process is being cancelled. + SyncStatusCancelling SyncStatus = "cancelling" + // SyncStatusFailed indicates that the synchronization process has encountered an unrecoverable failure + SyncStatusFailed SyncStatus = "failed" +) + +var ( + // ValidSyncStatusValues is a list of all possible sync status values. + ValidSyncStatusValues = []SyncStatus{ + SyncStatusInactive, + SyncStatusInitializing, + SyncStatusInitialSync, + SyncStatusRunning, + SyncStatusCancelling, + SyncStatusFailed, + } +) + +// Normalize converts an empty status to inactive. +func (s SyncStatus) Normalize() SyncStatus { + if s == "" { + return SyncStatusInactive + } + return s +} + +// Equals returns true when the other status is equal to the given +// status (both normalized). +func (s SyncStatus) Equals(other SyncStatus) bool { + return s.Normalize() == other.Normalize() +} + +// IsInactiveOrEmpty returns true if the given status equals inactive or is empty. +func (s SyncStatus) IsInactiveOrEmpty() bool { + return s == SyncStatusInactive || s == "" +} + +// IsInitialSyncOrRunning returns true if the given status equals initial-sync or running. +func (s SyncStatus) IsInitialSyncOrRunning() bool { + return s == SyncStatusInitialSync || s == SyncStatusRunning +} + +// IsActive returns true if the given status indicates an active state. +// The is: initializing, initial-sync or running +func (s SyncStatus) IsActive() bool { + return s == SyncStatusInitializing || s == SyncStatusInitialSync || s == SyncStatusRunning +} + +// +// TLSAuthentication contains configuration for using client certificates +// and TLS verification of the server. +type TLSAuthentication = tasks.TLSAuthentication + +type SynchronizationRequest struct { + // Endpoint of sync master of the source cluster + Source Endpoint `json:"source"` + // Authentication of the master + Authentication TLSAuthentication `json:"authentication"` +} + +// Clone returns a deep copy of the given request. +func (r SynchronizationRequest) Clone() SynchronizationRequest { + c := r + c.Source = r.Source.Clone() + return c +} + +// IsSame returns true if both requests contain the same values. +// The source is considered the same is the intersection of existing & given source is not empty. +// We consider an intersection because: +// - Servers can be down, resulting in a temporary missing endpoint +// - Customer can specify only 1 of all servers +func (r SynchronizationRequest) IsSame(other SynchronizationRequest) bool { + if r.Source.Intersection(other.Source).IsEmpty() { + return false + } + if r.Authentication.ClientCertificate != other.Authentication.ClientCertificate { + return false + } + if r.Authentication.ClientKey != other.Authentication.ClientKey { + return false + } + if r.Authentication.CACertificate != other.Authentication.CACertificate { + return false + } + return true +} + +// Validate checks the values of the given request and returns an error +// in case of improper values. +// Returns nil on success. +func (r SynchronizationRequest) Validate() error { + if len(r.Source) == 0 { + return errors.Wrap(BadRequestError, "source missing") + } + if err := r.Source.Validate(); err != nil { + return errors.Wrapf(BadRequestError, "Invalid source: %s", err.Error()) + } + if r.Authentication.ClientCertificate == "" { + return errors.Wrap(BadRequestError, "clientCertificate missing") + } + if r.Authentication.ClientKey == "" { + return errors.Wrap(BadRequestError, "clientKey missing") + } + if r.Authentication.CACertificate == "" { + return errors.Wrap(BadRequestError, "caCertificate missing") + } + return nil +} + +type CancelSynchronizationRequest struct { + // WaitTimeout is the amount of time the cancel function will wait + // until the synchronization has reached an `inactive` state. + // If this value is zero, the cancel function will only switch to the canceling state + // but not wait until the `inactive` state is reached. + WaitTimeout time.Duration `json:"wait_timeout,omitempty"` + // Force is set if you want to end the synchronization even if the source + // master cannot be reached. + Force bool `json:"force,omitempty"` + // ForceTimeout is the amount of time the syncmaster tries to contact + // the source master to notify it about cancelling the synchronization. + // This fields is only used when Force is true. + ForceTimeout time.Duration `json:"force_timeout,omitempty"` +} + +type CancelSynchronizationResponse struct { + // Aborted is set when synchronization has cancelled (state is now inactive) + // but the source sync master was not notified. + // This is only possible when the Force flags is set on the request. + Aborted bool `json:"aborted,omitempty"` + // Source is the endpoint of sync master on remote cluster that we used + // to be synchronizing from. + Source Endpoint `json:"source,omitempty"` + // ClusterID is the ID of the local synchronization cluster. + ClusterID string `json:"cluster_id,omitempty"` +} + +type SetMessageTimeoutRequest struct { + MessageTimeout time.Duration `json:"messageTimeout"` +} + +type EndpointsResponse struct { + Endpoints Endpoint `json:"endpoints"` +} + +type MastersResponse struct { + Masters []MasterInfo `json:"masters"` +} diff --git a/deps/github.com/arangodb/arangosync/client/api_internal.go b/deps/github.com/arangodb/arangosync/client/api_internal.go new file mode 100644 index 000000000..daa650911 --- /dev/null +++ b/deps/github.com/arangodb/arangosync/client/api_internal.go @@ -0,0 +1,448 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package client + +import ( + "context" + "encoding/json" + "fmt" + "math" + "time" + + "github.com/arangodb/arangosync/tasks" +) + +// InternalMasterAPI contains the internal API of the sync master. +type InternalMasterAPI interface { + // Worker -> Master + + // Load configuration data from the master + ConfigureWorker(ctx context.Context, endpoint string) (WorkerConfiguration, error) + // Return all registered workers + RegisteredWorkers(ctx context.Context) ([]WorkerRegistration, error) + // Return info about a specific worker + RegisteredWorker(ctx context.Context, id string) (WorkerRegistration, error) + // Register (or update registration of) a worker + RegisterWorker(ctx context.Context, endpoint, token, hostID string) (WorkerRegistrationResponse, error) + // Remove the registration of a worker + UnregisterWorker(ctx context.Context, id string) error + // Get info about a specific task + Task(ctx context.Context, id string) (TaskInfo, error) + // Get all known tasks + Tasks(ctx context.Context) ([]TaskInfo, error) + // Get all known tasks for a given channel + TasksByChannel(ctx context.Context, channelName string) ([]TaskInfo, error) + // Notify the master that a task with given ID has completed. + TaskCompleted(ctx context.Context, taskID string, info TaskCompletedRequest) error + // Create tasks to start synchronization of a shard in the given db+col. + SynchronizeShard(ctx context.Context, dbName, colName string, shardIndex int) error + // Stop tasks to synchronize a shard in the given db+col. + CancelSynchronizeShard(ctx context.Context, dbName, colName string, shardIndex int) error + // Report status of the synchronization of a shard back to the master. + SynchronizeShardStatus(ctx context.Context, entries []SynchronizationShardStatusRequestEntry) error + // IsChannelRelevant checks if a MQ channel is still relevant + IsChannelRelevant(ctx context.Context, channelName string) (bool, error) + + // Worker & Master -> Master + // GetDirectMQTopicEndpoint returns an endpoint that the caller can use to fetch direct MQ messages + // from. + // This method requires a directMQ token or client cert for authentication. + GetDirectMQTopicEndpoint(ctx context.Context, channelName string) (DirectMQTopicEndpoint, error) + // RenewDirectMQToken renews a given direct MQ token. + // This method requires a directMQ token for authentication. + RenewDirectMQToken(ctx context.Context, token string) (DirectMQToken, error) + // CloneDirectMQToken creates a clone of a given direct MQ token. + // When the given token is revoked, the newly cloned token is also revoked. + // This method requires a directMQ token for authentication. + CloneDirectMQToken(ctx context.Context, token string) (DirectMQToken, error) + // Add entire direct MQ API + InternalDirectMQAPI + + // Master -> Master + + // Start a task that sends inventory data to a receiving remote cluster. + OutgoingSynchronization(ctx context.Context, input OutgoingSynchronizationRequest) (OutgoingSynchronizationResponse, error) + // Cancel sending synchronization data to the remote cluster with given ID. + CancelOutgoingSynchronization(ctx context.Context, remoteID string) error + // Create tasks to send synchronization data of a shard in the given db+col to a remote cluster. + OutgoingSynchronizeShard(ctx context.Context, remoteID, dbName, colName string, shardIndex int, input OutgoingSynchronizeShardRequest) error + // Stop tasks to send synchronization data of a shard in the given db+col to a remote cluster. + CancelOutgoingSynchronizeShard(ctx context.Context, remoteID, dbName, colName string, shardIndex int) error + // Report status of the synchronization of a shard back to the master. + OutgoingSynchronizeShardStatus(ctx context.Context, entries []SynchronizationShardStatusRequestEntry) error + // Reset a failed shard synchronization. + OutgoingResetShardSynchronization(ctx context.Context, remoteID, dbName, colName string, shardIndex int, newControlChannel, newDataChannel string) error + + // Get a prefix for names of channels that contain message + // going to this master. + ChannelPrefix(ctx context.Context) (string, error) + // Get the local message queue configuration. + GetMessageQueueConfig(ctx context.Context) (MessageQueueConfig, error) +} + +// InternalWorkerAPI contains the internal API of the sync worker. +type InternalWorkerAPI interface { + // StartTask is called by the master to instruct the worker + // to run a task with given instructions. + StartTask(ctx context.Context, data StartTaskRequest) error + // StopTask is called by the master to instruct the worker + // to stop all work on the given task. + StopTask(ctx context.Context, taskID string) error + // SetDirectMQTopicToken configures the token used to access messages of a given channel. + SetDirectMQTopicToken(ctx context.Context, channelName, token string, tokenTTL time.Duration) error + // Add entire direct MQ API + InternalDirectMQAPI +} + +// InternalDirectMQAPI contains the internal API of the sync master/worker wrt direct MQ messages. +type InternalDirectMQAPI interface { + // GetDirectMQMessages return messages for a given MQ channel. + GetDirectMQMessages(ctx context.Context, channelName string) ([]DirectMQMessage, error) + // CommitDirectMQMessage removes all messages from the given channel up to an including the given offset. + CommitDirectMQMessage(ctx context.Context, channelName string, offset int64) error +} + +// MessageQueueConfig contains all deployment configuration info for the local MQ. +type MessageQueueConfig struct { + Type string `json:"type"` + Endpoints []string `json:"endpoints"` + Authentication TLSAuthentication `json:"authentication"` +} + +// Clone returns a deep copy of the given config +func (c MessageQueueConfig) Clone() MessageQueueConfig { + result := c + result.Endpoints = append([]string{}, c.Endpoints...) + return result +} + +// ConfigureWorkerRequest is the JSON body for the ConfigureWorker request. +type ConfigureWorkerRequest struct { + Endpoint string `json:"endpoint"` // Endpoint of the worker +} + +// WorkerConfiguration contains configuration data passed from +// the master to the worker. +type WorkerConfiguration struct { + Cluster struct { + Endpoints []string `json:"endpoints"` + JWTSecret string `json:"jwtSecret,omitempty"` + MaxDocumentSize int `json:"maxDocumentSize,omitempty"` + // Minimum replication factor of new/modified collections + MinReplicationFactor int `json:"min-replication-factor,omitempty"` + // Maximum replication factor of new/modified collections + MaxReplicationFactor int `json:"max-replication-factor,omitempty"` + } `json:"cluster"` + HTTPServer struct { + Certificate string `json:"certificate"` + Key string `json:"key"` + } `json:"httpServer"` + MessageQueue struct { + MessageQueueConfig // MQ configuration of local MQ + } `json:"mq"` +} + +// SetDefaults fills empty values with defaults +func (c *WorkerConfiguration) SetDefaults() { + if c.Cluster.MinReplicationFactor <= 0 { + c.Cluster.MinReplicationFactor = 1 + } + if c.Cluster.MaxReplicationFactor <= 0 { + c.Cluster.MaxReplicationFactor = math.MaxInt32 + } +} + +// Validate the given configuration. +// Return an error on validation errors, nil when all ok. +func (c WorkerConfiguration) Validate() error { + if c.Cluster.MinReplicationFactor < 1 { + return maskAny(fmt.Errorf("MinReplicationFactor must be >= 1")) + } + if c.Cluster.MaxReplicationFactor < 1 { + return maskAny(fmt.Errorf("MaxReplicationFactor must be >= 1")) + } + if c.Cluster.MaxReplicationFactor < c.Cluster.MinReplicationFactor { + return maskAny(fmt.Errorf("MaxReplicationFactor must be >= MinReplicationFactor")) + } + return nil +} + +type WorkerRegistrations struct { + Workers []WorkerRegistration `json:"workers"` +} + +type WorkerRegistration struct { + // ID of the worker assigned to it by the master + ID string `json:"id"` + // Endpoint of the worker + Endpoint string `json:"endpoint"` + // Expiration time of the last registration of the worker + ExpiresAt time.Time `json:"expiresAt"` + // ID of the worker when communicating with ArangoDB servers. + ServerID int64 `json:"serverID"` + // IF of the host the worker process is running on + HostID string `json:"host,omitempty"` +} + +// Validate the given registration. +// Return nil if ok, error otherwise. +func (wr WorkerRegistration) Validate() error { + if wr.ID == "" { + return maskAny(fmt.Errorf("ID empty")) + } + if wr.Endpoint == "" { + return maskAny(fmt.Errorf("Endpoint empty")) + } + if wr.ServerID == 0 { + return maskAny(fmt.Errorf("ServerID == 0")) + } + return nil +} + +// IsExpired returns true when the given worker is expired. +func (wr WorkerRegistration) IsExpired() bool { + return time.Now().After(wr.ExpiresAt) +} + +type WorkerRegistrationRequest struct { + Endpoint string `json:"endpoint"` + Token string `json:"token,omitempty"` + HostID string `json:host,omitempty"` +} + +type WorkerRegistrationResponse struct { + WorkerRegistration + // Maximum time between message in a task channel. + MessageTimeout time.Duration `json:"messageTimeout,omitempty"` +} + +type StartTaskRequest struct { + ID string `json:"id"` + tasks.TaskData + // MQ configuration of the remote cluster + RemoteMessageQueueConfig MessageQueueConfig `json:"remote-mq-config"` +} + +// OutgoingSynchronizationRequest holds the master->master request +// data for configuring an outgoing inventory stream. +type OutgoingSynchronizationRequest struct { + // ID of remote cluster + ID string `json:"id"` + // Endpoints of sync masters of the remote (target) cluster + Target Endpoint `json:"target"` + Channels struct { + // Name of MQ topic to send inventory data to. + Inventory string `json:"inventory"` + } `json:"channels"` + // MQ configuration of the remote (target) cluster + MessageQueueConfig MessageQueueConfig `json:"mq-config"` +} + +// Clone returns a deep copy of the given request. +func (r OutgoingSynchronizationRequest) Clone() OutgoingSynchronizationRequest { + c := r + c.Target = r.Target.Clone() + c.MessageQueueConfig = r.MessageQueueConfig.Clone() + return c +} + +// OutgoingSynchronizationResponse holds the answer to an +// master->master request for configuring an outgoing synchronization. +type OutgoingSynchronizationResponse struct { + // MQ configuration of the remote (source) cluster + MessageQueueConfig MessageQueueConfig `json:"mq-config"` +} + +// OutgoingSynchronizeShardRequest holds the master->master request +// data for configuring an outgoing shard synchronization stream. +type OutgoingSynchronizeShardRequest struct { + Channels struct { + // Name of MQ topic to receive control messages on. + Control string `json:"control"` + // Name of MQ topic to send data messages to. + Data string `json:"data"` + } `json:"channels"` +} + +// SynchronizationShardStatusRequest is the request body of a (Outgoing)SynchronizationShardStatus request. +type SynchronizationShardStatusRequest struct { + Entries []SynchronizationShardStatusRequestEntry `json:"entries"` +} + +// SynchronizationShardStatusRequestEntry is a single entry in a SynchronizationShardStatusRequest +type SynchronizationShardStatusRequestEntry struct { + RemoteID string `json:"remoteID"` + Database string `json:"database"` + Collection string `json:"collection"` + ShardIndex int `json:"shardIndex"` + Status SynchronizationShardStatus `json:"status"` +} + +type SynchronizationShardStatus struct { + // Current status + Status SyncStatus `json:"status"` + // Human readable status message + StatusMessage string `json:"status_message,omitempty"` + // Delay between us and other data center. + Delay time.Duration `json:"delay"` + // Time of last message received by the task handling this shard + LastMessage time.Time `json:"last_message"` + // Time of last message that resulted in a data change, received by the task handling this shard + LastDataChange time.Time `json:"last_data_change"` + // Time of when we last had a change in the status of the shard master + LastShardMasterChange time.Time `json:"last_shard_master_change"` + // Is the shard master known? + ShardMasterKnown bool `json:"shard_master_known"` +} + +// IsSame returns true when the Status & StatusMessage of both statuses +// are equal and the Delay is very close. +func (s SynchronizationShardStatus) IsSame(other SynchronizationShardStatus) bool { + if s.Status != other.Status || s.StatusMessage != other.StatusMessage || + s.LastMessage != other.LastMessage || s.LastDataChange != other.LastDataChange || + s.LastShardMasterChange != other.LastShardMasterChange || s.ShardMasterKnown != other.ShardMasterKnown { + return false + } + return !IsSignificantDelayDiff(s.Delay, other.Delay) +} + +// TaskCompletedRequest holds the info for a TaskCompleted request. +type TaskCompletedRequest struct { + Error bool `json:"error,omitempty"` +} + +// TaskAssignment contains information of the assignment of a +// task to a worker. +// It is serialized as JSON into the agency. +type TaskAssignment struct { + // ID of worker the task is assigned to + WorkerID string `json:"worker_id"` + // When the assignment was made + CreatedAt time.Time `json:"created_at"` + // How many assignments have been made + Counter int `json:"counter,omitempty"` +} + +// TaskInfo contains all information known about a task. +type TaskInfo struct { + ID string `json:"id"` + Task tasks.TaskData `json:"task"` + Assignment TaskAssignment `json:"assignment"` +} + +// IsAssigned returns true when the task in given info is assigned to a +// worker, false otherwise. +func (i TaskInfo) IsAssigned() bool { + return i.Assignment.WorkerID != "" +} + +// NeedsCleanup returns true when the entry is subject to cleanup. +func (i TaskInfo) NeedsCleanup() bool { + return i.Assignment.Counter > 0 && !i.Task.Persistent +} + +// TasksResponse is the JSON response for MasterAPI.Tasks method. +type TasksResponse struct { + Tasks []TaskInfo `json:"tasks,omitempty"` +} + +// IsSignificantDelayDiff returns true if there is a significant difference +// between the given delays. +func IsSignificantDelayDiff(d1, d2 time.Duration) bool { + if d2 == 0 { + return d1 != 0 + } + x := float64(d1) / float64(d2) + return x < 0.9 || x > 1.1 +} + +// IsChannelRelevantResponse is the JSON response for a MasterAPI.IsChannelRelevant call +type IsChannelRelevantResponse struct { + IsRelevant bool `json:"isRelevant"` +} + +// StatusAPI describes the API provided to task workers used to send status updates to the master. +type StatusAPI interface { + // SendIncomingStatus queues a given incoming synchronization status entry for sending. + SendIncomingStatus(entry SynchronizationShardStatusRequestEntry) + // SendOutgoingStatus queues a given outgoing synchronization status entry for sending. + SendOutgoingStatus(entry SynchronizationShardStatusRequestEntry) +} + +// DirectMQToken provides a token with its TTL +type DirectMQToken struct { + // Token used to authenticate with the server. + Token string `json:"token"` + // How long the token will be valid. + // Afterwards a new token has to be fetched. + TokenTTL time.Duration `json:"token-ttl"` +} + +// DirectMQTokenRequest is the JSON request body for Renew/Clone direct MQ token request. +type DirectMQTokenRequest struct { + // Token used to authenticate with the server. + Token string `json:"token"` +} + +// DirectMQTopicEndpoint provides information about an endpoint for Direct MQ messages. +type DirectMQTopicEndpoint struct { + // Endpoint of the server that can provide messages for a specific topic. + Endpoint Endpoint `json:"endpoint"` + // CA certificate used to sign the TLS connection of the server. + // This is used for verifying the server. + CACertificate string `json:"caCertificate"` + // Token used to authenticate with the server. + Token string `json:"token"` + // How long the token will be valid. + // Afterwards a new token has to be fetched. + TokenTTL time.Duration `json:"token-ttl"` +} + +// SetDirectMQTopicTokenRequest is the JSON request body for SetDirectMQTopicToken request. +type SetDirectMQTopicTokenRequest struct { + // Token used to authenticate with the server. + Token string `json:"token"` + // How long the token will be valid. + // Afterwards a new token has to be fetched. + TokenTTL time.Duration `json:"token-ttl"` +} + +// DirectMQMessage is a direct MQ message. +type DirectMQMessage struct { + Offset int64 `json:"offset"` + Message json.RawMessage `json:"message"` +} + +// GetDirectMQMessagesResponse is the JSON body for GetDirectMQMessages response. +type GetDirectMQMessagesResponse struct { + Messages []DirectMQMessage `json:"messages,omitempty"` +} + +// CommitDirectMQMessageRequest is the JSON request body for CommitDirectMQMessage request. +type CommitDirectMQMessageRequest struct { + Offset int64 `json:"offset"` +} diff --git a/deps/github.com/arangodb/arangosync/client/client.go b/deps/github.com/arangodb/arangosync/client/client.go new file mode 100644 index 000000000..9cad19b21 --- /dev/null +++ b/deps/github.com/arangodb/arangosync/client/client.go @@ -0,0 +1,389 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package client + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/base64" + "encoding/json" + "io" + "io/ioutil" + "math/rand" + "net/http" + "net/url" + "sync" + "sync/atomic" + "time" + + "github.com/arangodb/arangosync/pkg/jwt" + "github.com/pkg/errors" +) + +type AuthenticationConfig struct { + JWTSecret string + BearerToken string + UserName string + Password string +} + +var ( + sharedHTTPClient = DefaultHTTPClient(nil) +) + +const ( + // AllowForwardRequestHeaderKey is a request header key. + // If this header is set, the syncmaster will forward + // requests to the current leader instead of returning a + // 503. + AllowForwardRequestHeaderKey = "X-Allow-Forward-To-Leader" +) + +// NewArangoSyncClient creates a new client implementation. +func NewArangoSyncClient(endpoints []string, authConf AuthenticationConfig, tlsConfig *tls.Config) (API, error) { + httpClient := sharedHTTPClient + sharedClient := true + if tlsConfig != nil { + httpClient = DefaultHTTPClient(tlsConfig) + sharedClient = false + } + c := &client{ + auth: authConf, + client: httpClient, + sharedClient: sharedClient, + } + c.client.Timeout = 0 + c.endpoints.config = Endpoint(endpoints) + list, err := c.endpoints.config.URLs() + if err != nil { + return nil, maskAny(err) + } + c.endpoints.urls = list + return c, nil +} + +type client struct { + endpoints struct { + mutex sync.RWMutex + config Endpoint + urls []url.URL + preferred int32 + } + auth AuthenticationConfig + client *http.Client + sharedClient bool + clientID string +} + +const ( + contentTypeJSON = "application/json" +) + +// Returns the master API (only valid when Role returns master) +func (c *client) Master() MasterAPI { + return c +} + +// Returns the worker API (only valid when Role returns worker) +func (c *client) Worker() WorkerAPI { + return c +} + +// Set the ID of the client that is making requests. +func (c *client) SetClientID(id string) { + c.clientID = id +} + +// SetShared marks the client as shared. +// Closing a shared client will not close all idle connections. +func (c *client) SetShared() { + c.sharedClient = true +} + +// Close this client +func (c *client) Close() error { + if !c.sharedClient { + if transport, ok := c.client.Transport.(*http.Transport); ok { + transport.CloseIdleConnections() + } + } + return nil +} + +// Version requests the version of an arangosync instance. +func (c *client) Version(ctx context.Context) (VersionInfo, error) { + url := c.createURLs("/_api/version", nil) + + var result VersionInfo + req, err := c.newRequests("GET", url, nil) + if err != nil { + return VersionInfo{}, maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return VersionInfo{}, maskAny(err) + } + + return result, nil +} + +// Role requests the role of an arangosync instance. +func (c *client) Role(ctx context.Context) (Role, error) { + url := c.createURLs("/_api/role", nil) + + var result RoleInfo + req, err := c.newRequests("GET", url, nil) + if err != nil { + return "", maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return "", maskAny(err) + } + + return result.Role, nil +} + +// Endpoint returns the currently used endpoint for this client. +func (c *client) Endpoint() Endpoint { + c.endpoints.mutex.RLock() + defer c.endpoints.mutex.RUnlock() + + return c.endpoints.config +} + +// SynchronizeMasterEndpoints ensures that the client is using all known master +// endpoints. +// Do not use for connections to workers. +// Returns true when endpoints have changed. +func (c *client) SynchronizeMasterEndpoints(ctx context.Context) (bool, error) { + // Fetch all endpoints + update, err := c.GetEndpoints(ctx) + if err != nil { + return false, errors.Wrap(err, "Failed to get master endpoints") + } + c.endpoints.mutex.Lock() + defer c.endpoints.mutex.Unlock() + if !c.endpoints.config.Equals(update) { + // Load changed + list, err := update.URLs() + if err != nil { + return false, errors.Wrap(err, "Failed to parse master endpoints") + } + c.endpoints.config = update + c.endpoints.urls = list + return true, nil + } + return false, nil +} + +// createURLs creates a full URLs (for all endpoints) for a request with given local path & query. +func (c *client) createURLs(urlPath string, query url.Values) []string { + c.endpoints.mutex.RLock() + defer c.endpoints.mutex.RUnlock() + + result := make([]string, len(c.endpoints.urls)) + for i, ep := range c.endpoints.urls { + u := ep // Create copy + u.Path = urlPath + if query != nil { + u.RawQuery = query.Encode() + } + result[i] = u.String() + } + return result +} + +// newRequests creates new requests with optional body and context +// Returns: request, cancel, error +func (c *client) newRequests(method string, urls []string, body interface{}) ([]*http.Request, error) { + var encoded []byte + if body != nil { + var err error + encoded, err = json.Marshal(body) + if err != nil { + return nil, maskAny(err) + } + } + + result := make([]*http.Request, len(urls)) + for i, url := range urls { + var bodyRd io.Reader + if encoded != nil { + bodyRd = bytes.NewReader(encoded) + } + req, err := http.NewRequest(method, url, bodyRd) + if err != nil { + return nil, maskAny(err) + } + req.Header.Set(AllowForwardRequestHeaderKey, "true") + if c.auth.JWTSecret != "" { + jwt.AddArangoSyncJwtHeader(req, c.auth.JWTSecret) + } else if c.auth.BearerToken != "" { + req.Header.Set("Authorization", "Bearer "+c.auth.BearerToken) + } else if c.auth.UserName != "" { + plainText := c.auth.UserName + ":" + c.auth.Password + encoded := base64.StdEncoding.EncodeToString([]byte(plainText)) + req.Header.Set("Authorization", "Basic "+encoded) + } + if c.clientID != "" { + req.Header.Set(ClientIDHeaderKey, c.clientID) + } + result[i] = req + } + return result, nil +} + +type response struct { + Body []byte + StatusCode int + Request *http.Request +} + +// do performs the given requests all at once. +// The first request to answer with a success or permanent failure is returned. +func (c *client) do(ctx context.Context, reqs []*http.Request, result interface{}) error { + if ctx == nil { + ctx = context.Background() + } + var cancel func() + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + ctx, cancel = context.WithTimeout(ctx, defaultHTTPTimeout) + } else { + ctx, cancel = context.WithCancel(ctx) + } + defer cancel() + + // All requests sequencially + order := rand.Perm(len(reqs)) + var lastErr error + for _, idx := range order { + retryNext, err := c.doOnce(ctx, []*http.Request{reqs[idx]}, result) + if err == nil { + return nil + } + if retryNext { + lastErr = err + } else { + return maskAny(err) + } + } + if lastErr != nil { + return maskAny(lastErr) + } + return maskAny(errors.Wrapf(ServiceUnavailableError, "No requests available")) +} + +// doOnce performs the given requests all at once. +// The first request to answer with a success or permanent failure is returned. +// Return: retryNext, error +func (c *client) doOnce(ctx context.Context, reqs []*http.Request, result interface{}) (bool, error) { + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + resultChan := make(chan response, len(reqs)) + errorChan := make(chan error, len(reqs)) + wg := sync.WaitGroup{} + for regIdx, req := range reqs { + req = req.WithContext(ctx) + wg.Add(1) + go func(regIdx int, req *http.Request) { + defer wg.Done() + + if len(reqs) > 1 { + preferred := atomic.LoadInt32(&c.endpoints.preferred) + if int32(regIdx) != preferred { + select { + case <-time.After(time.Millisecond * 50): + // Continue + case <-ctx.Done(): + // Context cancelled + errorChan <- maskAny(ctx.Err()) + return + } + } + } + resp, err := c.client.Do(req) + if err != nil { + // Request failed + errorChan <- maskAny(err) + return + } + + // Check status + statusCode := resp.StatusCode + if statusCode >= 200 && statusCode < 500 && statusCode != 408 { + // Read content + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + errorChan <- maskAny(err) + return + } + + // Success or permanent error + resultChan <- response{ + Body: body, + StatusCode: statusCode, + Request: req, + } + // Cancel all other requests + cancel() + atomic.StoreInt32(&c.endpoints.preferred, int32(regIdx)) + return + } + // No permanent error, try next agent + }(regIdx+1, req) // regIdx+1 is intended. That way a preferred==0 results in all requests being fired at once. + } + + // Wait for go routines to finished + wg.Wait() + cancel() + close(resultChan) + close(errorChan) + if resp, ok := <-resultChan; ok { + // Use first valid response + // Read response body into memory + if resp.StatusCode != http.StatusOK { + // Unexpected status, try to parse error. + return false, maskAny(parseResponseError(resp.Body, resp.StatusCode)) + } + + // Got a success status + if result != nil { + if err := json.Unmarshal(resp.Body, result); err != nil { + method := resp.Request.Method + url := resp.Request.URL.String() + return false, errors.Wrapf(err, "Failed decoding response data from %s request to %s: %v", method, url, err) + } + } + return false, nil + } + if err, ok := <-errorChan; ok { + // Return first error + return false, maskAny(err) + } + return true, errors.Wrapf(ServiceUnavailableError, "All %d servers responded with temporary failure", len(reqs)) +} diff --git a/deps/github.com/arangodb/arangosync/client/client_cache.go b/deps/github.com/arangodb/arangosync/client/client_cache.go new file mode 100644 index 000000000..c066925c5 --- /dev/null +++ b/deps/github.com/arangodb/arangosync/client/client_cache.go @@ -0,0 +1,135 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package client + +import ( + "crypto/sha1" + "fmt" + "strings" + "sync" + + certificates "github.com/arangodb-helper/go-certificates" + + "github.com/arangodb/arangosync/pkg/errors" + "github.com/rs/zerolog" +) + +type ClientCache struct { + mutex sync.Mutex + clients map[string]API +} + +// GetClient returns a client used to access the source with given authentication. +func (cc *ClientCache) GetClient(log zerolog.Logger, source Endpoint, auth Authentication, insecureSkipVerify bool) (API, error) { + if len(source) == 0 { + return nil, errors.Wrapf(PreconditionFailedError, "Cannot create master client: no source configured") + } + keyData := strings.Join(source, ",") + ":" + auth.String() + key := fmt.Sprintf("%x", sha1.Sum([]byte(keyData))) + + cc.mutex.Lock() + defer cc.mutex.Unlock() + + if cc.clients == nil { + cc.clients = make(map[string]API) + } + + // Get existing client (if any) + if c, ok := cc.clients[key]; ok { + return c, nil + } + + // Client does not exist, create one + log.Debug().Msg("Creating new client") + c, err := cc.createClient(source, auth, insecureSkipVerify) + if err != nil { + return nil, maskAny(err) + } + + cc.clients[key] = c + c.SetShared() + return c, nil +} + +// createClient creates a client used to access the source with given authentication. +func (cc *ClientCache) createClient(source Endpoint, auth Authentication, insecureSkipVerify bool) (API, error) { + if len(source) == 0 { + return nil, errors.Wrapf(PreconditionFailedError, "Cannot create master client: no source configured") + } + tlsConfig, err := certificates.CreateTLSConfigFromAuthentication(AuthProxy{auth.TLSAuthentication}, insecureSkipVerify) + if err != nil { + return nil, maskAny(err) + } + ac := AuthenticationConfig{} + if auth.Username != "" { + ac.UserName = auth.Username + ac.Password = auth.Password + } else if auth.JWTSecret != "" { + ac.JWTSecret = auth.JWTSecret + } else if auth.ClientToken != "" { + ac.BearerToken = auth.ClientToken + } + c, err := NewArangoSyncClient(source, ac, tlsConfig) + if err != nil { + return nil, maskAny(err) + } + return c, nil +} + +// NewAuthentication creates a new Authentication from given arguments. +func NewAuthentication(tlsAuth TLSAuthentication, jwtSecret string) Authentication { + return Authentication{ + TLSAuthentication: tlsAuth, + JWTSecret: jwtSecret, + } +} + +// Authentication contains all possible authentication methods for a client. +// Order of authentication methods: +// - JWTSecret +// - ClientToken +// - ClientCertificate +type Authentication struct { + TLSAuthentication + JWTSecret string + Username string + Password string +} + +// String returns a string used to unique identify the authentication settings. +func (a Authentication) String() string { + return a.TLSAuthentication.String() + ":" + a.JWTSecret + ":" + a.Username + ":" + a.Password +} + +// AuthProxy is a helper that implements github.com/arangodb-helper/go-certificates#TLSAuthentication. +type AuthProxy struct { + TLSAuthentication +} + +func (a AuthProxy) CACertificate() string { return a.TLSAuthentication.CACertificate } +func (a AuthProxy) ClientCertificate() string { return a.TLSAuthentication.ClientCertificate } +func (a AuthProxy) ClientKey() string { return a.TLSAuthentication.ClientKey } diff --git a/deps/github.com/arangodb/arangosync/client/client_master.go b/deps/github.com/arangodb/arangosync/client/client_master.go new file mode 100644 index 000000000..246b0fd6a --- /dev/null +++ b/deps/github.com/arangodb/arangosync/client/client_master.go @@ -0,0 +1,582 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package client + +import ( + "context" + "net/url" + "path" + "strconv" + "time" +) + +// Get a prefix for names of channels that contain message +// going to this master. +func (c *client) ChannelPrefix(ctx context.Context) (string, error) { + url := c.createURLs("/_api/channels/prefix", nil) + + var result ChannelPrefixInfo + req, err := c.newRequests("GET", url, nil) + if err != nil { + return "", maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return "", maskAny(err) + } + + return result.Prefix, nil +} + +// Get the local message queue configuration. +func (c *client) GetMessageQueueConfig(ctx context.Context) (MessageQueueConfig, error) { + url := c.createURLs("/_api/mq/config", nil) + + var result MessageQueueConfig + req, err := c.newRequests("GET", url, nil) + if err != nil { + return MessageQueueConfig{}, maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return MessageQueueConfig{}, maskAny(err) + } + + return result, nil +} + +// Gets the current status of synchronization towards the local cluster. +func (c *client) Status(ctx context.Context) (SyncInfo, error) { + url := c.createURLs("/_api/sync", nil) + + var result SyncInfo + req, err := c.newRequests("GET", url, nil) + if err != nil { + return SyncInfo{}, maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return SyncInfo{}, maskAny(err) + } + + return result, nil +} + +// Health performs a quick health check. +// Returns an error when anything is wrong. If so, check Status. +func (c *client) Health(ctx context.Context) error { + url := c.createURLs("/_api/health", nil) + + req, err := c.newRequests("GET", url, nil) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// Return a list of all known master endpoints of this datacenter. +func (c *client) GetEndpoints(ctx context.Context) (Endpoint, error) { + url := c.createURLs("/_api/endpoints", nil) + + req, err := c.newRequests("GET", url, nil) + if err != nil { + return nil, maskAny(err) + } + var result EndpointsResponse + if err := c.do(ctx, req, &result); err != nil { + return nil, maskAny(err) + } + + return result.Endpoints, nil +} + +// Return a list of master endpoints of the leader (syncmaster) of this datacenter. +// Length of returned list will 1 or the call will fail because no master is available. +func (c *client) GetLeaderEndpoint(ctx context.Context) (Endpoint, error) { + url := c.createURLs("/_api/leader/endpoint", nil) + + req, err := c.newRequests("GET", url, nil) + if err != nil { + return nil, maskAny(err) + } + var result EndpointsResponse + if err := c.do(ctx, req, &result); err != nil { + return nil, maskAny(err) + } + + return result.Endpoints, nil +} + +// Return a list of known masters in this datacenter. +func (c *client) Masters(ctx context.Context) ([]MasterInfo, error) { + url := c.createURLs("/_api/masters", nil) + + req, err := c.newRequests("GET", url, nil) + if err != nil { + return nil, maskAny(err) + } + var result MastersResponse + if err := c.do(ctx, req, &result); err != nil { + return nil, maskAny(err) + } + + return result.Masters, nil +} + +// Synchronize configures the master to synchronize the local cluster from a given remote cluster. +func (c *client) Synchronize(ctx context.Context, input SynchronizationRequest) error { + url := c.createURLs("/_api/sync", nil) + + req, err := c.newRequests("POST", url, input) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// CancelSynchronization configures the master to stop & completely cancel the current synchronization of the +// local cluster from a remote cluster. +func (c *client) CancelSynchronization(ctx context.Context, input CancelSynchronizationRequest) (CancelSynchronizationResponse, error) { + q := make(url.Values) + + url := c.createURLs("/_api/sync", q) + var result CancelSynchronizationResponse + req, err := c.newRequests("DELETE", url, input) + if err != nil { + return result, maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return result, maskAny(err) + } + + return result, nil +} + +// Reset a failed shard synchronization. +func (c *client) ResetShardSynchronization(ctx context.Context, dbName, colName string, shardIndex int) error { + url := c.createURLs(path.Join("/_api/sync/database", dbName, "collection", colName, "shard", strconv.Itoa(shardIndex), "reset"), nil) + req, err := c.newRequests("PUT", url, nil) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// Update the maximum allowed time between messages in a task channel. +func (c *client) SetMessageTimeout(ctx context.Context, timeout time.Duration) error { + url := c.createURLs("/_api/message-timeout", nil) + input := SetMessageTimeoutRequest{ + MessageTimeout: timeout, + } + req, err := c.newRequests("PUT", url, input) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// Create tasks to start synchronization of a shard in the given db+col. +func (c *client) SynchronizeShard(ctx context.Context, dbName, colName string, shardIndex int) error { + url := c.createURLs(path.Join("/_api/sync/database", dbName, "collection", colName, "shard", strconv.Itoa(shardIndex)), nil) + + req, err := c.newRequests("POST", url, struct{}{}) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// Stop tasks to synchronize a shard in the given db+col. +func (c *client) CancelSynchronizeShard(ctx context.Context, dbName, colName string, shardIndex int) error { + url := c.createURLs(path.Join("/_api/sync/database", dbName, "collection", colName, "shard", strconv.Itoa(shardIndex)), nil) + + req, err := c.newRequests("DELETE", url, nil) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// Report status of the synchronization of a shard back to the master. +func (c *client) SynchronizeShardStatus(ctx context.Context, entries []SynchronizationShardStatusRequestEntry) error { + url := c.createURLs(path.Join("/_api/sync/multiple/status"), nil) + + input := SynchronizationShardStatusRequest{ + Entries: entries, + } + req, err := c.newRequests("PUT", url, input) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// IsChannelRelevant checks if a MQ channel is still relevant +func (c *client) IsChannelRelevant(ctx context.Context, channelName string) (bool, error) { + url := c.createURLs(path.Join("/_api/mq/channel", url.PathEscape(channelName), "is-relevant"), nil) + + req, err := c.newRequests("GET", url, nil) + if err != nil { + return false, maskAny(err) + } + var result IsChannelRelevantResponse + if err := c.do(ctx, req, &result); err != nil { + return false, maskAny(err) + } + + return result.IsRelevant, nil +} + +// GetDirectMQTopicEndpoint returns an endpoint that the caller can use to fetch direct MQ messages +// from. +func (c *client) GetDirectMQTopicEndpoint(ctx context.Context, channelName string) (DirectMQTopicEndpoint, error) { + url := c.createURLs(path.Join("/_api/mq/direct/channel", url.PathEscape(channelName), "endpoint"), nil) + + req, err := c.newRequests("GET", url, nil) + if err != nil { + return DirectMQTopicEndpoint{}, maskAny(err) + } + var result DirectMQTopicEndpoint + if err := c.do(ctx, req, &result); err != nil { + return DirectMQTopicEndpoint{}, maskAny(err) + } + + return result, nil +} + +// RenewDirectMQToken renews a given direct MQ token. +// This method requires a directMQ token for authentication. +func (c *client) RenewDirectMQToken(ctx context.Context, token string) (DirectMQToken, error) { + url := c.createURLs("/_api/mq/direct/token/renew", nil) + + input := DirectMQTokenRequest{ + Token: token, + } + req, err := c.newRequests("POST", url, input) + if err != nil { + return DirectMQToken{}, maskAny(err) + } + var result DirectMQToken + if err := c.do(ctx, req, &result); err != nil { + return DirectMQToken{}, maskAny(err) + } + + return result, nil +} + +// CloneDirectMQToken creates a clone of a given direct MQ token. +// When the given token is revoked, the newly cloned token is also revoked. +// This method requires a directMQ token for authentication. +func (c *client) CloneDirectMQToken(ctx context.Context, token string) (DirectMQToken, error) { + url := c.createURLs("/_api/mq/direct/token/clone", nil) + + input := DirectMQTokenRequest{ + Token: token, + } + req, err := c.newRequests("POST", url, input) + if err != nil { + return DirectMQToken{}, maskAny(err) + } + var result DirectMQToken + if err := c.do(ctx, req, &result); err != nil { + return DirectMQToken{}, maskAny(err) + } + + return result, nil +} + +// Start a task that sends inventory data to a receiving remote cluster. +func (c *client) OutgoingSynchronization(ctx context.Context, input OutgoingSynchronizationRequest) (OutgoingSynchronizationResponse, error) { + url := c.createURLs("/_api/sync/outgoing", nil) + + req, err := c.newRequests("POST", url, input) + if err != nil { + return OutgoingSynchronizationResponse{}, maskAny(err) + } + var result OutgoingSynchronizationResponse + if err := c.do(ctx, req, &result); err != nil { + return OutgoingSynchronizationResponse{}, maskAny(err) + } + + return result, nil +} + +// Cancel sending synchronization data to the remote cluster with given ID. +func (c *client) CancelOutgoingSynchronization(ctx context.Context, remoteID string) error { + url := c.createURLs(path.Join("/_api/sync/outgoing", remoteID), nil) + + req, err := c.newRequests("DELETE", url, nil) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// Create tasks to send synchronization data of a shard in the given db+col to a remote cluster. +func (c *client) OutgoingSynchronizeShard(ctx context.Context, remoteID, dbName, colName string, shardIndex int, input OutgoingSynchronizeShardRequest) error { + url := c.createURLs(path.Join("/_api/sync/outgoing", remoteID, "database", dbName, "collection", colName, "shard", strconv.Itoa(shardIndex)), nil) + + req, err := c.newRequests("POST", url, input) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// Stop tasks to send synchronization data of a shard in the given db+col to a remote cluster. +func (c *client) CancelOutgoingSynchronizeShard(ctx context.Context, remoteID, dbName, colName string, shardIndex int) error { + url := c.createURLs(path.Join("/_api/sync/outgoing", remoteID, "database", dbName, "collection", colName, "shard", strconv.Itoa(shardIndex)), nil) + + req, err := c.newRequests("DELETE", url, nil) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// Report status of the synchronization of a shard back to the master. +func (c *client) OutgoingSynchronizeShardStatus(ctx context.Context, entries []SynchronizationShardStatusRequestEntry) error { + url := c.createURLs(path.Join("/_api/sync/multiple/outgoing/status"), nil) + + input := SynchronizationShardStatusRequest{ + Entries: entries, + } + req, err := c.newRequests("PUT", url, input) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// Reset a failed shard synchronization. +func (c *client) OutgoingResetShardSynchronization(ctx context.Context, remoteID, dbName, colName string, shardIndex int, newControlChannel, newDataChannel string) error { + url := c.createURLs(path.Join("/_api/sync/outgoing", remoteID, "database", dbName, "collection", colName, "shard", strconv.Itoa(shardIndex), "reset"), nil) + + input := OutgoingSynchronizeShardRequest{} + input.Channels.Control = newControlChannel + input.Channels.Data = newDataChannel + req, err := c.newRequests("PUT", url, input) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// Load configuration data from the master +func (c *client) ConfigureWorker(ctx context.Context, endpoint string) (WorkerConfiguration, error) { + url := c.createURLs("/_api/worker/configure", nil) + + input := ConfigureWorkerRequest{ + Endpoint: endpoint, + } + req, err := c.newRequests("POST", url, input) + if err != nil { + return WorkerConfiguration{}, maskAny(err) + } + var result WorkerConfiguration + if err := c.do(ctx, req, &result); err != nil { + return WorkerConfiguration{}, maskAny(err) + } + + return result, nil +} + +// Return all registered workers +func (c *client) RegisteredWorkers(ctx context.Context) ([]WorkerRegistration, error) { + url := c.createURLs("/_api/worker", nil) + + var result WorkerRegistrations + req, err := c.newRequests("GET", url, nil) + if err != nil { + return nil, maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return nil, maskAny(err) + } + + return result.Workers, nil +} + +// Return info about a specific registered worker +func (c *client) RegisteredWorker(ctx context.Context, id string) (WorkerRegistration, error) { + url := c.createURLs(path.Join("/_api/worker", id), nil) + + var result WorkerRegistration + req, err := c.newRequests("GET", url, nil) + if err != nil { + return WorkerRegistration{}, maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return WorkerRegistration{}, maskAny(err) + } + + return result, nil +} + +// Register (or update registration of) a worker +func (c *client) RegisterWorker(ctx context.Context, endpoint, token, hostID string) (WorkerRegistrationResponse, error) { + url := c.createURLs("/_api/worker", nil) + + input := WorkerRegistrationRequest{ + Endpoint: endpoint, + Token: token, + HostID: hostID, + } + var result WorkerRegistrationResponse + req, err := c.newRequests("PUT", url, input) + if err != nil { + return WorkerRegistrationResponse{}, maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return WorkerRegistrationResponse{}, maskAny(err) + } + + return result, nil +} + +// Remove the registration of a worker +func (c *client) UnregisterWorker(ctx context.Context, id string) error { + url := c.createURLs(path.Join("/_api/worker", id), nil) + + req, err := c.newRequests("DELETE", url, nil) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// Get info about a specific task +func (c *client) Task(ctx context.Context, id string) (TaskInfo, error) { + url := c.createURLs(path.Join("/_api/task/id", id), nil) + + var result TaskInfo + req, err := c.newRequests("GET", url, nil) + if err != nil { + return TaskInfo{}, maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return TaskInfo{}, maskAny(err) + } + + return result, nil +} + +// Get all known tasks +func (c *client) Tasks(ctx context.Context) ([]TaskInfo, error) { + url := c.createURLs("/_api/task", nil) + + var result TasksResponse + req, err := c.newRequests("GET", url, nil) + if err != nil { + return nil, maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return nil, maskAny(err) + } + + return result.Tasks, nil +} + +// Get all known tasks for a given channel +func (c *client) TasksByChannel(ctx context.Context, channelName string) ([]TaskInfo, error) { + url := c.createURLs(path.Join("/_api/task/channel", channelName), nil) + + var result TasksResponse + req, err := c.newRequests("GET", url, nil) + if err != nil { + return nil, maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return nil, maskAny(err) + } + + return result.Tasks, nil +} + +// Notify the master that a task with given ID has completed. +func (c *client) TaskCompleted(ctx context.Context, taskID string, info TaskCompletedRequest) error { + url := c.createURLs(path.Join("/_api/task", taskID, "completed"), nil) + + req, err := c.newRequests("PUT", url, info) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} diff --git a/deps/github.com/arangodb/arangosync/client/client_worker.go b/deps/github.com/arangodb/arangosync/client/client_worker.go new file mode 100644 index 000000000..44abe82aa --- /dev/null +++ b/deps/github.com/arangodb/arangosync/client/client_worker.go @@ -0,0 +1,119 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package client + +import ( + "context" + "net/url" + "path" + "time" +) + +// StartTask is called by the master to instruct the worker +// to run a task with given instructions. +func (c *client) StartTask(ctx context.Context, data StartTaskRequest) error { + url := c.createURLs("/_api/task", nil) + + req, err := c.newRequests("POST", url, data) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// StopTask is called by the master to instruct the worker +// to stop all work on the given task. +func (c *client) StopTask(ctx context.Context, taskID string) error { + url := c.createURLs(path.Join("/_api/task", taskID), nil) + + req, err := c.newRequests("DELETE", url, nil) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// SetDirectMQTopicToken configures the token used to access messages of a given channel. +func (c *client) SetDirectMQTopicToken(ctx context.Context, channelName, token string, tokenTTL time.Duration) error { + url := c.createURLs(path.Join("/_api/mq/direct/channel", url.PathEscape(channelName), "token"), nil) + + data := SetDirectMQTopicTokenRequest{ + Token: token, + TokenTTL: tokenTTL, + } + req, err := c.newRequests("POST", url, data) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} + +// GetDirectMQMessages return messages for a given MQ channel. +func (c *client) GetDirectMQMessages(ctx context.Context, channelName string) ([]DirectMQMessage, error) { + url := c.createURLs(path.Join("/_api/mq/direct/channel", url.PathEscape(channelName), "messages"), nil) + + var result GetDirectMQMessagesResponse + req, err := c.newRequests("GET", url, nil) + if err != nil { + return nil, maskAny(err) + } + if err := c.do(ctx, req, &result); err != nil { + return nil, maskAny(err) + } + + return result.Messages, nil +} + +// CommitDirectMQMessage removes all messages from the given channel up to an including the given offset. +func (c *client) CommitDirectMQMessage(ctx context.Context, channelName string, offset int64) error { + url := c.createURLs(path.Join("/_api/mq/direct/channel", url.PathEscape(channelName), "commit"), nil) + + data := CommitDirectMQMessageRequest{ + Offset: offset, + } + req, err := c.newRequests("POST", url, data) + if err != nil { + return maskAny(err) + } + if err := c.do(ctx, req, nil); err != nil { + return maskAny(err) + } + + return nil +} diff --git a/deps/github.com/arangodb/arangosync/client/endpoint.go b/deps/github.com/arangodb/arangosync/client/endpoint.go new file mode 100644 index 000000000..271645d81 --- /dev/null +++ b/deps/github.com/arangodb/arangosync/client/endpoint.go @@ -0,0 +1,148 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package client + +import ( + "fmt" + "net/url" + "sort" +) + +// Endpoint is a list of URL's that are considered to be off the same service. +type Endpoint []string + +// Contains returns true when x is an element of ep. +func (ep Endpoint) Contains(x string) bool { + x = normalizeSingleEndpoint(x) + for _, y := range ep { + if x == normalizeSingleEndpoint(y) { + return true + } + } + return false +} + +// IsEmpty returns true ep has no elements. +func (ep Endpoint) IsEmpty() bool { + return len(ep) == 0 +} + +// Clone returns a deep clone of the given endpoint +func (ep Endpoint) Clone() Endpoint { + return append(Endpoint{}, ep...) +} + +// Equals returns true when a and b contain +// the same elements (perhaps in different order). +func (ep Endpoint) Equals(other Endpoint) bool { + if len(ep) != len(other) { + return false + } + // Clone lists so we can sort them without affecting the original lists. + a := append([]string{}, ep.normalized()...) + b := append([]string{}, other.normalized()...) + sort.Strings(a) + sort.Strings(b) + for i, x := range a { + if x != b[i] { + return false + } + } + return true +} + +// Intersection the endpoint containing all elements included in ep and in other. +func (ep Endpoint) Intersection(other Endpoint) Endpoint { + result := make([]string, 0, len(ep)+len(other)) + for _, x := range ep { + if other.Contains(x) { + result = append(result, x) + } + } + sort.Strings(result) + return result +} + +// Validate checks all URL's, returning the first error found. +func (ep Endpoint) Validate() error { + for _, x := range ep { + if u, err := url.Parse(x); err != nil { + return maskAny(fmt.Errorf("Endpoint '%s' is invalid: %s", x, err.Error())) + } else if u.Host == "" { + return maskAny(fmt.Errorf("Endpoint '%s' is missing a host", x)) + } + } + return nil +} + +// URLs returns all endpoints as parsed URL's +func (ep Endpoint) URLs() ([]url.URL, error) { + list := make([]url.URL, 0, len(ep)) + for _, x := range ep { + u, err := url.Parse(x) + if err != nil { + return nil, maskAny(err) + } + u.Path = "" + list = append(list, *u) + } + return list, nil +} + +// Merge adds the given endpoint to the endpoint, avoiding duplicates +func (ep Endpoint) Merge(args ...string) Endpoint { + m := make(map[string]struct{}) + for _, x := range ep { + m[x] = struct{}{} + } + for _, x := range args { + m[x] = struct{}{} + } + result := make([]string, 0, len(m)) + for x := range m { + result = append(result, x) + } + sort.Strings(result) + return result +} + +// normalized returns a clone of the given endpoint that contains normalized elements +func (ep Endpoint) normalized() Endpoint { + result := make(Endpoint, len(ep)) + for i, x := range ep { + result[i] = normalizeSingleEndpoint(x) + } + return result +} + +func normalizeSingleEndpoint(ep string) string { + if u, err := url.Parse(ep); err == nil { + u.Path = "" + return u.String() + } + return ep +} diff --git a/deps/github.com/arangodb/arangosync/client/endpoint_test.go b/deps/github.com/arangodb/arangosync/client/endpoint_test.go new file mode 100644 index 000000000..93be1ecf4 --- /dev/null +++ b/deps/github.com/arangodb/arangosync/client/endpoint_test.go @@ -0,0 +1,213 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package client + +import "testing" + +func TestEndpointContains(t *testing.T) { + ep := Endpoint{"http://a", "http://b", "http://c"} + for _, x := range []string{"http://a", "http://b", "http://c", "http://a/"} { + if !ep.Contains(x) { + t.Errorf("Expected endpoint to contain '%s' but it did not", x) + } + } + for _, x := range []string{"", "http://ab", "-", "http://abc"} { + if ep.Contains(x) { + t.Errorf("Expected endpoint to not contain '%s' but it did", x) + } + } +} + +func TestEndpointIsEmpty(t *testing.T) { + ep := Endpoint{"http://a", "http://b", "http://c"} + if ep.IsEmpty() { + t.Error("Expected endpoint to be not empty, but it is") + } + ep = nil + if !ep.IsEmpty() { + t.Error("Expected endpoint to be empty, but it is not") + } + ep = Endpoint{} + if !ep.IsEmpty() { + t.Error("Expected endpoint to be empty, but it is not") + } +} + +func TestEndpointEquals(t *testing.T) { + expectEqual := []Endpoint{ + Endpoint{}, Endpoint{}, + Endpoint{}, nil, + Endpoint{"http://a"}, Endpoint{"http://a"}, + Endpoint{"http://a", "http://b"}, Endpoint{"http://b", "http://a"}, + Endpoint{"http://foo:8529"}, Endpoint{"http://foo:8529/"}, + } + for i := 0; i < len(expectEqual); i += 2 { + epa := expectEqual[i] + epb := expectEqual[i+1] + if !epa.Equals(epb) { + t.Errorf("Expected endpoint %v to be equal to %v, but it is not", epa, epb) + } + if !epb.Equals(epa) { + t.Errorf("Expected endpoint %v to be equal to %v, but it is not", epb, epa) + } + } + + expectNotEqual := []Endpoint{ + Endpoint{"http://a"}, Endpoint{}, + Endpoint{"http://z"}, nil, + Endpoint{"http://aa"}, Endpoint{"http://a"}, + Endpoint{"http://a:100"}, Endpoint{"http://a:200"}, + Endpoint{"http://a", "http://b", "http://c"}, Endpoint{"http://b", "http://a"}, + } + for i := 0; i < len(expectNotEqual); i += 2 { + epa := expectNotEqual[i] + epb := expectNotEqual[i+1] + if epa.Equals(epb) { + t.Errorf("Expected endpoint %v to be not equal to %v, but it is", epa, epb) + } + if epb.Equals(epa) { + t.Errorf("Expected endpoint %v to be not equal to %v, but it is", epb, epa) + } + } +} + +func TestEndpointClone(t *testing.T) { + tests := []Endpoint{ + Endpoint{}, + Endpoint{"http://a"}, + Endpoint{"http://a", "http://b"}, + } + for _, orig := range tests { + c := orig.Clone() + if !orig.Equals(c) { + t.Errorf("Expected endpoint %v to be equal to clone %v, but it is not", orig, c) + } + if len(c) > 0 { + c[0] = "http://modified" + if orig.Equals(c) { + t.Errorf("Expected endpoint %v to be no longer equal to clone %v, but it is", orig, c) + } + } + } +} + +func TestEndpointIntersection(t *testing.T) { + expectIntersection := []Endpoint{ + Endpoint{"http://a"}, Endpoint{"http://a"}, + Endpoint{"http://a"}, Endpoint{"http://a", "http://b"}, + Endpoint{"http://a"}, Endpoint{"http://b", "http://a"}, + Endpoint{"http://a", "http://b"}, Endpoint{"http://b", "http://foo27"}, + Endpoint{"http://foo:8529"}, Endpoint{"http://foo:8529/"}, + } + for i := 0; i < len(expectIntersection); i += 2 { + epa := expectIntersection[i] + epb := expectIntersection[i+1] + if len(epa.Intersection(epb)) == 0 { + t.Errorf("Expected endpoint %v to have an intersection with %v, but it does not", epa, epb) + } + if len(epb.Intersection(epa)) == 0 { + t.Errorf("Expected endpoint %v to have an intersection with %v, but it does not", epb, epa) + } + } + + expectNoIntersection := []Endpoint{ + Endpoint{"http://a"}, Endpoint{}, + Endpoint{"http://z"}, nil, + Endpoint{"http://aa"}, Endpoint{"http://a"}, + Endpoint{"http://a", "http://b", "http://c"}, Endpoint{"http://e", "http://f"}, + } + for i := 0; i < len(expectNoIntersection); i += 2 { + epa := expectNoIntersection[i] + epb := expectNoIntersection[i+1] + if len(epa.Intersection(epb)) > 0 { + t.Errorf("Expected endpoint %v to have no intersection with %v, but it does", epa, epb) + } + if len(epb.Intersection(epa)) > 0 { + t.Errorf("Expected endpoint %v to havenoan intersection with %v, but it does", epb, epa) + } + } +} + +func TestEndpointValidate(t *testing.T) { + validTests := []Endpoint{ + Endpoint{}, + Endpoint{"http://a"}, + Endpoint{"http://a", "http://b"}, + } + for _, x := range validTests { + if err := x.Validate(); err != nil { + t.Errorf("Expected endpoint %v to be valid, but it is not because %s", x, err) + } + } + invalidTests := []Endpoint{ + Endpoint{":http::foo"}, + Endpoint{"http/a"}, + Endpoint{"http??"}, + Endpoint{"http:/"}, + Endpoint{"http:/foo"}, + } + for _, x := range invalidTests { + if err := x.Validate(); err == nil { + t.Errorf("Expected endpoint %v to be not valid, but it is", x) + } + } +} + +func TestEndpointURLs(t *testing.T) { + ep := Endpoint{"http://a", "http://b/rel"} + expected := []string{"http://a", "http://b"} + list, err := ep.URLs() + if err != nil { + t.Errorf("URLs expected to succeed, but got %s", err) + } else { + for i, x := range list { + found := x.String() + if found != expected[i] { + t.Errorf("Unexpected URL at index %d of %v, expected '%s', got '%s'", i, ep, expected[i], found) + } + } + } +} + +func TestEndpointMerge(t *testing.T) { + tests := []Endpoint{ + Endpoint{"http://a"}, Endpoint{}, Endpoint{"http://a"}, + Endpoint{"http://z"}, nil, Endpoint{"http://z"}, + Endpoint{"http://aa"}, Endpoint{"http://a"}, Endpoint{"http://aa", "http://a"}, + Endpoint{"http://a", "http://b", "http://c"}, Endpoint{"http://e", "http://f"}, Endpoint{"http://a", "http://b", "http://c", "http://e", "http://f"}, + Endpoint{"http://a", "http://b", "http://c"}, Endpoint{"http://a", "http://f"}, Endpoint{"http://a", "http://b", "http://c", "http://f"}, + } + for i := 0; i < len(tests); i += 3 { + epa := tests[i] + epb := tests[i+1] + expected := tests[i+2] + result := epa.Merge(epb...) + if !result.Equals(expected) { + t.Errorf("Expected merge of endpoints %v & %v to be %v, but got %v", epa, epb, expected, result) + } + } +} diff --git a/deps/github.com/arangodb/arangosync/client/error.go b/deps/github.com/arangodb/arangosync/client/error.go new file mode 100644 index 000000000..038ef489c --- /dev/null +++ b/deps/github.com/arangodb/arangosync/client/error.go @@ -0,0 +1,190 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package client + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/pkg/errors" + + "github.com/arangodb/arangosync/pkg/retry" +) + +var ( + maskAny = errors.WithStack + // NotFoundError indicates that an object does not exist. + NotFoundError = StatusError{StatusCode: http.StatusNotFound, message: "not found"} + // ServiceUnavailableError indicates that right now the service is not available, please retry later. + ServiceUnavailableError = StatusError{StatusCode: http.StatusServiceUnavailable, message: "service unavailable"} + // BadRequestError indicates invalid arguments. + BadRequestError = StatusError{StatusCode: http.StatusBadRequest, message: "bad request"} + // PreconditionFailedError indicates that the state of the system is such that the request cannot be executed. + PreconditionFailedError = StatusError{StatusCode: http.StatusPreconditionFailed, message: "precondition failed"} + // InternalServerError indicates an unspecified error inside the server, perhaps a bug. + InternalServerError = StatusError{StatusCode: http.StatusInternalServerError, message: "internal server error"} + // UnauthorizedError indicates that the request has not the correct authorization. + UnauthorizedError = StatusError{StatusCode: http.StatusUnauthorized, message: "unauthorized"} + // RequestTimeoutError indicates that the request is taken longer than we're prepared to wait. + RequestTimeoutError = StatusError{StatusCode: http.StatusRequestTimeout, message: "request timeout"} +) + +type StatusError struct { + StatusCode int + message string +} + +func (e StatusError) Error() string { + if e.message != "" { + return e.message + } + return fmt.Sprintf("Status %d", e.StatusCode) +} + +// IsStatusError returns the status code and true +// if the given error is caused by a StatusError. +func IsStatusError(err error) (int, bool) { + err = errors.Cause(err) + if serr, ok := err.(StatusError); ok { + return serr.StatusCode, true + } + return 0, false +} + +// IsStatusErrorWithCode returns true if the given error is caused +// by a StatusError with given code. +func IsStatusErrorWithCode(err error, code int) bool { + err = errors.Cause(err) + if serr, ok := err.(StatusError); ok { + return serr.StatusCode == code + } + return false +} + +type ErrorResponse struct { + Error string +} + +type RedirectToError struct { + Location string +} + +func (e RedirectToError) Error() string { + return fmt.Sprintf("Redirect to: %s", e.Location) +} + +// IsRedirectTo returns true when the given error is caused by an +// RedirectToError. If so, it also returns the redirect location. +func IsRedirectTo(err error) (string, bool) { + err = errors.Cause(err) + if rterr, ok := err.(RedirectToError); ok { + return rterr.Location, true + } + return "", false +} + +// IsNotFound returns true if the given error is caused by a NotFoundError. +func IsNotFound(err error) bool { + return IsStatusErrorWithCode(err, http.StatusNotFound) +} + +// IsServiceUnavailable returns true if the given error is caused by a ServiceUnavailableError. +func IsServiceUnavailable(err error) bool { + return IsStatusErrorWithCode(err, http.StatusServiceUnavailable) +} + +// IsBadRequest returns true if the given error is caused by a BadRequestError. +func IsBadRequest(err error) bool { + return IsStatusErrorWithCode(err, http.StatusBadRequest) +} + +// IsPreconditionFailed returns true if the given error is caused by a PreconditionFailedError. +func IsPreconditionFailed(err error) bool { + return IsStatusErrorWithCode(err, http.StatusPreconditionFailed) +} + +// IsInternalServer returns true if the given error is caused by a InternalServerError. +func IsInternalServer(err error) bool { + return IsStatusErrorWithCode(err, http.StatusInternalServerError) +} + +// IsUnauthorized returns true if the given error is caused by a UnauthorizedError. +func IsUnauthorized(err error) bool { + return IsStatusErrorWithCode(err, http.StatusUnauthorized) +} + +// IsRequestTimeout returns true if the given error is caused by a RequestTimeoutError. +func IsRequestTimeout(err error) bool { + return IsStatusErrorWithCode(err, http.StatusRequestTimeout) +} + +// IsCanceled returns true if the given error is caused by a context.Canceled. +func IsCanceled(err error) bool { + return errors.Cause(err) == context.Canceled +} + +// ParseResponseError returns an error from given response. +// It tries to parse the body (if given body is nil, will be read from response) +// for ErrorResponse. +func ParseResponseError(r *http.Response, body []byte) error { + // Read body (if needed) + if body == nil { + defer r.Body.Close() + body, _ = ioutil.ReadAll(r.Body) + } + return parseResponseError(body, r.StatusCode) +} + +// parseResponseError returns an error from given response. +// It tries to parse the body (if given body is nil, will be read from response) +// for ErrorResponse. +func parseResponseError(body []byte, statusCode int) error { + // Parse body (if available) + var result error + if len(body) > 0 { + var errRes ErrorResponse + if err := json.Unmarshal(body, &errRes); err == nil { + // Found ErrorResponse + result = StatusError{StatusCode: statusCode, message: errRes.Error} + } + } + + if result == nil { + // No ErrorResponse found, fallback to default message + result = StatusError{StatusCode: statusCode} + } + + // Is permanent error? + if statusCode >= 400 && statusCode < 500 { + result = retry.Permanent(result) + } + + return result +} diff --git a/deps/github.com/arangodb/arangosync/client/http.go b/deps/github.com/arangodb/arangosync/client/http.go new file mode 100644 index 000000000..146108ea6 --- /dev/null +++ b/deps/github.com/arangodb/arangosync/client/http.go @@ -0,0 +1,64 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package client + +import ( + "crypto/tls" + "net" + "net/http" + "time" +) + +const ( + defaultHTTPTimeout = time.Minute * 2 +) + +// DefaultHTTPClient creates a new HTTP client configured for accessing a starter. +func DefaultHTTPClient(tlsConfig *tls.Config) *http.Client { + if tlsConfig == nil { + tlsConfig = &tls.Config{ + InsecureSkipVerify: true, + } + } + return &http.Client{ + Timeout: defaultHTTPTimeout, + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 10, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 90 * time.Second, + TLSClientConfig: tlsConfig, + ExpectContinueTimeout: 1 * time.Second, + }, + } +} diff --git a/deps/github.com/arangodb/arangosync/pkg/errors/aggregate_error.go b/deps/github.com/arangodb/arangosync/pkg/errors/aggregate_error.go new file mode 100644 index 000000000..11b4a23a3 --- /dev/null +++ b/deps/github.com/arangodb/arangosync/pkg/errors/aggregate_error.go @@ -0,0 +1,63 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package errors + +// AggregateError is a helper to wrap zero or more errors as a go `error`. +type AggregateError struct { + errors []error +} + +// Add returns a new error with given error added. +func (ae AggregateError) Add(e error) AggregateError { + return AggregateError{ + errors: append(ae.errors, e), + } +} + +func (ae AggregateError) Error() string { + switch len(ae.errors) { + case 0: + return "no errors" + case 1: + return ae.errors[0].Error() + default: + return ae.errors[0].Error() + ", ..." + } +} + +// AsError returns the given aggregate error if it contains 2 or more errors. +// It returns the first error if it contains exactly 1 error. +// Otherwise nil is returned. +func (ae AggregateError) AsError() error { + if len(ae.errors) == 0 { + return nil + } + if len(ae.errors) == 1 { + return ae.errors[0] + } + return ae +} diff --git a/deps/github.com/arangodb/arangosync/pkg/errors/errors.go b/deps/github.com/arangodb/arangosync/pkg/errors/errors.go new file mode 100644 index 000000000..508dce0cd --- /dev/null +++ b/deps/github.com/arangodb/arangosync/pkg/errors/errors.go @@ -0,0 +1,207 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package errors + +import ( + "context" + "fmt" + "io" + "net" + "net/url" + "os" + "syscall" + + driver "github.com/arangodb/go-driver" + errs "github.com/pkg/errors" +) + +var ( + Cause = errs.Cause + New = errs.New + WithStack = errs.WithStack + Wrap = errs.Wrap + Wrapf = errs.Wrapf +) + +// WithMessage annotates err with a new message. +// The messages of given error is hidden. +// If err is nil, WithMessage returns nil. +func WithMessage(err error, message string) error { + if err == nil { + return nil + } + return &withMessage{ + cause: err, + msg: message, + } +} + +type withMessage struct { + cause error + msg string +} + +func (w *withMessage) Error() string { return w.msg } +func (w *withMessage) Cause() error { return w.cause } + +func (w *withMessage) Format(s fmt.State, verb rune) { + switch verb { + case 'v': + if s.Flag('+') { + fmt.Fprintf(s, "%+v\n", w.Cause()) + io.WriteString(s, w.msg) + return + } + fallthrough + case 's', 'q': + io.WriteString(s, w.Error()) + } +} + +type timeout interface { + Timeout() bool +} + +// IsTimeout returns true if the given error is caused by a timeout error. +func IsTimeout(err error) bool { + if err == nil { + return false + } + if t, ok := errs.Cause(err).(timeout); ok { + return t.Timeout() + } + return false +} + +type temporary interface { + Temporary() bool +} + +// IsTemporary returns true if the given error is caused by a temporary error. +func IsTemporary(err error) bool { + if err == nil { + return false + } + if t, ok := errs.Cause(err).(temporary); ok { + return t.Temporary() + } + return false +} + +// IsEOF returns true if the given error is caused by an EOF error. +func IsEOF(err error) bool { + err = errs.Cause(err) + if err == io.EOF { + return true + } + if ok, err := libCause(err); ok { + return IsEOF(err) + } + return false +} + +// IsConnectionRefused returns true if the given error is caused by an "connection refused" error. +func IsConnectionRefused(err error) bool { + err = errs.Cause(err) + if err, ok := err.(syscall.Errno); ok { + return err == syscall.ECONNREFUSED + } + if ok, err := libCause(err); ok { + return IsConnectionRefused(err) + } + return false +} + +// IsConnectionReset returns true if the given error is caused by an "connection reset by peer" error. +func IsConnectionReset(err error) bool { + err = errs.Cause(err) + if err, ok := err.(syscall.Errno); ok { + return err == syscall.ECONNRESET + } + if ok, err := libCause(err); ok { + return IsConnectionReset(err) + } + return false +} + +// IsContextCanceled returns true if the given error is caused by a context cancelation. +func IsContextCanceled(err error) bool { + err = errs.Cause(err) + if err == context.Canceled { + return true + } + if ok, err := libCause(err); ok { + return IsContextCanceled(err) + } + return false +} + +// IsContextDeadlineExpired returns true if the given error is caused by a context deadline expiration. +func IsContextDeadlineExpired(err error) bool { + err = errs.Cause(err) + if err == context.DeadlineExceeded { + return true + } + if ok, err := libCause(err); ok { + return IsContextDeadlineExpired(err) + } + return false +} + +// IsContextCanceledOrExpired returns true if the given error is caused by a context cancelation +// or deadline expiration. +func IsContextCanceledOrExpired(err error) bool { + err = errs.Cause(err) + if err == context.Canceled || err == context.DeadlineExceeded { + return true + } + if ok, err := libCause(err); ok { + return IsContextCanceledOrExpired(err) + } + return false +} + +// libCause returns the Cause of well known go library errors. +func libCause(err error) (bool, error) { + original := err + for { + switch e := err.(type) { + case *driver.ResponseError: + err = e.Err + case *net.DNSConfigError: + err = e.Err + case *net.OpError: + err = e.Err + case *os.SyscallError: + err = e.Err + case *url.Error: + err = e.Err + default: + return err != original, err + } + } +} diff --git a/deps/github.com/arangodb/arangosync/pkg/jwt/error.go b/deps/github.com/arangodb/arangosync/pkg/jwt/error.go new file mode 100644 index 000000000..e24fff9f0 --- /dev/null +++ b/deps/github.com/arangodb/arangosync/pkg/jwt/error.go @@ -0,0 +1,33 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package jwt + +import "github.com/pkg/errors" + +var ( + maskAny = errors.WithStack +) diff --git a/deps/github.com/arangodb/arangosync/pkg/jwt/jwt.go b/deps/github.com/arangodb/arangosync/pkg/jwt/jwt.go new file mode 100644 index 000000000..83c518b63 --- /dev/null +++ b/deps/github.com/arangodb/arangosync/pkg/jwt/jwt.go @@ -0,0 +1,137 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package jwt + +import ( + "fmt" + "net/http" + "strings" + + jg "github.com/dgrijalva/jwt-go" +) + +const ( + issArangod = "arangodb" + issArangoSync = "arangosync" +) + +// AddArangodJwtHeader calculates a JWT authorization header, for authorization +// of a request to an arangod server, based on the given secret +// and adds it to the given request. +// If the secret is empty, nothing is done. +func AddArangodJwtHeader(req *http.Request, jwtSecret string) error { + if jwtSecret == "" { + return nil + } + value, err := CreateArangodJwtAuthorizationHeader(jwtSecret) + if err != nil { + return maskAny(err) + } + + req.Header.Set("Authorization", value) + return nil +} + +// CreateArangodJwtAuthorizationHeader calculates a JWT authorization header, for authorization +// of a request to an arangod server, based on the given secret. +// If the secret is empty, nothing is done. +func CreateArangodJwtAuthorizationHeader(jwtSecret string) (string, error) { + if jwtSecret == "" { + return "", nil + } + // Create a new token object, specifying signing method and the claims + // you would like it to contain. + token := jg.NewWithClaims(jg.SigningMethodHS256, jg.MapClaims{ + "iss": issArangod, + "server_id": "foo", + }) + + // Sign and get the complete encoded token as a string using the secret + signedToken, err := token.SignedString([]byte(jwtSecret)) + if err != nil { + return "", maskAny(err) + } + + return "bearer " + signedToken, nil +} + +// AddArangoSyncJwtHeader calculates a JWT authorization header, for authorization +// of a request to an arangosync server, based on the given secret +// and adds it to the given request. +// If the secret is empty, nothing is done. +func AddArangoSyncJwtHeader(req *http.Request, jwtSecret string) error { + if jwtSecret == "" { + return nil + } + // Create a new token object, specifying signing method and the claims + // you would like it to contain. + token := jg.NewWithClaims(jg.SigningMethodHS256, jg.MapClaims{ + "iss": issArangoSync, + "server_id": "foo", + }) + + // Sign and get the complete encoded token as a string using the secret + signedToken, err := token.SignedString([]byte(jwtSecret)) + if err != nil { + return maskAny(err) + } + + req.Header.Set("Authorization", "bearer "+signedToken) + return nil +} + +// VerifyArangoSyncJwtHeader verifies the bearer token in the given request with +// the given secret. +// If returns nil when verification succeed, an error if verification fails. +// If the secret is empty, nothing is done. +func VerifyArangoSyncJwtHeader(req *http.Request, jwtSecret string) error { + if jwtSecret == "" { + return nil + } + // Extract Authorization header + authHdr := strings.TrimSpace(req.Header.Get("Authorization")) + if authHdr == "" { + return maskAny(fmt.Errorf("No Authorization found")) + } + prefix := "bearer " + if !strings.HasPrefix(strings.ToLower(authHdr), prefix) { + // Missing bearer prefix + return maskAny(fmt.Errorf("No bearer prefix")) + } + tokenStr := strings.TrimSpace(authHdr[len(prefix):]) + // Parse token + claims := jg.MapClaims{ + "iss": issArangoSync, + "server_id": "foo", + } + _, err := jg.ParseWithClaims(tokenStr, claims, func(*jg.Token) (interface{}, error) { return []byte(jwtSecret), nil }) + if err != nil { + return maskAny(err) + } + + return nil +} diff --git a/deps/github.com/arangodb/arangosync/pkg/retry/retry.go b/deps/github.com/arangodb/arangosync/pkg/retry/retry.go new file mode 100644 index 000000000..8245ed552 --- /dev/null +++ b/deps/github.com/arangodb/arangosync/pkg/retry/retry.go @@ -0,0 +1,152 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package retry + +import ( + "context" + "time" + + "github.com/cenkalti/backoff" + "github.com/pkg/errors" +) + +var ( + maskAny = errors.WithStack +) + +type permanentError struct { + Err error +} + +func (e *permanentError) Error() string { + return e.Err.Error() +} + +func (e *permanentError) Cause() error { + return e.Err +} + +// Permanent makes the given error a permanent failure +// that stops the Retry loop immediately. +func Permanent(err error) error { + return &permanentError{Err: err} +} + +func isPermanent(err error) (*permanentError, bool) { + type causer interface { + Cause() error + } + + for err != nil { + if pe, ok := err.(*permanentError); ok { + return pe, true + } + cause, ok := err.(causer) + if !ok { + break + } + err = cause.Cause() + } + return nil, false +} + +// retry the given operation until it succeeds, +// has a permanent failure or times out. +func retry(ctx context.Context, op func() error, timeout time.Duration) error { + var failure error + wrappedOp := func() error { + if err := op(); err == nil { + return nil + } else { + if pe, ok := isPermanent(err); ok { + // Detected permanent error + failure = pe.Err + return nil + } else { + return err + } + } + } + + eb := backoff.NewExponentialBackOff() + eb.MaxElapsedTime = timeout + eb.MaxInterval = timeout / 3 + + var b backoff.BackOff + if ctx != nil { + b = backoff.WithContext(eb, ctx) + } else { + b = eb + } + + if err := backoff.Retry(wrappedOp, b); err != nil { + return maskAny(err) + } + if failure != nil { + return maskAny(failure) + } + return nil +} + +// Retry the given operation until it succeeds, +// has a permanent failure or times out. +func Retry(op func() error, timeout time.Duration) error { + return retry(nil, op, timeout) +} + +// RetryWithContext retries the given operation until it succeeds, +// has a permanent failure or times out. +// The timeout is the minimum between the timeout of the context and the given timeout. +// The context given to the operation will have a timeout of a percentage of the overall timeout. +// The percentage is calculated from the given minimum number of attempts. +// If the given minimum number of attempts is 3, the timeout of each `op` call if the overall timeout / 3. +// The default minimum number of attempts is 2. +func RetryWithContext(ctx context.Context, op func(ctx context.Context) error, timeout time.Duration, minAttempts ...int) error { + deadline, ok := ctx.Deadline() + if ok { + ctxTimeout := time.Until(deadline) + if ctxTimeout < timeout { + timeout = ctxTimeout + } + } + divider := 2 + if len(minAttempts) == 1 { + divider = minAttempts[0] + } + ctxOp := func() error { + lctx, cancel := context.WithTimeout(ctx, timeout/time.Duration(divider)) + defer cancel() + if err := op(lctx); err != nil { + return maskAny(err) + } + return nil + } + if err := retry(ctx, ctxOp, timeout); err != nil { + return maskAny(err) + } + return nil +} diff --git a/deps/github.com/arangodb/arangosync/tasks/task.go b/deps/github.com/arangodb/arangosync/tasks/task.go new file mode 100644 index 000000000..92e1632ad --- /dev/null +++ b/deps/github.com/arangodb/arangosync/tasks/task.go @@ -0,0 +1,153 @@ +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// The Programs (which include both the software and documentation) contain +// proprietary information of ArangoDB GmbH; they are provided under a license +// agreement containing restrictions on use and disclosure and are also +// protected by copyright, patent and other intellectual and industrial +// property laws. Reverse engineering, disassembly or decompilation of the +// Programs, except to the extent required to obtain interoperability with +// other independently created software or as specified by law, is prohibited. +// +// It shall be the licensee's responsibility to take all appropriate fail-safe, +// backup, redundancy, and other measures to ensure the safe use of +// applications if the Programs are used for purposes such as nuclear, +// aviation, mass transit, medical, or other inherently dangerous applications, +// and ArangoDB GmbH disclaims liability for any damages caused by such use of +// the Programs. +// +// This software is the confidential and proprietary information of ArangoDB +// GmbH. You shall not disclose such confidential and proprietary information +// and shall use it only in accordance with the terms of the license agreement +// you entered into with ArangoDB GmbH. +// +// Author Ewout Prangsma +// + +package tasks + +import ( + "context" + "reflect" + "time" +) + +// TaskData contains persistent data of a task. +// This data is stored as JSON object in the agency. +type TaskData struct { + // Type of task + Type TaskType `json:"type"` + // If Persistent is set, this task should be re-assigned to another worker when + // the worker, that the task was assigned to, is unregistered (or expires). + Persistent bool `json:"persistent,omitempty"` + // Channels contains names of MQ channels used for this task + Channels struct { + // Data channel is used to send data messages from sync source to sync target + Data string `json:"data,omitempty"` + // Control channel is used to send control messages from sync target to sync source. + Control string `json:"control,omitempty"` + } `json:"channels"` + // If set, contains the ID of the remote cluster this task is targeting + TargetID string `json:"target_id,omitempty"` + // If set, contains the name of the database this task is working on. + Database string `json:"database,omitempty"` + // If set, contains the name of the collection this task is working on. + Collection string `json:"collection,omitempty"` + // If set, contains the index of the shard this task is working on. + ShardIndex int `json:"shardIndex,omitempty"` +} + +// IsShardSpecific returns true when the task is intended to operate on a specific +// shard. +func (t TaskData) IsShardSpecific() bool { + return t.Database != "" && t.Collection != "" +} + +// Equals returns true when both TaskData's are identical. +func (t TaskData) Equals(other TaskData) bool { + return reflect.DeepEqual(t, other) +} + +// TaskType is a type of task. +// Values are hardcoded and should not be changed. +type TaskType string + +const ( + // TaskTypeSendInventory is a task type that sends inventory updates to the sync target. + TaskTypeSendInventory TaskType = "send-inventory" + // TaskTypeReceiveInventory is a task type that received inventory updates from the sync source and updates the local + // structure accordingly. + TaskTypeReceiveInventory TaskType = "receive-inventory" + // TaskTypeSendShard is a task type that sends synchronization updates to the sync target for a specific shard. + TaskTypeSendShard TaskType = "send-shard" + // TaskTypeReceiveShard is a task type that received synchronization updates from the sync source for a specific shard. + TaskTypeReceiveShard TaskType = "receive-shard" +) + +func (t TaskType) String() string { + return string(t) +} + +// TaskWorker is a generic interface for the implementation of a task. +type TaskWorker interface { + // Run the task. + // Do not return until completion or a fatal error occurs + Run() error + + // Stop the task. + // If waitUntilFinished is set, do not return until the task has been stopped. + Stop(waitUntilFinished bool) error + + // Update the message timeout of this task. + // This timeout is the maximum time between messages + // in a task channel. + // If no messages have been received within the + // message timeout period, the channel is considered + // broken. + // If is up to the task implementation to cope + // with a broken channel. + SetMessageTimeout(timeout time.Duration) + + // Returns true if this task does not have a valid shard master, but does need it. + HasUnknownShardMaster() bool + + // RenewTokens is called once every 5 minutes. The task worker is expected to renew all + // authentication tokens it needs. + RenewTokens(ctx context.Context) error +} + +// TLSClientAuthentication contains configuration for using client certificates or client tokens. +type TLSClientAuthentication struct { + // Client certificate used to authenticate myself. + ClientCertificate string `json:"clientCertificate"` + // Private key of client certificate used to authentication. + ClientKey string `json:"clientKey"` + // Client token used to authenticate myself. + ClientToken string `json:"clientToken"` +} + +// String returns a string representation of the given object. +func (a TLSClientAuthentication) String() string { + return a.ClientCertificate + "/" + a.ClientKey + "/" + a.ClientToken +} + +// TLSAuthentication contains configuration for using client certificates +// and TLS verification of the server. +type TLSAuthentication struct { + TLSClientAuthentication + // CA certificate used to sign the TLS connection of the server. + // This is used for verifying the server. + CACertificate string `json:"caCertificate"` +} + +// String returns a string representation of the given object. +func (a TLSAuthentication) String() string { + return a.TLSClientAuthentication.String() + "/" + a.CACertificate +} + +// MessageQueueConfig contains all deployment configuration info for a MQ. +type MessageQueueConfig struct { + Type string `json:"type"` + Endpoints []string `json:"endpoints"` + Authentication TLSAuthentication `json:"authentication"` +} diff --git a/deps/github.com/arangodb/go-driver/.travis.yml b/deps/github.com/arangodb/go-driver/.travis.yml index 5ab1337f6..da58d729d 100644 --- a/deps/github.com/arangodb/go-driver/.travis.yml +++ b/deps/github.com/arangodb/go-driver/.travis.yml @@ -7,7 +7,7 @@ language: go env: - TEST_SUITE=run-tests-http - - TEST_SUITE=run-tests-single ARANGODB=arangodb:3.1 + - TEST_SUITE=run-tests-single ARANGODB=arangodb:3.2 - TEST_SUITE=run-tests-single ARANGODB=arangodb/arangodb:latest - TEST_SUITE=run-tests-single ARANGODB=arangodb/arangodb-preview:latest diff --git a/deps/github.com/arangodb/go-driver/Makefile b/deps/github.com/arangodb/go-driver/Makefile index d42e0aa3b..b8f6f00d2 100644 --- a/deps/github.com/arangodb/go-driver/Makefile +++ b/deps/github.com/arangodb/go-driver/Makefile @@ -3,7 +3,7 @@ SCRIPTDIR := $(shell pwd) ROOTDIR := $(shell cd $(SCRIPTDIR) && pwd) GOBUILDDIR := $(SCRIPTDIR)/.gobuild -GOVERSION := 1.9.2-alpine +GOVERSION := 1.10.1-alpine TMPDIR := $(GOBUILDDIR) ifndef ARANGODB @@ -58,13 +58,9 @@ else ifeq ("$(TEST_AUTH)", "jwt") ARANGOARGS := --server.jwt-secret=/jwtsecret endif -ifeq ("$(TEST_MODE)", "single") - TEST_NET := container:$(DBCONTAINER) - TEST_ENDPOINTS := http://localhost:8529 -else - TEST_NET := container:$(TESTCONTAINER)-ns - TEST_ENDPOINTS := http://localhost:7001 - TESTS := $(REPOPATH)/test +TEST_NET := container:$(TESTCONTAINER)-ns +TEST_ENDPOINTS := http://localhost:7001 +TESTS := $(REPOPATH)/test ifeq ("$(TEST_AUTH)", "rootpw") CLUSTERENV := JWTSECRET=testing TEST_AUTHENTICATION := basic:root: @@ -77,7 +73,6 @@ ifeq ("$(TEST_SSL)", "auto") CLUSTERENV := SSL=auto $(CLUSTERENV) TEST_ENDPOINTS = https://localhost:7001 endif -endif ifeq ("$(TEST_CONNECTION)", "vst") TESTS := $(REPOPATH)/test @@ -107,7 +102,7 @@ endif all: build build: $(GOBUILDDIR) $(SOURCES) - GOPATH=$(GOBUILDDIR) go build -v $(REPOPATH) $(REPOPATH)/http $(REPOPATH)/vst + GOPATH=$(GOBUILDDIR) go build -v $(REPOPATH) $(REPOPATH)/http $(REPOPATH)/vst $(REPOPATH)/agency $(REPOPATH)/jwt clean: rm -Rf $(GOBUILDDIR) @@ -116,6 +111,7 @@ $(GOBUILDDIR): @mkdir -p $(ORGDIR) @rm -f $(REPODIR) && ln -s ../../../.. $(REPODIR) GOPATH=$(GOBUILDDIR) go get github.com/arangodb/go-velocypack + GOPATH=$(GOBUILDDIR) go get github.com/dgrijalva/jwt-go run-tests: run-tests-http run-tests-single run-tests-resilientsingle run-tests-cluster @@ -132,13 +128,13 @@ run-tests-http: $(GOBUILDDIR) # Single server tests run-tests-single: run-tests-single-json run-tests-single-vpack run-tests-single-vst-1.0 $(VST11_SINGLE_TESTS) -run-tests-single-json: run-tests-single-json-with-auth run-tests-single-json-no-auth +run-tests-single-json: run-tests-single-json-with-auth run-tests-single-json-no-auth run-tests-single-json-ssl -run-tests-single-vpack: run-tests-single-vpack-with-auth run-tests-single-vpack-no-auth +run-tests-single-vpack: run-tests-single-vpack-with-auth run-tests-single-vpack-no-auth run-tests-single-vpack-ssl -run-tests-single-vst-1.0: run-tests-single-vst-1.0-with-auth run-tests-single-vst-1.0-no-auth +run-tests-single-vst-1.0: run-tests-single-vst-1.0-with-auth run-tests-single-vst-1.0-no-auth run-tests-single-vst-1.0-ssl -run-tests-single-vst-1.1: run-tests-single-vst-1.1-with-auth run-tests-single-vst-1.1-jwt-auth run-tests-single-vst-1.1-no-auth +run-tests-single-vst-1.1: run-tests-single-vst-1.1-with-auth run-tests-single-vst-1.1-jwt-auth run-tests-single-vst-1.1-no-auth run-tests-single-vst-1.1-ssl run-tests-single-vst-1.1-jwt-ssl run-tests-single-json-no-auth: @echo "Single server, HTTP+JSON, no authentication" @@ -176,6 +172,26 @@ run-tests-single-vst-1.1-jwt-auth: @echo "Single server, Velocystream 1.1, JWT authentication" @${MAKE} TEST_MODE="single" TEST_AUTH="jwt" TEST_CONNECTION="vst" TEST_CVERSION="1.1" __run_tests +run-tests-single-json-ssl: + @echo "Single server, HTTP+JSON, with authentication, SSL" + @${MAKE} TEST_MODE="single" TEST_AUTH="rootpw" TEST_SSL="auto" TEST_CONTENT_TYPE="json" __run_tests + +run-tests-single-vpack-ssl: + @echo "Single server, HTTP+Velocypack, with authentication, SSL" + @${MAKE} TEST_MODE="single" TEST_AUTH="rootpw" TEST_SSL="auto" TEST_CONTENT_TYPE="vpack" __run_tests + +run-tests-single-vst-1.0-ssl: + @echo "Single server, Velocystream 1.0, with authentication, SSL" + @${MAKE} TEST_MODE="single" TEST_AUTH="rootpw" TEST_SSL="auto" TEST_CONNECTION="vst" TEST_CVERSION="1.0" __run_tests + +run-tests-single-vst-1.1-ssl: + @echo "Single server, Velocystream 1.1, with authentication, SSL" + @${MAKE} TEST_MODE="single" TEST_AUTH="rootpw" TEST_SSL="auto" TEST_CONNECTION="vst" TEST_CVERSION="1.1" __run_tests + +run-tests-single-vst-1.1-jwt-ssl: + @echo "Single server, Velocystream 1.1, JWT authentication, SSL" + @${MAKE} TEST_MODE="single" TEST_AUTH="jwt" TEST_SSL="auto" TEST_CONNECTION="vst" TEST_CVERSION="1.1" __run_tests + # ResilientSingle server tests run-tests-resilientsingle: run-tests-resilientsingle-json run-tests-resilientsingle-vpack run-tests-resilientsingle-vst-1.0 $(VST11_RESILIENTSINGLE_TESTS) @@ -291,11 +307,13 @@ __test_go_test: --net=$(TEST_NET) \ -v $(ROOTDIR):/usr/code \ -e GOPATH=/usr/code/.gobuild \ + -e GOCACHE=off \ -e TEST_ENDPOINTS=$(TEST_ENDPOINTS) \ -e TEST_AUTHENTICATION=$(TEST_AUTHENTICATION) \ -e TEST_CONNECTION=$(TEST_CONNECTION) \ -e TEST_CVERSION=$(TEST_CVERSION) \ -e TEST_CONTENT_TYPE=$(TEST_CONTENT_TYPE) \ + -e TEST_PPROF=$(TEST_PPROF) \ -w /usr/code/ \ golang:$(GOVERSION) \ go test $(TAGS) $(TESTOPTIONS) $(TESTVERBOSEOPTIONS) $(TESTS) @@ -307,25 +325,14 @@ else ifdef JWTSECRET echo "$JWTSECRET" > "${JWTSECRETFILE}" endif -ifeq ("$(TEST_MODE)", "single") - @-docker rm -f -v $(DBCONTAINER) $(TESTCONTAINER) &> /dev/null - docker run -d --name $(DBCONTAINER) \ - $(ARANGOENV) $(ARANGOVOL) \ - $(ARANGODB) --log.level requests=debug --log.use-microtime true $(ARANGOARGS) -else @-docker rm -f -v $(TESTCONTAINER) &> /dev/null @TESTCONTAINER=$(TESTCONTAINER) ARANGODB=$(ARANGODB) STARTER=$(STARTER) STARTERMODE=$(TEST_MODE) TMPDIR=${GOBUILDDIR} $(CLUSTERENV) $(ROOTDIR)/test/cluster.sh start endif -endif __test_cleanup: @docker rm -f -v $(TESTCONTAINER) &> /dev/null ifndef TEST_ENDPOINTS_OVERRIDE -ifeq ("$(TEST_MODE)", "single") - @docker rm -f -v $(DBCONTAINER) &> /dev/null -else @TESTCONTAINER=$(TESTCONTAINER) ARANGODB=$(ARANGODB) STARTER=$(STARTER) STARTERMODE=$(TEST_MODE) $(ROOTDIR)/test/cluster.sh cleanup -endif endif @sleep 3 diff --git a/deps/github.com/arangodb/go-driver/README.md b/deps/github.com/arangodb/go-driver/README.md index a25ddad95..88726a21c 100644 --- a/deps/github.com/arangodb/go-driver/README.md +++ b/deps/github.com/arangodb/go-driver/README.md @@ -1,416 +1,13 @@ -# ArangoDB GO Driver. +![ArangoDB-Logo](https://docs.arangodb.com/assets/arangodb_logo_2016_inverted.png) +# ArangoDB GO Driver + +This project contains the official Go driver for the [ArangoDB database](https://arangodb.com). [![Build Status](https://travis-ci.org/arangodb/go-driver.svg?branch=master)](https://travis-ci.org/arangodb/go-driver) [![GoDoc](https://godoc.org/github.com/arangodb/g-driver?status.svg)](http://godoc.org/github.com/arangodb/go-driver) -API and implementation is considered stable, more protocols (Velocystream) are being added within the existing API. - -This project contains a Go driver for the [ArangoDB database](https://arangodb.com). - -## Supported versions - -- ArangoDB versions 3.1 and up. - - Single server & cluster setups - - With or without authentication -- Go 1.7 and up. - -## Go dependencies - -- None (Additional error libraries are supported). - -## Getting started - -To use the driver, first fetch the sources into your GOPATH. - -```sh -go get github.com/arangodb/go-driver -``` - -Using the driver, you always need to create a `Client`. -The following example shows how to create a `Client` for a single server -running on localhost. - -```go -import ( - "fmt" - - driver "github.com/arangodb/go-driver" - "github.com/arangodb/go-driver/http" -) - -... - -conn, err := http.NewConnection(http.ConnectionConfig{ - Endpoints: []string{"http://localhost:8529"}, -}) -if err != nil { - // Handle error -} -c, err := driver.NewClient(driver.ClientConfig{ - Connection: conn, -}) -if err != nil { - // Handle error -} -``` - -Once you have a `Client` you can access/create databases on the server, -access/create collections, graphs, documents and so on. - -The following example shows how to open an existing collection in an existing database -and create a new document in that collection. - -```go -// Open "examples_books" database -db, err := c.Database(nil, "examples_books") -if err != nil { - // Handle error -} - -// Open "books" collection -col, err := db.Collection(nil, "books") -if err != nil { - // Handle error -} - -// Create document -book := Book{ - Title: "ArangoDB Cookbook", - NoPages: 257, -} -meta, err := col.CreateDocument(nil, book) -if err != nil { - // Handle error -} -fmt.Printf("Created document in collection '%s' in database '%s'\n", col.Name(), db.Name()) -``` - -## API design - -### Concurrency - -All functions of the driver are stricly synchronous. They operate and only return a value (or error) -when they're done. - -If you want to run operations concurrently, use a go routine. All objects in the driver are designed -to be used from multiple concurrent go routines, except `Cursor`. - -All database objects (except `Cursor`) are considered static. After their creation they won't change. -E.g. after creating a `Collection` instance you can remove the collection, but the (Go) instance -will still be there. Calling functions on such a removed collection will of course fail. - -### Structured error handling & wrapping - -All functions of the driver that can fail return an `error` value. If that value is not `nil`, the -function call is considered to be failed. In that case all other return values are set to their `zero` -values. - -All errors are structured using error checking functions named `Is`. -E.g. `IsNotFound(error)` return true if the given error is of the category "not found". -There can be multiple internal error codes that all map onto the same category. - -All errors returned from any function of the driver (either internal or exposed) wrap errors -using the `WithStack` function. This can be used to provide detail stack trackes in case of an error. -All error checking functions use the `Cause` function to get the cause of an error instead of the error wrapper. - -Note that `WithStack` and `Cause` are actually variables to you can implement it using your own error -wrapper library. - -If you for example use https://github.com/pkg/errors, you want to initialize to go driver like this: -```go -import ( - driver "github.com/arangodb/go-driver" - "github.com/pkg/errors" -) - -func init() { - driver.WithStack = errors.WithStack - driver.Cause = errors.Cause -} -``` - -### Context aware - -All functions of the driver that involve some kind of long running operation or -support additional options not given as function arguments, have a `context.Context` argument. -This enables you cancel running requests, pass timeouts/deadlines and pass additional options. - -In all methods that take a `context.Context` argument you can pass `nil` as value. -This is equivalent to passing `context.Background()`. - -Many functions support 1 or more optional (and infrequently used) additional options. -These can be used with a `With` function. -E.g. to force a create document call to wait until the data is synchronized to disk, -use a prepared context like this: -```go -ctx := driver.WithWaitForSync(parentContext) -collection.CreateDocument(ctx, yourDocument) -``` - -### Failover - -The driver supports multiple endpoints to connect to. All request are in principle -send to the same endpoint until that endpoint fails to respond. -In that case a new endpoint is chosen and the operation is retried. - -The following example shows how to connect to a cluster of 3 servers. - -```go -conn, err := http.NewConnection(http.ConnectionConfig{ - Endpoints: []string{"http://server1:8529", "http://server2:8529", "http://server3:8529"}, -}) -if err != nil { - // Handle error -} -c, err := driver.NewClient(driver.ClientConfig{ - Connection: conn, -}) -if err != nil { - // Handle error -} -``` - -Note that a valid endpoint is an URL to either a standalone server, or a URL to a coordinator -in a cluster. - -### Failover: Exact behavior - -The driver monitors the request being send to a specific server (endpoint). -As soon as the request has been completely written, failover will no longer happen. -The reason for that is that several operations cannot be (safely) retried. -E.g. when a request to create a document has been send to a server and a timeout -occurs, the driver has no way of knowing if the server did or did not create -the document in the database. - -If the driver detects that a request has been completely written, but still gets -an error (other than an error response from Arango itself), it will wrap the -error in a `ResponseError`. The client can test for such an error using `IsResponseError`. - -If a client received a `ResponseError`, it can do one of the following: -- Retry the operation and be prepared for some kind of duplicate record / unique constraint violation. -- Perform a test operation to see if the "failed" operation did succeed after all. -- Simply consider the operation failed. This is risky, since it can still be the case that the operation did succeed. - -### Failover: Timeouts - -To control the timeout of any function in the driver, you must pass it a context -configured with `context.WithTimeout` (or `context.WithDeadline`). - -In the case of multiple endpoints, the actual timeout used for requests will be shorter than -the timeout given in the context. -The driver will divide the timeout by the number of endpoints with a maximum of 3. -This ensures that the driver can try up to 3 different endpoints (in case of failover) without -being canceled due to the timeout given by the client. -E.g. -- With 1 endpoint and a given timeout of 1 minute, the actual request timeout will be 1 minute. -- With 3 endpoints and a given timeout of 1 minute, the actual request timeout will be 20 seconds. -- With 8 endpoints and a given timeout of 1 minute, the actual request timeout will be 20 seconds. - -For most requests you want a actual request timeout of at least 30 seconds. - -### Secure connections (SSL) - -The driver supports endpoints that use SSL using the `https` URL scheme. - -The following example shows how to connect to a server that has a secure endpoint using -a self-signed certificate. - -```go -conn, err := http.NewConnection(http.ConnectionConfig{ - Endpoints: []string{"https://localhost:8529"}, - TLSConfig: &tls.Config{InsecureSkipVerify: true}, -}) -if err != nil { - // Handle error -} -c, err := driver.NewClient(driver.ClientConfig{ - Connection: conn, -}) -if err != nil { - // Handle error -} -``` - -# Sample requests - -## Connecting to ArangoDB - -```go -conn, err := http.NewConnection(http.ConnectionConfig{ - Endpoints: []string{"http://localhost:8529"}, - TLSConfig: &tls.Config{ /*...*/ }, -}) -if err != nil { - // Handle error -} -c, err := driver.NewClient(driver.ClientConfig{ - Connection: conn, - Authentication: driver.BasicAuthentication("user", "password"), -}) -if err != nil { - // Handle error -} -``` - -## Opening a database - -```go -ctx := context.Background() -db, err := client.Database(ctx, "myDB") -if err != nil { - // handle error -} -``` - -## Opening a collection - -```go -ctx := context.Background() -col, err := db.Collection(ctx, "myCollection") -if err != nil { - // handle error -} -``` - -## Checking if a collection exists - -```go -ctx := context.Background() -found, err := db.CollectionExists(ctx, "myCollection") -if err != nil { - // handle error -} -``` - -## Creating a collection - -```go -ctx := context.Background() -options := &driver.CreateCollectionOptions{ /* ... */ } -col, err := db.CreateCollection(ctx, "myCollection", options) -if err != nil { - // handle error -} -``` - -## Reading a document from a collection - -```go -var doc MyDocument -ctx := context.Background() -meta, err := col.ReadDocument(ctx, myDocumentKey, &doc) -if err != nil { - // handle error -} -``` - -## Reading a document from a collection with an explicit revision - -```go -var doc MyDocument -revCtx := driver.WithRevision(ctx, "mySpecificRevision") -meta, err := col.ReadDocument(revCtx, myDocumentKey, &doc) -if err != nil { - // handle error -} -``` - -## Creating a document - -```go -doc := MyDocument{ - Name: "jan", - Counter: 23, -} -ctx := context.Background() -meta, err := col.CreateDocument(ctx, doc) -if err != nil { - // handle error -} -fmt.Printf("Created document with key '%s', revision '%s'\n", meta.Key, meta.Rev) -``` - -## Removing a document - -```go -ctx := context.Background() -err := col.RemoveDocument(revCtx, myDocumentKey) -if err != nil { - // handle error -} -``` - -## Removing a document with an explicit revision - -```go -revCtx := driver.WithRevision(ctx, "mySpecificRevision") -err := col.RemoveDocument(revCtx, myDocumentKey) -if err != nil { - // handle error -} -``` - -## Updating a document - -```go -ctx := context.Background() -patch := map[string]interface{}{ - "Name": "Frank", -} -meta, err := col.UpdateDocument(ctx, myDocumentKey, patch) -if err != nil { - // handle error -} -``` - -## Querying documents, one document at a time - -```go -ctx := context.Background() -query := "FOR d IN myCollection LIMIT 10 RETURN d" -cursor, err := db.Query(ctx, query, nil) -if err != nil { - // handle error -} -defer cursor.Close() -for { - var doc MyDocument - meta, err := cursor.ReadDocument(ctx, &doc) - if driver.IsNoMoreDocuments(err) { - break - } else if err != nil { - // handle other errors - } - fmt.Printf("Got doc with key '%s' from query\n", meta.Key) -} -``` - -## Querying documents, fetching total count - -```go -ctx := driver.WithQueryCount(context.Background()) -query := "FOR d IN myCollection RETURN d" -cursor, err := db.Query(ctx, query, nil) -if err != nil { - // handle error -} -defer cursor.Close() -fmt.Printf("Query yields %d documents\n", cursor.Count()) -``` - -## Querying documents, with bind variables - -```go -ctx := context.Background() -query := "FOR d IN myCollection FILTER d.Name == @name RETURN d" -bindVars := map[string]interface{}{ - "name": "Some name", -} -cursor, err := db.Query(ctx, query, bindVars) -if err != nil { - // handle error -} -defer cursor.Close() -... -``` +- [Getting Started](docs/Drivers/GO/GettingStarted/README.md) +- [Example Requests](docs/Drivers/GO/ExampleRequests/README.md) +- [Connection Management](docs/Drivers/GO/ConnectionManagement/README.md) +- [Reference](https://godoc.org/github.com/arangodb/go-driver) diff --git a/deps/github.com/arangodb/go-driver/agency/agency.go b/deps/github.com/arangodb/go-driver/agency/agency.go new file mode 100644 index 000000000..cf666622d --- /dev/null +++ b/deps/github.com/arangodb/go-driver/agency/agency.go @@ -0,0 +1,116 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package agency + +import ( + "context" + "time" + + driver "github.com/arangodb/go-driver" +) + +// Agency provides API implemented by the ArangoDB agency. +type Agency interface { + // Connection returns the connection used by this api. + Connection() driver.Connection + + // ReadKey reads the value of a given key in the agency. + ReadKey(ctx context.Context, key []string, value interface{}) error + + // WriteKey writes the given value with the given key with a given TTL (unless TTL is zero). + // If you pass a condition (only 1 allowed), this condition has to be true, + // otherwise the write will fail with a ConditionFailed error. + WriteKey(ctx context.Context, key []string, value interface{}, ttl time.Duration, condition ...WriteCondition) error + + // WriteKeyIfEmpty writes the given value with the given key only if the key was empty before. + WriteKeyIfEmpty(ctx context.Context, key []string, value interface{}, ttl time.Duration) error + + // WriteKeyIfEqualTo writes the given new value with the given key only if the existing value for that key equals + // to the given old value. + WriteKeyIfEqualTo(ctx context.Context, key []string, newValue, oldValue interface{}, ttl time.Duration) error + + // RemoveKey removes the given key. + // If you pass a condition (only 1 allowed), this condition has to be true, + // otherwise the remove will fail with a ConditionFailed error. + RemoveKey(ctx context.Context, key []string, condition ...WriteCondition) error + + // RemoveKeyIfEqualTo removes the given key only if the existing value for that key equals + // to the given old value. + RemoveKeyIfEqualTo(ctx context.Context, key []string, oldValue interface{}) error + + // Register a URL to receive notification callbacks when the value of the given key changes + RegisterChangeCallback(ctx context.Context, key []string, cbURL string) error + // Register a URL to receive notification callbacks when the value of the given key changes + UnregisterChangeCallback(ctx context.Context, key []string, cbURL string) error +} + +// WriteCondition is a precondition before a write is accepted. +type WriteCondition struct { + conditions map[string]writeCondition +} + +// IfEmpty adds an empty check on the given key to the given condition +// and returns the updated condition. +func (c WriteCondition) add(key []string, updater func(wc *writeCondition)) WriteCondition { + if c.conditions == nil { + c.conditions = make(map[string]writeCondition) + } + fullKey := createFullKey(key) + wc := c.conditions[fullKey] + updater(&wc) + c.conditions[fullKey] = wc + return c +} + +// toMap convert the given condition to a map suitable for writeTransaction. +func (c WriteCondition) toMap() map[string]interface{} { + result := make(map[string]interface{}) + for k, v := range c.conditions { + result[k] = v + } + return result +} + +// IfEmpty adds an "is empty" check on the given key to the given condition +// and returns the updated condition. +func (c WriteCondition) IfEmpty(key []string) WriteCondition { + return c.add(key, func(wc *writeCondition) { + wc.OldEmpty = &condTrue + }) +} + +// IfIsArray adds an "is-array" check on the given key to the given condition +// and returns the updated condition. +func (c WriteCondition) IfIsArray(key []string) WriteCondition { + return c.add(key, func(wc *writeCondition) { + wc.IsArray = &condTrue + }) +} + +// IfEqualTo adds an "value equals oldValue" check to given old value on the +// given key to the given condition and returns the updated condition. +func (c WriteCondition) IfEqualTo(key []string, oldValue interface{}) WriteCondition { + return c.add(key, func(wc *writeCondition) { + wc.Old = oldValue + }) +} diff --git a/deps/github.com/arangodb/go-driver/agency/agency_connection.go b/deps/github.com/arangodb/go-driver/agency/agency_connection.go new file mode 100644 index 000000000..1ff703fb4 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/agency/agency_connection.go @@ -0,0 +1,315 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package agency + +import ( + "context" + "fmt" + "sync" + "time" + + driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/http" +) + +const ( + minAgencyTimeout = time.Second * 2 +) + +type agencyConnection struct { + mutex sync.RWMutex + config http.ConnectionConfig + connections []driver.Connection + auth driver.Authentication +} + +// NewAgencyConnection creates an agency connection for agents at the given endpoints. +// This type of connection differs from normal HTTP/VST connection in the way +// requests are executed. +// This type of connection makes use of the fact that only 1 agent will respond +// to requests at a time. All other agents will respond with an "I'm not the leader" error. +// A request will be send to all agents at the same time. +// The result of the first agent to respond with a normal response is used. +func NewAgencyConnection(config http.ConnectionConfig) (driver.Connection, error) { + c := &agencyConnection{ + config: config, + } + if err := c.UpdateEndpoints(config.Endpoints); err != nil { + return nil, driver.WithStack(err) + } + return c, nil +} + +// NewRequest creates a new request with given method and path. +func (c *agencyConnection) NewRequest(method, path string) (driver.Request, error) { + c.mutex.RLock() + defer c.mutex.RUnlock() + + if len(c.connections) == 0 { + return nil, driver.WithStack(fmt.Errorf("no connections")) + } + r, err := c.connections[0].NewRequest(method, path) + if err != nil { + return nil, driver.WithStack(err) + } + return r, nil +} + +// Do performs a given request, returning its response. +// In case of a termporary failure, the request is retried until +// the deadline is exceeded. +func (c *agencyConnection) Do(ctx context.Context, req driver.Request) (driver.Response, error) { + if ctx == nil { + ctx = context.Background() + } + deadline, ok := ctx.Deadline() + if !ok { + deadline = time.Now().Add(time.Second * 30) + } + timeout := time.Until(deadline) + if timeout < minAgencyTimeout { + timeout = minAgencyTimeout + } + attempt := 1 + delay := agencyConnectionFailureBackoff(0) + for { + lctx, cancel := context.WithTimeout(ctx, timeout/3) + resp, isPerm, err := c.doOnce(lctx, req) + cancel() + if err == nil { + // Success + return resp, nil + } else if isPerm { + // Permanent error + return nil, driver.WithStack(err) + } + // Is deadline exceeded? + if time.Now().After(deadline) { + return nil, driver.WithStack(fmt.Errorf("All %d attemps resulted in temporary failure", attempt)) + } + // Just retry + attempt++ + delay = agencyConnectionFailureBackoff(delay) + // Wait a bit so we don't hammer the agency + select { + case <-time.After(delay): + // Continue + case <-ctx.Done(): + // Context canceled + return nil, driver.WithStack(ctx.Err()) + } + } +} + +// Do performs a given request once, returning its response. +// Returns: Response, isPermanentError, Error +func (c *agencyConnection) doOnce(ctx context.Context, req driver.Request) (driver.Response, bool, error) { + c.mutex.RLock() + connections := c.connections + c.mutex.RUnlock() + + if len(c.connections) == 0 { + return nil, true, driver.WithStack(fmt.Errorf("no connections")) + } + + ctx, cancel := context.WithCancel(ctx) + results := make(chan driver.Response, len(connections)) + errors := make(chan error, len(connections)) + wg := sync.WaitGroup{} + for _, epConn := range connections { + wg.Add(1) + go func(epConn driver.Connection) { + defer wg.Done() + epReq := req.Clone() + result, err := epConn.Do(ctx, epReq) + if err == nil { + if err = isSuccess(result); err == nil { + // Success + results <- result + // Cancel all other requests + cancel() + return + } + } + // Check error + if statusCode, ok := isArangoError(err); ok { + // We have a status code, check it + if statusCode >= 400 && statusCode < 500 && statusCode != 408 { + // Permanent error, return it + errors <- driver.WithStack(err) + // Cancel all other requests + cancel() + return + } + } + // No permanent error. Are we the only endpoint? + if len(connections) == 1 { + errors <- driver.WithStack(err) + } + // No permanent error, try next agent + }(epConn) + } + + // Wait for go routines to finished + wg.Wait() + cancel() + close(results) + close(errors) + if result, ok := <-results; ok { + // Return first result + return result, false, nil + } + if err, ok := <-errors; ok { + // Return first error + return nil, true, driver.WithStack(err) + } + return nil, false, driver.WithStack(fmt.Errorf("All %d servers responded with temporary failure", len(connections))) +} + +func isSuccess(resp driver.Response) error { + if resp == nil { + return driver.WithStack(fmt.Errorf("Response is nil")) + } + statusCode := resp.StatusCode() + if statusCode >= 200 && statusCode < 300 { + return nil + } + return driver.ArangoError{ + HasError: true, + Code: statusCode, + } +} + +// isArangoError checks if the given error is (or is caused by) an ArangoError. +// If so it returned the Code and true, otherwise it returns 0, false. +func isArangoError(err error) (int, bool) { + if aerr, ok := driver.Cause(err).(driver.ArangoError); ok { + return aerr.Code, true + } + return 0, false +} + +// Unmarshal unmarshals the given raw object into the given result interface. +func (c *agencyConnection) Unmarshal(data driver.RawObject, result interface{}) error { + c.mutex.RLock() + defer c.mutex.RUnlock() + + if len(c.connections) == 0 { + return driver.WithStack(fmt.Errorf("no connections")) + } + if err := c.connections[0].Unmarshal(data, result); err != nil { + return driver.WithStack(err) + } + return nil +} + +// Endpoints returns the endpoints used by this connection. +func (c *agencyConnection) Endpoints() []string { + c.mutex.RLock() + defer c.mutex.RUnlock() + + var result []string + for _, x := range c.connections { + result = append(result, x.Endpoints()...) + } + return result +} + +// UpdateEndpoints reconfigures the connection to use the given endpoints. +func (c *agencyConnection) UpdateEndpoints(endpoints []string) error { + c.mutex.Lock() + defer c.mutex.Unlock() + + newConnections := make([]driver.Connection, len(endpoints)) + for i, ep := range endpoints { + config := c.config + config.Endpoints = []string{ep} + config.DontFollowRedirect = true + httpConn, err := http.NewConnection(config) + if err != nil { + return driver.WithStack(err) + } + if c.auth != nil { + httpConn, err = httpConn.SetAuthentication(c.auth) + if err != nil { + return driver.WithStack(err) + } + } + newConnections[i] = httpConn + } + c.connections = newConnections + return nil +} + +// Configure the authentication used for this connection. +func (c *agencyConnection) SetAuthentication(auth driver.Authentication) (driver.Connection, error) { + c.mutex.Lock() + defer c.mutex.Unlock() + + newConnections := make([]driver.Connection, len(c.connections)) + for i, x := range c.connections { + xAuth, err := x.SetAuthentication(auth) + if err != nil { + return nil, driver.WithStack(err) + } + newConnections[i] = xAuth + } + c.connections = newConnections + c.auth = auth + return c, nil +} + +// Protocols returns all protocols used by this connection. +func (c *agencyConnection) Protocols() driver.ProtocolSet { + c.mutex.RLock() + defer c.mutex.RUnlock() + + result := driver.ProtocolSet{} + for _, x := range c.connections { + for _, p := range x.Protocols() { + if !result.Contains(p) { + result = append(result, p) + } + } + } + return result +} + +// agencyConnectionFailureBackoff returns a backoff delay for cases where all +// agents responded with a non-fatal error. +func agencyConnectionFailureBackoff(lastDelay time.Duration) time.Duration { + return increaseDelay(lastDelay, 1.5, time.Millisecond, time.Second*2) +} + +// increaseDelay returns an delay, increased from an old delay with a given +// factor, limited to given min & max. +func increaseDelay(oldDelay time.Duration, factor float64, min, max time.Duration) time.Duration { + delay := time.Duration(float64(oldDelay) * factor) + if delay < min { + delay = min + } + if delay > max { + delay = max + } + return delay +} diff --git a/deps/github.com/arangodb/go-driver/agency/agency_health.go b/deps/github.com/arangodb/go-driver/agency/agency_health.go new file mode 100644 index 000000000..0dc826d73 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/agency/agency_health.go @@ -0,0 +1,141 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package agency + +import ( + "context" + "fmt" + "net/url" + "strings" + "sync" + "time" + + driver "github.com/arangodb/go-driver" +) + +const ( + maxAgentResponseTime = time.Second * 10 + keyAllowNoLeader driver.ContextKey = "arangodb-agency-allow-no-leader" +) + +// agentStatus is a helper structure used in AreAgentsHealthy. +type agentStatus struct { + IsLeader bool + LeaderEndpoint string + IsResponding bool +} + +// WithAllowNoLeader is used to configure a context to make AreAgentsHealthy +// accept the situation where it finds 0 leaders. +func WithAllowNoLeader(parent context.Context) context.Context { + if parent == nil { + parent = context.Background() + } + return context.WithValue(parent, keyAllowNoLeader, true) +} + +// hasAllowNoLeader returns true when the given context was +// prepared with WithAllowNoLeader. +func hasAllowNoLeader(ctx context.Context) bool { + return ctx != nil && ctx.Value(keyAllowNoLeader) != nil +} + +// AreAgentsHealthy performs a health check on all given agents. +// Of the given agents, 1 must respond as leader and all others must redirect to the leader. +// The function returns nil when all agents are healthy or an error when something is wrong. +func AreAgentsHealthy(ctx context.Context, clients []driver.Connection) error { + wg := sync.WaitGroup{} + invalidKey := []string{"does-not-exist-70ddb948-59ea-52f3-9a19-baaca18de7ae"} + statuses := make([]agentStatus, len(clients)) + for i, c := range clients { + wg.Add(1) + go func(i int, c driver.Connection) { + defer wg.Done() + lctx, cancel := context.WithTimeout(ctx, maxAgentResponseTime) + defer cancel() + var result interface{} + a, err := NewAgency(c) + if err == nil { + var resp driver.Response + lctx = driver.WithResponse(lctx, &resp) + if err := a.ReadKey(lctx, invalidKey, &result); err == nil || IsKeyNotFound(err) { + // We got a valid read from the leader + statuses[i].IsLeader = true + statuses[i].LeaderEndpoint = strings.Join(c.Endpoints(), ",") + statuses[i].IsResponding = true + } else { + if driver.IsArangoErrorWithCode(err, 307) && resp != nil { + location := resp.Header("Location") + // Valid response from a follower + statuses[i].IsLeader = false + statuses[i].LeaderEndpoint = location + statuses[i].IsResponding = true + } else { + // Unexpected / invalid response + statuses[i].IsResponding = false + } + } + } + }(i, c) + } + wg.Wait() + + // Check the results + noLeaders := 0 + for i, status := range statuses { + if !status.IsResponding { + return driver.WithStack(fmt.Errorf("Agent %s is not responding", strings.Join(clients[i].Endpoints(), ","))) + } + if status.IsLeader { + noLeaders++ + } + if i > 0 { + // Compare leader endpoint with previous + prev := statuses[i-1].LeaderEndpoint + if !IsSameEndpoint(prev, status.LeaderEndpoint) { + return driver.WithStack(fmt.Errorf("Not all agents report the same leader endpoint")) + } + } + } + if noLeaders != 1 && !hasAllowNoLeader(ctx) { + return driver.WithStack(fmt.Errorf("Unexpected number of agency leaders: %d", noLeaders)) + } + return nil +} + +// IsSameEndpoint returns true when the 2 given endpoints +// refer to the same server. +func IsSameEndpoint(a, b string) bool { + if a == b { + return true + } + ua, err := url.Parse(a) + if err != nil { + return false + } + ub, err := url.Parse(b) + if err != nil { + return false + } + return ua.Hostname() == ub.Hostname() +} diff --git a/deps/github.com/arangodb/go-driver/agency/agency_impl.go b/deps/github.com/arangodb/go-driver/agency/agency_impl.go new file mode 100644 index 000000000..fb1f6a050 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/agency/agency_impl.go @@ -0,0 +1,369 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package agency + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/arangodb/go-driver" +) + +var ( + condTrue = true // Do not change! + condFalse = false // Do not change! +) + +type agency struct { + conn driver.Connection +} + +// NewAgency creates an Agency accessor for the given connection. +// The connection must contain the endpoints of one or more agents, and only agents. +func NewAgency(conn driver.Connection) (Agency, error) { + return &agency{ + conn: conn, + }, nil +} + +// Connection returns the connection used by this api. +func (c *agency) Connection() driver.Connection { + return c.conn +} + +// ReadKey reads the value of a given key in the agency. +func (c *agency) ReadKey(ctx context.Context, key []string, value interface{}) error { + conn := c.conn + req, err := conn.NewRequest("POST", "_api/agency/read") + if err != nil { + return driver.WithStack(err) + } + fullKey := createFullKey(key) + input := [][]string{{fullKey}} + req, err = req.SetBody(input) + if err != nil { + return driver.WithStack(err) + } + //var raw []byte + //ctx = driver.WithRawResponse(ctx, &raw) + resp, err := conn.Do(ctx, req) + if err != nil { + return driver.WithStack(err) + } + if err := resp.CheckStatus(200, 201, 202); err != nil { + return driver.WithStack(err) + } + //fmt.Printf("Agent response: %s\n", string(raw)) + elems, err := resp.ParseArrayBody() + if err != nil { + return driver.WithStack(err) + } + if len(elems) != 1 { + return driver.WithStack(fmt.Errorf("Expected 1 element, got %d", len(elems))) + } + // If empty key parse directly + if len(key) == 0 { + if err := elems[0].ParseBody("", &value); err != nil { + return driver.WithStack(err) + } + } else { + // Now remove all wrapping objects for each key element + var rawObject map[string]interface{} + if err := elems[0].ParseBody("", &rawObject); err != nil { + return driver.WithStack(err) + } + var rawMsg interface{} + for keyIndex := 0; keyIndex < len(key); keyIndex++ { + if keyIndex > 0 { + var ok bool + rawObject, ok = rawMsg.(map[string]interface{}) + if !ok { + return driver.WithStack(fmt.Errorf("Data is not an object at key %s", key[:keyIndex+1])) + } + } + var found bool + rawMsg, found = rawObject[key[keyIndex]] + if !found { + return driver.WithStack(KeyNotFoundError{Key: key[:keyIndex+1]}) + } + } + // Encode to json ... + encoded, err := json.Marshal(rawMsg) + if err != nil { + return driver.WithStack(err) + } + // and decode back into result + if err := json.Unmarshal(encoded, &value); err != nil { + return driver.WithStack(err) + } + } + + // fmt.Printf("result as JSON: %s\n", rawResult) + return nil +} + +type writeUpdate struct { + Operation string `json:"op,omitempty"` + New interface{} `json:"new,omitempty"` + TTL int64 `json:"ttl,omitempty"` + URL string `json:"url,omitempty"` +} + +type writeCondition struct { + Old interface{} `json:"old,omitempty"` // Require old value to be equal to this + OldEmpty *bool `json:"oldEmpty,omitempty"` // Require old value to be empty + IsArray *bool `json:"isArray,omitempty"` // Require old value to be array +} + +type writeTransaction []map[string]interface{} +type writeTransactions []writeTransaction + +type writeResult struct { + Results []int64 `json:"results"` +} + +// WriteKey writes the given value with the given key with a given TTL (unless TTL is zero). +// If you pass a condition (only 1 allowed), this condition has to be true, +// otherwise the write will fail with a ConditionFailed error. +func (c *agency) WriteKey(ctx context.Context, key []string, value interface{}, ttl time.Duration, condition ...WriteCondition) error { + var cond WriteCondition + switch len(condition) { + case 0: + // No condition, do nothing + case 1: + cond = condition[0] + default: + return driver.WithStack(fmt.Errorf("too many conditions")) + } + if err := c.write(ctx, "set", key, value, cond, ttl); err != nil { + return driver.WithStack(err) + } + return nil +} + +// WriteKeyIfEmpty writes the given value with the given key only if the key was empty before. +func (c *agency) WriteKeyIfEmpty(ctx context.Context, key []string, value interface{}, ttl time.Duration) error { + var cond WriteCondition + cond = cond.IfEmpty(key) + if err := c.write(ctx, "set", key, value, cond, ttl); err != nil { + return driver.WithStack(err) + } + return nil +} + +// WriteKeyIfEqualTo writes the given new value with the given key only if the existing value for that key equals +// to the given old value. +func (c *agency) WriteKeyIfEqualTo(ctx context.Context, key []string, newValue, oldValue interface{}, ttl time.Duration) error { + var cond WriteCondition + cond = cond.IfEqualTo(key, oldValue) + if err := c.write(ctx, "set", key, newValue, cond, ttl); err != nil { + return driver.WithStack(err) + } + return nil +} + +// write writes the given value with the given key only if the given condition is fullfilled. +func (c *agency) write(ctx context.Context, operation string, key []string, value interface{}, condition WriteCondition, ttl time.Duration) error { + conn := c.conn + req, err := conn.NewRequest("POST", "_api/agency/write") + if err != nil { + return driver.WithStack(err) + } + + fullKey := createFullKey(key) + writeTxs := writeTransactions{ + writeTransaction{ + // Update + map[string]interface{}{ + fullKey: writeUpdate{ + Operation: operation, + New: value, + TTL: int64(ttl.Seconds()), + }, + }, + // Condition + condition.toMap(), + }, + } + req, err = req.SetBody(writeTxs) + if err != nil { + return driver.WithStack(err) + } + resp, err := conn.Do(ctx, req) + if err != nil { + return driver.WithStack(err) + } + + var result writeResult + if err := resp.CheckStatus(200, 201, 202); err != nil { + return driver.WithStack(err) + } + if err := resp.ParseBody("", &result); err != nil { + return driver.WithStack(err) + } + + // "results" should be 1 long + if len(result.Results) != 1 { + return driver.WithStack(fmt.Errorf("Expected results of 1 long, got %d", len(result.Results))) + } + + // If results[0] == 0, condition failed, otherwise success + if result.Results[0] == 0 { + // Condition failed + return driver.WithStack(preconditionFailedError) + } + + // Success + return nil +} + +// RemoveKey removes the given key. +// If you pass a condition (only 1 allowed), this condition has to be true, +// otherwise the remove will fail with a ConditionFailed error. +func (c *agency) RemoveKey(ctx context.Context, key []string, condition ...WriteCondition) error { + var cond WriteCondition + switch len(condition) { + case 0: + // No condition, do nothing + case 1: + cond = condition[0] + default: + return driver.WithStack(fmt.Errorf("too many conditions")) + } + if err := c.write(ctx, "delete", key, nil, cond, 0); err != nil { + return driver.WithStack(err) + } + return nil +} + +// RemoveKeyIfEqualTo removes the given key only if the existing value for that key equals +// to the given old value. +func (c *agency) RemoveKeyIfEqualTo(ctx context.Context, key []string, oldValue interface{}) error { + var cond WriteCondition + cond = cond.IfEqualTo(key, oldValue) + if err := c.write(ctx, "delete", key, nil, cond, 0); err != nil { + return driver.WithStack(err) + } + return nil +} + +// Register a URL to receive notification callbacks when the value of the given key changes +func (c *agency) RegisterChangeCallback(ctx context.Context, key []string, cbURL string) error { + conn := c.conn + req, err := conn.NewRequest("POST", "_api/agency/write") + if err != nil { + return driver.WithStack(err) + } + + fullKey := createFullKey(key) + writeTxs := writeTransactions{ + writeTransaction{ + // Update + map[string]interface{}{ + fullKey: writeUpdate{ + Operation: "observe", + URL: cbURL, + }, + }, + }, + } + + req, err = req.SetBody(writeTxs) + if err != nil { + return driver.WithStack(err) + } + resp, err := conn.Do(ctx, req) + if err != nil { + return driver.WithStack(err) + } + + var result writeResult + if err := resp.CheckStatus(200, 201, 202); err != nil { + return driver.WithStack(err) + } + if err := resp.ParseBody("", &result); err != nil { + return driver.WithStack(err) + } + + // "results" should be 1 long + if len(result.Results) != 1 { + return driver.WithStack(fmt.Errorf("Expected results of 1 long, got %d", len(result.Results))) + } + + // Success + return nil +} + +// Register a URL to receive notification callbacks when the value of the given key changes +func (c *agency) UnregisterChangeCallback(ctx context.Context, key []string, cbURL string) error { + conn := c.conn + req, err := conn.NewRequest("POST", "_api/agency/write") + if err != nil { + return driver.WithStack(err) + } + + fullKey := createFullKey(key) + writeTxs := writeTransactions{ + writeTransaction{ + // Update + map[string]interface{}{ + fullKey: writeUpdate{ + Operation: "unobserve", + URL: cbURL, + }, + }, + }, + } + + req, err = req.SetBody(writeTxs) + if err != nil { + return driver.WithStack(err) + } + resp, err := conn.Do(ctx, req) + if err != nil { + return driver.WithStack(err) + } + + var result writeResult + if err := resp.CheckStatus(200, 201, 202); err != nil { + return driver.WithStack(err) + } + if err := resp.ParseBody("", &result); err != nil { + return driver.WithStack(err) + } + + // "results" should be 1 long + if len(result.Results) != 1 { + return driver.WithStack(fmt.Errorf("Expected results of 1 long, got %d", len(result.Results))) + } + + // Success + return nil +} + +func createFullKey(key []string) string { + return "/" + strings.Join(key, "/") +} diff --git a/deps/github.com/arangodb/go-driver/agency/doc.go b/deps/github.com/arangodb/go-driver/agency/doc.go new file mode 100644 index 000000000..76412a159 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/agency/doc.go @@ -0,0 +1,38 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package agency provides an API to access the ArangoDB agency (it is unlikely that you need this package directly). + +The Agency is fault-tolerant and highly-available key-value store +that is used to store critical, low-level information about +an ArangoDB cluster. + +THIS API IS NOT USED FOR NORMAL DATABASE ACCESS. + +Reasons for using this API are: +- You want to make use of an indepent Agency as your own HA key-value store. +- You want access to low-level information of your database. USE WITH GREAT CARE! + +WARNING: Messing around in the Agency can quickly lead to a corrupt database! +*/ +package agency diff --git a/deps/github.com/arangodb/go-driver/agency/error.go b/deps/github.com/arangodb/go-driver/agency/error.go new file mode 100644 index 000000000..ddbc3fdfa --- /dev/null +++ b/deps/github.com/arangodb/go-driver/agency/error.go @@ -0,0 +1,55 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package agency + +import ( + "fmt" + "net/http" + "strings" + + driver "github.com/arangodb/go-driver" +) + +var ( + // preconditionFailedError indicates that a precondition for the request is not existing. + preconditionFailedError = driver.ArangoError{ + HasError: true, + Code: http.StatusPreconditionFailed, + } +) + +// KeyNotFoundError indicates that a key was not found. +type KeyNotFoundError struct { + Key []string +} + +// Error returns a human readable error string +func (e KeyNotFoundError) Error() string { + return fmt.Sprintf("Key '%s' not found", strings.Join(e.Key, "/")) +} + +// IsKeyNotFound returns true if the given error is (or is caused by) a KeyNotFoundError. +func IsKeyNotFound(err error) bool { + _, ok := driver.Cause(err).(KeyNotFoundError) + return ok +} diff --git a/deps/github.com/arangodb/go-driver/agency/lock.go b/deps/github.com/arangodb/go-driver/agency/lock.go new file mode 100644 index 000000000..11fc3414f --- /dev/null +++ b/deps/github.com/arangodb/go-driver/agency/lock.go @@ -0,0 +1,204 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package agency + +import ( + "context" + "crypto/rand" + "encoding/hex" + "sync" + "time" + + driver "github.com/arangodb/go-driver" +) + +const ( + minLockTTL = time.Second * 5 +) + +// Lock is an agency backed exclusive lock. +type Lock interface { + // Lock tries to lock the lock. + // If it is not possible to lock, an error is returned. + // If the lock is already held by me, an error is returned. + Lock(ctx context.Context) error + + // Unlock tries to unlock the lock. + // If it is not possible to unlock, an error is returned. + // If the lock is not held by me, an error is returned. + Unlock(ctx context.Context) error + + // IsLocked return true if the lock is held by me. + IsLocked() bool +} + +// Logger abstracts a logger. +type Logger interface { + Errorf(msg string, args ...interface{}) +} + +// NewLock creates a new lock on the given key. +func NewLock(log Logger, api Agency, key []string, id string, ttl time.Duration) (Lock, error) { + if ttl < minLockTTL { + ttl = minLockTTL + } + if id == "" { + randBytes := make([]byte, 16) + rand.Read(randBytes) + id = hex.EncodeToString(randBytes) + } + return &lock{ + log: log, + api: api, + key: key, + id: id, + ttl: ttl, + }, nil +} + +type lock struct { + mutex sync.Mutex + log Logger + api Agency + key []string + id string + ttl time.Duration + locked bool + cancelRenewal func() +} + +// Lock tries to lock the lock. +// If it is not possible to lock, an error is returned. +// If the lock is already held by me, an error is returned. +func (l *lock) Lock(ctx context.Context) error { + l.mutex.Lock() + defer l.mutex.Unlock() + + if l.locked { + return driver.WithStack(AlreadyLockedError) + } + + // Try to claim lock + ctx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + if err := l.api.WriteKeyIfEmpty(ctx, l.key, l.id, l.ttl); err != nil { + if driver.IsPreconditionFailed(err) { + return driver.WithStack(AlreadyLockedError) + } + return driver.WithStack(err) + } + + // Success + l.locked = true + + // Keep renewing + renewCtx, renewCancel := context.WithCancel(context.Background()) + go l.renewLock(renewCtx) + l.cancelRenewal = renewCancel + + return nil +} + +// Unlock tries to unlock the lock. +// If it is not possible to unlock, an error is returned. +// If the lock is not held by me, an error is returned. +func (l *lock) Unlock(ctx context.Context) error { + l.mutex.Lock() + defer l.mutex.Unlock() + + if !l.locked { + return driver.WithStack(NotLockedError) + } + + // Release the lock + ctx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + if err := l.api.RemoveKeyIfEqualTo(ctx, l.key, l.id); err != nil { + return driver.WithStack(err) + } + + // Cleanup + l.locked = false + if l.cancelRenewal != nil { + l.cancelRenewal() + l.cancelRenewal = nil + } + + return nil +} + +// IsLocked return true if the lock is held by me. +func (l *lock) IsLocked() bool { + l.mutex.Lock() + defer l.mutex.Unlock() + return l.locked +} + +// renewLock keeps renewing the lock until the given context is canceled. +func (l *lock) renewLock(ctx context.Context) { + // op performs a renewal once. + // returns stop, error + op := func() (bool, error) { + l.mutex.Lock() + defer l.mutex.Unlock() + + if !l.locked { + return true, driver.WithStack(NotLockedError) + } + + // Update key in agency + ctx, cancel := context.WithTimeout(ctx, time.Second*10) + defer cancel() + if err := l.api.WriteKeyIfEqualTo(ctx, l.key, l.id, l.id, l.ttl); err != nil { + if driver.IsPreconditionFailed(err) { + // We're not longer the leader + l.locked = false + l.cancelRenewal = nil + return true, driver.WithStack(err) + } + return false, driver.WithStack(err) + } + return false, nil + } + for { + delay := l.ttl / 2 + stop, err := op() + if stop || driver.Cause(err) == context.Canceled { + return + } + if err != nil { + if l.log != nil { + l.log.Errorf("Failed to renew lock %s. %v", l.key, err) + } + delay = time.Second + } + + select { + case <-ctx.Done(): + // we're done + return + case <-time.After(delay): + // Try to renew + } + } +} diff --git a/deps/github.com/arangodb/go-driver/agency/lock_errors.go b/deps/github.com/arangodb/go-driver/agency/lock_errors.go new file mode 100644 index 000000000..d6b468cbd --- /dev/null +++ b/deps/github.com/arangodb/go-driver/agency/lock_errors.go @@ -0,0 +1,46 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package agency + +import ( + "errors" + + driver "github.com/arangodb/go-driver" +) + +var ( + // AlreadyLockedError indicates that the lock is already locked. + AlreadyLockedError = errors.New("already locked") + // NotLockedError indicates that the lock is not locked when trying to unlock. + NotLockedError = errors.New("not locked") +) + +// IsAlreadyLocked returns true if the given error is or is caused by an AlreadyLockedError. +func IsAlreadyLocked(err error) bool { + return driver.Cause(err) == AlreadyLockedError +} + +// IsNotLocked returns true if the given error is or is caused by an NotLockedError. +func IsNotLocked(err error) bool { + return driver.Cause(err) == NotLockedError +} diff --git a/deps/github.com/arangodb/go-driver/client_server_info.go b/deps/github.com/arangodb/go-driver/client_server_info.go index 179f89873..30fe0ccc7 100644 --- a/deps/github.com/arangodb/go-driver/client_server_info.go +++ b/deps/github.com/arangodb/go-driver/client_server_info.go @@ -34,6 +34,10 @@ type ClientServerInfo interface { // ServerRole returns the role of the server that answers the request. ServerRole(ctx context.Context) (ServerRole, error) + + // Gets the ID of this server in the cluster. + // An error is returned when calling this to a server that is not part of a cluster. + ServerID(ctx context.Context) (string, error) } // ServerRole is the role of an arangod server diff --git a/deps/github.com/arangodb/go-driver/client_server_info_impl.go b/deps/github.com/arangodb/go-driver/client_server_info_impl.go index 611fc86c5..6fee52a51 100644 --- a/deps/github.com/arangodb/go-driver/client_server_info_impl.go +++ b/deps/github.com/arangodb/go-driver/client_server_info_impl.go @@ -107,6 +107,32 @@ func (c *client) ServerRole(ctx context.Context) (ServerRole, error) { return role, nil } +type idResponse struct { + ID string `json:"id,omitempty"` +} + +// Gets the ID of this server in the cluster. +// An error is returned when calling this to a server that is not part of a cluster. +func (c *client) ServerID(ctx context.Context) (string, error) { + req, err := c.conn.NewRequest("GET", "_admin/server/id") + if err != nil { + return "", WithStack(err) + } + applyContextSettings(ctx, req) + resp, err := c.conn.Do(ctx, req) + if err != nil { + return "", WithStack(err) + } + if err := resp.CheckStatus(200); err != nil { + return "", WithStack(err) + } + var data idResponse + if err := resp.ParseBody("", &data); err != nil { + return "", WithStack(err) + } + return data.ID, nil +} + // clusterEndpoints returns the endpoints of a cluster. func (c *client) echo(ctx context.Context) error { req, err := c.conn.NewRequest("GET", "_admin/echo") diff --git a/deps/github.com/arangodb/go-driver/cluster.go b/deps/github.com/arangodb/go-driver/cluster.go index d51a674fd..fc414bc83 100644 --- a/deps/github.com/arangodb/go-driver/cluster.go +++ b/deps/github.com/arangodb/go-driver/cluster.go @@ -45,6 +45,11 @@ type Cluster interface { // IsCleanedOut checks if the dbserver with given ID has been cleaned out. IsCleanedOut(ctx context.Context, serverID string) (bool, error) + + // RemoveServer is a low-level option to remove a server from a cluster. + // This function is suitable for servers of type coordinator or dbserver. + // The use of `ClientServerAdmin.Shutdown` is highly recommended above this function. + RemoveServer(ctx context.Context, serverID ServerID) error } // ServerID identifies an arangod server in a cluster. diff --git a/deps/github.com/arangodb/go-driver/cluster/doc.go b/deps/github.com/arangodb/go-driver/cluster/doc.go new file mode 100644 index 000000000..53abf2920 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/cluster/doc.go @@ -0,0 +1,26 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package cluster implements a driver.Connection that provides cluster failover support (it is not intended to be used directly). +*/ +package cluster diff --git a/deps/github.com/arangodb/go-driver/cluster_impl.go b/deps/github.com/arangodb/go-driver/cluster_impl.go index 3bf795f82..0ddd307f0 100644 --- a/deps/github.com/arangodb/go-driver/cluster_impl.go +++ b/deps/github.com/arangodb/go-driver/cluster_impl.go @@ -123,6 +123,10 @@ type cleanOutServerRequest struct { Server string `json:"server"` } +type cleanOutServerResponse struct { + JobID string `json:"id"` +} + // CleanOutServer triggers activities to clean out a DBServers. func (c *cluster) CleanOutServer(ctx context.Context, serverID string) error { req, err := c.conn.NewRequest("POST", "_admin/cluster/cleanOutServer") @@ -135,7 +139,7 @@ func (c *cluster) CleanOutServer(ctx context.Context, serverID string) error { if _, err := req.SetBody(input); err != nil { return WithStack(err) } - applyContextSettings(ctx, req) + cs := applyContextSettings(ctx, req) resp, err := c.conn.Do(ctx, req) if err != nil { return WithStack(err) @@ -143,6 +147,13 @@ func (c *cluster) CleanOutServer(ctx context.Context, serverID string) error { if err := resp.CheckStatus(200, 202); err != nil { return WithStack(err) } + var result cleanOutServerResponse + if err := resp.ParseBody("", &result); err != nil { + return WithStack(err) + } + if cs.JobIDResponse != nil { + *cs.JobIDResponse = result.JobID + } return nil } @@ -188,3 +199,25 @@ func (c *cluster) NumberOfServers(ctx context.Context) (NumberOfServersResponse, } return result, nil } + +// RemoveServer is a low-level option to remove a server from a cluster. +// This function is suitable for servers of type coordinator or dbserver. +// The use of `ClientServerAdmin.Shutdown` is highly recommended above this function. +func (c *cluster) RemoveServer(ctx context.Context, serverID ServerID) error { + req, err := c.conn.NewRequest("POST", "_admin/cluster/removeServer") + if err != nil { + return WithStack(err) + } + if _, err := req.SetBody(serverID); err != nil { + return WithStack(err) + } + applyContextSettings(ctx, req) + resp, err := c.conn.Do(ctx, req) + if err != nil { + return WithStack(err) + } + if err := resp.CheckStatus(200, 202); err != nil { + return WithStack(err) + } + return nil +} diff --git a/deps/github.com/arangodb/go-driver/collection_document_impl.go b/deps/github.com/arangodb/go-driver/collection_document_impl.go index b9a83eb0a..bc09aa5f7 100644 --- a/deps/github.com/arangodb/go-driver/collection_document_impl.go +++ b/deps/github.com/arangodb/go-driver/collection_document_impl.go @@ -103,7 +103,7 @@ func (c *collection) CreateDocument(ctx context.Context, document interface{}) ( if err != nil { return DocumentMeta{}, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, WithStack(err) } if cs.Silent { @@ -155,7 +155,7 @@ func (c *collection) CreateDocuments(ctx context.Context, documents interface{}) if err != nil { return nil, nil, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return nil, nil, WithStack(err) } if cs.Silent { @@ -196,7 +196,7 @@ func (c *collection) UpdateDocument(ctx context.Context, key string, update inte if err != nil { return DocumentMeta{}, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, WithStack(err) } if cs.Silent { @@ -264,7 +264,7 @@ func (c *collection) UpdateDocuments(ctx context.Context, keys []string, updates if err != nil { return nil, nil, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return nil, nil, WithStack(err) } if cs.Silent { @@ -305,7 +305,7 @@ func (c *collection) ReplaceDocument(ctx context.Context, key string, document i if err != nil { return DocumentMeta{}, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, WithStack(err) } if cs.Silent { @@ -373,7 +373,7 @@ func (c *collection) ReplaceDocuments(ctx context.Context, keys []string, docume if err != nil { return nil, nil, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return nil, nil, WithStack(err) } if cs.Silent { @@ -407,7 +407,7 @@ func (c *collection) RemoveDocument(ctx context.Context, key string) (DocumentMe if err != nil { return DocumentMeta{}, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(200, 202)); err != nil { + if err := resp.CheckStatus(200, 202); err != nil { return DocumentMeta{}, WithStack(err) } if cs.Silent { @@ -456,7 +456,7 @@ func (c *collection) RemoveDocuments(ctx context.Context, keys []string) (Docume if err != nil { return nil, nil, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(200, 202); err != nil { return nil, nil, WithStack(err) } if cs.Silent { diff --git a/deps/github.com/arangodb/go-driver/context.go b/deps/github.com/arangodb/go-driver/context.go index 3b39bea05..170e2d5dd 100644 --- a/deps/github.com/arangodb/go-driver/context.go +++ b/deps/github.com/arangodb/go-driver/context.go @@ -55,6 +55,8 @@ const ( keyConfigured ContextKey = "arangodb-configured" keyFollowLeaderRedirect ContextKey = "arangodb-followLeaderRedirect" keyDBServerID ContextKey = "arangodb-dbserverID" + keyBatchID ContextKey = "arangodb-batchID" + keyJobIDResponse ContextKey = "arangodb-jobIDResponse" ) // WithRevision is used to configure a context to make document @@ -206,6 +208,19 @@ func WithDBServerID(parent context.Context, id string) context.Context { return context.WithValue(contextOrBackground(parent), keyDBServerID, id) } +// WithBatchID is used to configure a context that includes an ID of a Batch. +// This is used in replication functions. +func WithBatchID(parent context.Context, id string) context.Context { + return context.WithValue(contextOrBackground(parent), keyBatchID, id) +} + +// WithJobIDResponse is used to configure a context that includes a reference to a JobID +// that is filled on a error-free response. +// This is used in cluster functions. +func WithJobIDResponse(parent context.Context, jobID *string) context.Context { + return context.WithValue(contextOrBackground(parent), keyJobIDResponse, jobID) +} + type contextSettings struct { Silent bool WaitForSync bool @@ -221,6 +236,8 @@ type contextSettings struct { Configured *bool FollowLeaderRedirect *bool DBServerID string + BatchID string + JobIDResponse *string } // applyContextSettings returns the settings configured in the context in the given request. @@ -341,17 +358,20 @@ func applyContextSettings(ctx context.Context, req Request) contextSettings { result.DBServerID = id } } - return result -} - -// okStatus returns one of the given status codes depending on the WaitForSync field value. -// If WaitForSync==true, statusWithWaitForSync is returned, otherwise statusWithoutWaitForSync is returned. -func (cs contextSettings) okStatus(statusWithWaitForSync, statusWithoutWaitForSync int) int { - if cs.WaitForSync { - return statusWithWaitForSync - } else { - return statusWithoutWaitForSync + // BatchID + if v := ctx.Value(keyBatchID); v != nil { + if id, ok := v.(string); ok { + req.SetQuery("batchId", id) + result.BatchID = id + } } + // JobIDResponse + if v := ctx.Value(keyJobIDResponse); v != nil { + if idRef, ok := v.(*string); ok { + result.JobIDResponse = idRef + } + } + return result } // contextOrBackground returns the given context if it is not nil. diff --git a/deps/github.com/arangodb/go-driver/cursor.go b/deps/github.com/arangodb/go-driver/cursor.go index da0deeedd..6ee03dc4b 100644 --- a/deps/github.com/arangodb/go-driver/cursor.go +++ b/deps/github.com/arangodb/go-driver/cursor.go @@ -25,8 +25,30 @@ package driver import ( "context" "io" + "time" ) +// Statistics returned with the query cursor +type QueryStatistics interface { + // the total number of data-modification operations successfully executed. + WritesExecuted() int64 + // The total number of data-modification operations that were unsuccessful + WritesIgnored() int64 + // The total number of documents iterated over when scanning a collection without an index. + ScannedFull() int64 + // The total number of documents iterated over when scanning a collection using an index. + ScannedIndex() int64 + // the total number of documents that were removed after executing a filter condition in a FilterNode + Filtered() int64 + // Returns the numer of results before the last LIMIT in the query was applied. + // A valid return value is only available when the has been created with a context that was + // prepared with `WithFullCount`. Additionally this will also not return a valid value if + // the context was prepared with `WithStream`. + FullCount() int64 + // Execution time of the query (wall-clock time). value will be set from the outside + ExecutionTime() time.Duration +} + // Cursor is returned from a query, used to iterate over a list of documents. // Note that a Cursor must always be closed to avoid holding on to resources in the server while they are no longer needed. type Cursor interface { @@ -44,6 +66,11 @@ type Cursor interface { // Count returns the total number of result documents available. // A valid return value is only available when the cursor has been created with a context that was - // prepare with `WithQueryCount`. + // prepared with `WithQueryCount` and not with `WithQueryStream`. Count() int64 + + // Statistics returns the query execution statistics for this cursor. + // This might not be valid if the cursor has been created with a context that was + // prepared with `WithQueryStream` + Statistics() QueryStatistics } diff --git a/deps/github.com/arangodb/go-driver/cursor_impl.go b/deps/github.com/arangodb/go-driver/cursor_impl.go index 62c7edc3e..805d4baf3 100644 --- a/deps/github.com/arangodb/go-driver/cursor_impl.go +++ b/deps/github.com/arangodb/go-driver/cursor_impl.go @@ -24,9 +24,12 @@ package driver import ( "context" + "encoding/json" "path" + "reflect" "sync" "sync/atomic" + "time" ) // newCursor creates a new Cursor implementation. @@ -52,11 +55,31 @@ type cursor struct { closeMutex sync.Mutex } +type cursorStats struct { + // The total number of data-modification operations successfully executed. + WritesExecutedInt int64 `json:"writesExecuted,omitempty"` + // The total number of data-modification operations that were unsuccessful + WritesIgnoredInt int64 `json:"writesIgnored,omitempty"` + // The total number of documents iterated over when scanning a collection without an index. + ScannedFullInt int64 `json:"scannedFull,omitempty"` + // The total number of documents iterated over when scanning a collection using an index. + ScannedIndexInt int64 `json:"scannedIndex,omitempty"` + // The total number of documents that were removed after executing a filter condition in a FilterNode + FilteredInt int64 `json:"filtered,omitempty"` + // The total number of documents that matched the search condition if the query's final LIMIT statement were not present. + FullCountInt int64 `json:"fullCount,omitempty"` + // Query execution time (wall-clock time). value will be set from the outside + ExecutionTimeInt float64 `json:"executionTime,omitempty"` +} + type cursorData struct { Count int64 `json:"count,omitempty"` // the total number of result documents available (only available if the query was executed with the count attribute set) ID string `json:"id"` // id of temporary cursor created on the server (optional, see above) Result []*RawObject `json:"result,omitempty"` // an array of result documents (might be empty if query has no results) HasMore bool `json:"hasMore,omitempty"` // A boolean indicator whether there are more results available for the cursor on the server + Extra struct { + Stats cursorStats `json:"stats,omitempty"` + } `json:"extra"` } // relPath creates the relative path to this cursor (`_db//_api/cursor`) @@ -144,12 +167,68 @@ func (c *cursor) ReadDocument(ctx context.Context, result interface{}) (Document } c.resultIndex++ var meta DocumentMeta - if err := c.conn.Unmarshal(*c.Result[index], &meta); err != nil { - // If a cursor returns something other than a document, this will fail. - // Just ignore it. - } - if err := c.conn.Unmarshal(*c.Result[index], result); err != nil { - return DocumentMeta{}, WithStack(err) + resultPtr := c.Result[index] + if resultPtr == nil { + // Got NULL result + rv := reflect.ValueOf(result) + if rv.Kind() != reflect.Ptr || rv.IsNil() { + return DocumentMeta{}, WithStack(&json.InvalidUnmarshalError{Type: reflect.TypeOf(result)}) + } + e := rv.Elem() + e.Set(reflect.Zero(e.Type())) + } else { + if err := c.conn.Unmarshal(*resultPtr, &meta); err != nil { + // If a cursor returns something other than a document, this will fail. + // Just ignore it. + } + if err := c.conn.Unmarshal(*resultPtr, result); err != nil { + return DocumentMeta{}, WithStack(err) + } } return meta, nil } + +// Return execution statistics for this cursor. This might not +// be valid if the cursor has been created with a context that was +// prepared with `WithStream` +func (c *cursor) Statistics() QueryStatistics { + return c.cursorData.Extra.Stats +} + +// the total number of data-modification operations successfully executed. +func (cs cursorStats) WritesExecuted() int64 { + return cs.WritesExecutedInt +} + +// The total number of data-modification operations that were unsuccessful +func (cs cursorStats) WritesIgnored() int64 { + return cs.WritesIgnoredInt +} + +// The total number of documents iterated over when scanning a collection without an index. +func (cs cursorStats) ScannedFull() int64 { + return cs.ScannedFullInt +} + +// The total number of documents iterated over when scanning a collection using an index. +func (cs cursorStats) ScannedIndex() int64 { + return cs.ScannedIndexInt +} + +// the total number of documents that were removed after executing a filter condition in a FilterNode +func (cs cursorStats) Filtered() int64 { + return cs.FilteredInt +} + +// Returns the numer of results before the last LIMIT in the query was applied. +// A valid return value is only available when the has been created with a context that was +// prepared with `WithFullCount`. Additionally this will also not return a valid value if +// the context was prepared with `WithStream`. +func (cs cursorStats) FullCount() int64 { + return cs.FullCountInt +} + +// query execution time (wall-clock time). value will be set from the outside +func (cs cursorStats) ExecutionTime() time.Duration { + return time.Duration(cs.ExecutionTimeInt) * time.Second +} diff --git a/deps/github.com/arangodb/go-driver/docs/Drivers/GO/ConnectionManagement/README.md b/deps/github.com/arangodb/go-driver/docs/Drivers/GO/ConnectionManagement/README.md new file mode 100644 index 000000000..e9ee5af26 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/docs/Drivers/GO/ConnectionManagement/README.md @@ -0,0 +1,84 @@ +# ArangoDB GO Driver - Connection Management +## Failover + +The driver supports multiple endpoints to connect to. All request are in principle +send to the same endpoint until that endpoint fails to respond. +In that case a new endpoint is chosen and the operation is retried. + +The following example shows how to connect to a cluster of 3 servers. + +```go +conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"http://server1:8529", "http://server2:8529", "http://server3:8529"}, +}) +if err != nil { + // Handle error +} +c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, +}) +if err != nil { + // Handle error +} +``` + +Note that a valid endpoint is an URL to either a standalone server, or a URL to a coordinator +in a cluster. + +## Failover: Exact behavior + +The driver monitors the request being send to a specific server (endpoint). +As soon as the request has been completely written, failover will no longer happen. +The reason for that is that several operations cannot be (safely) retried. +E.g. when a request to create a document has been send to a server and a timeout +occurs, the driver has no way of knowing if the server did or did not create +the document in the database. + +If the driver detects that a request has been completely written, but still gets +an error (other than an error response from Arango itself), it will wrap the +error in a `ResponseError`. The client can test for such an error using `IsResponseError`. + +If a client received a `ResponseError`, it can do one of the following: +- Retry the operation and be prepared for some kind of duplicate record / unique constraint violation. +- Perform a test operation to see if the "failed" operation did succeed after all. +- Simply consider the operation failed. This is risky, since it can still be the case that the operation did succeed. + +## Failover: Timeouts + +To control the timeout of any function in the driver, you must pass it a context +configured with `context.WithTimeout` (or `context.WithDeadline`). + +In the case of multiple endpoints, the actual timeout used for requests will be shorter than +the timeout given in the context. +The driver will divide the timeout by the number of endpoints with a maximum of 3. +This ensures that the driver can try up to 3 different endpoints (in case of failover) without +being canceled due to the timeout given by the client. +E.g. +- With 1 endpoint and a given timeout of 1 minute, the actual request timeout will be 1 minute. +- With 3 endpoints and a given timeout of 1 minute, the actual request timeout will be 20 seconds. +- With 8 endpoints and a given timeout of 1 minute, the actual request timeout will be 20 seconds. + +For most requests you want a actual request timeout of at least 30 seconds. + +## Secure connections (SSL) + +The driver supports endpoints that use SSL using the `https` URL scheme. + +The following example shows how to connect to a server that has a secure endpoint using +a self-signed certificate. + +```go +conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"https://localhost:8529"}, + TLSConfig: &tls.Config{InsecureSkipVerify: true}, +}) +if err != nil { + // Handle error +} +c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, +}) +if err != nil { + // Handle error +} +``` diff --git a/deps/github.com/arangodb/go-driver/docs/Drivers/GO/ExampleRequests/README.md b/deps/github.com/arangodb/go-driver/docs/Drivers/GO/ExampleRequests/README.md new file mode 100644 index 000000000..e7f3f8937 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/docs/Drivers/GO/ExampleRequests/README.md @@ -0,0 +1,182 @@ +# ArangoDB GO Driver - Example requests + +## Connecting to ArangoDB + +```go +conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, + TLSConfig: &tls.Config{ /*...*/ }, +}) +if err != nil { + // Handle error +} +c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + Authentication: driver.BasicAuthentication("user", "password"), +}) +if err != nil { + // Handle error +} +``` + +## Opening a database + +```go +ctx := context.Background() +db, err := client.Database(ctx, "myDB") +if err != nil { + // handle error +} +``` + +## Opening a collection + +```go +ctx := context.Background() +col, err := db.Collection(ctx, "myCollection") +if err != nil { + // handle error +} +``` + +## Checking if a collection exists + +```go +ctx := context.Background() +found, err := db.CollectionExists(ctx, "myCollection") +if err != nil { + // handle error +} +``` + +## Creating a collection + +```go +ctx := context.Background() +options := &driver.CreateCollectionOptions{ /* ... */ } +col, err := db.CreateCollection(ctx, "myCollection", options) +if err != nil { + // handle error +} +``` + +## Reading a document from a collection + +```go +var doc MyDocument +ctx := context.Background() +meta, err := col.ReadDocument(ctx, myDocumentKey, &doc) +if err != nil { + // handle error +} +``` + +## Reading a document from a collection with an explicit revision + +```go +var doc MyDocument +revCtx := driver.WithRevision(ctx, "mySpecificRevision") +meta, err := col.ReadDocument(revCtx, myDocumentKey, &doc) +if err != nil { + // handle error +} +``` + +## Creating a document + +```go +doc := MyDocument{ + Name: "jan", + Counter: 23, +} +ctx := context.Background() +meta, err := col.CreateDocument(ctx, doc) +if err != nil { + // handle error +} +fmt.Printf("Created document with key '%s', revision '%s'\n", meta.Key, meta.Rev) +``` + +## Removing a document + +```go +ctx := context.Background() +err := col.RemoveDocument(revCtx, myDocumentKey) +if err != nil { + // handle error +} +``` + +## Removing a document with an explicit revision + +```go +revCtx := driver.WithRevision(ctx, "mySpecificRevision") +err := col.RemoveDocument(revCtx, myDocumentKey) +if err != nil { + // handle error +} +``` + +## Updating a document + +```go +ctx := context.Background() +patch := map[string]interface{}{ + "Name": "Frank", +} +meta, err := col.UpdateDocument(ctx, myDocumentKey, patch) +if err != nil { + // handle error +} +``` + +## Querying documents, one document at a time + +```go +ctx := context.Background() +query := "FOR d IN myCollection LIMIT 10 RETURN d" +cursor, err := db.Query(ctx, query, nil) +if err != nil { + // handle error +} +defer cursor.Close() +for { + var doc MyDocument + meta, err := cursor.ReadDocument(ctx, &doc) + if driver.IsNoMoreDocuments(err) { + break + } else if err != nil { + // handle other errors + } + fmt.Printf("Got doc with key '%s' from query\n", meta.Key) +} +``` + +## Querying documents, fetching total count + +```go +ctx := driver.WithQueryCount(context.Background()) +query := "FOR d IN myCollection RETURN d" +cursor, err := db.Query(ctx, query, nil) +if err != nil { + // handle error +} +defer cursor.Close() +fmt.Printf("Query yields %d documents\n", cursor.Count()) +``` + +## Querying documents, with bind variables + +```go +ctx := context.Background() +query := "FOR d IN myCollection FILTER d.Name == @name RETURN d" +bindVars := map[string]interface{}{ + "name": "Some name", +} +cursor, err := db.Query(ctx, query, bindVars) +if err != nil { + // handle error +} +defer cursor.Close() +... +``` diff --git a/deps/github.com/arangodb/go-driver/docs/Drivers/GO/GettingStarted/README.md b/deps/github.com/arangodb/go-driver/docs/Drivers/GO/GettingStarted/README.md new file mode 100644 index 000000000..59e1172d9 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/docs/Drivers/GO/GettingStarted/README.md @@ -0,0 +1,141 @@ +# ArangoDB GO Driver - Getting Started + +## Supported versions + +- ArangoDB versions 3.1 and up. + - Single server & cluster setups + - With or without authentication +- Go 1.7 and up. + +## Go dependencies + +- None (Additional error libraries are supported). + +## Configuration + +To use the driver, first fetch the sources into your GOPATH. + +```sh +go get github.com/arangodb/go-driver +``` + +Using the driver, you always need to create a `Client`. +The following example shows how to create a `Client` for a single server +running on localhost. + +```go +import ( + "fmt" + + driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/http" +) + +... + +conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, +}) +if err != nil { + // Handle error +} +c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, +}) +if err != nil { + // Handle error +} +``` + +Once you have a `Client` you can access/create databases on the server, +access/create collections, graphs, documents and so on. + +The following example shows how to open an existing collection in an existing database +and create a new document in that collection. + +```go +// Open "examples_books" database +db, err := c.Database(nil, "examples_books") +if err != nil { + // Handle error +} + +// Open "books" collection +col, err := db.Collection(nil, "books") +if err != nil { + // Handle error +} + +// Create document +book := Book{ + Title: "ArangoDB Cookbook", + NoPages: 257, +} +meta, err := col.CreateDocument(nil, book) +if err != nil { + // Handle error +} +fmt.Printf("Created document in collection '%s' in database '%s'\n", col.Name(), db.Name()) +``` + +## API design + +### Concurrency + +All functions of the driver are stricly synchronous. They operate and only return a value (or error) +when they're done. + +If you want to run operations concurrently, use a go routine. All objects in the driver are designed +to be used from multiple concurrent go routines, except `Cursor`. + +All database objects (except `Cursor`) are considered static. After their creation they won't change. +E.g. after creating a `Collection` instance you can remove the collection, but the (Go) instance +will still be there. Calling functions on such a removed collection will of course fail. + +### Structured error handling & wrapping + +All functions of the driver that can fail return an `error` value. If that value is not `nil`, the +function call is considered to be failed. In that case all other return values are set to their `zero` +values. + +All errors are structured using error checking functions named `Is`. +E.g. `IsNotFound(error)` return true if the given error is of the category "not found". +There can be multiple internal error codes that all map onto the same category. + +All errors returned from any function of the driver (either internal or exposed) wrap errors +using the `WithStack` function. This can be used to provide detail stack trackes in case of an error. +All error checking functions use the `Cause` function to get the cause of an error instead of the error wrapper. + +Note that `WithStack` and `Cause` are actually variables to you can implement it using your own error +wrapper library. + +If you for example use https://github.com/pkg/errors, you want to initialize to go driver like this: +```go +import ( + driver "github.com/arangodb/go-driver" + "github.com/pkg/errors" +) + +func init() { + driver.WithStack = errors.WithStack + driver.Cause = errors.Cause +} +``` + +### Context aware + +All functions of the driver that involve some kind of long running operation or +support additional options not given as function arguments, have a `context.Context` argument. +This enables you cancel running requests, pass timeouts/deadlines and pass additional options. + +In all methods that take a `context.Context` argument you can pass `nil` as value. +This is equivalent to passing `context.Background()`. + +Many functions support 1 or more optional (and infrequently used) additional options. +These can be used with a `With` function. +E.g. to force a create document call to wait until the data is synchronized to disk, +use a prepared context like this: +```go +ctx := driver.WithWaitForSync(parentContext) +collection.CreateDocument(ctx, yourDocument) +``` diff --git a/deps/github.com/arangodb/go-driver/docs/Drivers/GO/README.md b/deps/github.com/arangodb/go-driver/docs/Drivers/GO/README.md new file mode 100644 index 000000000..b98c1e2e5 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/docs/Drivers/GO/README.md @@ -0,0 +1,8 @@ +# ArangoDB GO Driver + +The official [ArangoDB](https://arangodb.com) GO Driver + +- [Getting Started](GettingStarted/README.md) +- [Example Requests](ExampleRequests/README.md) +- [Connection Management](ConnectionManagement/README.md) +- [Reference](https://godoc.org/github.com/arangodb/go-driver) diff --git a/deps/github.com/arangodb/go-driver/edge_collection_documents_impl.go b/deps/github.com/arangodb/go-driver/edge_collection_documents_impl.go index c09d5d7b7..100526953 100644 --- a/deps/github.com/arangodb/go-driver/edge_collection_documents_impl.go +++ b/deps/github.com/arangodb/go-driver/edge_collection_documents_impl.go @@ -103,7 +103,7 @@ func (c *edgeCollection) createDocument(ctx context.Context, document interface{ if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { @@ -318,7 +318,7 @@ func (c *edgeCollection) replaceDocument(ctx context.Context, key string, docume if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { @@ -433,7 +433,7 @@ func (c *edgeCollection) removeDocument(ctx context.Context, key string) (Docume if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(200, 202)); err != nil { + if err := resp.CheckStatus(200, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { diff --git a/deps/github.com/arangodb/go-driver/encode-go_1_8.go b/deps/github.com/arangodb/go-driver/encode-go_1_8.go index 1fc2d8348..1f2247177 100644 --- a/deps/github.com/arangodb/go-driver/encode-go_1_8.go +++ b/deps/github.com/arangodb/go-driver/encode-go_1_8.go @@ -20,7 +20,7 @@ // Author Ewout Prangsma // -// +build "go1.8" +// +build go1.8 package driver @@ -33,5 +33,6 @@ func pathEscape(s string) string { // pathUnescape unescapes the given value for use in a URL path. func pathUnescape(s string) string { - return url.PathUnescape(s) + r, _ := url.PathUnescape(s) + return r } diff --git a/deps/github.com/arangodb/go-driver/encode.go b/deps/github.com/arangodb/go-driver/encode.go index a490ab94e..af8d4bb20 100644 --- a/deps/github.com/arangodb/go-driver/encode.go +++ b/deps/github.com/arangodb/go-driver/encode.go @@ -20,7 +20,7 @@ // Author Ewout Prangsma // -// +build !"go1.8" +// +build !go1.8 package driver diff --git a/deps/github.com/arangodb/go-driver/encode_test.go b/deps/github.com/arangodb/go-driver/encode_test.go new file mode 100644 index 000000000..25539ce08 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/encode_test.go @@ -0,0 +1,38 @@ +// +// DISCLAIMER +// +// Copyright 2017 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package driver + +import "testing" + +func TestPathEscape(t *testing.T) { + tests := map[string]string{ // Input : Expected-Output + "abc": "abc", + "The Donald": "The%20Donald", + } + for input, expected := range tests { + result := pathEscape(input) + if result != expected { + t.Errorf("pathEscapse failed for '%s': Expected '%s', got '%s'", input, expected, result) + } + } +} diff --git a/deps/github.com/arangodb/go-driver/http/connection.go b/deps/github.com/arangodb/go-driver/http/connection.go index 92ee9f4b1..e4c428894 100644 --- a/deps/github.com/arangodb/go-driver/http/connection.go +++ b/deps/github.com/arangodb/go-driver/http/connection.go @@ -43,6 +43,7 @@ import ( const ( DefaultMaxIdleConnsPerHost = 64 + DefaultConnLimit = 32 keyRawResponse driver.ContextKey = "arangodb-rawResponse" keyResponse driver.ContextKey = "arangodb-response" @@ -76,6 +77,10 @@ type ConnectionConfig struct { cluster.ConnectionConfig // ContentType specified type of content encoding to use. ContentType driver.ContentType + // ConnLimit is the upper limit to the number of connections to a single server. + // The default is 32 (DefaultConnLimit). + // Set this value to -1 if you do not want any upper limit. + ConnLimit int } // NewConnection creates a new HTTP connection based on the given configuration settings. @@ -95,6 +100,9 @@ func NewConnection(config ConnectionConfig) (driver.Connection, error) { // newHTTPConnection creates a new HTTP connection for a single endpoint and the remainder of the given configuration settings. func newHTTPConnection(endpoint string, config ConnectionConfig) (driver.Connection, error) { + if config.ConnLimit == 0 { + config.ConnLimit = DefaultConnLimit + } endpoint = util.FixupEndpointURLScheme(endpoint) u, err := url.Parse(endpoint) if err != nil { @@ -154,10 +162,19 @@ func newHTTPConnection(endpoint string, config ConnectionConfig) (driver.Connect } } } + var connPool chan int + if config.ConnLimit > 0 { + connPool = make(chan int, config.ConnLimit) + // Fill with available tokens + for i := 0; i < config.ConnLimit; i++ { + connPool <- i + } + } c := &httpConnection{ endpoint: *u, contentType: config.ContentType, client: httpClient, + connPool: connPool, } return c, nil } @@ -167,6 +184,7 @@ type httpConnection struct { endpoint url.URL contentType driver.ContentType client *http.Client + connPool chan int } // String returns the endpoint as string @@ -225,6 +243,22 @@ func (c *httpConnection) Do(ctx context.Context, req driver.Request) (driver.Res if err != nil { return nil, driver.WithStack(err) } + + // Block on too many concurrent connections + if c.connPool != nil { + select { + case t := <-c.connPool: + // Ok, we're allowed to continue + defer func() { + // Give back token + c.connPool <- t + }() + case <-rctx.Done(): + // Context cancelled or expired + return nil, driver.WithStack(rctx.Err()) + } + } + resp, err := c.client.Do(r) if err != nil { return nil, driver.WithStack(err) diff --git a/deps/github.com/arangodb/go-driver/http/doc.go b/deps/github.com/arangodb/go-driver/http/doc.go new file mode 100644 index 000000000..0cffc2b01 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/http/doc.go @@ -0,0 +1,69 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package http implements driver.Connection using an HTTP connection. + +This connection uses HTTP or HTTPS to connect to the ArangoDB database and +encodes its content as JSON or Velocypack, depending on the value +of the `ContentType` fields in the `http.ConnectionConfig`. + +Creating an Insecure Connection + +To create an HTTP connection, use code like this. + + // Create an HTTP connection to the database + conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, + }) + if err != nil { + // Handle error + } + +The resulting connection is used to create a client which you will use +for normal database requests. + + // Create a client + c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + }) + if err != nil { + // Handle error + } + +Creating a Secure Connection + +To create a secure HTTPS connection, use code like this. + + // Create an HTTPS connection to the database + conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"https://localhost:8529"}, + TLSConfig: &tls.Config{ + InsecureSkipVerify: trueWhenUsingNonPublicCertificates, + }, + }) + if err != nil { + // Handle error + } + +*/ +package http diff --git a/deps/github.com/arangodb/go-driver/jwt/doc.go b/deps/github.com/arangodb/go-driver/jwt/doc.go new file mode 100644 index 000000000..7d226ee37 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/jwt/doc.go @@ -0,0 +1,57 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package jwt provides a helper function used to access ArangoDB +servers using a JWT secret. + +Authenticating with a JWT secret results in "super-user" access +to the database. + +To use a JWT secret to access your database, use code like this: + + // Create an HTTP connection to the database + conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, + }) + if err != nil { + // Handle error + } + + // Prepare authentication + hdr, err := CreateArangodJwtAuthorizationHeader("yourJWTSecret", "yourUniqueServerID") + if err != nil { + // Handle error + } + auth := driver.RawAuthentication(hdr) + + // Create a client + c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + Authentication: auth, + }) + if err != nil { + // Handle error + } + +*/ +package jwt diff --git a/pkg/util/arangod/jwt.go b/deps/github.com/arangodb/go-driver/jwt/jwt.go similarity index 81% rename from pkg/util/arangod/jwt.go rename to deps/github.com/arangodb/go-driver/jwt/jwt.go index 96f0db2be..3e328dd90 100644 --- a/pkg/util/arangod/jwt.go +++ b/deps/github.com/arangodb/go-driver/jwt/jwt.go @@ -20,9 +20,10 @@ // Author Ewout Prangsma // -package arangod +package jwt import ( + driver "github.com/arangodb/go-driver" jg "github.com/dgrijalva/jwt-go" ) @@ -33,21 +34,22 @@ const ( // CreateArangodJwtAuthorizationHeader calculates a JWT authorization header, for authorization // of a request to an arangod server, based on the given secret. // If the secret is empty, nothing is done. -func CreateArangodJwtAuthorizationHeader(jwtSecret string) (string, error) { - if jwtSecret == "" { +// Use the result of this function as input for driver.RawAuthentication. +func CreateArangodJwtAuthorizationHeader(jwtSecret, serverID string) (string, error) { + if jwtSecret == "" || serverID == "" { return "", nil } // Create a new token object, specifying signing method and the claims // you would like it to contain. token := jg.NewWithClaims(jg.SigningMethodHS256, jg.MapClaims{ "iss": issArangod, - "server_id": "foo", + "server_id": serverID, }) // Sign and get the complete encoded token as a string using the secret signedToken, err := token.SignedString([]byte(jwtSecret)) if err != nil { - return "", maskAny(err) + return "", driver.WithStack(err) } return "bearer " + signedToken, nil diff --git a/deps/github.com/arangodb/go-driver/query.go b/deps/github.com/arangodb/go-driver/query.go index 43af0152f..47c5a2cfc 100644 --- a/deps/github.com/arangodb/go-driver/query.go +++ b/deps/github.com/arangodb/go-driver/query.go @@ -28,11 +28,14 @@ import ( ) const ( - keyQueryCount = "arangodb-query-count" - keyQueryBatchSize = "arangodb-query-batchSize" - keyQueryCache = "arangodb-query-cache" - keyQueryMemoryLimit = "arangodb-query-memoryLimit" - keyQueryTTL = "arangodb-query-ttl" + keyQueryCount = "arangodb-query-count" + keyQueryBatchSize = "arangodb-query-batchSize" + keyQueryCache = "arangodb-query-cache" + keyQueryMemoryLimit = "arangodb-query-memoryLimit" + keyQueryTTL = "arangodb-query-ttl" + keyQueryOptSatSyncWait = "arangodb-query-opt-satSyncWait" + keyQueryOptFullCount = "arangodb-query-opt-fullCount" + keyQueryOptStream = "arangodb-query-opt-stream" ) // WithQueryCount is used to configure a context that will set the Count of a query request, @@ -70,6 +73,34 @@ func WithQueryTTL(parent context.Context, value time.Duration) context.Context { return context.WithValue(contextOrBackground(parent), keyQueryTTL, value) } +// WithQuerySatelliteSyncWait sets the satelliteSyncWait query value on the query cursor request +func WithQuerySatelliteSyncWait(parent context.Context, value time.Duration) context.Context { + return context.WithValue(contextOrBackground(parent), keyQueryOptSatSyncWait, value) +} + +// WithQueryFullCount is used to configure whether the query returns the full count of results +// before the last LIMIT statement +func WithQueryFullCount(parent context.Context, value ...bool) context.Context { + v := true + if len(value) > 0 { + v = value[0] + } + return context.WithValue(contextOrBackground(parent), keyQueryOptFullCount, v) +} + +// WithQueryStream is used to configure whether this becomes a stream query. +// A stream query is not executed right away, but continually evaluated +// when the client is requesting more results. Should the cursor expire +// the query transaction is canceled. This means for writing queries clients +// have to read the query-cursor until the HasMore() method returns false. +func WithQueryStream(parent context.Context, value ...bool) context.Context { + v := true + if len(value) > 0 { + v = value[0] + } + return context.WithValue(contextOrBackground(parent), keyQueryOptStream, v) +} + type queryRequest struct { // indicates whether the number of documents in the result set should be returned in the "count" attribute of the result. // Calculating the "count" attribute might have a performance impact for some queries in the future so this option is @@ -113,6 +144,11 @@ type queryRequest struct { FullCount bool `json:"fullCount,omitempty"` // Limits the maximum number of plans that are created by the AQL query optimizer. MaxPlans int `json:"maxPlans,omitempty"` + // Specify true and the query will be executed in a streaming fashion. The query result is not stored on + // the server, but calculated on the fly. Beware: long-running queries will need to hold the collection + // locks for as long as the query cursor exists. When set to false a query will be executed right away in + // its entirety. + Stream bool `json:"stream,omitempty"` } `json:"options,omitempty"` } @@ -146,6 +182,21 @@ func (q *queryRequest) applyContextSettings(ctx context.Context) { q.TTL = value.Seconds() } } + if rawValue := ctx.Value(keyQueryOptSatSyncWait); rawValue != nil { + if value, ok := rawValue.(time.Duration); ok { + q.Options.SatelliteSyncWait = value.Seconds() + } + } + if rawValue := ctx.Value(keyQueryOptFullCount); rawValue != nil { + if value, ok := rawValue.(bool); ok { + q.Options.FullCount = value + } + } + if rawValue := ctx.Value(keyQueryOptStream); rawValue != nil { + if value, ok := rawValue.(bool); ok { + q.Options.Stream = value + } + } } type parseQueryRequest struct { diff --git a/deps/github.com/arangodb/go-driver/replication.go b/deps/github.com/arangodb/go-driver/replication.go index 18769a01f..88109bd40 100644 --- a/deps/github.com/arangodb/go-driver/replication.go +++ b/deps/github.com/arangodb/go-driver/replication.go @@ -24,10 +24,31 @@ package driver import ( "context" + "time" ) +// Tick is represent a place in either the Write-Ahead Log, +// journals and datafiles value reported by the server +type Tick string + +// Batch represents state on the server used during +// certain replication operations to keep state required +// by the client (such as Write-Ahead Log, inventory and data-files) +type Batch interface { + // id of this batch + BatchID() string + // LastTick reported by the server for this batch + LastTick() Tick + // Extend the lifetime of an existing batch on the server + Extend(ctx context.Context, ttl time.Duration) error + // DeleteBatch deletes an existing batch on the server + Delete(ctx context.Context) error +} + // Replication provides access to replication related operations. type Replication interface { + // CreateBatch creates a "batch" to prevent removal of state required for replication + CreateBatch(ctx context.Context, db Database, serverID int64, ttl time.Duration) (Batch, error) // Get the inventory of the server containing all collections (with entire details) of a database. // When this function is called on a coordinator is a cluster, an ID of a DBServer must be provided // using a context that is prepare with `WithDBServerID`. diff --git a/deps/github.com/arangodb/go-driver/replication_impl.go b/deps/github.com/arangodb/go-driver/replication_impl.go index 3ea72dba1..9005290f8 100644 --- a/deps/github.com/arangodb/go-driver/replication_impl.go +++ b/deps/github.com/arangodb/go-driver/replication_impl.go @@ -24,9 +24,57 @@ package driver import ( "context" + "errors" "path" + "strconv" + "sync/atomic" + "time" ) +// Content of the create batch resp +type batchMetadata struct { + // Id of the batch + ID string `json:"id"` + // Last Tick reported by the server + LastTickInt Tick `json:"lastTick,omitempty"` + + cl *client + serverID int64 + database string + closed int32 +} + +// CreateBatch creates a "batch" to prevent WAL file removal and to take a snapshot +func (c *client) CreateBatch(ctx context.Context, db Database, serverID int64, ttl time.Duration) (Batch, error) { + req, err := c.conn.NewRequest("POST", path.Join("_db", db.Name(), "_api/replication/batch")) + if err != nil { + return nil, WithStack(err) + } + req = req.SetQuery("serverId", strconv.FormatInt(serverID, 10)) + params := struct { + TTL float64 `json:"ttl"` + }{TTL: ttl.Seconds()} // just use a default ttl value + req, err = req.SetBody(params) + if err != nil { + return nil, WithStack(err) + } + resp, err := c.conn.Do(ctx, req) + if err != nil { + return nil, WithStack(err) + } + if err := resp.CheckStatus(200); err != nil { + return nil, WithStack(err) + } + var batch batchMetadata + if err := resp.ParseBody("", &batch); err != nil { + return nil, WithStack(err) + } + batch.cl = c + batch.serverID = serverID + batch.database = db.Name() + return &batch, nil +} + // Get the inventory of a server containing all collections (with entire details) of a database. func (c *client) DatabaseInventory(ctx context.Context, db Database) (DatabaseInventory, error) { req, err := c.conn.NewRequest("GET", path.Join("_db", db.Name(), "_api/replication/inventory")) @@ -47,3 +95,63 @@ func (c *client) DatabaseInventory(ctx context.Context, db Database) (DatabaseIn } return result, nil } + +// BatchID reported by the server +func (b batchMetadata) BatchID() string { + return b.ID +} + +// LastTick reported by the server for this batch +func (b batchMetadata) LastTick() Tick { + return b.LastTickInt +} + +// Extend the lifetime of an existing batch on the server +func (b batchMetadata) Extend(ctx context.Context, ttl time.Duration) error { + if !atomic.CompareAndSwapInt32(&b.closed, 0, 0) { + return WithStack(errors.New("Batch already closed")) + } + + req, err := b.cl.conn.NewRequest("PUT", path.Join("_db", b.database, "_api/replication/batch", b.ID)) + if err != nil { + return WithStack(err) + } + req = req.SetQuery("serverId", strconv.FormatInt(b.serverID, 10)) + input := struct { + TTL int64 `json:"ttl"` + }{ + TTL: int64(ttl.Seconds()), + } + req, err = req.SetBody(input) + if err != nil { + return WithStack(err) + } + resp, err := b.cl.conn.Do(ctx, req) + if err != nil { + return WithStack(err) + } + if err := resp.CheckStatus(204); err != nil { + return WithStack(err) + } + return nil +} + +// Delete an existing dump batch +func (b *batchMetadata) Delete(ctx context.Context) error { + if !atomic.CompareAndSwapInt32(&b.closed, 0, 1) { + return WithStack(errors.New("Batch already closed")) + } + + req, err := b.cl.conn.NewRequest("DELETE", path.Join("_db", b.database, "_api/replication/batch", b.ID)) + if err != nil { + return WithStack(err) + } + resp, err := b.cl.conn.Do(ctx, req) + if err != nil { + return WithStack(err) + } + if err := resp.CheckStatus(204); err != nil { + return WithStack(err) + } + return nil +} diff --git a/deps/github.com/arangodb/go-driver/test/agency_lock_test.go b/deps/github.com/arangodb/go-driver/test/agency_lock_test.go new file mode 100644 index 000000000..d5d694010 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/test/agency_lock_test.go @@ -0,0 +1,82 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package test + +import ( + "context" + "testing" + "time" + + driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/agency" +) + +// TestAgencyLock tests the agency.Lock interface. +func TestAgencyLock(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + if a, err := getAgencyConnection(ctx, t, c); driver.IsPreconditionFailed(err) { + t.Skip("Not a cluster") + } else if err != nil { + t.Fatalf("Cluster failed: %s", describe(err)) + } else { + key := []string{"go-driver", "TestAgencyLock"} + l, err := agency.NewLock(t, a, key, "2b2173ae-6684-501c-b8b1-c8b754b7fd40", time.Minute) + if err != nil { + t.Fatalf("NewLock failed: %s", describe(err)) + } + if l.IsLocked() { + t.Error("IsLocked must be false, got true") + } + if err := l.Lock(ctx); err != nil { + t.Fatalf("Lock failed: %s", describe(err)) + } + if !l.IsLocked() { + t.Error("IsLocked must be true, got false") + } + if err := l.Lock(ctx); !agency.IsAlreadyLocked(err) { + t.Fatalf("AlreadyLockedError expected, got %s", describe(err)) + } + if err := l.Unlock(ctx); err != nil { + t.Fatalf("Unlock failed: %s", describe(err)) + } + if l.IsLocked() { + t.Error("IsLocked must be false, got true") + } + if err := l.Unlock(ctx); !agency.IsNotLocked(err) { + t.Fatalf("NotLockedError expected, got %s", describe(err)) + } + if err := l.Lock(ctx); err != nil { + t.Fatalf("Lock failed: %s", describe(err)) + } + if !l.IsLocked() { + t.Error("IsLocked must be true, got false") + } + if err := l.Unlock(ctx); err != nil { + t.Fatalf("Unlock failed: %s", describe(err)) + } + if l.IsLocked() { + t.Error("IsLocked must be false, got true") + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/agency_test.go b/deps/github.com/arangodb/go-driver/test/agency_test.go new file mode 100644 index 000000000..8d895df5b --- /dev/null +++ b/deps/github.com/arangodb/go-driver/test/agency_test.go @@ -0,0 +1,289 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package test + +import ( + "context" + "crypto/tls" + "os" + "reflect" + "testing" + + driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/agency" + "github.com/arangodb/go-driver/http" + "github.com/arangodb/go-driver/util" +) + +// getAgencyEndpoints queries the cluster to get all agency endpoints. +func getAgencyEndpoints(ctx context.Context, c driver.Client) ([]string, error) { + cl, err := c.Cluster(ctx) + if err != nil { + return nil, err + } + h, err := cl.Health(ctx) + if err != nil { + return nil, err + } + result := []string{} + for _, entry := range h.Health { + if entry.Role == driver.ServerRoleAgent { + ep := util.FixupEndpointURLScheme(entry.Endpoint) + result = append(result, ep) + } + } + return result, nil +} + +// getAgencyConnection queries the cluster and creates an agency accessor using an agency.AgencyConnection for the entire agency. +func getAgencyConnection(ctx context.Context, t testEnv, c driver.Client) (agency.Agency, error) { + if os.Getenv("TEST_CONNECTION") == "vst" { + // These tests assume an HTTP connetion, so we skip under this condition + return nil, driver.ArangoError{HasError: true, Code: 412} + } + endpoints, err := getAgencyEndpoints(ctx, c) + if err != nil { + return nil, err + } + conn, err := agency.NewAgencyConnection(http.ConnectionConfig{ + Endpoints: endpoints, + TLSConfig: &tls.Config{InsecureSkipVerify: true}, + }) + if auth := createAuthenticationFromEnv(t); auth != nil { + // This requires a JWT token, which we not always have in this test, so we skip under this condition + return nil, driver.ArangoError{HasError: true, Code: 412} + } + result, err := agency.NewAgency(conn) + if err != nil { + return nil, err + } + return result, nil +} + +// getIndividualAgencyConnections queries the cluster and creates an agency accessor using a single http.Connection for each agent. +func getIndividualAgencyConnections(ctx context.Context, t testEnv, c driver.Client) ([]agency.Agency, error) { + if os.Getenv("TEST_CONNECTION") == "vst" { + // These tests assume an HTTP connetion, so we skip under this condition + return nil, driver.ArangoError{HasError: true, Code: 412} + } + endpoints, err := getAgencyEndpoints(ctx, c) + if err != nil { + return nil, err + } + if auth := createAuthenticationFromEnv(t); auth != nil { + // This requires a JWT token, which we not always have in this test, so we skip under this condition + return nil, driver.ArangoError{HasError: true, Code: 412} + } + result := make([]agency.Agency, len(endpoints)) + for i, ep := range endpoints { + conn, err := http.NewConnection(http.ConnectionConfig{ + Endpoints: []string{ep}, + TLSConfig: &tls.Config{InsecureSkipVerify: true}, + DontFollowRedirect: true, + }) + if err != nil { + return nil, err + } + result[i], err = agency.NewAgency(conn) + } + return result, nil +} + +// TestAgencyRead tests the Agency.ReadKey method. +func TestAgencyRead(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + if a, err := getAgencyConnection(ctx, t, c); driver.IsPreconditionFailed(err) { + t.Skip("Not a cluster") + } else if err != nil { + t.Fatalf("Cluster failed: %s", describe(err)) + } else { + var result interface{} + if err := a.ReadKey(ctx, []string{"not-found-b1d534b1-26d8-5ad0-b22d-23d49d3ea92c"}, &result); !agency.IsKeyNotFound(err) { + t.Errorf("Expected KeyNotFoundError, got %s", describe(err)) + } + if err := a.ReadKey(ctx, []string{"arango"}, &result); err != nil { + t.Errorf("Expected success, got %s", describe(err)) + } + } +} + +// TestAgencyWrite tests the Agency.WriteKey method. +func TestAgencyWrite(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + if a, err := getAgencyConnection(ctx, t, c); driver.IsPreconditionFailed(err) { + t.Skip("Not a cluster") + } else if err != nil { + t.Fatalf("Cluster failed: %s", describe(err)) + } else { + op := func(key []string, value, result interface{}) { + if err := a.WriteKey(ctx, key, value, 0); err != nil { + t.Fatalf("WriteKey failed: %s", describe(err)) + } + if err := a.ReadKey(ctx, key, result); err != nil { + t.Fatalf("ReadKey failed: %s", describe(err)) + } + if !reflect.DeepEqual(value, reflect.ValueOf(result).Elem().Interface()) { + t.Errorf("Expected '%v', got '%v'", value, result) + } + } + op([]string{"go-driver", "TestAgencyWrite", "string"}, "hello world", new(string)) + op([]string{"go-driver", "TestAgencyWrite", "int"}, 55, new(int)) + op([]string{"go-driver", "TestAgencyWrite", "bool"}, true, new(bool)) + op([]string{"go-driver", "TestAgencyWrite", "object"}, struct{ Field string }{Field: "hello world"}, &struct{ Field string }{}) + op([]string{"go-driver", "TestAgencyWrite", "string-array"}, []string{"hello", "world"}, new([]string)) + op([]string{"go-driver", "TestAgencyWrite", "int-array"}, []int{-5, 34, 11}, new([]int)) + } +} + +// TestAgencyWriteIfEmpty tests the Agency.WriteKeyIfEmpty method. +func TestAgencyWriteIfEmpty(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + if a, err := getAgencyConnection(ctx, t, c); driver.IsPreconditionFailed(err) { + t.Skip("Not a cluster") + } else if err != nil { + t.Fatalf("Cluster failed: %s", describe(err)) + } else { + key := []string{"go-driver", "TestAgencyWriteIfEmpty"} + if err := a.WriteKey(ctx, key, "foo", 0); err != nil { + t.Fatalf("WriteKey failed: %s", describe(err)) + } + var result string + if err := a.ReadKey(ctx, key, &result); err != nil { + t.Errorf("ReadKey failed: %s", describe(err)) + } + if err := a.WriteKeyIfEmpty(ctx, key, "not-foo", 0); !driver.IsPreconditionFailed(err) { + t.Errorf("Expected PreconditionFailedError, got %s", describe(err)) + } + if err := a.RemoveKey(ctx, key); err != nil { + t.Fatalf("RemoveKey failed: %s", describe(err)) + } + if err := a.ReadKey(ctx, key, &result); !agency.IsKeyNotFound(err) { + t.Errorf("Expected KeyNotFoundError, got %s", describe(err)) + } + if err := a.WriteKeyIfEmpty(ctx, key, "again-foo", 0); err != nil { + t.Errorf("WriteKeyIfEmpty failed: %s", describe(err)) + } + } +} + +// TestAgencyWriteIfEqualTo tests the Agency.WriteIfEqualTo method. +func TestAgencyWriteIfEqualTo(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + if a, err := getAgencyConnection(ctx, t, c); driver.IsPreconditionFailed(err) { + t.Skip("Not a cluster") + } else if err != nil { + t.Fatalf("Cluster failed: %s", describe(err)) + } else { + key := []string{"go-driver", "TestAgencyWriteIfEqualTo"} + if err := a.WriteKey(ctx, key, "foo", 0); err != nil { + t.Fatalf("WriteKey failed: %s", describe(err)) + } + var result string + if err := a.ReadKey(ctx, key, &result); err != nil { + t.Errorf("ReadKey failed: %s", describe(err)) + } + if result != "foo" { + t.Errorf("Expected 'foo', got '%s", result) + } + if err := a.WriteKeyIfEqualTo(ctx, key, "not-foo", "incorrect", 0); !driver.IsPreconditionFailed(err) { + t.Errorf("Expected PreconditionFailedError, got %s", describe(err)) + } + if err := a.ReadKey(ctx, key, &result); err != nil { + t.Errorf("ReadKey failed: %s", describe(err)) + } + if result != "foo" { + t.Errorf("Expected 'foo', got '%s", result) + } + if err := a.WriteKeyIfEqualTo(ctx, key, "not-foo", "foo", 0); err != nil { + t.Fatalf("WriteKeyIfEqualTo failed: %s", describe(err)) + } + if err := a.ReadKey(ctx, key, &result); err != nil { + t.Errorf("ReadKey failed: %s", describe(err)) + } + if result != "not-foo" { + t.Errorf("Expected 'not-foo', got '%s", result) + } + } +} + +// TestAgencyRemove tests the Agency.RemoveKey method. +func TestAgencyRemove(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + if a, err := getAgencyConnection(ctx, t, c); driver.IsPreconditionFailed(err) { + t.Skip("Not a cluster") + } else if err != nil { + t.Fatalf("Cluster failed: %s", describe(err)) + } else { + key := []string{"go-driver", "TestAgencyRemove"} + if err := a.WriteKey(ctx, key, "foo", 0); err != nil { + t.Fatalf("WriteKey failed: %s", describe(err)) + } + var result string + if err := a.ReadKey(ctx, key, &result); err != nil { + t.Errorf("ReadKey failed: %s", describe(err)) + } + if err := a.RemoveKey(ctx, key); err != nil { + t.Fatalf("RemoveKey failed: %s", describe(err)) + } + if err := a.ReadKey(ctx, key, &result); !agency.IsKeyNotFound(err) { + t.Errorf("Expected KeyNotFoundError, got %s", describe(err)) + } + } +} + +// TestAgencyRemoveIfEqualTo tests the Agency.RemoveKeyIfEqualTo method. +func TestAgencyRemoveIfEqualTo(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + if a, err := getAgencyConnection(ctx, t, c); driver.IsPreconditionFailed(err) { + t.Skip("Not a cluster") + } else if err != nil { + t.Fatalf("Cluster failed: %s", describe(err)) + } else { + key := []string{"go-driver", "RemoveKeyIfEqualTo"} + if err := a.WriteKey(ctx, key, "foo", 0); err != nil { + t.Fatalf("WriteKey failed: %s", describe(err)) + } + var result string + if err := a.ReadKey(ctx, key, &result); err != nil { + t.Errorf("ReadKey failed: %s", describe(err)) + } + if err := a.RemoveKeyIfEqualTo(ctx, key, "incorrect"); !driver.IsPreconditionFailed(err) { + t.Errorf("Expected PreconditionFailedError, got %s", describe(err)) + } + if err := a.ReadKey(ctx, key, &result); err != nil { + t.Errorf("ReadKey failed: %s", describe(err)) + } + if err := a.RemoveKeyIfEqualTo(ctx, key, "foo"); err != nil { + t.Fatalf("RemoveKeyIfEqualTo failed: %s", describe(err)) + } + if err := a.ReadKey(ctx, key, &result); !agency.IsKeyNotFound(err) { + t.Errorf("Expected KeyNotFoundError, got %s", describe(err)) + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/client_test.go b/deps/github.com/arangodb/go-driver/test/client_test.go index 6e6d9b892..986237e89 100644 --- a/deps/github.com/arangodb/go-driver/test/client_test.go +++ b/deps/github.com/arangodb/go-driver/test/client_test.go @@ -25,6 +25,7 @@ package test import ( "context" "crypto/tls" + "log" httplib "net/http" "os" "strconv" @@ -33,6 +34,8 @@ import ( "testing" "time" + _ "net/http/pprof" + driver "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/http" "github.com/arangodb/go-driver/vst" @@ -40,7 +43,8 @@ import ( ) var ( - logEndpointsOnce sync.Once + logEndpointsOnce sync.Once + runPProfServerOnce sync.Once ) // skipBelowVersion skips the test if the current server version is less than @@ -152,6 +156,16 @@ func createConnectionFromEnv(t testEnv) driver.Connection { // createClientFromEnv initializes a Client from information specified in environment variables. func createClientFromEnv(t testEnv, waitUntilReady bool, connection ...*driver.Connection) driver.Client { + runPProfServerOnce.Do(func() { + if os.Getenv("TEST_PPROF") != "" { + go func() { + // Start pprof server on port 6060 + // To use it in the test, run a command like: + // docker exec -it go-driver-test sh -c "apk add -U curl && curl http://localhost:6060/debug/pprof/goroutine?debug=1" + log.Println(httplib.ListenAndServe("localhost:6060", nil)) + }() + } + }) conn := createConnectionFromEnv(t) if len(connection) == 1 { *connection[0] = conn @@ -193,11 +207,10 @@ func waitUntilServerAvailable(ctx context.Context, c driver.Client, t testEnv) b cancel() instanceUp <- true return - } else { - cancel() - //t.Logf("Version failed: %s %#v", describe(err), err) - time.Sleep(time.Second) } + cancel() + //t.Logf("Version failed: %s %#v", describe(err), err) + time.Sleep(time.Second) } }() select { diff --git a/deps/github.com/arangodb/go-driver/test/cluster.sh b/deps/github.com/arangodb/go-driver/test/cluster.sh index b3c5fa209..0bc80ff5d 100755 --- a/deps/github.com/arangodb/go-driver/test/cluster.sh +++ b/deps/github.com/arangodb/go-driver/test/cluster.sh @@ -50,5 +50,5 @@ if [ "$CMD" == "start" ]; then ${STARTER} \ --starter.port=7000 --starter.address=127.0.0.1 \ --docker.image=${ARANGODB} \ - --starter.local --starter.mode=${STARTERMODE} $STARTERARGS + --starter.local --starter.mode=${STARTERMODE} --all.log.output=+ $STARTERARGS fi diff --git a/deps/github.com/arangodb/go-driver/test/cluster_test.go b/deps/github.com/arangodb/go-driver/test/cluster_test.go index 3a9a253ef..3a321a592 100644 --- a/deps/github.com/arangodb/go-driver/test/cluster_test.go +++ b/deps/github.com/arangodb/go-driver/test/cluster_test.go @@ -37,6 +37,8 @@ func TestClusterHealth(t *testing.T) { cl, err := c.Cluster(ctx) if driver.IsPreconditionFailed(err) { t.Skip("Not a cluster") + } else if err != nil { + t.Fatalf("Health failed: %s", describe(err)) } else { h, err := cl.Health(ctx) if err != nil { diff --git a/deps/github.com/arangodb/go-driver/test/concurrency_test.go b/deps/github.com/arangodb/go-driver/test/concurrency_test.go new file mode 100644 index 000000000..6af0d4840 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/test/concurrency_test.go @@ -0,0 +1,205 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package test + +import ( + "context" + "crypto/rand" + "encoding/hex" + "os" + "strconv" + "sync" + "testing" + + driver "github.com/arangodb/go-driver" +) + +// TestConcurrentCreateSmallDocuments make a lot of concurrent CreateDocument calls. +// It then verifies that all documents "have arrived". +func TestConcurrentCreateSmallDocuments(t *testing.T) { + if testing.Short() { + t.Skip("Skip on short tests") + } + c := createClientFromEnv(t, true) + + version, err := c.Version(nil) + if err != nil { + t.Fatalf("Version failed: %s", describe(err)) + } + isv33p := version.Version.CompareTo("3.3") >= 0 + if !isv33p && os.Getenv("TEST_CONNECTION") == "vst" { + t.Skip("Skipping VST load test on 3.2") + } else { + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestConcurrentCreateSmallDocuments", nil, t) + + docChan := make(chan driver.DocumentMeta, 16*1024) + + creator := func(limit, interval int) { + for i := 0; i < limit; i++ { + ctx := context.Background() + doc := UserDoc{ + "Jan", + i * interval, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + docChan <- meta + } + } + + reader := func() { + for { + meta, ok := <-docChan + if !ok { + return + } + // Document must exists now + if found, err := col.DocumentExists(nil, meta.Key); err != nil { + t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err)) + } else if !found { + t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key) + } + // Read document + var readDoc UserDoc + if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + } + } + + noCreators := getIntFromEnv("NOCREATORS", 25) + noReaders := getIntFromEnv("NOREADERS", 50) + noDocuments := getIntFromEnv("NODOCUMENTS", 1000) // per creator + + wgCreators := sync.WaitGroup{} + // Run N concurrent creators + for i := 0; i < noCreators; i++ { + wgCreators.Add(1) + go func() { + defer wgCreators.Done() + creator(noDocuments, noCreators) + }() + } + wgReaders := sync.WaitGroup{} + // Run M readers + for i := 0; i < noReaders; i++ { + wgReaders.Add(1) + go func() { + defer wgReaders.Done() + reader() + }() + } + wgCreators.Wait() + close(docChan) + wgReaders.Wait() + } +} + +// TestConcurrentCreateBigDocuments make a lot of concurrent CreateDocument calls. +// It then verifies that all documents "have arrived". +func TestConcurrentCreateBigDocuments(t *testing.T) { + if testing.Short() { + t.Skip("Skip on short tests") + } + c := createClientFromEnv(t, true) + + version, err := c.Version(nil) + if err != nil { + t.Fatalf("Version failed: %s", describe(err)) + } + isv33p := version.Version.CompareTo("3.3") >= 0 + if !isv33p && os.Getenv("TEST_CONNECTION") == "vst" { + t.Skip("Skipping VST load test on 3.2") + } else { + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestConcurrentCreateBigDocuments", nil, t) + + docChan := make(chan driver.DocumentMeta, 16*1024) + + creator := func(limit, interval int) { + data := make([]byte, 1024) + for i := 0; i < limit; i++ { + rand.Read(data) + ctx := context.Background() + doc := UserDoc{ + "Jan" + strconv.Itoa(i) + hex.EncodeToString(data), + i * interval, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + docChan <- meta + } + } + + reader := func() { + for { + meta, ok := <-docChan + if !ok { + return + } + // Document must exists now + if found, err := col.DocumentExists(nil, meta.Key); err != nil { + t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err)) + } else if !found { + t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key) + } + // Read document + var readDoc UserDoc + if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + } + } + + noCreators := getIntFromEnv("NOCREATORS", 25) + noReaders := getIntFromEnv("NOREADERS", 50) + noDocuments := getIntFromEnv("NODOCUMENTS", 100) // per creator + + wgCreators := sync.WaitGroup{} + // Run N concurrent creators + for i := 0; i < noCreators; i++ { + wgCreators.Add(1) + go func() { + defer wgCreators.Done() + creator(noDocuments, noCreators) + }() + } + wgReaders := sync.WaitGroup{} + // Run M readers + for i := 0; i < noReaders; i++ { + wgReaders.Add(1) + go func() { + defer wgReaders.Done() + reader() + }() + } + wgCreators.Wait() + close(docChan) + wgReaders.Wait() + } +} diff --git a/deps/github.com/arangodb/go-driver/test/cursor_test.go b/deps/github.com/arangodb/go-driver/test/cursor_test.go index 7fc86bfcd..2c806faeb 100644 --- a/deps/github.com/arangodb/go-driver/test/cursor_test.go +++ b/deps/github.com/arangodb/go-driver/test/cursor_test.go @@ -228,3 +228,153 @@ func TestCreateCursor(t *testing.T) { } } } + +// TestCreateCursorReturnNull creates a cursor with a `RETURN NULL` query. +func TestCreateCursorReturnNull(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "cursor_test", nil, t) + + var result interface{} + query := "return null" + cursor, err := db.Query(ctx, query, nil) + if err != nil { + t.Fatalf("Query(return null) failed: %s", describe(err)) + } + defer cursor.Close() + if _, err := cursor.ReadDocument(ctx, &result); err != nil { + t.Fatalf("ReadDocument failed: %s", describe(err)) + } + if result != nil { + t.Errorf("Expected result to be nil, got %#v", result) + } +} + +// Test stream query cursors. The goroutines are technically only +// relevant for the MMFiles engine, but don't hurt on rocksdb either +func TestCreateStreamCursor(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + c := createClientFromEnv(t, true) + + version, err := c.Version(nil) + if err != nil { + t.Fatalf("Version failed: %s", describe(err)) + } + if version.Version.CompareTo("3.4") < 0 { + t.Skip("This test requires version 3.4") + return + } + + db := ensureDatabase(ctx, c, "cursor_stream_test", nil, t) + col := ensureCollection(ctx, db, "cursor_stream_test", nil, t) + + // Query engine info (on rocksdb, JournalSize is always 0) + info, err := db.EngineInfo(nil) + if err != nil { + t.Fatalf("Failed to get engine info: %s", describe(err)) + } + + // This might take a few seconds + for i := 0; i < 10000; i++ { + user := UserDoc{Name: "John", Age: i} + if _, err := col.CreateDocument(ctx, user); err != nil { + t.Fatalf("Expected success, got %s", describe(err)) + } + } + t.Log("Completed inserting 10k docs") + + const expectedResults int = 10 * 10000 + query := "FOR doc IN cursor_stream_test RETURN doc" + ctx2 := driver.WithQueryStream(ctx, true) + var cursors []driver.Cursor + + // create a bunch of read-only cursors + for i := 0; i < 10; i++ { + cursor, err := db.Query(ctx2, query, nil) + if err != nil { + t.Fatalf("Expected success in query %d (%s), got '%s'", i, query, describe(err)) + } + defer cursor.Close() + count := cursor.Count() + if count != 0 { + t.Errorf("Expected count of 0, got %d in query %d (%s)", count, i, query) + } + stats := cursor.Statistics() + count = stats.FullCount() + if count != 0 { + t.Errorf("Expected fullCount of 0, got %d in query %d (%s)", count, i, query) + } + if !cursor.HasMore() { + t.Errorf("Expected cursor %d to have more documents", i) + } + + cursors = append(cursors, cursor) + } + + t.Logf("Created %d cursors", len(cursors)) + + // start a write query on the same collection inbetween + // contrary to normal cursors which are executed right + // away this will block until all read cursors are resolved + testReady := make(chan bool) + go func() { + query = "FOR doc IN 1..5 LET y = SLEEP(0.01) INSERT {name:'Peter', age:0} INTO cursor_stream_test" + cursor, err := db.Query(ctx2, query, nil) // should not return immediately + if err != nil { + t.Fatalf("Expected success in write-query %s, got '%s'", query, describe(err)) + } + defer cursor.Close() + + for cursor.HasMore() { + var data interface{} + if _, err := cursor.ReadDocument(ctx2, &data); err != nil { + t.Fatalf("Failed to read document, err: %s", describe(err)) + } + } + testReady <- true // signal write done + }() + + readCount := 0 + go func() { + // read all cursors until the end, server closes them automatically + for i, cursor := range cursors { + for cursor.HasMore() { + var user UserDoc + if _, err := cursor.ReadDocument(ctx2, &user); err != nil { + t.Fatalf("Failed to result document %d: %s", i, describe(err)) + } + readCount++ + } + } + testReady <- false // signal read done + }() + + writeDone := false + readDone := false + for { + select { + case <-ctx.Done(): + t.Fatal("Timeout") + case v := <-testReady: + if v { + writeDone = true + } else { + readDone = true + } + } + // On MMFiles the read-cursors have to finish first + if writeDone && !readDone && info.Type == driver.EngineTypeMMFiles { + t.Error("Write cursor was able to complete before read cursors") + } + + if writeDone && readDone { + close(testReady) + break + } + } + + if readCount != expectedResults { + t.Errorf("Expected to read %d documents, instead got %d", expectedResults, readCount) + } +} diff --git a/deps/github.com/arangodb/go-driver/test/doc.go b/deps/github.com/arangodb/go-driver/test/doc.go new file mode 100644 index 000000000..45574733e --- /dev/null +++ b/deps/github.com/arangodb/go-driver/test/doc.go @@ -0,0 +1,26 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package test implements add tests for the go-driver. +*/ +package test diff --git a/deps/github.com/arangodb/go-driver/test/document_create_test.go b/deps/github.com/arangodb/go-driver/test/document_create_test.go index 3d5b2e178..c9a2faddd 100644 --- a/deps/github.com/arangodb/go-driver/test/document_create_test.go +++ b/deps/github.com/arangodb/go-driver/test/document_create_test.go @@ -156,3 +156,35 @@ func TestCreateDocumentNil(t *testing.T) { t.Fatalf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestCreateDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled, +// and then checks that it exists. +func TestCreateDocumentInWaitForSyncCollection(t *testing.T) { + c := createClientFromEnv(t, true) + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestCreateDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + doc := UserDoc{ + "Jan", + 40, + } + meta, err := col.CreateDocument(nil, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + // Document must exists now + if found, err := col.DocumentExists(nil, meta.Key); err != nil { + t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err)) + } else if !found { + t.Errorf("DocumentExists returned false for '%s', expected true", meta.Key) + } + // Read document + var readDoc UserDoc + if _, err := col.ReadDocument(nil, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + if !reflect.DeepEqual(doc, readDoc) { + t.Errorf("Got wrong document. Expected %+v, got %+v", doc, readDoc) + } +} diff --git a/deps/github.com/arangodb/go-driver/test/document_remove_test.go b/deps/github.com/arangodb/go-driver/test/document_remove_test.go index 8b3b227d1..bd01d1996 100644 --- a/deps/github.com/arangodb/go-driver/test/document_remove_test.go +++ b/deps/github.com/arangodb/go-driver/test/document_remove_test.go @@ -169,3 +169,36 @@ func TestRemoveDocumentKeyEmpty(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestReplaceDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled, +// removes it and then checks the removal has succeeded. +func TestRemoveDocumentInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestRemoveDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + doc := UserDoc{ + "Piere", + 23, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + if _, err := col.RemoveDocument(ctx, meta.Key); err != nil { + t.Fatalf("Failed to remove document '%s': %s", meta.Key, describe(err)) + } + // Should not longer exist + var readDoc Account + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); !driver.IsNotFound(err) { + t.Fatalf("Expected NotFoundError, got %s", describe(err)) + } + // Document must exists now + if found, err := col.DocumentExists(ctx, meta.Key); err != nil { + t.Fatalf("DocumentExists failed for '%s': %s", meta.Key, describe(err)) + } else if found { + t.Errorf("DocumentExists returned true for '%s', expected false", meta.Key) + } +} diff --git a/deps/github.com/arangodb/go-driver/test/document_replace_test.go b/deps/github.com/arangodb/go-driver/test/document_replace_test.go index f6d7bbeb9..d033e219c 100644 --- a/deps/github.com/arangodb/go-driver/test/document_replace_test.go +++ b/deps/github.com/arangodb/go-driver/test/document_replace_test.go @@ -214,3 +214,38 @@ func TestReplaceDocumentUpdateNil(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestReplaceDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled, +// replaces it and then checks the replacement has succeeded. +func TestReplaceDocumentInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestReplaceDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + doc := UserDoc{ + "Piere", + 23, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + // Replacement doc + replacement := Account{ + ID: "foo", + User: &UserDoc{}, + } + if _, err := col.ReplaceDocument(ctx, meta.Key, replacement); err != nil { + t.Fatalf("Failed to replace document '%s': %s", meta.Key, describe(err)) + } + // Read replaces document + var readDoc Account + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + if !reflect.DeepEqual(replacement, readDoc) { + t.Errorf("Got wrong document. Expected %+v, got %+v", replacement, readDoc) + } +} diff --git a/deps/github.com/arangodb/go-driver/test/document_update_test.go b/deps/github.com/arangodb/go-driver/test/document_update_test.go index 6c3a7fe5d..ab70f4004 100644 --- a/deps/github.com/arangodb/go-driver/test/document_update_test.go +++ b/deps/github.com/arangodb/go-driver/test/document_update_test.go @@ -296,3 +296,38 @@ func TestUpdateDocumentUpdateNil(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestUpdateDocumentInWaitForSyncCollection creates a document in a collection with waitForSync enabled, +// updates it and then checks the update has succeeded. +func TestUpdateDocumentInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestUpdateDocumentInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + doc := UserDoc{ + "Piere", + 23, + } + meta, err := col.CreateDocument(ctx, doc) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } + // Update document + update := map[string]interface{}{ + "name": "Updated", + } + if _, err := col.UpdateDocument(ctx, meta.Key, update); err != nil { + t.Fatalf("Failed to update document '%s': %s", meta.Key, describe(err)) + } + // Read updated document + var readDoc UserDoc + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + doc.Name = "Updated" + if !reflect.DeepEqual(doc, readDoc) { + t.Errorf("Got wrong document. Expected %+v, got %+v", doc, readDoc) + } +} diff --git a/deps/github.com/arangodb/go-driver/test/documents_create_test.go b/deps/github.com/arangodb/go-driver/test/documents_create_test.go index e10b36075..30ae1e4af 100644 --- a/deps/github.com/arangodb/go-driver/test/documents_create_test.go +++ b/deps/github.com/arangodb/go-driver/test/documents_create_test.go @@ -167,3 +167,47 @@ func TestCreateDocumentsNonSlice(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestCreateDocumentsInWaitForSyncCollection creates a few documents in a collection with waitForSync enabled and then checks that it exists. +func TestCreateDocumentsInWaitForSyncCollection(t *testing.T) { + c := createClientFromEnv(t, true) + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestCreateDocumentsInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDoc{ + UserDoc{ + "Jan", + 40, + }, + UserDoc{ + "Foo", + 41, + }, + UserDoc{ + "Frank", + 42, + }, + } + metas, errs, err := col.CreateDocuments(nil, docs) + if err != nil { + t.Fatalf("Failed to create new documents: %s", describe(err)) + } else if len(metas) != len(docs) { + t.Errorf("Expected %d metas, got %d", len(docs), len(metas)) + } else { + for i := 0; i < len(docs); i++ { + if err := errs[i]; err != nil { + t.Errorf("Expected no error at index %d, got %s", i, describe(err)) + } + + // Document must exists now + var readDoc UserDoc + if _, err := col.ReadDocument(nil, metas[i].Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", metas[i].Key, describe(err)) + } + if !reflect.DeepEqual(docs[i], readDoc) { + t.Errorf("Got wrong document. Expected %+v, got %+v", docs[i], readDoc) + } + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/documents_import_test.go b/deps/github.com/arangodb/go-driver/test/documents_import_test.go index 1d0840412..414fb7933 100644 --- a/deps/github.com/arangodb/go-driver/test/documents_import_test.go +++ b/deps/github.com/arangodb/go-driver/test/documents_import_test.go @@ -559,3 +559,47 @@ func TestImportDocumentsOverwriteNo(t *testing.T) { } } } + +// TestImportDocumentsWithKeysInWaitForSyncCollection imports documents into a collection with waitForSync enabled +// and then checks that it exists. +func TestImportDocumentsWithKeysInWaitForSyncCollection(t *testing.T) { + c := createClientFromEnv(t, true) + db := ensureDatabase(nil, c, "document_test", nil, t) + col := ensureCollection(nil, db, "TestImportDocumentsWithKeysInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDocWithKey{ + UserDocWithKey{ + "jan", + "Jan", + 40, + }, + UserDocWithKey{ + "foo", + "Foo", + 41, + }, + UserDocWithKey{ + "frank", + "Frank", + 42, + }, + } + + var raw []byte + ctx := driver.WithRawResponse(nil, &raw) + stats, err := col.ImportDocuments(ctx, docs, nil) + if err != nil { + t.Fatalf("Failed to import documents: %s %#v", describe(err), err) + } else { + if stats.Created != int64(len(docs)) { + t.Errorf("Expected %d created documents, got %d (json %s)", len(docs), stats.Created, formatRawResponse(raw)) + } + if stats.Errors != 0 { + t.Errorf("Expected %d error documents, got %d (json %s)", 0, stats.Errors, formatRawResponse(raw)) + } + if stats.Empty != 0 { + t.Errorf("Expected %d empty documents, got %d (json %s)", 0, stats.Empty, formatRawResponse(raw)) + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/documents_remove_test.go b/deps/github.com/arangodb/go-driver/test/documents_remove_test.go index c32391817..a52f69365 100644 --- a/deps/github.com/arangodb/go-driver/test/documents_remove_test.go +++ b/deps/github.com/arangodb/go-driver/test/documents_remove_test.go @@ -227,3 +227,36 @@ func TestRemoveDocumentsKeyEmpty(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestRemoveDocumentsInWaitForSyncCollection creates documents in a collection with waitForSync enabled, +// removes them and then checks the removal has succeeded. +func TestRemoveDocumentsInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestRemoveDocumentsInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDoc{ + UserDoc{ + "Piere", + 23, + }, + } + metas, errs, err := col.CreateDocuments(ctx, docs) + if err != nil { + t.Fatalf("Failed to create new documents: %s", describe(err)) + } else if err := errs.FirstNonNil(); err != nil { + t.Fatalf("Expected no errors, got first: %s", describe(err)) + } + if _, _, err := col.RemoveDocuments(ctx, metas.Keys()); err != nil { + t.Fatalf("Failed to remove documents: %s", describe(err)) + } + // Should not longer exist + for i, meta := range metas { + var readDoc Account + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); !driver.IsNotFound(err) { + t.Fatalf("Expected NotFoundError at %d, got %s", i, describe(err)) + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/documents_replace_test.go b/deps/github.com/arangodb/go-driver/test/documents_replace_test.go index 2240a9eb4..19748560a 100644 --- a/deps/github.com/arangodb/go-driver/test/documents_replace_test.go +++ b/deps/github.com/arangodb/go-driver/test/documents_replace_test.go @@ -329,3 +329,54 @@ func TestReplaceDocumentsUpdateLenDiff(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestReplaceDocumentsInWaitForSyncCollection creates documents into a collection with waitForSync enabled, +// replaces them and then checks the replacements have succeeded. +func TestReplaceDocumentsInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestReplaceDocumentsInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDoc{ + UserDoc{ + "Piere", + 23, + }, + UserDoc{ + "Pioter", + 45, + }, + } + metas, errs, err := col.CreateDocuments(ctx, docs) + if err != nil { + t.Fatalf("Failed to create new document: %s", describe(err)) + } else if err := errs.FirstNonNil(); err != nil { + t.Fatalf("Expected no errors, got first: %s", describe(err)) + } + // Replacement docs + replacements := []Account{ + Account{ + ID: "foo", + User: &UserDoc{}, + }, + Account{ + ID: "foo2", + User: &UserDoc{}, + }, + } + if _, _, err := col.ReplaceDocuments(ctx, metas.Keys(), replacements); err != nil { + t.Fatalf("Failed to replace documents: %s", describe(err)) + } + // Read replaced documents + for i, meta := range metas { + var readDoc Account + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + if !reflect.DeepEqual(replacements[i], readDoc) { + t.Errorf("Got wrong document %d. Expected %+v, got %+v", i, replacements[i], readDoc) + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/documents_update_test.go b/deps/github.com/arangodb/go-driver/test/documents_update_test.go index f601b3758..a4ccad15a 100644 --- a/deps/github.com/arangodb/go-driver/test/documents_update_test.go +++ b/deps/github.com/arangodb/go-driver/test/documents_update_test.go @@ -452,3 +452,54 @@ func TestUpdateDocumentsUpdateLenDiff(t *testing.T) { t.Errorf("Expected InvalidArgumentError, got %s", describe(err)) } } + +// TestUpdateDocumentsInWaitForSyncCollection creates documents in a collection with waitForSync enabled, +// updates them and then checks the updates have succeeded. +func TestUpdateDocumentsInWaitForSyncCollection(t *testing.T) { + ctx := context.Background() + c := createClientFromEnv(t, true) + db := ensureDatabase(ctx, c, "document_test", nil, t) + col := ensureCollection(ctx, db, "TestUpdateDocumentsInWaitForSyncCollection", &driver.CreateCollectionOptions{ + WaitForSync: true, + }, t) + docs := []UserDoc{ + UserDoc{ + "Piere", + 23, + }, + UserDoc{ + "Otto", + 43, + }, + } + metas, errs, err := col.CreateDocuments(ctx, docs) + if err != nil { + t.Fatalf("Failed to create new documents: %s", describe(err)) + } else if err := errs.FirstNonNil(); err != nil { + t.Fatalf("Expected no errors, got first: %s", describe(err)) + } + // Update documents + updates := []map[string]interface{}{ + map[string]interface{}{ + "name": "Updated1", + }, + map[string]interface{}{ + "name": "Updated2", + }, + } + if _, _, err := col.UpdateDocuments(ctx, metas.Keys(), updates); err != nil { + t.Fatalf("Failed to update documents: %s", describe(err)) + } + // Read updated documents + for i, meta := range metas { + var readDoc UserDoc + if _, err := col.ReadDocument(ctx, meta.Key, &readDoc); err != nil { + t.Fatalf("Failed to read document '%s': %s", meta.Key, describe(err)) + } + doc := docs[i] + doc.Name = fmt.Sprintf("Updated%d", i+1) + if !reflect.DeepEqual(doc, readDoc) { + t.Errorf("Got wrong document %d. Expected %+v, got %+v", i, doc, readDoc) + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/replication_test.go b/deps/github.com/arangodb/go-driver/test/replication_test.go index 9e64961ae..25671d227 100644 --- a/deps/github.com/arangodb/go-driver/test/replication_test.go +++ b/deps/github.com/arangodb/go-driver/test/replication_test.go @@ -25,6 +25,7 @@ package test import ( "context" "testing" + "time" driver "github.com/arangodb/go-driver" ) @@ -45,7 +46,25 @@ func TestReplicationDatabaseInventory(t *testing.T) { if err != nil { t.Fatalf("Failed to open _system database: %s", describe(err)) } - inv, err := rep.DatabaseInventory(ctx, db) + + version, err := c.Version(nil) + if err != nil { + t.Fatalf("Version failed: %s", describe(err)) + } + + ctx2 := ctx + if version.Version.CompareTo("3.2") >= 0 { + var serverID int64 = 1337 // Random test value + // RocksDB requires batchID + batch, err := rep.CreateBatch(ctx, db, serverID, time.Second*60) + if err != nil { + t.Fatalf("CreateBatch failed: %s", describe(err)) + } + ctx2 = driver.WithBatchID(ctx, batch.BatchID()) + defer batch.Delete(ctx) + } + + inv, err := rep.DatabaseInventory(ctx2, db) if err != nil { t.Fatalf("DatabaseInventory failed: %s", describe(err)) } diff --git a/deps/github.com/arangodb/go-driver/test/server_info_test.go b/deps/github.com/arangodb/go-driver/test/server_info_test.go new file mode 100644 index 000000000..d10d5ffb7 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/test/server_info_test.go @@ -0,0 +1,59 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package test + +import ( + "context" + "testing" + + driver "github.com/arangodb/go-driver" +) + +// TestServerID tests ClientServerInfo.ServerID. +func TestServerID(t *testing.T) { + c := createClientFromEnv(t, true) + ctx := context.Background() + + var isCluster bool + if _, err := c.Cluster(ctx); driver.IsPreconditionFailed(err) { + isCluster = false + } else if err != nil { + t.Fatalf("Health failed: %s", describe(err)) + } else { + isCluster = true + } + + if isCluster { + id, err := c.ServerID(ctx) + if err != nil { + t.Fatalf("ServerID failed: %s", describe(err)) + } + if id == "" { + t.Error("Expected ID to be non-empty") + } + } else { + if _, err := c.ServerID(ctx); err == nil { + t.Fatalf("ServerID succeeded, expected error") + } + } +} diff --git a/deps/github.com/arangodb/go-driver/test/user_auth_test.go b/deps/github.com/arangodb/go-driver/test/user_auth_test.go index 29db97099..cc60a12bb 100644 --- a/deps/github.com/arangodb/go-driver/test/user_auth_test.go +++ b/deps/github.com/arangodb/go-driver/test/user_auth_test.go @@ -210,12 +210,6 @@ func TestGrantUserDefaultDatabase(t *testing.T) { t.Skipf("This test requires 3.2 or higher, got %s", version.Version) } - // We skip this test until Feb-1 - startTestDate := time.Date(2018, time.February, 1, 0, 0, 0, 0, time.UTC) - if time.Now().Before(startTestDate) { - t.Skipf("This test is skipped until %s", startTestDate) - } - u := ensureUser(nil, c, "grant_user_def", &driver.UserOptions{Password: "foo"}, t) db := ensureDatabase(nil, c, "grant_user_def_test", nil, t) // Grant read/write access to default database @@ -258,39 +252,74 @@ func TestGrantUserDefaultDatabase(t *testing.T) { t.Fatalf("Expected success, got %s", describe(err)) } - // wait for change to propagate (TODO add a check to the coordinators) - time.Sleep(time.Second * 5) - - // Try to create document in collection, should fail because there are no collection grants for this user and/or collection. - if _, err := authCol.CreateDocument(nil, Book{Title: "I cannot write"}); !driver.IsForbidden(err) { - t.Errorf("Expected failure, got %s", describe(err)) - } - // Grant read-only access to default database if err := u.SetDatabaseAccess(nil, nil, driver.GrantReadOnly); err != nil { t.Fatalf("SetDatabaseAccess failed: %s", describe(err)) } - // Try to create collection, should fail - if _, err := authDb.CreateCollection(nil, "books_def_ro_db", nil); !driver.IsForbidden(err) { - t.Errorf("Expected failure, got %s", describe(err)) + + // wait for change to propagate + { + deadline := time.Now().Add(time.Minute) + for { + // Try to create document in collection, should fail because there are no collection grants for this user and/or collection. + if _, err := authCol.CreateDocument(nil, Book{Title: "I cannot write"}); err == nil { + if time.Now().Before(deadline) { + t.Logf("Expected failure, got %s, trying again...", describe(err)) + time.Sleep(time.Second * 2) + continue + } + t.Errorf("Expected failure, got %s", describe(err)) + } + + // Try to create collection, should fail + if _, err := authDb.CreateCollection(nil, "books_def_ro_db", nil); err == nil { + t.Errorf("Expected failure, got %s", describe(err)) + } + break + } } // Grant no access to default database if err := u.SetDatabaseAccess(nil, nil, driver.GrantNone); err != nil { t.Fatalf("SetDatabaseAccess failed: %s", describe(err)) } - // Try to create collection, should fail - if _, err := authDb.CreateCollection(nil, "books_def_none_db", nil); !driver.IsUnauthorized(err) { - t.Errorf("Expected failure, got %s", describe(err)) + + // wait for change to propagate + { + deadline := time.Now().Add(time.Minute) + for { + // Try to create collection, should fail + if _, err := authDb.CreateCollection(nil, "books_def_none_db", nil); err == nil { + if time.Now().Before(deadline) { + t.Logf("Expected failure, got %s, trying again...", describe(err)) + time.Sleep(time.Second * 2) + continue + } + t.Errorf("Expected failure, got %s", describe(err)) + } + break + } } // Remove default database access, should fallback to "no-access" then if err := u.RemoveDatabaseAccess(nil, nil); err != nil { t.Fatalf("RemoveDatabaseAccess failed: %s", describe(err)) } - // Try to create collection, should fail - if _, err := authDb.CreateCollection(nil, "books_def_star_db", nil); !driver.IsUnauthorized(err) { - t.Errorf("Expected failure, got %s", describe(err)) + // wait for change to propagate + { + deadline := time.Now().Add(time.Minute) + for { + // Try to create collection, should fail + if _, err := authDb.CreateCollection(nil, "books_def_star_db", nil); err == nil { + if time.Now().Before(deadline) { + t.Logf("Expected failure, got %s, trying again...", describe(err)) + time.Sleep(time.Second * 2) + continue + } + t.Errorf("Expected failure, got %s", describe(err)) + } + break + } } } @@ -301,17 +330,13 @@ func TestGrantUserCollection(t *testing.T) { if err != nil { t.Fatalf("Version failed: %s", describe(err)) } + // 3.3.4 changes behaviour to better support LDAP isv32p := version.Version.CompareTo("3.2") >= 0 + isv334 := version.Version.CompareTo("3.3.4") >= 0 if !isv32p { t.Skipf("This test requires 3.2 or higher, got %s", version.Version) } - // We skip this test until Feb-1 - startTestDate := time.Date(2018, time.February, 1, 0, 0, 0, 0, time.UTC) - if time.Now().Before(startTestDate) { - t.Skipf("This test is skipped until %s", startTestDate) - } - u := ensureUser(nil, c, "grant_user_col", &driver.UserOptions{Password: "foo"}, t) db := ensureDatabase(nil, c, "grant_user_col_test", nil, t) // Grant read/write access to database @@ -390,21 +415,42 @@ func TestGrantUserCollection(t *testing.T) { if _, err := authCol.ReadDocument(nil, meta1.Key, &doc); !driver.IsForbidden(err) { t.Errorf("Expected failure, got %s", describe(err)) } - // Now remove explicit collection access if err := u.RemoveCollectionAccess(nil, col); err != nil { t.Fatalf("RemoveCollectionAccess failed: %s", describe(err)) } + expected := driver.GrantNone + if isv334 { + expected = driver.GrantReadWrite + } // Read back collection access if grant, err := u.GetCollectionAccess(nil, col); err != nil { t.Fatalf("GetCollectionAccess failed: %s", describe(err)) - } else if grant != driver.GrantNone { - t.Errorf("Collection access invalid, expected 'none', got '%s'", grant) + } else if grant != expected { + t.Errorf("Collection access invalid, expected '%s', got '%s'", expected, grant) + } + // Grant read-only access to database + if err := u.SetDatabaseAccess(nil, db, driver.GrantReadOnly); err != nil { + t.Fatalf("SetDatabaseAccess failed: %s", describe(err)) + } + expected = driver.GrantNone + if isv334 { + expected = driver.GrantReadOnly + } + // Read back collection access + if grant, err := u.GetCollectionAccess(nil, col); err != nil { + t.Fatalf("GetCollectionAccess failed: %s", describe(err)) + } else if grant != expected { + t.Errorf("Collection access invalid, expected '%s', got '%s'", expected, grant) } // Try to create another document, should fail if _, err := authCol.CreateDocument(nil, Book{Title: "I should not be able to write"}); !driver.IsForbidden(err) { t.Errorf("Expected failure, got: %s", describe(err)) } + // Grant no access to collection + if err := u.SetCollectionAccess(nil, col, driver.GrantNone); err != nil { + t.Fatalf("SetDatabaseAccess failed: %s", describe(err)) + } // Try to read back first document, should fail if _, err := authCol.ReadDocument(nil, meta1.Key, &doc); !driver.IsForbidden(err) { t.Errorf("Expected failure, got %s", describe(err)) diff --git a/deps/github.com/arangodb/go-driver/test/util.go b/deps/github.com/arangodb/go-driver/test/util.go index d764d8ce4..40ea6d031 100644 --- a/deps/github.com/arangodb/go-driver/test/util.go +++ b/deps/github.com/arangodb/go-driver/test/util.go @@ -26,6 +26,9 @@ import ( "encoding/hex" "encoding/json" "fmt" + "os" + "strconv" + "strings" "testing" driver "github.com/arangodb/go-driver" @@ -67,9 +70,8 @@ func describe(err error) string { } if cause.Error() != err.Error() { return fmt.Sprintf("%v caused by %v (%v)", err, cause, msg) - } else { - return fmt.Sprintf("%v (%v)", err, msg) } + return fmt.Sprintf("%v (%v)", err, msg) } func formatRawResponse(raw []byte) string { @@ -82,3 +84,16 @@ func formatRawResponse(raw []byte) string { } return hex.EncodeToString(raw) } + +// getIntFromEnv looks for an environment variable with given key. +// If found, it parses the value to an int, if success that value is returned. +// In all other cases, the given default value is returned. +func getIntFromEnv(envKey string, defaultValue int) int { + v := strings.TrimSpace(os.Getenv(envKey)) + if v != "" { + if result, err := strconv.Atoi(v); err == nil { + return result + } + } + return defaultValue +} diff --git a/deps/github.com/arangodb/go-driver/util/doc.go b/deps/github.com/arangodb/go-driver/util/doc.go new file mode 100644 index 000000000..b16feff9f --- /dev/null +++ b/deps/github.com/arangodb/go-driver/util/doc.go @@ -0,0 +1,26 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package util provides some helper methods for the go-driver (it is unlikely that you need this package directly). +*/ +package util diff --git a/deps/github.com/arangodb/go-driver/vertex_collection_documents_impl.go b/deps/github.com/arangodb/go-driver/vertex_collection_documents_impl.go index 233231dc1..323e2c5b0 100644 --- a/deps/github.com/arangodb/go-driver/vertex_collection_documents_impl.go +++ b/deps/github.com/arangodb/go-driver/vertex_collection_documents_impl.go @@ -102,7 +102,7 @@ func (c *vertexCollection) createDocument(ctx context.Context, document interfac if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { @@ -317,7 +317,7 @@ func (c *vertexCollection) replaceDocument(ctx context.Context, key string, docu if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(201, 202)); err != nil { + if err := resp.CheckStatus(201, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { @@ -432,7 +432,7 @@ func (c *vertexCollection) removeDocument(ctx context.Context, key string) (Docu if err != nil { return DocumentMeta{}, cs, WithStack(err) } - if err := resp.CheckStatus(cs.okStatus(200, 202)); err != nil { + if err := resp.CheckStatus(200, 202); err != nil { return DocumentMeta{}, cs, WithStack(err) } if cs.Silent { diff --git a/deps/github.com/arangodb/go-driver/vst/connection.go b/deps/github.com/arangodb/go-driver/vst/connection.go index 1250bd2d1..cb8254d37 100644 --- a/deps/github.com/arangodb/go-driver/vst/connection.go +++ b/deps/github.com/arangodb/go-driver/vst/connection.go @@ -142,6 +142,9 @@ func (c *vstConnection) Do(ctx context.Context, req driver.Request) (driver.Resp // Do performs a given request, returning its response. func (c *vstConnection) do(ctx context.Context, req driver.Request, transport messageTransport) (driver.Response, error) { + if ctx == nil { + ctx = context.Background() + } vstReq, ok := req.(*vstRequest) if !ok { return nil, driver.WithStack(driver.InvalidArgumentError{Message: "request is not a *vstRequest"}) @@ -158,10 +161,16 @@ func (c *vstConnection) do(ctx context.Context, req driver.Request, transport me vstReq.WroteRequest() // Wait for response - msg, ok := <-resp - if !ok { - // Message was cancelled / timeout - return nil, driver.WithStack(context.DeadlineExceeded) + var msg protocol.Message + select { + case msg, ok = <-resp: + if !ok { + // Message was canceled / timeout + return nil, driver.WithStack(context.DeadlineExceeded) + } + case <-ctx.Done(): + // Context canceled while waiting here + return nil, driver.WithStack(ctx.Err()) } //fmt.Printf("Received msg: %d\n", msg.ID) diff --git a/deps/github.com/arangodb/go-driver/vst/doc.go b/deps/github.com/arangodb/go-driver/vst/doc.go new file mode 100644 index 000000000..2337aea23 --- /dev/null +++ b/deps/github.com/arangodb/go-driver/vst/doc.go @@ -0,0 +1,68 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package vst implements driver.Connection using a VelocyStream connection. + +This connection uses VelocyStream (with optional TLS) to connect to the ArangoDB database. +It encodes its contents as Velocypack. + +Creating an Insecure Connection + +To create a VST connection, use code like this. + + // Create a VST connection to the database + conn, err := vst.NewConnection(vst.ConnectionConfig{ + Endpoints: []string{"http://localhost:8529"}, + }) + if err != nil { + // Handle error + } + +The resulting connection is used to create a client which you will use +for normal database requests. + + // Create a client + c, err := driver.NewClient(driver.ClientConfig{ + Connection: conn, + }) + if err != nil { + // Handle error + } + +Creating a Secure Connection + +To create a secure VST connection, use code like this. + + // Create a VST over TLS connection to the database + conn, err := vst.NewConnection(vst.ConnectionConfig{ + Endpoints: []string{"https://localhost:8529"}, + TLSConfig: &tls.Config{ + InsecureSkipVerify: trueWhenUsingNonPublicCertificates, + }, + }) + if err != nil { + // Handle error + } + +*/ +package vst diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/chunk.go b/deps/github.com/arangodb/go-driver/vst/protocol/chunk.go index eb1e59bb4..d16559adb 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/chunk.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/chunk.go @@ -87,6 +87,10 @@ func buildChunks(messageID uint64, maxChunkSize uint32, messageParts ...[]byte) func readBytes(dst []byte, r io.Reader) error { offset := 0 remaining := len(dst) + if remaining == 0 { + // Nothing left to read + return nil + } for { n, err := r.Read(dst[offset:]) offset += n diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/chunk_1_0.go b/deps/github.com/arangodb/go-driver/vst/protocol/chunk_1_0.go index 974b1dbeb..6458c344b 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/chunk_1_0.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/chunk_1_0.go @@ -24,7 +24,6 @@ package protocol import ( "encoding/binary" - "fmt" "io" driver "github.com/arangodb/go-driver" @@ -45,7 +44,7 @@ func readChunkVST1_0(r io.Reader) (chunk, error) { if (1 == (chunkX & 0x1)) && ((chunkX >> 1) > 1) { // First chunk, numberOfChunks>1 -> read messageLength - fmt.Println("Reading maxHdr") + //fmt.Println("Reading maxHdr") if err := readBytes(hdr[minChunkHeaderSize:], r); err != nil { return chunk{}, driver.WithStack(err) } diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/connection.go b/deps/github.com/arangodb/go-driver/vst/protocol/connection.go index 927f1aae2..d1af33c4a 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/connection.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/connection.go @@ -43,12 +43,14 @@ type Connection struct { msgStore messageStore conn net.Conn writeMutex sync.Mutex - closing bool + closing int32 lastActivity time.Time + configured int32 // Set to 1 after the configuration callback has finished without errors. } const ( defaultMaxChunkSize = 30000 + maxRecentErrors = 64 ) var ( @@ -58,13 +60,8 @@ var ( // dial opens a new connection to the server on the given address. func dial(version Version, addr string, tlsConfig *tls.Config) (*Connection, error) { - var conn net.Conn - var err error - if tlsConfig != nil { - conn, err = tls.Dial("tcp", addr, tlsConfig) - } else { - conn, err = net.Dial("tcp", addr) - } + // Create TCP connection + conn, err := net.Dial("tcp", addr) if err != nil { return nil, driver.WithStack(err) } @@ -75,6 +72,12 @@ func dial(version Version, addr string, tlsConfig *tls.Config) (*Connection, err tcpConn.SetNoDelay(true) } + // Add TLS if needed + if tlsConfig != nil { + tlsConn := tls.Client(conn, tlsConfig) + conn = tlsConn + } + // Send protocol header switch version { case Version1_0: @@ -103,18 +106,20 @@ func dial(version Version, addr string, tlsConfig *tls.Config) (*Connection, err return c, nil } +// load returns an indication of the amount of work this connection has. +// 0 means no work at all, >0 means some work. +func (c *Connection) load() int { + return c.msgStore.Size() +} + // Close the connection to the server func (c *Connection) Close() error { - if !c.closing { - c.closing = true + if atomic.CompareAndSwapInt32(&c.closing, 0, 1) { if err := c.conn.Close(); err != nil { return driver.WithStack(err) } c.msgStore.ForEach(func(m *Message) { - if m.response != nil { - close(m.response) - m.response = nil - } + m.closeResponseChan() }) } return nil @@ -122,7 +127,12 @@ func (c *Connection) Close() error { // IsClosed returns true when the connection is closed, false otherwise. func (c *Connection) IsClosed() bool { - return c.closing + return atomic.LoadInt32(&c.closing) == 1 +} + +// IsConfigured returns true when the configuration callback has finished on this connection, without errors. +func (c *Connection) IsConfigured() bool { + return atomic.LoadInt32(&c.configured) == 1 } // Send sends a message (consisting of given parts) to the server and returns @@ -140,6 +150,7 @@ func (c *Connection) Send(ctx context.Context, messageParts ...[]byte) (<-chan M } // Prepare for receiving a response m := c.msgStore.Add(msgID) + responseChan := m.responseChan //panic(fmt.Sprintf("chunks: %d, messageParts: %d, first: %s", len(chunks), len(messageParts), hex.EncodeToString(messageParts[0]))) @@ -168,7 +179,7 @@ func (c *Connection) Send(ctx context.Context, messageParts ...[]byte) (<-chan M if err != nil { return nil, driver.WithStack(err) } - return m.response, nil + return responseChan, nil case <-ctx.Done(): return nil, ctx.Err() } @@ -198,8 +209,10 @@ func (c *Connection) sendChunk(deadline time.Time, chunk chunk) error { // readChunkLoop reads chunks from the connection until it is closed. func (c *Connection) readChunkLoop() { + recentErrors := 0 + goodChunks := 0 for { - if c.closing { + if c.IsClosed() { // Closing, we're done return } @@ -215,17 +228,27 @@ func (c *Connection) readChunkLoop() { } c.updateLastActivity() if err != nil { - if !c.closing { + if !c.IsClosed() { // Handle error if err == io.EOF { // Connection closed c.Close() } else { - fmt.Printf("readChunkLoop error: %#v\n", err) + recentErrors++ + fmt.Printf("readChunkLoop error: %#v (goodChunks=%d)\n", err, goodChunks) + if recentErrors > maxRecentErrors { + // When we get to many errors in a row, close this connection + c.Close() + } else { + // Backoff a bit, so we allow things to settle. + time.Sleep(time.Millisecond * time.Duration(recentErrors*5)) + } } } } else { // Process chunk + recentErrors = 0 + goodChunks++ go c.processChunk(chunk) } } @@ -252,10 +275,7 @@ func (c *Connection) processChunk(chunk chunk) { //fmt.Println("Chunk: " + hex.EncodeToString(chunk.Data) + "\nMessage: " + hex.EncodeToString(m.Data)) // Notify listener - if m.response != nil { - m.response <- *m - close(m.response) - } + m.notifyListener() } } diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/doc.go b/deps/github.com/arangodb/go-driver/vst/protocol/doc.go new file mode 100644 index 000000000..84f503dca --- /dev/null +++ b/deps/github.com/arangodb/go-driver/vst/protocol/doc.go @@ -0,0 +1,26 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +/* +Package protocol implements the VelocyStream protocol (it is not intended to be used directly). +*/ +package protocol diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/message.go b/deps/github.com/arangodb/go-driver/vst/protocol/message.go index efbaccc8e..b7ba1a234 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/message.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/message.go @@ -22,21 +22,51 @@ package protocol -import "sort" +import ( + "sort" + "sync" + "sync/atomic" +) // Message is what is send back to the client in response to a request. type Message struct { ID uint64 Data []byte - chunks []chunk - numberOfChunks uint32 - response chan Message + chunksMutex sync.Mutex + chunks []chunk + numberOfChunks uint32 + responseChanClosed int32 + responseChan chan Message +} + +// closes the response channel if needed. +func (m *Message) closeResponseChan() { + if atomic.CompareAndSwapInt32(&m.responseChanClosed, 0, 1) { + if ch := m.responseChan; ch != nil { + m.responseChan = nil + close(ch) + } + } +} + +// notifyListener pushes itself onto its response channel and closes the response channel afterwards. +func (m *Message) notifyListener() { + if atomic.CompareAndSwapInt32(&m.responseChanClosed, 0, 1) { + if ch := m.responseChan; ch != nil { + m.responseChan = nil + ch <- *m + close(ch) + } + } } // addChunk adds the given chunks to the list of chunks of the message. // If the given chunk is the first chunk, the expected number of chunks is recorded. func (m *Message) addChunk(c chunk) { + m.chunksMutex.Lock() + defer m.chunksMutex.Unlock() + m.chunks = append(m.chunks, c) if c.IsFirst() { m.numberOfChunks = c.NumberOfChunks() @@ -48,6 +78,9 @@ func (m *Message) addChunk(c chunk) { // is returned. // If all chunks are available, the Data field is build and set and true is returned. func (m *Message) assemble() bool { + m.chunksMutex.Lock() + defer m.chunksMutex.Unlock() + if m.Data != nil { // Already assembled return true diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/message_store.go b/deps/github.com/arangodb/go-driver/vst/protocol/message_store.go index 5c60d4bf5..e45da2cc1 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/message_store.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/message_store.go @@ -32,6 +32,14 @@ type messageStore struct { messages map[uint64]*Message } +// Size returns the number of messages in this store. +func (s *messageStore) Size() int { + s.mutex.RLock() + defer s.mutex.RUnlock() + + return len(s.messages) +} + // Get returns the message with given id, or nil if not found func (s *messageStore) Get(id uint64) *Message { s.mutex.RLock() @@ -58,8 +66,8 @@ func (s *messageStore) Add(id uint64) *Message { } m := &Message{ - ID: id, - response: make(chan Message), + ID: id, + responseChan: make(chan Message), } s.messages[id] = m return m diff --git a/deps/github.com/arangodb/go-driver/vst/protocol/transport.go b/deps/github.com/arangodb/go-driver/vst/protocol/transport.go index a943bc60f..36192c889 100644 --- a/deps/github.com/arangodb/go-driver/vst/protocol/transport.go +++ b/deps/github.com/arangodb/go-driver/vst/protocol/transport.go @@ -26,6 +26,7 @@ import ( "context" "crypto/tls" "sync" + "sync/atomic" "time" driver "github.com/arangodb/go-driver" @@ -33,6 +34,7 @@ import ( const ( DefaultIdleConnTimeout = time.Minute + DefaultConnLimit = 3 ) // TransportConfig contains configuration options for Transport. @@ -43,6 +45,11 @@ type TransportConfig struct { // Zero means no limit. IdleConnTimeout time.Duration + // ConnLimit is the upper limit to the number of connections to a single server. + // Due to the nature of the VST protocol, this value does not have to be high. + // The default is 3 (DefaultConnLimit). + ConnLimit int + // Version specifies the version of the Velocystream protocol Version Version } @@ -63,6 +70,9 @@ func NewTransport(hostAddr string, tlsConfig *tls.Config, config TransportConfig if config.IdleConnTimeout == 0 { config.IdleConnTimeout = DefaultIdleConnTimeout } + if config.ConnLimit == 0 { + config.ConnLimit = DefaultConnLimit + } return &Transport{ TransportConfig: config, hostAddr: hostAddr, @@ -91,13 +101,16 @@ func (c *Transport) CloseIdleConnections() (closed, remaining int) { c.connMutex.Lock() defer c.connMutex.Unlock() - for i, conn := range c.connections { + for i := 0; i < len(c.connections); { + conn := c.connections[i] if conn.IsClosed() || conn.IsIdle(c.IdleConnTimeout) { // Remove connection from list c.connections = append(c.connections[:i], c.connections[i+1:]...) // Close connection go conn.Close() closed++ + } else { + i++ } } @@ -141,10 +154,14 @@ func (c *Transport) getConnection(ctx context.Context) (*Connection, error) { // Invoke callback if cb := c.onConnectionCreated; cb != nil { if err := cb(ctx, conn); err != nil { + conn.Close() return nil, driver.WithStack(err) } } + // Mark the connection as ready + atomic.StoreInt32(&conn.configured, 1) + return conn, nil } @@ -154,15 +171,36 @@ func (c *Transport) getAvailableConnection() *Connection { c.connMutex.Lock() defer c.connMutex.Unlock() + // Select the connection with the least amount of traffic + var bestConn *Connection + bestConnLoad := 0 + activeConnCount := 0 for _, conn := range c.connections { if !conn.IsClosed() { - conn.updateLastActivity() - return conn + activeConnCount++ + if conn.IsConfigured() { + connLoad := conn.load() + if bestConn == nil || connLoad < bestConnLoad { + bestConn = conn + bestConnLoad = connLoad + } + } } } - // No connections available - return nil + if bestConn == nil { + // No connections available + return nil + } + + // Is load is >0 AND the number of connections is below the limit, create a new one + if bestConnLoad > 0 && activeConnCount < c.ConnLimit { + return nil + } + + // Use the best connection found + bestConn.updateLastActivity() + return bestConn } // createConnection creates a new connection. diff --git a/deps/github.com/pkg/errors/.travis.yml b/deps/github.com/pkg/errors/.travis.yml index 7ca408d1b..15e5a1926 100644 --- a/deps/github.com/pkg/errors/.travis.yml +++ b/deps/github.com/pkg/errors/.travis.yml @@ -7,6 +7,7 @@ go: - 1.7.x - 1.8.x - 1.9.x + - 1.10.x - tip script: diff --git a/docs/Manual/Deployment/Kubernetes/Authentication.md b/docs/Manual/Deployment/Kubernetes/Authentication.md new file mode 100644 index 000000000..d9bff945f --- /dev/null +++ b/docs/Manual/Deployment/Kubernetes/Authentication.md @@ -0,0 +1,18 @@ +# Authentication + +The ArangoDB Kubernetes Operator will by default create ArangoDB deployments +that require authentication to access the database. + +It uses a single JWT secret (stored in a Kubernetes secret) +to provide *super-user* access between all servers of the deployment +as well as access from the ArangoDB Operator to the deployment. + +To disable authentication, set `spec.auth.jwtSecretName` to `None`. + +Initially the deployment is accessible through the web user-interface and +API's, using the user `root` with an empty password. +Make sure to change this password immediately after starting the deployment! + +## See also + +- [Secure connections (TLS)](./Tls.md) diff --git a/docs/Manual/Deployment/Kubernetes/DeploymentReplicationResource.md b/docs/Manual/Deployment/Kubernetes/DeploymentReplicationResource.md new file mode 100644 index 000000000..2a1a23cc2 --- /dev/null +++ b/docs/Manual/Deployment/Kubernetes/DeploymentReplicationResource.md @@ -0,0 +1,207 @@ +# ArangoDeploymentReplication Custom Resource + +The ArangoDB Replication Operator creates and maintains ArangoDB +`arangosync` configurations in a Kubernetes cluster, given a replication specification. +This replication specification is a `CustomResource` following +a `CustomResourceDefinition` created by the operator. + +Example minimal replication definition for 2 ArangoDB cluster with sync in the same Kubernetes cluster: + +```yaml +apiVersion: "replication.database.arangodb.com/v1alpha" +kind: "ArangoDeploymentReplication" +metadata: + name: "replication-from-a-to-b" +spec: + source: + deploymentName: cluster-a + auth: + keyfileSecretName: cluster-a-sync-auth + destination: + deploymentName: cluster-b +``` + +This definition results in: + +- the arangosync `SyncMaster` in deployment `cluster-b` is called to configure a synchronization + from the syncmasters in `cluster-a` to the syncmasters in `cluster-b`, + using the client authentication certificate stored in `Secret` `cluster-a-sync-auth`. + To access `cluster-a`, the JWT secret found in the deployment of `cluster-a` is used. + To access `cluster-b`, the JWT secret found in the deployment of `cluster-b` is used. + +Example replication definition for replicating from a source that is outside the current Kubernetes cluster +to a destination that is in the same Kubernetes cluster: + +```yaml +apiVersion: "replication.database.arangodb.com/v1alpha" +kind: "ArangoDeploymentReplication" +metadata: + name: "replication-from-a-to-b" +spec: + source: + masterEndpoint: ["https://163.172.149.229:31888", "https://51.15.225.110:31888", "https://51.15.229.133:31888"] + auth: + keyfileSecretName: cluster-a-sync-auth + tls: + caSecretName: cluster-a-sync-ca + destination: + deploymentName: cluster-b +``` + +This definition results in: + +- the arangosync `SyncMaster` in deployment `cluster-b` is called to configure a synchronization + from the syncmasters located at the given list of endpoint URL's to the syncmasters `cluster-b`, + using the client authentication certificate stored in `Secret` `cluster-a-sync-auth`. + To access `cluster-a`, the keyfile (containing a client authentication certificate) is used. + To access `cluster-b`, the JWT secret found in the deployment of `cluster-b` is used. + +## Specification reference + +Below you'll find all settings of the `ArangoDeploymentReplication` custom resource. + +### `spec.source.deploymentName: string` + +This setting specifies the name of an `ArangoDeployment` resource that runs a cluster +with sync enabled. + +This cluster configured as the replication source. + +### `spec.source.masterEndpoint: []string` + +This setting specifies zero or more master endpoint URL's of the source cluster. + +Use this setting if the source cluster is not running inside a Kubernetes cluster +that is reachable from the Kubernetes cluster the `ArangoDeploymentReplication` resource is deployed in. + +Specifying this setting and `spec.source.deploymentName` at the same time is not allowed. + +### `spec.source.auth.keyfileSecretName: string` + +This setting specifies the name of a `Secret` containing a client authentication certificate called `tls.keyfile` used to authenticate +with the SyncMaster at the specified source. + +If `spec.source.auth.userSecretName` has not been set, +the client authentication certificate found in the secret with this name is also used to configure +the synchronization and fetch the synchronization status. + +This setting is required. + +### `spec.source.auth.userSecretName: string` + +This setting specifies the name of a `Secret` containing a `username` & `password` used to authenticate +with the SyncMaster at the specified source in order to configure synchronization and fetch synchronization status. + +The user identified by the username must have write access in the `_system` database of the source ArangoDB cluster. + +### `spec.source.tls.caSecretName: string` + +This setting specifies the name of a `Secret` containing a TLS CA certificate `ca.crt` used to verify +the TLS connection created by the SyncMaster at the specified source. + +This setting is required, unless `spec.source.deploymentName` has been set. + +### `spec.destination.deploymentName: string` + +This setting specifies the name of an `ArangoDeployment` resource that runs a cluster +with sync enabled. + +This cluster configured as the replication destination. + +### `spec.destination.masterEndpoint: []string` + +This setting specifies zero or more master endpoint URL's of the destination cluster. + +Use this setting if the destination cluster is not running inside a Kubernetes cluster +that is reachable from the Kubernetes cluster the `ArangoDeploymentReplication` resource is deployed in. + +Specifying this setting and `spec.destination.deploymentName` at the same time is not allowed. + +### `spec.destination.auth.keyfileSecretName: string` + +This setting specifies the name of a `Secret` containing a client authentication certificate called `tls.keyfile` used to authenticate +with the SyncMaster at the specified destination. + +If `spec.destination.auth.userSecretName` has not been set, +the client authentication certificate found in the secret with this name is also used to configure +the synchronization and fetch the synchronization status. + +This setting is required, unless `spec.destination.deploymentName` or `spec.destination.auth.userSecretName` has been set. + +Specifying this setting and `spec.destination.userSecretName` at the same time is not allowed. + +### `spec.destination.auth.userSecretName: string` + +This setting specifies the name of a `Secret` containing a `username` & `password` used to authenticate +with the SyncMaster at the specified destination in order to configure synchronization and fetch synchronization status. + +The user identified by the username must have write access in the `_system` database of the destination ArangoDB cluster. + +Specifying this setting and `spec.destination.keyfileSecretName` at the same time is not allowed. + +### `spec.destination.tls.caSecretName: string` + +This setting specifies the name of a `Secret` containing a TLS CA certificate `ca.crt` used to verify +the TLS connection created by the SyncMaster at the specified destination. + +This setting is required, unless `spec.destination.deploymentName` has been set. + +## Authentication details + +The authentication settings in a `ArangoDeploymentReplication` resource are used for two distinct purposes. + +The first use is the authentication of the syncmasters at the destination with the syncmasters at the source. +This is always done using a client authentication certificate which is found in a `tls.keyfile` field +in a secret identified by `spec.source.auth.keyfileSecretName`. + +The second use is the authentication of the ArangoDB Replication operator with the syncmasters at the source +or destination. These connections are made to configure synchronization, stop configuration and fetch the status +of the configuration. +The method used for this authentication is derived as follows (where `X` is either `source` or `destination`): + +- If `spec.X.userSecretName` is set, the username + password found in the `Secret` identified by this name is used. +- If `spec.X.keyfileSecretName` is set, the client authentication certificate (keyfile) found in the `Secret` identifier by this name is used. +- If `spec.X.deploymentName` is set, the JWT secret found in the deployment is used. + +## Creating client authentication certificate keyfiles + +The client authentication certificates needed for the `Secrets` identified by `spec.source.auth.keyfileSecretName` & `spec.destination.auth.keyfileSecretName` +are normal ArangoDB keyfiles that can be created by the `arangosync create client-auth keyfile` command. +In order to do so, you must have access to the client authentication CA of the source/destination. + +If the client authentication CA at the source/destination also contains a private key (`ca.key`), the ArangoDeployment operator +can be used to create such a keyfile for you, without the need to have `arangosync` installed locally. +Read the following paragraphs for instructions on how to do that. + +## Creating and using access packages + +An access package is a YAML file that contains: + +- A client authentication certificate, wrapped in a `Secret` in a `tls.keyfile` data field. +- A TLS certificate authority public key, wrapped in a `Secret` in a `ca.crt` data field. + +The format of the access package is such that it can be inserted into a Kubernetes cluster using the standard `kubectl` tool. + +To create an access package that can be used to authenticate with the ArangoDB SyncMasters of an `ArangoDeployment`, +add a name of a non-existing `Secret` to the `spec.sync.externalAccess.accessPackageSecretNames` field of the `ArangoDeployment`. +In response, a `Secret` is created in that Kubernetes cluster, with the given name, that contains a `accessPackage.yaml` data field +that contains a Kubernetes resource specification that can be inserted into the other Kubernetes cluster. + +The process for creating and using an access package for authentication at the source cluster is as follows: + +- Edit the `ArangoDeployment` resource of the source cluster, set `spec.sync.externalAccess.accessPackageSecretNames` to `["my-access-package"]` +- Wait for the `ArangoDeployment` operator to create a `Secret` named `my-access-package`. +- Extract the access package from the Kubernetes source cluster using: + +```bash +kubectl get secret my-access-package --template='{{index .data "accessPackage.yaml"}}' | base64 -D > accessPackage.yaml +``` + +- Insert the secrets found in the access package in the Kubernetes destination cluster using: + +```bash +kubectl apply -f accessPackage.yaml +``` + +As a result, the destination Kubernetes cluster will have 2 additional `Secrets`. One contains a client authentication certificate +formatted as a keyfile. Another contains the public key of the TLS CA certificate of the source cluster. diff --git a/docs/Manual/Deployment/Kubernetes/DeploymentResource.md b/docs/Manual/Deployment/Kubernetes/DeploymentResource.md index 390b14e54..d0fbf6e83 100644 --- a/docs/Manual/Deployment/Kubernetes/DeploymentResource.md +++ b/docs/Manual/Deployment/Kubernetes/DeploymentResource.md @@ -51,10 +51,10 @@ Below you'll find all settings of the `ArangoDeployment` custom resource. Several settings are for various groups of servers. These are indicated with `` where `` can be any of: -- `agents` for all agents of a `Cluster` or `ResilientSingle` pair. +- `agents` for all agents of a `Cluster` or `ActiveFailover` pair. - `dbservers` for all dbservers of a `Cluster`. - `coordinators` for all coordinators of a `Cluster`. -- `single` for all single servers of a `Single` instance or `ResilientSingle` pair. +- `single` for all single servers of a `Single` instance or `ActiveFailover` pair. - `syncmasters` for all syncmasters of a `Cluster`. - `syncworkers` for all syncworkers of a `Cluster`. @@ -64,7 +64,7 @@ This setting specifies the type of deployment you want to create. Possible values are: - `Cluster` (default) Full cluster. Defaults to 3 agents, 3 dbservers & 3 coordinators. -- `ResilientSingle` Resilient single pair. Defaults to 3 agents and 2 single servers. +- `ActiveFailover` Active-failover single pair. Defaults to 3 agents and 2 single servers. - `Single` Single server only (note this does not provide high availability or reliability). This setting cannot be changed after the deployment has been created. @@ -107,6 +107,23 @@ Possible values are: This setting cannot be changed after the cluster has been created. +### `spec.downtimeAllowed: bool` + +This setting is used to allow automatic reconciliation actions that yield +some downtime of the ArangoDB deployment. +When this setting is set to `false` (the default), no automatic action that +may result in downtime is allowed. +If the need for such an action is detected, an event is added to the `ArangoDeployment`. + +Once this setting is set to `true`, the automatic action is executed. + +Operations that may result in downtime are: + +- Rotating TLS CA certificate + +Note: It is still possible that there is some downtime when the Kubernetes +cluster is down, or in a bad state, irrespective of the value of this setting. + ### `spec.rocksdb.encryption.keySecretName` This setting specifies the name of a kubernetes `Secret` that contains @@ -122,6 +139,32 @@ The encryption key cannot be changed after the cluster has been created. The secret specified by this setting, must have a data field named 'key' containing an encryption key that is exactly 32 bytes long. +### `spec.externalAccess.type: string` + +This setting specifies the type of `Service` that will be created to provide +access to the ArangoDB deployment from outside the Kubernetes cluster. +Possible values are: + +- `None` To limit access to application running inside the Kubernetes cluster. +- `LoadBalancer` To create a `Service` of type `LoadBalancer` for the ArangoDB deployment. +- `NodePort` To create a `Service` of type `NodePort` for the ArangoDB deployment. +- `Auto` (default) To create a `Service` of type `LoadBalancer` and fallback to a `Service` or type `NodePort` when the + `LoadBalancer` is not assigned an IP address. + +### `spec.externalAccess.loadBalancerIP: string` + +This setting specifies the IP used to for the LoadBalancer to expose the ArangoDB deployment on. +This setting is used when `spec.externalAccess.type` is set to `LoadBalancer` or `Auto`. + +If you do not specify this setting, an IP will be chosen automatically by the load-balancer provisioner. + +### `spec.externalAccess.nodePort: int` + +This setting specifies the port used to expose the ArangoDB deployment on. +This setting is used when `spec.externalAccess.type` is set to `NodePort` or `Auto`. + +If you do not specify this setting, a random port will be chosen automatically. + ### `spec.auth.jwtSecretName: string` This setting specifies the name of a kubernetes `Secret` that contains @@ -179,16 +222,54 @@ replication in the cluster. When enabled, the cluster will contain a number of `syncmaster` & `syncworker` servers. The default value is `false`. -### `spec.sync.image: string` +### `spec.sync.externalAccess.type: string` + +This setting specifies the type of `Service` that will be created to provide +access to the ArangoSync syncMasters from outside the Kubernetes cluster. +Possible values are: + +- `None` To limit access to applications running inside the Kubernetes cluster. +- `LoadBalancer` To create a `Service` of type `LoadBalancer` for the ArangoSync SyncMasters. +- `NodePort` To create a `Service` of type `NodePort` for the ArangoSync SyncMasters. +- `Auto` (default) To create a `Service` of type `LoadBalancer` and fallback to a `Service` or type `NodePort` when the + `LoadBalancer` is not assigned an IP address. + +Note that when you specify a value of `None`, a `Service` will still be created, but of type `ClusterIP`. -This setting specifies the docker image to use for all ArangoSync servers. -When not specified, the `spec.image` value is used. +### `spec.sync.externalAccess.loadBalancerIP: string` -### `spec.sync.imagePullPolicy: string` +This setting specifies the IP used for the LoadBalancer to expose the ArangoSync SyncMasters on. +This setting is used when `spec.sync.externalAccess.type` is set to `LoadBalancer` or `Auto`. -This setting specifies the pull policy for the docker image to use for all ArangoSync servers. -For possible values, see `spec.imagePullPolicy`. -When not specified, the `spec.imagePullPolicy` value is used. +If you do not specify this setting, an IP will be chosen automatically by the load-balancer provisioner. + +### `spec.sync.externalAccess.nodePort: int` + +This setting specifies the port used to expose the ArangoSync SyncMasters on. +This setting is used when `spec.sync.externalAccess.type` is set to `NodePort` or `Auto`. + +If you do not specify this setting, a random port will be chosen automatically. + +### `spec.sync.externalAccess.masterEndpoint: []string` + +This setting specifies the master endpoint(s) advertised by the ArangoSync SyncMasters. +If not set, this setting defaults to: + +- If `spec.sync.externalAccess.loadBalancerIP` is set, it defaults to `https://:<8629>`. +- Otherwise it defaults to `https://:<8629>`. + +### `spec.sync.externalAccess.accessPackageSecretNames: []string` + +This setting specifies the names of zero of more `Secrets` that will be created by the deployment +operator containing "access packages". An access package contains those `Secrets` that are needed +to access the SyncMasters of this `ArangoDeployment`. + +By removing a name from this setting, the corresponding `Secret` is also deleted. +Note that to remove all access packages, leave an empty array in place (`[]`). +Completely removing the setting results in not modifying the list. + +See [the `ArangoDeploymentReplication` specification](./DeploymentReplicationResource.md) for more information +on access packages. ### `spec.sync.auth.jwtSecretName: string` @@ -254,7 +335,7 @@ The default is `false`. This setting specifies the number of servers to start for the given group. For the agent group, this value must be a positive, odd number. The default value is `3` for all groups except `single` (there the default is `1` -for `spec.mode: single` and `2` for `spec.mode: resilientsingle`). +for `spec.mode: Single` and `2` for `spec.mode: ActiveFailover`). For the `syncworkers` group, it is highly recommended to use the same number as for the `dbservers` group. @@ -284,6 +365,14 @@ The default value is `8Gi`. This setting is not available for group `coordinators`, `syncmasters` & `syncworkers` because servers in these groups do not need persistent storage. +### `spec..serviceAccountName: string` + +This setting specifies the `serviceAccountName` for the `Pods` created +for each server of this group. + +Using an alternative `ServiceAccount` is typically used to separate access rights. +The ArangoDB deployments do not require any special rights. + ### `spec..storageClassName: string` This setting specifies the `storageClass` for the `PersistentVolume`s created @@ -291,3 +380,16 @@ for each server of this group. This setting is not available for group `coordinators`, `syncmasters` & `syncworkers` because servers in these groups do not need persistent storage. + +### `spec..tolerations: [Toleration]` + +This setting specifies the `tolerations` for the `Pod`s created +for each server of this group. + +By default, suitable tolerations are set for the following keys with the `NoExecute` effect: + +- `node.kubernetes.io/not-ready` +- `node.kubernetes.io/unreachable` +- `node.alpha.kubernetes.io/unreachable` (will be removed in future version) + +For more information on tolerations, consult the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/). diff --git a/docs/Manual/Deployment/Kubernetes/DriverConfiguration.md b/docs/Manual/Deployment/Kubernetes/DriverConfiguration.md new file mode 100644 index 000000000..483d9de92 --- /dev/null +++ b/docs/Manual/Deployment/Kubernetes/DriverConfiguration.md @@ -0,0 +1,128 @@ +# Configuring your driver for ArangoDB access + +In this chapter you'll learn how to configure a driver for accessing +an ArangoDB deployment in Kubernetes. + +The exact methods to configure a driver are specific to that driver. + +## Database endpoint(s) + +The endpoint(s) (or URLs) to communicate with is the most important +parameter your need to configure in your driver. + +Finding the right endpoints depend on wether your client application is running in +the same Kubernetes cluster as the ArangoDB deployment or not. + +### Client application in same Kubernetes cluster + +If your client application is running in the same Kubernetes cluster as +the ArangoDB deployment, you should configure your driver to use the +following endpoint: + +```text +https://..svc:8529 +``` + +Only if your deployment has set `spec.tls.caSecretName` to `None`, should +you use `http` instead of `https`. + +### Client application outside Kubernetes cluster + +If your client application is running outside the Kubernetes cluster in which +the ArangoDB deployment is running, your driver endpoint depends on the +external-access configuration of your ArangoDB deployment. + +If the external-access of the ArangoDB deployment is of type `LoadBalancer`, +then use the IP address of that `LoadBalancer` like this: + +```text +https://:8529 +``` + +If the external-access of the ArangoDB deployment is of type `NodePort`, +then use the IP address(es) of the `Nodes` of the Kubernetes cluster, +combined with the `NodePort` that is used by the external-access service. + +For example: + +```text +https://:30123 +``` + +You can find the type of external-access by inspecting the external-access `Service`. +To do so, run the following command: + +```bash +kubectl get service -n -ea +``` + +The output looks like this: + +```bash +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR +example-simple-cluster-ea LoadBalancer 10.106.175.38 192.168.10.208 8529:31890/TCP 1s app=arangodb,arango_deployment=example-simple-cluster,role=coordinator +``` + +In this case the external-access is of type `LoadBalancer` with a load-balancer IP address +of `192.168.10.208`. +This results in an endpoint of `https://192.168.10.208:8529`. + +## TLS settings + +As mentioned before the ArangoDB deployment managed by the ArangoDB operator +will use a secure (TLS) connection unless you set `spec.tls.caSecretName` to `None` +in your `ArangoDeployment`. + +When using a secure connection, you can choose to verify the server certificates +provides by the ArangoDB servers or not. + +If you want to verify these certificates, configure your driver with the CA certificate +found in a Kubernetes `Secret` found in the same namespace as the `ArangoDeployment`. + +The name of this `Secret` is stored in the `spec.tls.caSecretName` setting of +the `ArangoDeployment`. If you don't set this setting explicitly, it will be +set automatically. + +Then fetch the CA secret using the following command (or use a Kubernetes client library to fetch it): + +```bash +kubectl get secret -n --template='{{index .data "ca.crt"}}' | base64 -D > ca.crt +``` + +This results in a file called `ca.crt` containing a PEM encoded, x509 CA certificate. + +## Query requests + +For most client requests made by a driver, it does not matter if there is any kind +of load-balancer between your client application and the ArangoDB deployment. + +{% hint 'info' %} +Note that even a simple `Service` of type `ClusterIP` already behaves as a load-balancer. +{% endhint %} + +The exception to this is cursor related requests made to an ArangoDB `Cluster` deployment. +The coordinator that handles an initial query request (that results in a `Cursor`) +will save some in-memory state in that coordinator, if the result of the query +is too big to be transfer back in the response of the initial request. + +Follow-up requests have to be made to fetch the remaining data. +These follow-up requests must be handled by the same coordinator to which the initial +request was made. + +As soon as there is a load-balancer between your client application and the ArangoDB cluster, +it is uncertain which coordinator will actually handle the follow-up request. + +To resolve this uncertainty, make sure to run your client application in the same +Kubernetes cluster and synchronize your endpoints before making the +initial query request. +This will result in the use (by the driver) of internal DNS names of all coordinators. +A follow-up request can then be sent to exactly the same coordinator. + +If your client application is running outside the Kubernetes cluster this is much harder +to solve. +The easiest way to work around it, is by making sure that the query results are small +enough. +When that is not feasible, it is also possible to resolve this +when the internal DNS names of your Kubernetes cluster are exposed to your client application +and the resuling IP addresses are routeable from your client application. +To expose internal DNS names of your Kubernetes cluster, your can use [CoreDNS](https://coredns.io). diff --git a/docs/Manual/Deployment/Kubernetes/README.md b/docs/Manual/Deployment/Kubernetes/README.md index 28849d67c..e88328853 100644 --- a/docs/Manual/Deployment/Kubernetes/README.md +++ b/docs/Manual/Deployment/Kubernetes/README.md @@ -1,6 +1,21 @@ # ArangoDB Kubernetes Operator -The ArangoDB Kubernetes Operator (`kube-arangodb`) is a set of two operators -that you deploy in your Kubernetes cluster to manage deployments of the -ArangoDB database and provide `PersistentVolumes` on local storage of your -nodes for optimal storage performance. +The ArangoDB Kubernetes Operator (`kube-arangodb`) is a set of operators +that you deploy in your Kubernetes cluster to: + +- Manage deployments of the ArangoDB database +- Provide `PersistentVolumes` on local storage of your nodes for optimal storage performance. +- Configure ArangoDB Datacenter to Datacenter replication + +Each of these uses involves a different custom resource. + +- Use an [`ArangoDeployment` resource](./DeploymentResource.md) to + create an ArangoDB database deployment. +- Use an [`ArangoLocalStorage` resource](./StorageResource.md) to + provide local `PersistentVolumes` for optimal I/O performance. +- Use an [`ArangoDeploymentReplication` resource](./DeploymentReplicationResource.md) to + configure ArangoDB Datacenter to Datacenter replication. + +Continue with [Using the ArangoDB Kubernetes Operator](./Usage.md) +to learn how to install the ArangoDB Kubernetes operator and create +your first deployment. \ No newline at end of file diff --git a/docs/Manual/Deployment/Kubernetes/ServicesAndLoadBalancer.md b/docs/Manual/Deployment/Kubernetes/ServicesAndLoadBalancer.md index e6191ac0a..da8a8bdad 100644 --- a/docs/Manual/Deployment/Kubernetes/ServicesAndLoadBalancer.md +++ b/docs/Manual/Deployment/Kubernetes/ServicesAndLoadBalancer.md @@ -3,26 +3,54 @@ The ArangoDB Kubernetes Operator will create services that can be used to reach the ArangoDB servers from inside the Kubernetes cluster. +By default, the ArangoDB Kubernetes Operator will also create an additional +service to reach the ArangoDB deployment from outside the Kubernetes cluster. + +For exposing the ArangoDB deployment to the outside, there are 2 options: + +- Using a `NodePort` service. This will expose the deployment on a specific port (above 30.000) + on all nodes of the Kubernetes cluster. +- Using a `LoadBalancer` service. This will expose the deployment on a load-balancer + that is provisioned by the Kubernetes cluster. + +The `LoadBalancer` option is the most convenient, but not all Kubernetes clusters +are able to provision a load-balancer. Therefore we offer a third (and default) option: `Auto`. +In this option, the ArangoDB Kubernetes Operator tries to create a `LoadBalancer` +service. It then waits for up to a minute for the Kubernetes cluster to provision +a load-balancer for it. If that has not happened after a minute, the service +is replaced by a service of type `NodePort`. + +To inspect the created service, run: + +```bash +kubectl get services -ea +``` + To use the ArangoDB servers from outside the Kubernetes cluster you have to add another service as explained below. ## Services +If you do not want the ArangoDB Kubernetes Operator to create an external-access +service for you, set `spec.externalAccess.Type` to `None`. + +If you want to create external access services manually, follow the instructions below. + ### Single server For a single server deployment, the operator creates a single -`Service` named ``. This service has a normal cluster IP +`Service` named ``. This service has a normal cluster IP address. ### Full cluster For a full cluster deployment, the operator creates two `Services`. -- `_servers` a headless `Service` intended to provide +- `-int` a headless `Service` intended to provide DNS names for all pods created by the operator. It selects all ArangoDB & ArangoSync servers in the cluster. -- `` a normal `Service` that selects only the coordinators +- `` a normal `Service` that selects only the coordinators of the cluster. This `Service` is configured with `ClientIP` session affinity. This is needed for cursor requests, since they are bound to a specific coordinator. @@ -30,7 +58,7 @@ For a full cluster deployment, the operator creates two `Services`. When the coordinators are asked to provide endpoints of the cluster (e.g. when calling `client.SynchronizeEndpoints()` in the go driver) the DNS names of the individual `Pods` will be returned -(`._servers..svc`) +(`.-int..svc`) ### Full cluster with DC2DC @@ -38,23 +66,46 @@ For a full cluster with datacenter replication deployment, the same `Services` are created as for a Full cluster, with the following additions: -- `_sync` a normal `Service` that selects only the syncmasters +- `-sync` a normal `Service` that selects only the syncmasters of the cluster. ## Load balancer -To reach the ArangoDB servers from outside the Kubernetes cluster, you -have to deploy additional services. +If you want full control of the `Services` needed to access the ArangoDB deployment +from outside your Kubernetes cluster, set `spec.externalAccess.Type` of the `ArangoDeployment` to `None` +and create a `Service` as specified below. -You can use `LoadBalancer` or `NodePort` services, depending on your +Create a `Service` of type `LoadBalancer` or `NodePort`, depending on your Kubernetes deployment. This service should select: -- `arangodb_cluster_name: ` +- `arango_deployment: ` - `role: coordinator` -For example: +The following example yields a service of type `LoadBalancer` with a specific +load balancer IP address. +With this service, the ArangoDB cluster can now be reached on `https://1.2.3.4:8529`. + +```yaml +kind: Service +apiVersion: v1 +metadata: + name: arangodb-cluster-exposed +spec: + selector: + arango_deployment: arangodb-cluster + role: coordinator + type: LoadBalancer + loadBalancerIP: 1.2.3.4 + ports: + - protocol: TCP + port: 8529 + targetPort: 8529 +``` + +The following example yields a service of type `NodePort` with the ArangoDB +cluster exposed on port 30529 of all nodes of the Kubernetes cluster. ```yaml kind: Service @@ -63,7 +114,7 @@ metadata: name: arangodb-cluster-exposed spec: selector: - arangodb_cluster_name: arangodb-cluster + arango_deployment: arangodb-cluster role: coordinator type: NodePort ports: diff --git a/docs/Manual/Deployment/Kubernetes/Storage.md b/docs/Manual/Deployment/Kubernetes/Storage.md index e6c5d9207..2b4b6b687 100644 --- a/docs/Manual/Deployment/Kubernetes/Storage.md +++ b/docs/Manual/Deployment/Kubernetes/Storage.md @@ -10,6 +10,22 @@ In the `ArangoDeployment` resource, one can specify the type of storage used by groups of servers using the `spec..storageClassName` setting. +This is an example of a `Cluster` deployment that stores its agent & dbserver +data on `PersistentVolumes` that use the `my-local-ssd` `StorageClass` + +```yaml +apiVersion: "database.arangodb.com/v1alpha" +kind: "ArangoDeployment" +metadata: + name: "cluster-using-local-ssh" +spec: + mode: Cluster + agents: + storageClassName: my-local-ssd + dbservers: + storageClassName: my-local-ssd +``` + The amount of storage needed is configured using the `spec..resources.requests.storage` setting. @@ -17,6 +33,22 @@ Note that configuring storage is done per group of servers. It is not possible to configure storage per individual server. +This is an example of a `Cluster` deployment that requests volumes of 80GB +for every dbserver, resulting in a total storage capacity of 240GB (with 3 dbservers). + +```yaml +apiVersion: "database.arangodb.com/v1alpha" +kind: "ArangoDeployment" +metadata: + name: "cluster-using-local-ssh" +spec: + mode: Cluster + dbservers: + resources: + requests: + storage: 80Gi +``` + ## Local storage For optimal performance, ArangoDB should be configured with locally attached @@ -26,6 +58,28 @@ The easiest way to accomplish this is to deploy an [`ArangoLocalStorage` resource](./StorageResource.md). The ArangoDB Storage Operator will use it to provide `PersistentVolumes` for you. +This is an example of an `ArangoLocalStorage` resource that will result in +`PersistentVolumes` created on any node of the Kubernetes cluster +under the directory `/mnt/big-ssd-disk`. + +```yaml +apiVersion: "storage.arangodb.com/v1alpha" +kind: "ArangoLocalStorage" +metadata: + name: "example-arangodb-storage" +spec: + storageClass: + name: my-local-ssd + localPath: + - /mnt/big-ssd-disk +``` + +Note that using local storage required `VolumeScheduling` to be enabled in your +Kubernetes cluster. ON Kubernetes 1.10 this is enabled by default, on version +1.9 you have to enable it with a `--feature-gate` setting. + +### Manually creating `PersistentVolumes` + The alternative is to create `PersistentVolumes` manually, for all servers that need persistent storage (single, agents & dbservers). E.g. for a `Cluster` with 3 agents and 5 dbservers, you must create 8 volumes. @@ -54,14 +108,14 @@ metadata: ]} }' spec: - capacity: - storage: 100Gi - accessModes: - - ReadWriteOnce - persistentVolumeReclaimPolicy: Delete - storageClassName: local-ssd - local: - path: /mnt/disks/ssd1 + capacity: + storage: 100Gi + accessModes: + - ReadWriteOnce + persistentVolumeReclaimPolicy: Delete + storageClassName: local-ssd + local: + path: /mnt/disks/ssd1 ``` For Kubernetes 1.9 and up, you should create a `StorageClass` which is configured diff --git a/docs/Manual/Deployment/Kubernetes/Tls.md b/docs/Manual/Deployment/Kubernetes/Tls.md index 127c664fd..be298fb68 100644 --- a/docs/Manual/Deployment/Kubernetes/Tls.md +++ b/docs/Manual/Deployment/Kubernetes/Tls.md @@ -1,4 +1,4 @@ -# TLS +# Secure connections (TLS) The ArangoDB Kubernetes Operator will by default create ArangoDB deployments that use secure TLS connections. @@ -23,7 +23,8 @@ kubectl get secret -ca --template='{{index .data "ca.crt"}}' | base ### Windows -TODO +To install a CA certificate in Windows, follow the +[procedure described here](http://wiki.cacert.org/HowTo/InstallCAcertRoots). ### MacOS @@ -41,4 +42,13 @@ sudo /usr/bin/security remove-trusted-cert -d ca.crt ### Linux -TODO +To install a CA certificate in Linux, on Ubuntu, run: + +```bash +sudo cp ca.crt /usr/local/share/ca-certificates/.crt +sudo update-ca-certificates +``` + +## See also + +- [Authentication](./Authentication.md) diff --git a/docs/Manual/Deployment/Kubernetes/Troubleshooting.md b/docs/Manual/Deployment/Kubernetes/Troubleshooting.md new file mode 100644 index 000000000..25363301f --- /dev/null +++ b/docs/Manual/Deployment/Kubernetes/Troubleshooting.md @@ -0,0 +1,115 @@ +# Troubleshooting + +While Kubernetes and the ArangoDB Kubernetes operator will automatically +resolve a lot of issues, there are always cases where human attention +is needed. + +This chapter gives your tips & tricks to help you troubleshoot deployments. + +## Where to look + +In Kubernetes all resources can be inspected using `kubectl` using either +the `get` or `describe` command. + +To get all details of the resource (both specification & status), +run the following command: + +```bash +kubectl get -n -o yaml +``` + +For example, to get the entire specification and status +of an `ArangoDeployment` resource named `my-arangodb` in the `default` namespace, +run: + +```bash +kubectl get ArangoDeployment my-arango -n default -o yaml +# or shorter +kubectl get arango my-arango -o yaml +``` + +Several types of resources (including all ArangoDB custom resources) support +events. These events show what happened to the resource over time. + +To show the events (and most important resource data) of a resource, +run the following command: + +```bash +kubectl describe -n +``` + +## Getting logs + +Another invaluable source of information is the log of containers being run +in Kubernetes. +These logs are accessible through the `Pods` that group these containers. + +To fetch the logs of the default container running in a `Pod`, run: + +```bash +kubectl logs -n +# or with follow option to keep inspecting logs while they are written +kubectl logs -n -f +``` + +To inspect the logs of a specific container in `Pod`, add `-c `. +You can find the names of the containers in the `Pod`, using `kubectl describe pod ...`. + +{% hint 'info' %} +Note that the ArangoDB operators are being deployed themselves as a Kubernetes `Deployment` +with 2 replicas. This means that you will have to fetch the logs of 2 `Pods` running +those replicas. +{% endhint %} + +## What if + +### The `Pods` of a deployment stay in `Pending` state + +There are two common causes for this. + +1) The `Pods` cannot be scheduled because there are not enough nodes available. + This is usally only the case with a `spec.environment` setting that has a value of `Production`. + + Solution: +Add more nodes. + +1) There are no `PersistentVolumes` available to be bound to the `PersistentVolumeClaims` + created by the operator. + + Solution: +Use `kubectl get persistentvolumes` to inspect the available `PersistentVolumes` +and if needed, use the [`ArangoLocalStorage` operator](./StorageResource.md) to provision `PersistentVolumes`. + +### When restarting a `Node`, the `Pods` scheduled on that node remain in `Terminating` state + +When a `Node` no longer makes regular calls to the Kubernetes API server, it is +marked as not available. Depending on specific settings in your `Pods`, Kubernetes +will at some point decide to terminate the `Pod`. As long as the `Node` is not +completely removed from the Kubernetes API server, Kubernetes will try to use +the `Node` itself to terminate the `Pod`. + +The `ArangoDeployment` operator recognizes this condition and will try to replace those +`Pods` with `Pods` on different nodes. The exact behavior differs per type of server. + +### What happens when a `Node` with local data is broken + +When a `Node` with `PersistentVolumes` hosted on that `Node` is broken and +cannot be repaired, the data in those `PersistentVolumes` is lost. + +If an `ArangoDeployment` of type `Single` was using one of those `PersistentVolumes` +the database is lost and must be restored from a backup. + +If an `ArangoDeployment` of type `ActiveFailover` or `Cluster` was using one of +those `PersistentVolumes`, it depends on the type of server that was using the volume. + +- If an `Agent` was using the volume, it can be repaired as long as 2 other agents are still healthy. +- If a `DBServer` was using the volume, and the replication factor of all database + collections is 2 or higher, and the remaining dbservers are still healthy, + the cluster will duplicate the remaining replicas to + bring the number of replicases back to the original number. +- If a `DBServer` was using the volume, and the replication factor of a database + collection is 1 and happens to be stored on that dbserver, the data is lost. +- If a single server of an `ActiveFailover` deployment was using the volume, and the + other single server is still healthy, the other single server will become leader. + After replacing the failed single server, the new follower will synchronize with + the leader. diff --git a/docs/Manual/Deployment/Kubernetes/Upgrading.md b/docs/Manual/Deployment/Kubernetes/Upgrading.md index 56fe7989a..90e9b7e73 100644 --- a/docs/Manual/Deployment/Kubernetes/Upgrading.md +++ b/docs/Manual/Deployment/Kubernetes/Upgrading.md @@ -3,6 +3,8 @@ The ArangoDB Kubernetes Operator supports upgrading an ArangoDB from one version to the next. +## Upgrade an ArangoDB deployment + To upgrade a cluster, change the version by changing the `spec.image` setting and the apply the updated custom resource using: @@ -11,6 +13,21 @@ custom resource using: kubectl apply -f yourCustomResourceFile.yaml ``` +The ArangoDB operator will perform an sequential upgrade +of all servers in your deployment. Only one server is upgraded +at a time. + +For patch level upgrades (e.g. 3.3.9 to 3.3.10) each server +is stopped and restarted with the new version. + +For minor level upgrades (e.g. 3.3.9 to 3.4.0) each server +is stopped, then the new version is started with `--database.auto-upgrade` +and once that is finish the new version is started with the normal arguments. + +The process for major level upgrades depends on the specific version. + +## Upgrade the operator itself + To update the ArangoDB Kubernetes Operator itself to a new version, update the image version of the deployment resource and apply it using: @@ -18,3 +35,7 @@ and apply it using: ```bash kubectl apply -f examples/yourUpdatedDeployment.yaml ``` + +## See also + +- [Scaling](./Scaling.md) \ No newline at end of file diff --git a/docs/Manual/Deployment/Kubernetes/Usage.md b/docs/Manual/Deployment/Kubernetes/Usage.md index 5e02b338b..627bcb676 100644 --- a/docs/Manual/Deployment/Kubernetes/Usage.md +++ b/docs/Manual/Deployment/Kubernetes/Usage.md @@ -8,23 +8,31 @@ cluster first. To do so, run (replace `` with the version of the operator that you want to install): ```bash -kubectl apply -f https://raw.githubusercontent.com/arangodb/kube-arangodb//manifests/crd.yaml -kubectl apply -f https://raw.githubusercontent.com/arangodb/kube-arangodb//manifests/arango-deployment.yaml +export URLPREFIX=https://raw.githubusercontent.com/arangodb/kube-arangodb//manifests +kubectl apply -f $URLPREFIX/crd.yaml +kubectl apply -f $URLPREFIX/arango-deployment.yaml ``` -To use `ArangoLocalStorage`, also run: +To use `ArangoLocalStorage` resources, also run: ```bash -kubectl apply -f https://raw.githubusercontent.com/arangodb/kube-arangodb//manifests/arango-storage.yaml +kubectl apply -f $URLPREFIX/arango-storage.yaml +``` + +To use `ArangoDeploymentReplication` resources, also run: + +```bash +kubectl apply -f $URLPREFIX/arango-deployment-replication.yaml ``` You can find the latest release of the ArangoDB Kubernetes Operator [in the kube-arangodb repository](https://github.com/arangodb/kube-arangodb/releases/latest). -## Cluster creation +## ArangoDB deployment creation -Once the operator is running, you can create your ArangoDB cluster -by creating a custom resource and deploying it. +Once the operator is running, you can create your ArangoDB database deployment +by creating a `ArangoDeployment` custom resource and deploying it into your +Kubernetes cluster. For example (all examples can be found [in the kube-arangodb repository](https://github.com/arangodb/kube-arangodb/tree/master/examples)): @@ -32,9 +40,9 @@ For example (all examples can be found [in the kube-arangodb repository](https:/ kubectl apply -f examples/simple-cluster.yaml ``` -## Cluster removal +## Deployment removal -To remove an existing cluster, delete the custom +To remove an existing ArangoDB deployment, delete the custom resource. The operator will then delete all created resources. For example: @@ -43,6 +51,10 @@ For example: kubectl delete -f examples/simple-cluster.yaml ``` +**Note that this will also delete all data in your ArangoDB deployment!** + +If you want to keep your data, make sure to create a backup before removing the deployment. + ## Operator removal To remove the entire ArangoDB Kubernetes Operator, remove all @@ -50,6 +62,14 @@ clusters first and then remove the operator by running: ```bash kubectl delete deployment arango-deployment-operator -# If `ArangoLocalStorage` is installed +# If `ArangoLocalStorage` operator is installed kubectl delete deployment -n kube-system arango-storage-operator +# If `ArangoDeploymentReplication` operator is installed +kubectl delete deployment arango-deployment-replication-operator ``` + +## See also + +- [Driver configuration](./DriverConfiguration.md) +- [Scaling](./Scaling.md) +- [Upgrading](./Upgrading.md) diff --git a/docs/Manual/Tutorials/Kubernetes/DC2DC.md b/docs/Manual/Tutorials/Kubernetes/DC2DC.md new file mode 100644 index 000000000..edd5a24c0 --- /dev/null +++ b/docs/Manual/Tutorials/Kubernetes/DC2DC.md @@ -0,0 +1,137 @@ +# Start ArangoDB Cluster to Cluster Synchronization on Kubernetes + +This tutorial guides you through the steps needed to configure +an ArangoDB datacenter to datacenter replication between two ArangoDB +clusters running in Kubernetes. + +## Requirements + +1. This tutorial assumes that you have 2 ArangoDB clusters running in 2 different Kubernetes clusters. +1. Both Kubernetes clusters are equipped with support for `Services` of type `LoadBalancer`. +1. You can create (global) DNS names for configured `Services` with low propagation times. E.g. use Cloudflare. +1. You have 4 DNS names available: + - One for the database in the source ArangoDB cluster. E.g. `src-db.mycompany.com` + - One for the ArangoDB syncmasters in the source ArangoDB cluster. E.g. `src-sync.mycompany.com` + - One for the database in the destination ArangoDB cluster. E.g. `dst-db.mycompany.com` + - One for the ArangoDB syncmasters in the destination ArangoDB cluster. E.g. `dst-sync.mycompany.com` + +## Step 1: Enable Datacenter Replication Support on source ArangoDB cluster + +Set your current Kubernetes context to the Kubernetes source cluster. + +Edit the `ArangoDeployment` of the source ArangoDB clusters. + +Set: + +- `spec.tls.altNames` to `["src-db.mycompany.com"]` (can include more names / IP addresses) +- `spec.sync.enabled` to `true` +- `spec.sync.externalAccess.masterEndpoint` to `["https://src-sync.mycompany.com:8629"]` +- `spec.sync.externalAccess.accessPackageSecretNames` to `["src-accesspackage"]` + +## Step 2: Extract access-package from source ArangoDB cluster + +Run: + +```bash +kubectl get secret src-accesspackage --template='{{index .data "accessPackage.yaml"}}' | \ + base64 -D > accessPackage.yaml +``` + +## Step 3: Configure source DNS names + +Run: + +```bash +kubectl get service +``` + +Find the IP address contained in the `LoadBalancer` column for the following `Services`: + +- `-ea` Use this IP address for the `src-db.mycompany.com` DNS name. +- `-sync` Use this IP address for the `src-sync.mycompany.com` DNS name. + +The process for configuring DNS names is specific to each DNS provider. + +## Step 4: Enable Datacenter Replication Support on destination ArangoDB cluster + +Set your current Kubernetes context to the Kubernetes destination cluster. + +Edit the `ArangoDeployment` of the source ArangoDB clusters. + +Set: + +- `spec.tls.altNames` to `["dst-db.mycompany.com"]` (can include more names / IP addresses) +- `spec.sync.enabled` to `true` +- `spec.sync.externalAccess.masterEndpoint` to `["https://dst-sync.mycompany.com:8629"]` + +## Step 5: Import access package in destination cluster + +Run: + +```bash +kubectl apply -f accessPackage.yaml +``` + +Note: This imports two `Secrets`, containing TLS information about the source cluster, +into the destination cluster + +## Step 6: Configure destination DNS names + +Run: + +```bash +kubectl get service +``` + +Find the IP address contained in the `LoadBalancer` column for the following `Services`: + +- `-ea` Use this IP address for the `dst-db.mycompany.com` DNS name. +- `-sync` Use this IP address for the `dst-sync.mycompany.com` DNS name. + +The process for configuring DNS names is specific to each DNS provider. + +## Step 7: Create an `ArangoDeploymentReplication` resource + +Create a yaml file (e.g. called `src-to-dst-repl.yaml`) with the following content: + +```yaml +apiVersion: "replication.database.arangodb.com/v1alpha" +kind: "ArangoDeploymentReplication" +metadata: + name: "replication-src-to-dst" +spec: + source: + masterEndpoint: ["https://src-sync.mycompany.com:8629"] + auth: + keyfileSecretName: src-accesspackage-auth + tls: + caSecretName: src-accesspackage-ca + destination: + deploymentName: +``` + +## Step 8: Wait for DNS names to propagate + +Wait until the DNS names configured in step 3 and 6 resolve to their configured +IP addresses. + +Depending on your DNS provides this can take a few minutes up to 24 hours. + +## Step 9: Activate replication + +Run: + +```bash +kubectl apply -f src-to-dst-repl.yaml +``` + +Replication from the source cluster to the destination cluster will now be configured. + +Check the status of the replication by inspecting the status of the `ArangoDeploymentReplication` resource using: + +```bash +kubectl describe ArangoDeploymentReplication replication-src-to-dst +``` + +As soon as the replication is configured, the `Add collection` button in the `Collections` +page of the web UI (of the destination cluster) will be grayed out. diff --git a/docs/Manual/Tutorials/Kubernetes/README.md b/docs/Manual/Tutorials/Kubernetes/README.md index 7399c0013..a776fc6f0 100644 --- a/docs/Manual/Tutorials/Kubernetes/README.md +++ b/docs/Manual/Tutorials/Kubernetes/README.md @@ -107,44 +107,26 @@ your database s available. ## Connecting to your database The single server database you deployed in the previous chapter is now -available, but only from within the Kubernetes cluster. +available from within the Kubernetes cluster as well as outside it. -To make the database available outside your Kubernetes cluster (e.g. for browser access) -you must deploy an additional `Service`. +Access to the database from outside the Kubernetes cluster is provided +using an external-access service. +By default this service is of type `LoadBalancer`. If this type of service +is not supported by your Kubernetes cluster, it will be replaced by +a service of type `NodePort` after a minute. -There are several possible types of `Service` to choose from. -We are going to use the `NodePort` type to expose the database on port 30529 of -every node of your Kubernetes cluster. - -Create a file called `single-server-service.yaml` with the following content. - -```yaml -kind: Service -apiVersion: v1 -metadata: - name: single-server-service -spec: - selector: - app: arangodb - arango_deployment: single-server - role: single - type: NodePort - ports: - - protocol: TCP - port: 8529 - targetPort: 8529 - nodePort: 30529 -``` - -Deploy the `Service` into your Kubernetes cluster using: +To see the type of service that has been created, run: ```bash -kubectl apply -f single-server-service.yaml +kubectl get service single-server-ea ``` -Now you can connect your browser to `https://:30529/`, -where `` is the name or IP address of any of the nodes -of your Kubernetes cluster. +When the service is of the `LoadBalancer` type, use the IP address +listed in the `EXTERNAL-IP` column with port 8529. +When the service is of the `NodePort` type, use the IP address +of any of the nodes of the cluster, combine with the high (>30000) port listed in the `PORT(S)` column. + +Now you can connect your browser to `https://:/`. Your browser will show a warning about an unknown certificate. Accept the certificate for now. @@ -183,34 +165,6 @@ kubectl apply -f cluster.yaml The same commands used in the single server deployment can be used to inspect your cluster. Just use the correct deployment name (`cluster` instead of `single-server`). -Connecting to your cluster requires a different `Service` since the -selector now has to select your `cluster` deployment and instead -of selecting all `Pods` with role `single` it will have to select -all coordinator pods. - -The service looks like this: - -```yaml -kind: Service -apiVersion: v1 -metadata: - name: cluster-service -spec: - selector: - app: arangodb - arango_deployment: cluster - role: coordinator - type: NodePort - ports: - - protocol: TCP - port: 8529 - targetPort: 8529 - nodePort: 31529 -``` - -Note that we have chosen a different node port (31529) for this `Service` -to avoid conflicts with the port used in `single-server-service`. - ## Where to go from here - [ArangoDB Kubernetes Operator](../../Deployment/Kubernetes/README.md) diff --git a/docs/design/acceptance_test.md b/docs/design/acceptance_test.md new file mode 100644 index 000000000..3bbe6649f --- /dev/null +++ b/docs/design/acceptance_test.md @@ -0,0 +1,474 @@ +# Acceptance test for kube-arangodb operator on specific Kubernetes platform + +This acceptance test plan describes all test scenario's that must be executed +successfully in order to consider the kube-arangodb operator production ready +on a specific Kubernetes setup (from now on we'll call a Kubernetes setup a platform). + +## Platform parameters + +Before the test, record the following parameters for the platform the test is executed on. + +- Name of the platform +- Version of the platform +- Upstream Kubernetes version used by the platform +- Number of nodes used by the Kubernetes cluster +- `StorageClasses` provided by the platform (run `kubectl get storageclass`) +- Does the platform use RBAC? +- Does the platform support services of type `LoadBalancer`? + +If one of the above questions can have multiple answers (e.g. different Kubernetes versions) +then make the platform more specific. E.g. consider "GKE with Kubernetes 1.10.2" a platform +instead of "GKE" which can have version "1.8", "1.9" & "1.10.2". + +## Platform preparations + +Before the tests can be run, the platform has to be prepared. + +### Deploy the ArangoDB operators + +Deploy the following ArangoDB operators: + +- `ArangoDeployment` operator +- `ArangoDeploymentReplication` operator +- `ArangoLocalStorage` operator + +To do so, follow the [instructions in the manual](../Manual/Deployment/Kubernetes/Usage.md). + +### `PersistentVolume` provider + +If the platform does not provide a `PersistentVolume` provider, create one by running: + +```bash +kubectl apply -f examples/arango-local-storage.yaml +``` + +## Basis tests + +The basis tests are executed on every platform with various images: + +Run the following tests with the following images: + +- Community 3.3.10 +- Enterprise 3.3.10 + +For every tests, one of these images can be chosen, as long as each image +is used in a test at least once. + +### Test 1a: Create single server deployment + +Create an `ArangoDeployment` of mode `Single`. + +Hint: Use `tests/acceptance/single.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 1 `Pod` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +### Test 1b: Create active failover deployment + +Create an `ArangoDeployment` of mode `ActiveFailover`. + +Hint: Use `tests/acceptance/activefailover.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 5 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +### Test 1c: Create cluster deployment + +Create an `ArangoDeployment` of mode `Cluster`. + +Hint: Use `tests/acceptance/cluster.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 9 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +### Test 1d: Create cluster deployment with dc2dc + +This test requires the use of the enterprise image. + +Create an `ArangoDeployment` of mode `Cluster` and dc2dc enabled. + +Hint: Derive from `tests/acceptance/cluster-sync.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 15 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The deployment must yield a `Service` named `-sync` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +### Test 2a: Scale an active failover deployment + +Create an `ArangoDeployment` of mode `ActiveFailover`. + +- [ ] The deployment must start +- [ ] The deployment must yield 5 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Change the value of `spec.single.count` from 2 to 3. + +- [ ] A single server is added +- [ ] The deployment must yield 6 `Pods` + +Change the value of `spec.single.count` from 3 to 2. + +- [ ] A single server is removed +- [ ] The deployment must yield 5 `Pods` + +### Test 2b: Scale a cluster deployment + +Create an `ArangoDeployment` of mode `Cluster`. + +Hint: Use `tests/acceptance/cluster.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 9 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Change the value of `spec.dbservers.count` from 3 to 5. + +- [ ] Two dbservers are added +- [ ] The deployment must yield 11 `Pods` + +Change the value of `spec.coordinators.count` from 3 to 4. + +- [ ] A coordinator is added +- [ ] The deployment must yield 12 `Pods` + +Change the value of `spec.dbservers.count` from 5 to 2. + +- [ ] Three dbservers are removed (one by one) +- [ ] The deployment must yield 9 `Pods` + +Change the value of `spec.coordinators.count` from 4 to 1. + +- [ ] Three coordinators are removed (one by one) +- [ ] The deployment must yield 6 `Pods` + +### Test 3: Production environment + +Production environment tests are only relevant if there are enough nodes +available that `Pods` can be scheduled on. + +The number of available nodes must be >= the maximum server count in +any group. + +### Test 3a: Create single server deployment in production environment + +Create an `ArangoDeployment` of mode `Single` with an environment of `Production`. + +Hint: Derive from `tests/acceptance/single.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 1 `Pod` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +### Test 3b: Create active failover deployment in production environment + +Create an `ArangoDeployment` of mode `ActiveFailover` with an environment of `Production`. + +Hint: Derive from `tests/acceptance/activefailover.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 5 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +### Test 3c: Create cluster deployment in production environment + +Create an `ArangoDeployment` of mode `Cluster` with an environment of `Production`. + +Hint: Derive from `tests/acceptance/cluster.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 9 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +### Test 3d: Create cluster deployment in production environment and scale it + +Create an `ArangoDeployment` of mode `Cluster` with an environment of `Production`. + +Hint: Derive from `tests/acceptance/cluster.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 9 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Change the value of `spec.dbservers.count` from 3 to 4. + +- [ ] Two dbservers are added +- [ ] The deployment must yield 10 `Pods` + +Change the value of `spec.coordinators.count` from 3 to 4. + +- [ ] A coordinator is added +- [ ] The deployment must yield 11 `Pods` + +Change the value of `spec.dbservers.count` from 4 to 2. + +- [ ] Three dbservers are removed (one by one) +- [ ] The deployment must yield 9 `Pods` + +Change the value of `spec.coordinators.count` from 4 to 1. + +- [ ] Three coordinators are removed (one by one) +- [ ] The deployment must yield 6 `Pods` + +### Test 4a: Create cluster deployment with `ArangoLocalStorage` provided volumes + +Ensure an `ArangoLocalStorage` is deployed. + +Hint: Use from `tests/acceptance/local-storage.yaml`. + +Create an `ArangoDeployment` of mode `Cluster` with a `StorageClass` that is +mapped to an `ArangoLocalStorage` provider. + +Hint: Derive from `tests/acceptance/cluster.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 9 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +### Test 4b: Create cluster deployment with a platform provided `StorageClass` + +This test only applies to platforms that provide their own `StorageClasses`. + +Create an `ArangoDeployment` of mode `Cluster` with a `StorageClass` that is +provided by the platform. + +Hint: Derive from `tests/acceptance/cluster.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 9 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +### Test 5a: Test `Pod` resilience on single servers + +Create an `ArangoDeployment` of mode `Single`. + +Hint: Use from `tests/acceptance/single.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 1 `Pod` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Delete the `Pod` of the deployment that contains the single server. + +- [ ] The `Pod` must be restarted +- [ ] After the `Pod` has restarted, the server must have the same data and be responsive again + +### Test 5b: Test `Pod` resilience on active failover + +Create an `ArangoDeployment` of mode `ActiveFailover`. + +Hint: Use from `tests/acceptance/activefailover.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 5 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Delete a `Pod` of the deployment that contains an agent. + +- [ ] While the `Pod` is gone & restarted, the cluster must still respond to requests (R/W) +- [ ] The `Pod` must be restarted + +Delete a `Pod` of the deployment that contains a single server. + +- [ ] While the `Pod` is gone & restarted, the cluster must still respond to requests (R/W) +- [ ] The `Pod` must be restarted + +### Test 5c: Test `Pod` resilience on clusters + +Create an `ArangoDeployment` of mode `Cluster`. + +Hint: Use from `tests/acceptance/cluster.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 9 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Delete a `Pod` of the deployment that contains an agent. + +- [ ] While the `Pod` is gone & restarted, the cluster must still respond to requests (R/W) +- [ ] The `Pod` must be restarted + +Delete a `Pod` of the deployment that contains a dbserver. + +- [ ] While the `Pod` is gone & restarted, the cluster must still respond to requests (R/W), except + for requests to collections with a replication factor of 1. +- [ ] The `Pod` must be restarted + +Delete a `Pod` of the deployment that contains an coordinator. + +- [ ] While the `Pod` is gone & restarted, the cluster must still respond to requests (R/W), except + requests targeting the restarting coordinator. +- [ ] The `Pod` must be restarted + +### Test 6a: Test `Node` reboot on single servers + +Create an `ArangoDeployment` of mode `Single`. + +Hint: Use from `tests/acceptance/single.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 1 `Pod` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Reboot the `Node` of the deployment that contains the single server. + +- [ ] The `Pod` running on the `Node` must be restarted +- [ ] After the `Pod` has restarted, the server must have the same data and be responsive again + +### Test 6b: Test `Node` reboot on active failover + +Create an `ArangoDeployment` of mode `ActiveFailover` with an environment of `Production`. + +Hint: Use from `tests/acceptance/activefailover.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 5 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Reboot a `Node`. + +- [ ] While the `Node` is restarting, the cluster must still respond to requests (R/W) +- [ ] All `Pods` on the `Node` must be restarted + +### Test 6c: Test `Node` reboot on clusters + +Create an `ArangoDeployment` of mode `Cluster` with an environment of `Production`. + +Hint: Use from `tests/acceptance/cluster.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 9 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Reboot a `Node`. + +- [ ] While the `Node` is restarting, the cluster must still respond to requests (R/W) +- [ ] All `Pods` on the `Node` must be restarted + +### Test 6d: Test `Node` removal on single servers + +This test is only valid when `StorageClass` is used that provides network attached `PersistentVolumes`. + +Create an `ArangoDeployment` of mode `Single`. + +Hint: Use from `tests/acceptance/single.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 1 `Pod` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Remove the `Node` containing the deployment from the Kubernetes cluster. + +- [ ] The `Pod` running on the `Node` must be restarted on another `Node` +- [ ] After the `Pod` has restarted, the server must have the same data and be responsive again + +### Test 6e: Test `Node` removal on active failover + +Create an `ArangoDeployment` of mode `ActiveFailover` with an environment of `Production`. + +Hint: Use from `tests/acceptance/activefailover.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 5 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Remove a `Node` containing the `Pods` of the deployment from the Kubernetes cluster. + +- [ ] While the `Pods` are being restarted on new `Nodes`, the cluster must still respond to requests (R/W) +- [ ] The `Pods` running on the `Node` must be restarted on another `Node` +- [ ] After the `Pods` have restarted, the server must have the same data and be responsive again + +### Test 6f: Test `Node` removal on clusters + +This test is only valid when: + +- A `StorageClass` is used that provides network attached `PersistentVolumes` +- or all collections have a replication factor of 2 or higher + +Create an `ArangoDeployment` of mode `Cluster` with an environment of `Production`. + +Hint: Use from `tests/acceptance/cluster.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 9 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Remove a `Node` containing the `Pods` of the deployment from the Kubernetes cluster. + +- [ ] While the `Pods` are being restarted on new `Nodes`, the cluster must still respond to requests (R/W) +- [ ] The `Pods` running on the `Node` must be restarted on another `Node` +- [ ] After the `Pods` have restarted, the server must have the same data and be responsive again + +### Test 6g: Test `Node` removal on clusters with replication factor 1 + +This test is only valid when: + +- A `StorageClass` is used that provides `Node` local `PersistentVolumes` +- and at least some collections have a replication factor of 1 + +Create an `ArangoDeployment` of mode `Cluster` with an environment of `Production`. + +Hint: Use from `tests/acceptance/cluster.yaml`. + +- [ ] The deployment must start +- [ ] The deployment must yield 9 `Pods` +- [ ] The deployment must yield a `Service` named `` +- [ ] The deployment must yield a `Service` named `-ea` +- [ ] The `Service` named `-ea` must be accessible from outside (LoadBalancer or NodePort) and show WebUI + +Remove a `Node`, containing the dbserver `Pod` that holds a collection with replication factor 1, +from the Kubernetes cluster. + +- [ ] While the `Pods` are being restarted on new `Nodes`, the cluster must still respond to requests (R/W), + except requests involving collections with a replication factor of 1 +- [ ] The `Pod` running the dbserver with a collection that has a replication factor of 1 must NOT be restarted on another `Node` + +Remove the collections with the replication factor of 1 + +- [ ] The remaining `Pods` running on the `Node` must be restarted on another `Node` +- [ ] After the `Pods` have restarted, the server must have the same data, except for the removed collections, and be responsive again diff --git a/docs/design/acceptance_test_platforms.md b/docs/design/acceptance_test_platforms.md new file mode 100644 index 000000000..61f31807d --- /dev/null +++ b/docs/design/acceptance_test_platforms.md @@ -0,0 +1,13 @@ +# Acceptance test platforms + +The [kube-arangodb acceptance tests](./acceptance_test.md) must be +executed on the following platforms: + +- Google GKE, with Kubernetes version 1.10 +- Amazon EKS, with Kubernetes version 1.10 +- Amazon & Kops, with Kubernetes version 1.10 +- Azure AKS, with Kubernetes version 1.10 +- Openshift, based on Kubernetes version 1.10 +- Bare metal with kubeadm 1.10 +- Minikube with Kubernetes version 1.10 +- Kubernetes on docker for Mac, with Kubernetes version 1.10 diff --git a/docs/design/lifecycle_hooks_and_finalizers.md b/docs/design/lifecycle_hooks_and_finalizers.md new file mode 100644 index 000000000..d30b4723d --- /dev/null +++ b/docs/design/lifecycle_hooks_and_finalizers.md @@ -0,0 +1,37 @@ +# Lifecycle hooks & Finalizers + +The ArangoDB operator expects full control of the `Pods` and `PersistentVolumeClaims` it creates. +Therefore it takes measures to prevent the removal of those resources +until it is safe to do so. + +To achieve this, the server containers in the `Pods` have +a `preStop` hook configured and finalizers are added to the `Pods` +and `PersistentVolumeClaims`. + +The `preStop` hook executes a binary that waits until all finalizers of +the current pod have been removed. +Until this `preStop` hook terminates, Kubernetes will not send a `TERM` signal +to the processes inside the container, which ensures that the server remains running +until it is safe to stop them. + +The operator performs all actions needed when a delete of a `Pod` or +`PersistentVolumeClaims` has been triggered. +E.g. for a dbserver it cleans out the server if the `Pod` and `PersistentVolumeClaim` are being deleted. + +## Lifecycle init-container + +Because the binary that is called in the `preStop` hook is not part of a standard +ArangoDB docker image, it has to be brought into the filesystem of a `Pod`. +This is done by an initial container that copies the binary to an `emptyDir` volume that +is shared between the init-container and the server container. + +## Finalizers + +The ArangoDB operators adds the following finalizers to `Pods`. + +- `dbserver.database.arangodb.com/drain`: Added to DBServers, removed only when the dbserver can be restarted or is completely drained +- `agent.database.arangodb.com/agency-serving`: Added to Agents, removed only when enough agents are left to keep the agency serving + +The ArangoDB operators adds the following finalizers to `PersistentVolumeClaims`. + +- `pvc.database.arangodb.com/member-exists`: removed only when its member exists no longer exists or can be safely rebuild diff --git a/docs/design/pod_evication_and_replacement.md b/docs/design/pod_evication_and_replacement.md new file mode 100644 index 000000000..8a5fa5e94 --- /dev/null +++ b/docs/design/pod_evication_and_replacement.md @@ -0,0 +1,124 @@ +# Pod Eviction & Replacement + +This chapter specifies the rules around evicting pods from nodes and +restarting or replacing them. + +## Eviction + +Eviction is the process of removing a pod that is running on a node from that node. + +This is typically the result of a drain action (`kubectl drain`) or +from a taint being added to a node (either automatically by Kubernetes or manually by an operator). + +## Replacement + +Replacement is the process of replacing a pod by another pod that takes over the responsibilities +of the original pod. + +The replacement pod has a new ID and new (read empty) persistent data. + +Note that replacing a pod is different from restarting a pod. A pod is restarted when it has been reported +to have termined. + +## NoExecute Tolerations + +NoExecute tolerations are used to control the behavior of Kubernetes (wrt. to a Pod) when the node +that the pod is running on is no longer reachable or becomes not-ready. + +See the applicable [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/) for more info. + +## Rules + +The rules for eviction & replacement are specified per type of pod. + +### Image ID Pods + +The Image ID pods are started to fetch the ArangoDB version of a specific +ArangoDB image and fetch the docker sha256 of that image. +They have no persistent state. + +- Image ID pods can always be evicted from any node +- Image ID pods can always be restarted on a different node. + There is no need to replace an image ID pod, nor will it cause problems when + 2 image ID pods run at the same time. +- `node.kubernetes.io/unreachable:NoExecute` toleration time is set very low (5sec) +- `node.kubernetes.io/not-ready:NoExecute` toleration time is set very low (5sec) + +### Coordinator Pods + +Coordinator pods run an ArangoDB coordinator as part of an ArangoDB cluster. +They have no persistent state, but do have a unique ID. + +- Coordinator pods can always be evicted from any node +- Coordinator pods can always be replaced with another coordinator pod with a different ID on a different node +- `node.kubernetes.io/unreachable:NoExecute` toleration time is set low (15sec) +- `node.kubernetes.io/not-ready:NoExecute` toleration time is set low (15sec) + +### DBServer Pods + +DBServer pods run an ArangoDB dbserver as part of an ArangoDB cluster. +It has persistent state potentially tied to the node it runs on and it has a unique ID. + +- DBServer pods can be evicted from any node as soon as: + - It has been completely drained AND + - It is no longer the shard master for any shard +- DBServer pods can be replaced with another dbserver pod with a different ID on a different node when: + - It is not the shard master for any shard OR + - For every shard it is the master for, there is an in-sync follower +- `node.kubernetes.io/unreachable:NoExecute` toleration time is set high to "wait it out a while" (5min) +- `node.kubernetes.io/not-ready:NoExecute` toleration time is set high to "wait it out a while" (5min) + +### Agent Pods + +Agent pods run an ArangoDB dbserver as part of an ArangoDB agency. +It has persistent state potentially tight to the node it runs on and it has a unique ID. + +- Agent pods can be evicted from any node as soon as: + - It is no longer the agency leader AND + - There is at least an agency leader that is responding AND + - There is at least an agency follower that is responding +- Agent pods can be replaced with another agent pod with the same ID but wiped persistent state on a different node when: + - The old pod is known to be deleted (e.g. explicit eviction) +- `node.kubernetes.io/unreachable:NoExecute` toleration time is set high to "wait it out a while" (5min) +- `node.kubernetes.io/not-ready:NoExecute` toleration time is set high to "wait it out a while" (5min) + +### Single Server Pods + +Single server pods run an ArangoDB server as part of an ArangoDB single server deployment. +It has persistent state potentially tied to the node. + +- Single server pods cannot be evicted from any node. +- Single server pods cannot be replaced with another pod. +- `node.kubernetes.io/unreachable:NoExecute` toleration time is not set to "wait it out forever" +- `node.kubernetes.io/not-ready:NoExecute` toleration time is not set "wait it out forever" + +### Single Pods in Active Failover Deployment + +Single pods run an ArangoDB single server as part of an ArangoDB active failover deployment. +It has persistent state potentially tied to the node it runs on and it has a unique ID. + +- Single pods can be evicted from any node as soon as: + - It is a follower of an active-failover deployment (Q: can we trigger this failover to another server?) +- Single pods can always be replaced with another single pod with a different ID on a different node. +- `node.kubernetes.io/unreachable:NoExecute` toleration time is set high to "wait it out a while" (5min) +- `node.kubernetes.io/not-ready:NoExecute` toleration time is set high to "wait it out a while" (5min) + +### SyncMaster Pods + +SyncMaster pods run an ArangoSync as master as part of an ArangoDB DC2DC cluster. +They have no persistent state, but do have a unique address. + +- SyncMaster pods can always be evicted from any node +- SyncMaster pods can always be replaced with another syncmaster pod on a different node +- `node.kubernetes.io/unreachable:NoExecute` toleration time is set low (15sec) +- `node.kubernetes.io/not-ready:NoExecute` toleration time is set low (15sec) + +### SyncWorker Pods + +SyncWorker pods run an ArangoSync as worker as part of an ArangoDB DC2DC cluster. +They have no persistent state, but do have in-memory state and a unique address. + +- SyncWorker pods can always be evicted from any node +- SyncWorker pods can always be replaced with another syncworker pod on a different node +- `node.kubernetes.io/unreachable:NoExecute` toleration time is set a bit higher to try to avoid resynchronization (1min) +- `node.kubernetes.io/not-ready:NoExecute` toleration time is set a bit higher to try to avoid resynchronization (1min) diff --git a/docs/design/testing.md b/docs/design/testing.md index 786ca9646..25e4680ab 100644 --- a/docs/design/testing.md +++ b/docs/design/testing.md @@ -23,6 +23,14 @@ The following test scenario's must be covered by automated tests: - Restart Node - API server unavailable +- Persistent Volumes: + - hint: RBAC file might need to be changed + - hint: get info via - client-go.CoreV1() + - Number of volumes should stay in reasonable bounds + - For some cases it might be possible to check that, the amount before and after the test stays the same + - A Cluster start should need 6 Volumes (DBServer + Agents) + - The release of a volume-claim should result in a release of the volume + ## Test environments - Kubernetes clusters diff --git a/examples/cluster1-with-sync.yaml b/examples/cluster1-with-sync.yaml new file mode 100644 index 000000000..a2d545496 --- /dev/null +++ b/examples/cluster1-with-sync.yaml @@ -0,0 +1,17 @@ +apiVersion: "database.arangodb.com/v1alpha" +kind: "ArangoDeployment" +metadata: + name: "cluster1-with-sync" +spec: + mode: Cluster + image: ewoutp/arangodb:3.3.8 + tls: + altNames: ["kube-01", "kube-02", "kube-03"] + sync: + enabled: true + auth: + clientCASecretName: client-auth-ca + externalAccess: + type: LoadBalancer + loadBalancerIP: 192.168.140.210 + diff --git a/examples/cluster2-with-sync.yaml b/examples/cluster2-with-sync.yaml new file mode 100644 index 000000000..7e5c3fe95 --- /dev/null +++ b/examples/cluster2-with-sync.yaml @@ -0,0 +1,17 @@ +apiVersion: "database.arangodb.com/v1alpha" +kind: "ArangoDeployment" +metadata: + name: "cluster2-with-sync" +spec: + mode: Cluster + image: ewoutp/arangodb:3.3.8 + tls: + altNames: ["kube-01", "kube-02", "kube-03"] + sync: + enabled: true + auth: + clientCASecretName: client-auth-ca + externalAccess: + type: LoadBalancer + loadBalancerIP: 192.168.140.211 + diff --git a/examples/production-cluster.yaml b/examples/production-cluster.yaml new file mode 100644 index 000000000..350963229 --- /dev/null +++ b/examples/production-cluster.yaml @@ -0,0 +1,8 @@ +apiVersion: "database.arangodb.com/v1alpha" +kind: "ArangoDeployment" +metadata: + name: "production-cluster" +spec: + mode: Cluster + image: arangodb/arangodb:3.3.10 + environment: Production diff --git a/examples/simple-cluster.yaml b/examples/simple-cluster.yaml index c924c297a..8d40025ab 100644 --- a/examples/simple-cluster.yaml +++ b/examples/simple-cluster.yaml @@ -4,9 +4,4 @@ metadata: name: "example-simple-cluster" spec: mode: Cluster - image: arangodb/arangodb:3.3.4 - tls: - altNames: ["kube-01", "kube-02", "kube-03"] - coordinators: - args: - - --log.level=true + image: arangodb/arangodb:3.3.10 diff --git a/examples/single-server.yaml b/examples/single-server.yaml index 710773b7c..b148b69b1 100644 --- a/examples/single-server.yaml +++ b/examples/single-server.yaml @@ -4,5 +4,4 @@ metadata: name: "example-simple-single" spec: mode: Single - single: - storageClassName: my-local-ssd + image: arangodb/arangodb-preview:3.3 diff --git a/lifecycle.go b/lifecycle.go new file mode 100644 index 000000000..7b5561388 --- /dev/null +++ b/lifecycle.go @@ -0,0 +1,150 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package main + +import ( + "io" + "os" + "path/filepath" + "time" + + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +var ( + cmdLifecycle = &cobra.Command{ + Use: "lifecycle", + Run: cmdUsage, + Hidden: true, + } + + cmdLifecyclePreStop = &cobra.Command{ + Use: "preStop", + Run: cmdLifecyclePreStopRun, + Hidden: true, + } + cmdLifecycleCopy = &cobra.Command{ + Use: "copy", + Run: cmdLifecycleCopyRun, + Hidden: true, + } + + lifecycleCopyOptions struct { + TargetDir string + } +) + +func init() { + cmdMain.AddCommand(cmdLifecycle) + cmdLifecycle.AddCommand(cmdLifecyclePreStop) + cmdLifecycle.AddCommand(cmdLifecycleCopy) + + cmdLifecycleCopy.Flags().StringVar(&lifecycleCopyOptions.TargetDir, "target", "", "Target directory to copy the executable to") +} + +// Wait until all finalizers of the current pod have been removed. +func cmdLifecyclePreStopRun(cmd *cobra.Command, args []string) { + cliLog.Info().Msgf("Starting arangodb-operator, lifecycle preStop, version %s build %s", projectVersion, projectBuild) + + // Get environment + namespace := os.Getenv(constants.EnvOperatorPodNamespace) + if len(namespace) == 0 { + cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodNamespace) + } + name := os.Getenv(constants.EnvOperatorPodName) + if len(name) == 0 { + cliLog.Fatal().Msgf("%s environment variable missing", constants.EnvOperatorPodName) + } + + // Create kubernetes client + kubecli, err := k8sutil.NewKubeClient() + if err != nil { + cliLog.Fatal().Err(err).Msg("Failed to create Kubernetes client") + } + + pods := kubecli.CoreV1().Pods(namespace) + recentErrors := 0 + for { + p, err := pods.Get(name, metav1.GetOptions{}) + if k8sutil.IsNotFound(err) { + cliLog.Warn().Msg("Pod not found") + return + } else if err != nil { + recentErrors++ + cliLog.Error().Err(err).Msg("Failed to get pod") + if recentErrors > 20 { + cliLog.Fatal().Err(err).Msg("Too many recent errors") + return + } + } else { + // We got our pod + finalizerCount := len(p.GetFinalizers()) + if finalizerCount == 0 { + // No more finalizers, we're done + cliLog.Info().Msg("All finalizers gone, we can stop now") + return + } + cliLog.Info().Msgf("Waiting for %d more finalizers to be removed", finalizerCount) + } + // Wait a bit + time.Sleep(time.Second) + } +} + +// Copy the executable to a given place. +func cmdLifecycleCopyRun(cmd *cobra.Command, args []string) { + cliLog.Info().Msgf("Starting arangodb-operator, lifecycle copy, version %s build %s", projectVersion, projectBuild) + + exePath, err := os.Executable() + if err != nil { + cliLog.Fatal().Err(err).Msg("Failed to get executable path") + } + + // Open source + rd, err := os.Open(exePath) + if err != nil { + cliLog.Fatal().Err(err).Msg("Failed to open executable file") + } + defer rd.Close() + + // Open target + targetPath := filepath.Join(lifecycleCopyOptions.TargetDir, filepath.Base(exePath)) + wr, err := os.Create(targetPath) + if err != nil { + cliLog.Fatal().Err(err).Msg("Failed to create target file") + } + defer wr.Close() + + if _, err := io.Copy(wr, rd); err != nil { + cliLog.Fatal().Err(err).Msg("Failed to copy") + } + + // Set file mode + if err := os.Chmod(targetPath, 0755); err != nil { + cliLog.Fatal().Err(err).Msg("Failed to chmod") + } +} diff --git a/main.go b/main.go index 7a80c339f..d93bb5d62 100644 --- a/main.go +++ b/main.go @@ -39,11 +39,11 @@ import ( "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - "k8s.io/client-go/kubernetes/scheme" v1core "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" "github.com/arangodb/kube-arangodb/pkg/client" + scheme "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/scheme" "github.com/arangodb/kube-arangodb/pkg/logging" "github.com/arangodb/kube-arangodb/pkg/operator" "github.com/arangodb/kube-arangodb/pkg/server" @@ -79,15 +79,17 @@ var ( tlsSecretName string } operatorOptions struct { - enableDeployment bool // Run deployment operator - enableStorage bool // Run deployment operator + enableDeployment bool // Run deployment operator + enableDeploymentReplication bool // Run deployment-replication operator + enableStorage bool // Run local-storage operator } chaosOptions struct { allowed bool } - livenessProbe probe.LivenessProbe - deploymentProbe probe.ReadyProbe - storageProbe probe.ReadyProbe + livenessProbe probe.LivenessProbe + deploymentProbe probe.ReadyProbe + deploymentReplicationProbe probe.ReadyProbe + storageProbe probe.ReadyProbe ) func init() { @@ -97,6 +99,7 @@ func init() { f.StringVar(&serverOptions.tlsSecretName, "server.tls-secret-name", "", "Name of secret containing tls.crt & tls.key for HTTPS server (if empty, self-signed certificate is used)") f.StringVar(&logLevel, "log.level", defaultLogLevel, "Set initial log level") f.BoolVar(&operatorOptions.enableDeployment, "operator.deployment", false, "Enable to run the ArangoDeployment operator") + f.BoolVar(&operatorOptions.enableDeploymentReplication, "operator.deployment-replication", false, "Enable to run the ArangoDeploymentReplication operator") f.BoolVar(&operatorOptions.enableStorage, "operator.storage", false, "Enable to run the ArangoLocalStorage operator") f.BoolVar(&chaosOptions.allowed, "chaos.allowed", false, "Set to allow chaos in deployments. Only activated when allowed and enabled in deployment") } @@ -121,8 +124,8 @@ func cmdMainRun(cmd *cobra.Command, args []string) { } // Check operating mode - if !operatorOptions.enableDeployment && !operatorOptions.enableStorage { - cliLog.Fatal().Err(err).Msg("Turn on --operator.deployment or --operator.storage or both") + if !operatorOptions.enableDeployment && !operatorOptions.enableDeploymentReplication && !operatorOptions.enableStorage { + cliLog.Fatal().Err(err).Msg("Turn on --operator.deployment, --operator.deployment-replication, --operator.storage or any combination of these") } // Log version @@ -157,6 +160,7 @@ func cmdMainRun(cmd *cobra.Command, args []string) { mux := http.NewServeMux() mux.HandleFunc("/health", livenessProbe.LivenessHandler) mux.HandleFunc("/ready/deployment", deploymentProbe.ReadyHandler) + mux.HandleFunc("/ready/deployment-replication", deploymentReplicationProbe.ReadyHandler) mux.HandleFunc("/ready/storage", storageProbe.ReadyHandler) mux.Handle("/metrics", prometheus.Handler()) listenAddr := net.JoinHostPort(serverOptions.host, strconv.Itoa(serverOptions.port)) @@ -193,7 +197,7 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper return operator.Config{}, operator.Dependencies{}, maskAny(err) } - serviceAccount, err := getMyPodServiceAccount(kubecli, namespace, name) + image, serviceAccount, err := getMyPodInfo(kubecli, namespace, name) if err != nil { return operator.Config{}, operator.Dependencies{}, maskAny(fmt.Errorf("Failed to get my pod's service account: %s", err)) } @@ -209,31 +213,35 @@ func newOperatorConfigAndDeps(id, namespace, name string) (operator.Config, oper eventRecorder := createRecorder(cliLog, kubecli, name, namespace) cfg := operator.Config{ - ID: id, - Namespace: namespace, - PodName: name, - ServiceAccount: serviceAccount, - EnableDeployment: operatorOptions.enableDeployment, - EnableStorage: operatorOptions.enableStorage, - AllowChaos: chaosOptions.allowed, + ID: id, + Namespace: namespace, + PodName: name, + ServiceAccount: serviceAccount, + LifecycleImage: image, + EnableDeployment: operatorOptions.enableDeployment, + EnableDeploymentReplication: operatorOptions.enableDeploymentReplication, + EnableStorage: operatorOptions.enableStorage, + AllowChaos: chaosOptions.allowed, } deps := operator.Dependencies{ - LogService: logService, - KubeCli: kubecli, - KubeExtCli: kubeExtCli, - CRCli: crCli, - EventRecorder: eventRecorder, - LivenessProbe: &livenessProbe, - DeploymentProbe: &deploymentProbe, - StorageProbe: &storageProbe, + LogService: logService, + KubeCli: kubecli, + KubeExtCli: kubeExtCli, + CRCli: crCli, + EventRecorder: eventRecorder, + LivenessProbe: &livenessProbe, + DeploymentProbe: &deploymentProbe, + DeploymentReplicationProbe: &deploymentReplicationProbe, + StorageProbe: &storageProbe, } return cfg, deps, nil } -// getMyPodServiceAccount looks up the service account of the pod with given name in given namespace -func getMyPodServiceAccount(kubecli kubernetes.Interface, namespace, name string) (string, error) { - var sa string +// getMyPodInfo looks up the image & service account of the pod with given name in given namespace +// Returns image, serviceAccount, error. +func getMyPodInfo(kubecli kubernetes.Interface, namespace, name string) (string, string, error) { + var image, sa string op := func() error { pod, err := kubecli.CoreV1().Pods(namespace).Get(name, metav1.GetOptions{}) if err != nil { @@ -244,12 +252,17 @@ func getMyPodServiceAccount(kubecli kubernetes.Interface, namespace, name string return maskAny(err) } sa = pod.Spec.ServiceAccountName + image = k8sutil.ConvertImageID2Image(pod.Status.ContainerStatuses[0].ImageID) + if image == "" { + // Fallback in case we don't know the id. + image = pod.Spec.Containers[0].Image + } return nil } if err := retry.Retry(op, time.Minute*5); err != nil { - return "", maskAny(err) + return "", "", maskAny(err) } - return sa, nil + return image, sa, nil } func createRecorder(log zerolog.Logger, kubecli kubernetes.Interface, name, namespace string) record.EventRecorder { diff --git a/manifests/.gitignore b/manifests/.gitignore index 9118a229f..0c8b9cdf1 100644 --- a/manifests/.gitignore +++ b/manifests/.gitignore @@ -1,3 +1,4 @@ arango-deployment-dev.yaml +arango-deployment-replication-dev.yaml arango-storage-dev.yaml arango-test-dev.yaml diff --git a/manifests/arango-deployment.yaml b/manifests/arango-deployment.yaml index 9bfcab96c..9adb0f0ac 100644 --- a/manifests/arango-deployment.yaml +++ b/manifests/arango-deployment.yaml @@ -26,6 +26,9 @@ rules: - apiGroups: [""] resources: ["pods", "services", "endpoints", "persistentvolumeclaims", "events", "secrets"] verbs: ["*"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] - apiGroups: ["apps"] resources: ["deployments"] verbs: ["*"] @@ -35,7 +38,7 @@ rules: --- -## Bind the cluster role granting access to ArangoLocalStorage resources +## Bind the cluster role granting access to ArangoDeployment resources ## to the default service account of the configured namespace. apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding @@ -79,7 +82,7 @@ metadata: name: arango-deployment-operator namespace: default spec: - replicas: 1 + replicas: 2 strategy: type: Recreate template: @@ -91,7 +94,7 @@ spec: containers: - name: operator imagePullPolicy: IfNotPresent - image: arangodb/kube-arangodb@sha256:4b58b71c0a53b7652ca6e9b518bdb1393bf3a0c79d3c666a0914dd079f8f018e + image: arangodb/kube-arangodb@sha256:43bdc14d072fb1912d885536c189a631076e13e5c3e8a87b06e5ddbe60c66a6d args: - --operator.deployment - --chaos.allowed=false @@ -125,4 +128,13 @@ spec: scheme: HTTPS initialDelaySeconds: 5 periodSeconds: 10 + tolerations: + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 5 + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 5 diff --git a/manifests/arango-storage.yaml b/manifests/arango-storage.yaml index c0980928f..fde46cb60 100644 --- a/manifests/arango-storage.yaml +++ b/manifests/arango-storage.yaml @@ -90,7 +90,7 @@ metadata: name: arango-storage-operator namespace: kube-system spec: - replicas: 1 + replicas: 2 strategy: type: Recreate template: @@ -103,7 +103,7 @@ spec: containers: - name: operator imagePullPolicy: IfNotPresent - image: arangodb/kube-arangodb@sha256:4b58b71c0a53b7652ca6e9b518bdb1393bf3a0c79d3c666a0914dd079f8f018e + image: arangodb/kube-arangodb@sha256:43bdc14d072fb1912d885536c189a631076e13e5c3e8a87b06e5ddbe60c66a6d args: - --operator.storage env: @@ -136,4 +136,13 @@ spec: scheme: HTTPS initialDelaySeconds: 5 periodSeconds: 10 - + tolerations: + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 5 + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 5 + diff --git a/manifests/arango-test.yaml b/manifests/arango-test.yaml new file mode 100644 index 000000000..5cad77481 --- /dev/null +++ b/manifests/arango-test.yaml @@ -0,0 +1,34 @@ +## test/rbac.yaml +## Cluster role granting access to resources needed by the integration tests. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: arango-operator-test +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list"] +- apiGroups: [""] + resources: ["pods", "services", "persistentvolumes", "persistentvolumeclaims", "secrets", "serviceaccounts"] + verbs: ["*"] +- apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["*"] + +--- + +## Bind the cluster role granting access to ArangoLocalStorage resources +## to the default service account of the configured namespace. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: arango-operator-test + namespace: default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: arango-operator-test +subjects: +- kind: ServiceAccount + name: default + namespace: default diff --git a/manifests/crd.yaml b/manifests/crd.yaml index 169262939..7cce03231 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -17,6 +17,24 @@ spec: --- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: arangodeploymentreplications.replication.database.arangodb.com +spec: + group: replication.database.arangodb.com + names: + kind: ArangoDeploymentReplication + listKind: ArangoDeploymentReplicationList + plural: arangodeploymentreplications + shortNames: + - arangorepl + singular: arangodeploymentreplication + scope: Namespaced + version: v1alpha + +--- + apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: diff --git a/manifests/templates/deployment-replication/deployment-replication.yaml b/manifests/templates/deployment-replication/deployment-replication.yaml new file mode 100644 index 000000000..7b5301127 --- /dev/null +++ b/manifests/templates/deployment-replication/deployment-replication.yaml @@ -0,0 +1,61 @@ + +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: {{ .DeploymentReplication.OperatorDeploymentName }} + namespace: {{ .DeploymentReplication.Operator.Namespace }} +spec: + replicas: 2 + strategy: + type: Recreate + template: + metadata: + labels: + name: {{ .DeploymentReplication.OperatorDeploymentName }} + app: arango-deployment-replication-operator + spec: + containers: + - name: operator + imagePullPolicy: {{ .ImagePullPolicy }} + image: {{ .Image }} + args: + - --operator.deployment-replication + env: + - name: MY_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + ports: + - name: metrics + containerPort: 8528 + livenessProbe: + httpGet: + path: /health + port: 8528 + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /ready/deployment-replication + port: 8528 + scheme: HTTPS + initialDelaySeconds: 5 + periodSeconds: 10 + tolerations: + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 5 + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 5 diff --git a/manifests/templates/deployment-replication/rbac.yaml b/manifests/templates/deployment-replication/rbac.yaml new file mode 100644 index 000000000..90df2baf4 --- /dev/null +++ b/manifests/templates/deployment-replication/rbac.yaml @@ -0,0 +1,75 @@ +{{- if .RBAC -}} +## Cluster role granting access to ArangoDeploymentReplication resources. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ .DeploymentReplication.User.RoleName }} +rules: +- apiGroups: ["replication.database.arangodb.com"] + resources: ["arangodeploymentreplications"] + verbs: ["*"] + +--- + +## Cluster role granting access to all resources needed by the ArangoDeploymentReplication operator. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: {{ .DeploymentReplication.Operator.RoleName }} +rules: +- apiGroups: ["replication.database.arangodb.com"] + resources: ["arangodeploymentreplications"] + verbs: ["*"] +- apiGroups: ["database.arangodb.com"] + resources: ["arangodeployments"] + verbs: ["get"] +- apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get"] +- apiGroups: [""] + resources: ["pods", "services", "endpoints", "persistentvolumeclaims", "events", "secrets"] + verbs: ["*"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +- apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["*"] + +--- + +## Bind the cluster role granting access to ArangoDeploymentReplication resources +## to the default service account of the configured namespace. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: {{ .DeploymentReplication.User.RoleBindingName }} + namespace: {{ .DeploymentReplication.User.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .DeploymentReplication.User.RoleName }} +subjects: +- kind: ServiceAccount + name: {{ .DeploymentReplication.User.ServiceAccountName }} + namespace: {{ .DeploymentReplication.User.Namespace }} + +--- + +## Bind the cluster role granting access to all resources needed by +## the ArangoDeploymentReplication operator to the default service account +## the is being used to run the operator deployment. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: {{ .DeploymentReplication.Operator.RoleBindingName }}-{{ .DeploymentReplication.Operator.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ .DeploymentReplication.Operator.RoleName }} +subjects: +- kind: ServiceAccount + name: {{ .DeploymentReplication.Operator.ServiceAccountName }} + namespace: {{ .DeploymentReplication.Operator.Namespace }} + +{{- end -}} diff --git a/manifests/templates/deployment/deployment.yaml b/manifests/templates/deployment/deployment.yaml index c74fb69d0..01b6a14a6 100644 --- a/manifests/templates/deployment/deployment.yaml +++ b/manifests/templates/deployment/deployment.yaml @@ -5,7 +5,7 @@ metadata: name: {{ .Deployment.OperatorDeploymentName }} namespace: {{ .Deployment.Operator.Namespace }} spec: - replicas: 1 + replicas: 2 strategy: type: Recreate template: @@ -51,3 +51,12 @@ spec: scheme: HTTPS initialDelaySeconds: 5 periodSeconds: 10 + tolerations: + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 5 + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 5 diff --git a/manifests/templates/deployment/rbac.yaml b/manifests/templates/deployment/rbac.yaml index 78145c714..b9362d931 100644 --- a/manifests/templates/deployment/rbac.yaml +++ b/manifests/templates/deployment/rbac.yaml @@ -26,6 +26,9 @@ rules: - apiGroups: [""] resources: ["pods", "services", "endpoints", "persistentvolumeclaims", "events", "secrets"] verbs: ["*"] +- apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] - apiGroups: ["apps"] resources: ["deployments"] verbs: ["*"] @@ -35,7 +38,7 @@ rules: --- -## Bind the cluster role granting access to ArangoLocalStorage resources +## Bind the cluster role granting access to ArangoDeployment resources ## to the default service account of the configured namespace. apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding diff --git a/manifests/templates/storage/deployment.yaml b/manifests/templates/storage/deployment.yaml index 1a1e16715..f6ffbf51b 100644 --- a/manifests/templates/storage/deployment.yaml +++ b/manifests/templates/storage/deployment.yaml @@ -13,7 +13,7 @@ metadata: name: {{ .Storage.OperatorDeploymentName }} namespace: {{ .Storage.Operator.Namespace }} spec: - replicas: 1 + replicas: 2 strategy: type: Recreate template: @@ -59,4 +59,12 @@ spec: scheme: HTTPS initialDelaySeconds: 5 periodSeconds: 10 - \ No newline at end of file + tolerations: + - key: "node.kubernetes.io/unreachable" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 5 + - key: "node.kubernetes.io/not-ready" + operator: "Exists" + effect: "NoExecute" + tolerationSeconds: 5 diff --git a/manifests/templates/test/rbac.yaml b/manifests/templates/test/rbac.yaml index a54c17d9a..16848c872 100644 --- a/manifests/templates/test/rbac.yaml +++ b/manifests/templates/test/rbac.yaml @@ -10,7 +10,7 @@ rules: resources: ["nodes"] verbs: ["list"] - apiGroups: [""] - resources: ["pods", "services", "persistentvolumes", "persistentvolumeclaims", "secrets"] + resources: ["pods", "services", "persistentvolumes", "persistentvolumeclaims", "secrets", "serviceaccounts"] verbs: ["*"] - apiGroups: ["apps"] resources: ["daemonsets"] diff --git a/pkg/apis/deployment/v1alpha/conditions.go b/pkg/apis/deployment/v1alpha/conditions.go index b4b023803..bbdf74969 100644 --- a/pkg/apis/deployment/v1alpha/conditions.go +++ b/pkg/apis/deployment/v1alpha/conditions.go @@ -37,12 +37,17 @@ const ( ConditionTypeTerminated ConditionType = "Terminated" // ConditionTypeAutoUpgrade indicates that the member has to be started with `--database.auto-upgrade` once. ConditionTypeAutoUpgrade ConditionType = "AutoUpgrade" + // ConditionTypeCleanedOut indicates that the member (dbserver) has been cleaned out. + // Always check in combination with ConditionTypeTerminated. + ConditionTypeCleanedOut ConditionType = "CleanedOut" // ConditionTypePodSchedulingFailure indicates that one or more pods belonging to the deployment cannot be schedule. ConditionTypePodSchedulingFailure ConditionType = "PodSchedulingFailure" // ConditionTypeSecretsChanged indicates that the value of one of more secrets used by // the deployment have changed. Once that is the case, the operator will no longer // touch the deployment, until the original secrets have been restored. ConditionTypeSecretsChanged ConditionType = "SecretsChanged" + // ConditionTypeMemberOfCluster indicates that the member is a known member of the ArangoDB cluster. + ConditionTypeMemberOfCluster ConditionType = "MemberOfCluster" ) // Condition represents one current condition of a deployment or deployment member. diff --git a/pkg/apis/deployment/v1alpha/deployment.go b/pkg/apis/deployment/v1alpha/deployment.go index d5ccd5663..7c2fba55b 100644 --- a/pkg/apis/deployment/v1alpha/deployment.go +++ b/pkg/apis/deployment/v1alpha/deployment.go @@ -50,11 +50,15 @@ type ArangoDeployment struct { // AsOwner creates an OwnerReference for the given deployment func (d *ArangoDeployment) AsOwner() metav1.OwnerReference { + trueVar := true return metav1.OwnerReference{ APIVersion: SchemeGroupVersion.String(), Kind: ArangoDeploymentResourceKind, Name: d.Name, UID: d.UID, + Controller: &trueVar, + // For now BlockOwnerDeletion does not work on OpenShift, so we leave it out. + //BlockOwnerDeletion: &trueVar, } } diff --git a/pkg/apis/deployment/v1alpha/deployment_mode.go b/pkg/apis/deployment/v1alpha/deployment_mode.go index c7d40f9a3..ebf06b28c 100644 --- a/pkg/apis/deployment/v1alpha/deployment_mode.go +++ b/pkg/apis/deployment/v1alpha/deployment_mode.go @@ -32,8 +32,8 @@ type DeploymentMode string const ( // DeploymentModeSingle yields a single server DeploymentModeSingle DeploymentMode = "Single" - // DeploymentModeResilientSingle yields an agency and a resilient-single server pair - DeploymentModeResilientSingle DeploymentMode = "ResilientSingle" + // DeploymentModeActiveFailover yields an agency and a active-failover server pair + DeploymentModeActiveFailover DeploymentMode = "ActiveFailover" // DeploymentModeCluster yields an full cluster (agency, dbservers & coordinators) DeploymentModeCluster DeploymentMode = "Cluster" ) @@ -42,21 +42,21 @@ const ( // Return errors when validation fails, nil on success. func (m DeploymentMode) Validate() error { switch m { - case DeploymentModeSingle, DeploymentModeResilientSingle, DeploymentModeCluster: + case DeploymentModeSingle, DeploymentModeActiveFailover, DeploymentModeCluster: return nil default: return maskAny(errors.Wrapf(ValidationError, "Unknown deployment mode: '%s'", string(m))) } } -// HasSingleServers returns true when the given mode is "Single" or "ResilientSingle". +// HasSingleServers returns true when the given mode is "Single" or "ActiveFailover". func (m DeploymentMode) HasSingleServers() bool { - return m == DeploymentModeSingle || m == DeploymentModeResilientSingle + return m == DeploymentModeSingle || m == DeploymentModeActiveFailover } -// HasAgents returns true when the given mode is "ResilientSingle" or "Cluster". +// HasAgents returns true when the given mode is "ActiveFailover" or "Cluster". func (m DeploymentMode) HasAgents() bool { - return m == DeploymentModeResilientSingle || m == DeploymentModeCluster + return m == DeploymentModeActiveFailover || m == DeploymentModeCluster } // HasDBServers returns true when the given mode is "Cluster". diff --git a/pkg/apis/deployment/v1alpha/deployment_mode_test.go b/pkg/apis/deployment/v1alpha/deployment_mode_test.go index b655281b7..da60c1782 100644 --- a/pkg/apis/deployment/v1alpha/deployment_mode_test.go +++ b/pkg/apis/deployment/v1alpha/deployment_mode_test.go @@ -31,7 +31,7 @@ import ( func TestDeploymentModeValidate(t *testing.T) { // Valid assert.Nil(t, DeploymentMode("Single").Validate()) - assert.Nil(t, DeploymentMode("ResilientSingle").Validate()) + assert.Nil(t, DeploymentMode("ActiveFailover").Validate()) assert.Nil(t, DeploymentMode("Cluster").Validate()) // Not valid @@ -39,30 +39,30 @@ func TestDeploymentModeValidate(t *testing.T) { assert.Error(t, DeploymentMode(" cluster").Validate()) assert.Error(t, DeploymentMode("singles").Validate()) assert.Error(t, DeploymentMode("single").Validate()) - assert.Error(t, DeploymentMode("resilientsingle").Validate()) + assert.Error(t, DeploymentMode("activefailover").Validate()) assert.Error(t, DeploymentMode("cluster").Validate()) } func TestDeploymentModeHasX(t *testing.T) { assert.True(t, DeploymentModeSingle.HasSingleServers()) - assert.True(t, DeploymentModeResilientSingle.HasSingleServers()) + assert.True(t, DeploymentModeActiveFailover.HasSingleServers()) assert.False(t, DeploymentModeCluster.HasSingleServers()) assert.False(t, DeploymentModeSingle.HasAgents()) - assert.True(t, DeploymentModeResilientSingle.HasAgents()) + assert.True(t, DeploymentModeActiveFailover.HasAgents()) assert.True(t, DeploymentModeCluster.HasAgents()) assert.False(t, DeploymentModeSingle.HasDBServers()) - assert.False(t, DeploymentModeResilientSingle.HasDBServers()) + assert.False(t, DeploymentModeActiveFailover.HasDBServers()) assert.True(t, DeploymentModeCluster.HasDBServers()) assert.False(t, DeploymentModeSingle.HasCoordinators()) - assert.False(t, DeploymentModeResilientSingle.HasCoordinators()) + assert.False(t, DeploymentModeActiveFailover.HasCoordinators()) assert.True(t, DeploymentModeCluster.HasCoordinators()) } func TestDeploymentModeSupportsSync(t *testing.T) { assert.False(t, DeploymentModeSingle.SupportsSync()) - assert.False(t, DeploymentModeResilientSingle.SupportsSync()) + assert.False(t, DeploymentModeActiveFailover.SupportsSync()) assert.True(t, DeploymentModeCluster.SupportsSync()) } diff --git a/pkg/apis/deployment/v1alpha/deployment_spec.go b/pkg/apis/deployment/v1alpha/deployment_spec.go index 7f8485cb3..aff2041a0 100644 --- a/pkg/apis/deployment/v1alpha/deployment_spec.go +++ b/pkg/apis/deployment/v1alpha/deployment_spec.go @@ -50,7 +50,9 @@ type DeploymentSpec struct { StorageEngine *StorageEngine `json:"storageEngine,omitempty"` Image *string `json:"image,omitempty"` ImagePullPolicy *v1.PullPolicy `json:"imagePullPolicy,omitempty"` + DowntimeAllowed *bool `json:"downtimeAllowed,omitempty"` + ExternalAccess ExternalAccessSpec `json:"externalAccess"` RocksDB RocksDBSpec `json:"rocksdb"` Authentication AuthenticationSpec `json:"auth"` TLS TLSSpec `json:"tls"` @@ -91,6 +93,11 @@ func (s DeploymentSpec) GetImagePullPolicy() v1.PullPolicy { return util.PullPolicyOrDefault(s.ImagePullPolicy) } +// IsDowntimeAllowed returns the value of downtimeAllowed. +func (s DeploymentSpec) IsDowntimeAllowed() bool { + return util.BoolOrDefault(s.DowntimeAllowed) +} + // IsAuthenticated returns true when authentication is enabled func (s DeploymentSpec) IsAuthenticated() bool { return s.Authentication.IsAuthenticated() @@ -139,10 +146,11 @@ func (s *DeploymentSpec) SetDefaults(deploymentName string) { if s.GetImagePullPolicy() == "" { s.ImagePullPolicy = util.NewPullPolicy(v1.PullIfNotPresent) } + s.ExternalAccess.SetDefaults() s.RocksDB.SetDefaults() s.Authentication.SetDefaults(deploymentName + "-jwt") s.TLS.SetDefaults(deploymentName + "-ca") - s.Sync.SetDefaults(s.GetImage(), s.GetImagePullPolicy(), deploymentName+"-sync-jwt", deploymentName+"-sync-ca") + s.Sync.SetDefaults(deploymentName+"-sync-jwt", deploymentName+"-sync-client-auth-ca", deploymentName+"-sync-ca", deploymentName+"-sync-mt") s.Single.SetDefaults(ServerGroupSingle, s.GetMode().HasSingleServers(), s.GetMode()) s.Agents.SetDefaults(ServerGroupAgents, s.GetMode().HasAgents(), s.GetMode()) s.DBServers.SetDefaults(ServerGroupDBServers, s.GetMode().HasDBServers(), s.GetMode()) @@ -169,6 +177,10 @@ func (s *DeploymentSpec) SetDefaultsFrom(source DeploymentSpec) { if s.ImagePullPolicy == nil { s.ImagePullPolicy = util.NewPullPolicyOrNil(source.ImagePullPolicy) } + if s.DowntimeAllowed == nil { + s.DowntimeAllowed = util.NewBoolOrNil(source.DowntimeAllowed) + } + s.ExternalAccess.SetDefaultsFrom(source.ExternalAccess) s.RocksDB.SetDefaultsFrom(source.RocksDB) s.Authentication.SetDefaultsFrom(source.Authentication) s.TLS.SetDefaultsFrom(source.TLS) @@ -200,6 +212,9 @@ func (s *DeploymentSpec) Validate() error { if s.GetImage() == "" { return maskAny(errors.Wrapf(ValidationError, "spec.image must be set")) } + if err := s.ExternalAccess.Validate(); err != nil { + return maskAny(errors.Wrap(err, "spec.externalAccess")) + } if err := s.RocksDB.Validate(); err != nil { return maskAny(errors.Wrap(err, "spec.rocksdb")) } @@ -254,6 +269,9 @@ func (s DeploymentSpec) ResetImmutableFields(target *DeploymentSpec) []string { target.StorageEngine = NewStorageEngineOrNil(s.StorageEngine) resetFields = append(resetFields, "storageEngine") } + if l := s.ExternalAccess.ResetImmutableFields("externalAccess", &target.ExternalAccess); l != nil { + resetFields = append(resetFields, l...) + } if l := s.RocksDB.ResetImmutableFields("rocksdb", &target.RocksDB); l != nil { resetFields = append(resetFields, l...) } diff --git a/pkg/apis/deployment/v1alpha/deployment_status_members.go b/pkg/apis/deployment/v1alpha/deployment_status_members.go index b13d9a932..7344b7425 100644 --- a/pkg/apis/deployment/v1alpha/deployment_status_members.go +++ b/pkg/apis/deployment/v1alpha/deployment_status_members.go @@ -23,8 +23,6 @@ package v1alpha import ( - "fmt" - "github.com/pkg/errors" ) @@ -75,23 +73,23 @@ func (ds DeploymentStatusMembers) ElementByID(id string) (MemberStatus, ServerGr // ForeachServerGroup calls the given callback for all server groups. // If the callback returns an error, this error is returned and the callback is // not called for the remaining groups. -func (ds DeploymentStatusMembers) ForeachServerGroup(cb func(group ServerGroup, list *MemberStatusList) error) error { - if err := cb(ServerGroupSingle, &ds.Single); err != nil { +func (ds DeploymentStatusMembers) ForeachServerGroup(cb func(group ServerGroup, list MemberStatusList) error) error { + if err := cb(ServerGroupSingle, ds.Single); err != nil { return maskAny(err) } - if err := cb(ServerGroupAgents, &ds.Agents); err != nil { + if err := cb(ServerGroupAgents, ds.Agents); err != nil { return maskAny(err) } - if err := cb(ServerGroupDBServers, &ds.DBServers); err != nil { + if err := cb(ServerGroupDBServers, ds.DBServers); err != nil { return maskAny(err) } - if err := cb(ServerGroupCoordinators, &ds.Coordinators); err != nil { + if err := cb(ServerGroupCoordinators, ds.Coordinators); err != nil { return maskAny(err) } - if err := cb(ServerGroupSyncMasters, &ds.SyncMasters); err != nil { + if err := cb(ServerGroupSyncMasters, ds.SyncMasters); err != nil { return maskAny(err) } - if err := cb(ServerGroupSyncWorkers, &ds.SyncWorkers); err != nil { + if err := cb(ServerGroupSyncWorkers, ds.SyncWorkers); err != nil { return maskAny(err) } return nil @@ -121,22 +119,63 @@ func (ds DeploymentStatusMembers) MemberStatusByPodName(podName string) (MemberS return MemberStatus{}, 0, false } -// UpdateMemberStatus updates the given status in the given group. -func (ds *DeploymentStatusMembers) UpdateMemberStatus(status MemberStatus, group ServerGroup) error { +// MemberStatusByPVCName returns a reference to the element in the given set of lists that has the given PVC name. +// If no such element exists, nil is returned. +func (ds DeploymentStatusMembers) MemberStatusByPVCName(pvcName string) (MemberStatus, ServerGroup, bool) { + if result, found := ds.Single.ElementByPVCName(pvcName); found { + return result, ServerGroupSingle, true + } + if result, found := ds.Agents.ElementByPVCName(pvcName); found { + return result, ServerGroupAgents, true + } + if result, found := ds.DBServers.ElementByPVCName(pvcName); found { + return result, ServerGroupDBServers, true + } + // Note: Other server groups do not have PVC's so we can skip them. + return MemberStatus{}, 0, false +} + +// Add adds the given status in the given group. +func (ds *DeploymentStatusMembers) Add(status MemberStatus, group ServerGroup) error { var err error switch group { case ServerGroupSingle: - err = ds.Single.Update(status) + err = ds.Single.add(status) case ServerGroupAgents: - err = ds.Agents.Update(status) + err = ds.Agents.add(status) case ServerGroupDBServers: - err = ds.DBServers.Update(status) + err = ds.DBServers.add(status) case ServerGroupCoordinators: - err = ds.Coordinators.Update(status) + err = ds.Coordinators.add(status) case ServerGroupSyncMasters: - err = ds.SyncMasters.Update(status) + err = ds.SyncMasters.add(status) case ServerGroupSyncWorkers: - err = ds.SyncWorkers.Update(status) + err = ds.SyncWorkers.add(status) + default: + return maskAny(errors.Wrapf(NotFoundError, "ServerGroup %d is not known", group)) + } + if err != nil { + return maskAny(err) + } + return nil +} + +// Update updates the given status in the given group. +func (ds *DeploymentStatusMembers) Update(status MemberStatus, group ServerGroup) error { + var err error + switch group { + case ServerGroupSingle: + err = ds.Single.update(status) + case ServerGroupAgents: + err = ds.Agents.update(status) + case ServerGroupDBServers: + err = ds.DBServers.update(status) + case ServerGroupCoordinators: + err = ds.Coordinators.update(status) + case ServerGroupSyncMasters: + err = ds.SyncMasters.update(status) + case ServerGroupSyncWorkers: + err = ds.SyncWorkers.update(status) default: return maskAny(errors.Wrapf(NotFoundError, "ServerGroup %d is not known", group)) } @@ -152,17 +191,17 @@ func (ds *DeploymentStatusMembers) RemoveByID(id string, group ServerGroup) erro var err error switch group { case ServerGroupSingle: - err = ds.Single.RemoveByID(id) + err = ds.Single.removeByID(id) case ServerGroupAgents: - err = ds.Agents.RemoveByID(id) + err = ds.Agents.removeByID(id) case ServerGroupDBServers: - err = ds.DBServers.RemoveByID(id) + err = ds.DBServers.removeByID(id) case ServerGroupCoordinators: - err = ds.Coordinators.RemoveByID(id) + err = ds.Coordinators.removeByID(id) case ServerGroupSyncMasters: - err = ds.SyncMasters.RemoveByID(id) + err = ds.SyncMasters.removeByID(id) case ServerGroupSyncWorkers: - err = ds.SyncWorkers.RemoveByID(id) + err = ds.SyncWorkers.removeByID(id) default: return maskAny(errors.Wrapf(NotFoundError, "ServerGroup %d is not known", group)) } @@ -172,17 +211,25 @@ func (ds *DeploymentStatusMembers) RemoveByID(id string, group ServerGroup) erro return nil } -// AllMembersReady returns true when all members are in the Ready state. -func (ds DeploymentStatusMembers) AllMembersReady() bool { - if err := ds.ForeachServerGroup(func(group ServerGroup, list *MemberStatusList) error { - for _, x := range *list { - if !x.Conditions.IsTrue(ConditionTypeReady) { - return fmt.Errorf("not ready") - } +// AllMembersReady returns true when all members, that must be ready for the given mode, are in the Ready state. +func (ds DeploymentStatusMembers) AllMembersReady(mode DeploymentMode, syncEnabled bool) bool { + syncReady := func() bool { + if syncEnabled { + return ds.SyncMasters.AllMembersReady() && ds.SyncWorkers.AllMembersReady() } - return nil - }); err != nil { + return true + } + switch mode { + case DeploymentModeSingle: + return ds.Single.MembersReady() > 0 + case DeploymentModeActiveFailover: + return ds.Agents.AllMembersReady() && ds.Single.MembersReady() > 0 + case DeploymentModeCluster: + return ds.Agents.AllMembersReady() && + ds.DBServers.AllMembersReady() && + ds.Coordinators.AllMembersReady() && + syncReady() + default: return false } - return true } diff --git a/pkg/apis/deployment/v1alpha/environment.go b/pkg/apis/deployment/v1alpha/environment.go index 458c829c7..8803842ce 100644 --- a/pkg/apis/deployment/v1alpha/environment.go +++ b/pkg/apis/deployment/v1alpha/environment.go @@ -47,6 +47,11 @@ func (e Environment) Validate() error { } } +// IsProduction returns true when the given environment is a production environment. +func (e Environment) IsProduction() bool { + return e == EnvironmentProduction +} + // NewEnvironment returns a reference to a string with given value. func NewEnvironment(input Environment) *Environment { return &input diff --git a/pkg/apis/deployment/v1alpha/external_access_spec.go b/pkg/apis/deployment/v1alpha/external_access_spec.go new file mode 100644 index 000000000..dd226750d --- /dev/null +++ b/pkg/apis/deployment/v1alpha/external_access_spec.go @@ -0,0 +1,84 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + "github.com/arangodb/kube-arangodb/pkg/util" +) + +// ExternalAccessSpec holds configuration for the external access provided for the deployment. +type ExternalAccessSpec struct { + // Type of external access + Type *ExternalAccessType `json:"type,omitempty"` + // Optional port used in case of Auto or NodePort type. + NodePort *int `json:"nodePort,omitempty"` + // Optional IP used to configure a load-balancer on, in case of Auto or LoadBalancer type. + LoadBalancerIP *string `json:"loadBalancerIP,omitempty"` +} + +// GetType returns the value of type. +func (s ExternalAccessSpec) GetType() ExternalAccessType { + return ExternalAccessTypeOrDefault(s.Type, ExternalAccessTypeAuto) +} + +// GetNodePort returns the value of nodePort. +func (s ExternalAccessSpec) GetNodePort() int { + return util.IntOrDefault(s.NodePort) +} + +// GetLoadBalancerIP returns the value of loadBalancerIP. +func (s ExternalAccessSpec) GetLoadBalancerIP() string { + return util.StringOrDefault(s.LoadBalancerIP) +} + +// Validate the given spec +func (s ExternalAccessSpec) Validate() error { + if err := s.GetType().Validate(); err != nil { + return maskAny(err) + } + return nil +} + +// SetDefaults fills in missing defaults +func (s *ExternalAccessSpec) SetDefaults() { +} + +// SetDefaultsFrom fills unspecified fields with a value from given source spec. +func (s *ExternalAccessSpec) SetDefaultsFrom(source ExternalAccessSpec) { + if s.Type == nil { + s.Type = NewExternalAccessTypeOrNil(source.Type) + } + if s.NodePort == nil { + s.NodePort = util.NewIntOrNil(source.NodePort) + } + if s.LoadBalancerIP == nil { + s.LoadBalancerIP = util.NewStringOrNil(source.LoadBalancerIP) + } +} + +// ResetImmutableFields replaces all immutable fields in the given target with values from the source spec. +// It returns a list of fields that have been reset. +// Field names are relative to given field prefix. +func (s ExternalAccessSpec) ResetImmutableFields(fieldPrefix string, target *ExternalAccessSpec) []string { + return nil +} diff --git a/pkg/apis/deployment/v1alpha/external_access_type.go b/pkg/apis/deployment/v1alpha/external_access_type.go new file mode 100644 index 000000000..a7a07545f --- /dev/null +++ b/pkg/apis/deployment/v1alpha/external_access_type.go @@ -0,0 +1,95 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + "github.com/pkg/errors" + "k8s.io/api/core/v1" +) + +// ExternalAccessType specifies the type of external access provides for the deployment +type ExternalAccessType string + +const ( + // ExternalAccessTypeNone yields a cluster with no external access + ExternalAccessTypeNone ExternalAccessType = "None" + // ExternalAccessTypeAuto yields a cluster with an automatic selection for external access + ExternalAccessTypeAuto ExternalAccessType = "Auto" + // ExternalAccessTypeLoadBalancer yields a cluster with a service of type `LoadBalancer` to provide external access + ExternalAccessTypeLoadBalancer ExternalAccessType = "LoadBalancer" + // ExternalAccessTypeNodePort yields a cluster with a service of type `NodePort` to provide external access + ExternalAccessTypeNodePort ExternalAccessType = "NodePort" +) + +func (t ExternalAccessType) IsNone() bool { return t == ExternalAccessTypeNone } +func (t ExternalAccessType) IsAuto() bool { return t == ExternalAccessTypeAuto } +func (t ExternalAccessType) IsLoadBalancer() bool { return t == ExternalAccessTypeLoadBalancer } +func (t ExternalAccessType) IsNodePort() bool { return t == ExternalAccessTypeNodePort } + +// AsServiceType returns the k8s ServiceType for this ExternalAccessType. +// If type is "Auto", ServiceTypeLoadBalancer is returned. +func (t ExternalAccessType) AsServiceType() v1.ServiceType { + switch t { + case ExternalAccessTypeLoadBalancer, ExternalAccessTypeAuto: + return v1.ServiceTypeLoadBalancer + case ExternalAccessTypeNodePort: + return v1.ServiceTypeNodePort + default: + return "" + } +} + +// Validate the type. +// Return errors when validation fails, nil on success. +func (t ExternalAccessType) Validate() error { + switch t { + case ExternalAccessTypeNone, ExternalAccessTypeAuto, ExternalAccessTypeLoadBalancer, ExternalAccessTypeNodePort: + return nil + default: + return maskAny(errors.Wrapf(ValidationError, "Unknown external access type: '%s'", string(t))) + } +} + +// NewExternalAccessType returns a reference to a string with given value. +func NewExternalAccessType(input ExternalAccessType) *ExternalAccessType { + return &input +} + +// NewExternalAccessTypeOrNil returns nil if input is nil, otherwise returns a clone of the given value. +func NewExternalAccessTypeOrNil(input *ExternalAccessType) *ExternalAccessType { + if input == nil { + return nil + } + return NewExternalAccessType(*input) +} + +// ExternalAccessTypeOrDefault returns the default value (or empty string) if input is nil, otherwise returns the referenced value. +func ExternalAccessTypeOrDefault(input *ExternalAccessType, defaultValue ...ExternalAccessType) ExternalAccessType { + if input == nil { + if len(defaultValue) > 0 { + return defaultValue[0] + } + return "" + } + return *input +} diff --git a/pkg/apis/deployment/v1alpha/image_info.go b/pkg/apis/deployment/v1alpha/image_info.go index 0b26f25c7..34c7f8a88 100644 --- a/pkg/apis/deployment/v1alpha/image_info.go +++ b/pkg/apis/deployment/v1alpha/image_info.go @@ -29,6 +29,7 @@ type ImageInfo struct { Image string `json:"image"` // Human provided name of the image ImageID string `json:"image-id,omitempty"` // Unique ID (with SHA256) of the image ArangoDBVersion driver.Version `json:"arangodb-version,omitempty"` // ArangoDB version within the image + Enterprise bool `json:"enterprise,omitempty"` // If set, this is an enterprise image } // ImageInfoList is a list of image infos diff --git a/pkg/apis/deployment/v1alpha/member_status.go b/pkg/apis/deployment/v1alpha/member_status.go index 38342dfd5..a1c18e56a 100644 --- a/pkg/apis/deployment/v1alpha/member_status.go +++ b/pkg/apis/deployment/v1alpha/member_status.go @@ -50,6 +50,13 @@ type MemberStatus struct { // IsInitialized is set after the very first time a pod was created for this member. // After that, DBServers must have a UUID field or fail. IsInitialized bool `json:"initialized"` + // CleanoutJobID holds the ID of the agency job for cleaning out this server + CleanoutJobID string `json:"cleanout-job-id,omitempty"` +} + +// Age returns the duration since the creation timestamp of this member. +func (s MemberStatus) Age() time.Duration { + return time.Since(s.CreatedAt.Time) } // RemoveTerminationsBefore removes all recent terminations before the given timestamp. diff --git a/pkg/apis/deployment/v1alpha/member_status_list.go b/pkg/apis/deployment/v1alpha/member_status_list.go index fff418e25..bc8ca9f21 100644 --- a/pkg/apis/deployment/v1alpha/member_status_list.go +++ b/pkg/apis/deployment/v1alpha/member_status_list.go @@ -63,9 +63,20 @@ func (l MemberStatusList) ElementByPodName(podName string) (MemberStatus, bool) return MemberStatus{}, false } +// ElementByPVCName returns the element in the given list that has the given PVC name and true. +// If no such element exists, an empty element and false is returned. +func (l MemberStatusList) ElementByPVCName(pvcName string) (MemberStatus, bool) { + for i, x := range l { + if x.PersistentVolumeClaimName == pvcName { + return l[i], true + } + } + return MemberStatus{}, false +} + // Add a member to the list. // Returns an AlreadyExistsError if the ID of the given member already exists. -func (l *MemberStatusList) Add(m MemberStatus) error { +func (l *MemberStatusList) add(m MemberStatus) error { src := *l for _, x := range src { if x.ID == m.ID { @@ -78,7 +89,7 @@ func (l *MemberStatusList) Add(m MemberStatus) error { // Update a member in the list. // Returns a NotFoundError if the ID of the given member cannot be found. -func (l MemberStatusList) Update(m MemberStatus) error { +func (l MemberStatusList) update(m MemberStatus) error { for i, x := range l { if x.ID == m.ID { l[i] = m @@ -90,7 +101,7 @@ func (l MemberStatusList) Update(m MemberStatus) error { // RemoveByID a member with given ID from the list. // Returns a NotFoundError if the ID of the given member cannot be found. -func (l *MemberStatusList) RemoveByID(id string) error { +func (l *MemberStatusList) removeByID(id string) error { src := *l for i, x := range src { if x.ID == id { @@ -123,3 +134,19 @@ func (l MemberStatusList) SelectMemberToRemove() (MemberStatus, error) { } return MemberStatus{}, maskAny(errors.Wrap(NotFoundError, "No member available for removal")) } + +// MembersReady returns the number of members that are in the Ready state. +func (l MemberStatusList) MembersReady() int { + readyCount := 0 + for _, x := range l { + if x.Conditions.IsTrue(ConditionTypeReady) { + readyCount++ + } + } + return readyCount +} + +// AllMembersReady returns the true if all members are in the Ready state. +func (l MemberStatusList) AllMembersReady() bool { + return len(l) == l.MembersReady() +} diff --git a/pkg/apis/deployment/v1alpha/member_status_list_test.go b/pkg/apis/deployment/v1alpha/member_status_list_test.go new file mode 100644 index 000000000..2cf583e95 --- /dev/null +++ b/pkg/apis/deployment/v1alpha/member_status_list_test.go @@ -0,0 +1,69 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestMemberStatusList tests modifying a MemberStatusList. +func TestMemberStatusList(t *testing.T) { + list := &MemberStatusList{} + m1 := MemberStatus{ID: "m1"} + m2 := MemberStatus{ID: "m2"} + m3 := MemberStatus{ID: "m3"} + assert.Equal(t, 0, len(*list)) + + assert.NoError(t, list.add(m1)) + assert.Equal(t, 1, len(*list)) + + assert.NoError(t, list.add(m2)) + assert.NoError(t, list.add(m3)) + assert.Equal(t, 3, len(*list)) + + assert.Error(t, list.add(m2)) + assert.Equal(t, 3, len(*list)) + + assert.NoError(t, list.removeByID(m3.ID)) + assert.Equal(t, 2, len(*list)) + assert.False(t, list.ContainsID(m3.ID)) + assert.Equal(t, m1.ID, (*list)[0].ID) + assert.Equal(t, m2.ID, (*list)[1].ID) + + m2.PodName = "foo" + assert.NoError(t, list.update(m2)) + assert.Equal(t, 2, len(*list)) + assert.True(t, list.ContainsID(m2.ID)) + x, found := list.ElementByPodName("foo") + assert.True(t, found) + assert.Equal(t, "foo", x.PodName) + assert.Equal(t, m2.ID, x.ID) + + assert.NoError(t, list.add(m3)) + assert.Equal(t, 3, len(*list)) + assert.Equal(t, m1.ID, (*list)[0].ID) + assert.Equal(t, m2.ID, (*list)[1].ID) + assert.Equal(t, m3.ID, (*list)[2].ID) +} diff --git a/pkg/apis/deployment/v1alpha/monitoring_spec.go b/pkg/apis/deployment/v1alpha/monitoring_spec.go index 719468cd7..5c11b590e 100644 --- a/pkg/apis/deployment/v1alpha/monitoring_spec.go +++ b/pkg/apis/deployment/v1alpha/monitoring_spec.go @@ -46,8 +46,12 @@ func (s MonitoringSpec) Validate() error { } // SetDefaults fills in missing defaults -func (s *MonitoringSpec) SetDefaults() { - // Nothing needed +func (s *MonitoringSpec) SetDefaults(defaultTokenSecretName string) { + if s.GetTokenSecretName() == "" { + // Note that we don't check for nil here, since even a specified, but empty + // string should result in the default value. + s.TokenSecretName = util.NewString(defaultTokenSecretName) + } } // SetDefaultsFrom fills unspecified fields with a value from given source spec. diff --git a/pkg/apis/deployment/v1alpha/monitoring_spec_test.go b/pkg/apis/deployment/v1alpha/monitoring_spec_test.go index 8ece949d0..855f74252 100644 --- a/pkg/apis/deployment/v1alpha/monitoring_spec_test.go +++ b/pkg/apis/deployment/v1alpha/monitoring_spec_test.go @@ -42,10 +42,15 @@ func TestMonitoringSpecValidate(t *testing.T) { func TestMonitoringSpecSetDefaults(t *testing.T) { def := func(spec MonitoringSpec) MonitoringSpec { - spec.SetDefaults() + spec.SetDefaults("") + return spec + } + def2 := func(spec MonitoringSpec) MonitoringSpec { + spec.SetDefaults("def2") return spec } assert.Equal(t, "", def(MonitoringSpec{}).GetTokenSecretName()) + assert.Equal(t, "def2", def2(MonitoringSpec{}).GetTokenSecretName()) assert.Equal(t, "foo", def(MonitoringSpec{TokenSecretName: util.NewString("foo")}).GetTokenSecretName()) } diff --git a/pkg/apis/deployment/v1alpha/plan.go b/pkg/apis/deployment/v1alpha/plan.go index 3d04c59a8..fd0ad77a2 100644 --- a/pkg/apis/deployment/v1alpha/plan.go +++ b/pkg/apis/deployment/v1alpha/plan.go @@ -47,6 +47,14 @@ const ( ActionTypeWaitForMemberUp ActionType = "WaitForMemberUp" // ActionTypeRenewTLSCertificate causes the TLS certificate of a member to be renewed. ActionTypeRenewTLSCertificate ActionType = "RenewTLSCertificate" + // ActionTypeRenewTLSCACertificate causes the TLS CA certificate of the entire deployment to be renewed. + ActionTypeRenewTLSCACertificate ActionType = "RenewTLSCACertificate" +) + +const ( + // MemberIDPreviousAction is used for Action.MemberID when the MemberID + // should be derived from the previous action. + MemberIDPreviousAction = "@previous" ) // Action represents a single action to be taken to update a deployment. diff --git a/pkg/apis/deployment/v1alpha/server_group.go b/pkg/apis/deployment/v1alpha/server_group.go index 8d8725693..4ccb7f21b 100644 --- a/pkg/apis/deployment/v1alpha/server_group.go +++ b/pkg/apis/deployment/v1alpha/server_group.go @@ -22,6 +22,8 @@ package v1alpha +import time "time" + type ServerGroup int const ( @@ -85,6 +87,30 @@ func (g ServerGroup) AsRoleAbbreviated() string { } } +// DefaultTerminationGracePeriod returns the default period between SIGTERM & SIGKILL for a server in the given group. +func (g ServerGroup) DefaultTerminationGracePeriod() time.Duration { + switch g { + case ServerGroupSingle: + return time.Minute + case ServerGroupAgents: + return time.Minute + case ServerGroupDBServers: + return time.Hour + default: + return time.Second * 30 + } +} + +// IsStateless returns true when the groups runs servers without a persistent volume. +func (g ServerGroup) IsStateless() bool { + switch g { + case ServerGroupCoordinators, ServerGroupSyncMasters, ServerGroupSyncWorkers: + return true + default: + return false + } +} + // IsArangod returns true when the groups runs servers of type `arangod`. func (g ServerGroup) IsArangod() bool { switch g { diff --git a/pkg/apis/deployment/v1alpha/server_group_spec.go b/pkg/apis/deployment/v1alpha/server_group_spec.go index f73bc3788..6d199599a 100644 --- a/pkg/apis/deployment/v1alpha/server_group_spec.go +++ b/pkg/apis/deployment/v1alpha/server_group_spec.go @@ -28,6 +28,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) // ServerGroupSpec contains the specification for all servers in a specific group (e.g. all agents) @@ -40,6 +41,10 @@ type ServerGroupSpec struct { StorageClassName *string `json:"storageClassName,omitempty"` // Resources holds resource requests & limits Resources v1.ResourceRequirements `json:"resources,omitempty"` + // Tolerations specifies the tolerations added to Pods in this group. + Tolerations []v1.Toleration `json:"tolerations,omitempty"` + // ServiceAccountName specifies the name of the service account used for Pods in this group. + ServiceAccountName *string `json:"serviceAccountName,omitempty"` } // GetCount returns the value of count. @@ -57,14 +62,25 @@ func (s ServerGroupSpec) GetStorageClassName() string { return util.StringOrDefault(s.StorageClassName) } +// GetTolerations returns the value of tolerations. +func (s ServerGroupSpec) GetTolerations() []v1.Toleration { + return s.Tolerations +} + +// GetServiceAccountName returns the value of serviceAccountName. +func (s ServerGroupSpec) GetServiceAccountName() string { + return util.StringOrDefault(s.ServiceAccountName) +} + // Validate the given group spec func (s ServerGroupSpec) Validate(group ServerGroup, used bool, mode DeploymentMode, env Environment) error { if used { minCount := 1 if env == EnvironmentProduction { + // Set validation boundaries for production mode switch group { case ServerGroupSingle: - if mode == DeploymentModeResilientSingle { + if mode == DeploymentModeActiveFailover { minCount = 2 } case ServerGroupAgents: @@ -72,6 +88,16 @@ func (s ServerGroupSpec) Validate(group ServerGroup, used bool, mode DeploymentM case ServerGroupDBServers, ServerGroupCoordinators, ServerGroupSyncMasters, ServerGroupSyncWorkers: minCount = 2 } + } else { + // Set validation boundaries for development mode + switch group { + case ServerGroupSingle: + if mode == DeploymentModeActiveFailover { + minCount = 2 + } + case ServerGroupDBServers: + minCount = 2 + } } if s.GetCount() < minCount { return maskAny(errors.Wrapf(ValidationError, "Invalid count value %d. Expected >= %d", s.GetCount(), minCount)) @@ -79,6 +105,16 @@ func (s ServerGroupSpec) Validate(group ServerGroup, used bool, mode DeploymentM if s.GetCount() > 1 && group == ServerGroupSingle && mode == DeploymentModeSingle { return maskAny(errors.Wrapf(ValidationError, "Invalid count value %d. Expected 1", s.GetCount())) } + if name := s.GetServiceAccountName(); name != "" { + if err := k8sutil.ValidateOptionalResourceName(name); err != nil { + return maskAny(errors.Wrapf(ValidationError, "Invalid serviceAccountName: %s", err)) + } + } + if name := s.GetStorageClassName(); name != "" { + if err := k8sutil.ValidateOptionalResourceName(name); err != nil { + return maskAny(errors.Wrapf(ValidationError, "Invalid storageClassName: %s", err)) + } + } } else if s.GetCount() != 0 { return maskAny(errors.Wrapf(ValidationError, "Invalid count value %d for un-used group. Expected 0", s.GetCount())) } @@ -93,11 +129,13 @@ func (s *ServerGroupSpec) SetDefaults(group ServerGroup, used bool, mode Deploym if mode == DeploymentModeSingle { s.Count = util.NewInt(1) // Single server } else { - s.Count = util.NewInt(2) // Resilient single + s.Count = util.NewInt(2) // ActiveFailover } default: s.Count = util.NewInt(3) } + } else if s.GetCount() > 0 && !used { + s.Count = util.NewInt(0) } if _, found := s.Resources.Requests[v1.ResourceStorage]; !found { switch group { @@ -133,6 +171,12 @@ func (s *ServerGroupSpec) SetDefaultsFrom(source ServerGroupSpec) { if s.StorageClassName == nil { s.StorageClassName = util.NewStringOrNil(source.StorageClassName) } + if s.Tolerations == nil { + s.Tolerations = source.Tolerations + } + if s.ServiceAccountName == nil { + s.ServiceAccountName = util.NewStringOrNil(source.ServiceAccountName) + } setDefaultsFromResourceList(&s.Resources.Limits, source.Resources.Limits) setDefaultsFromResourceList(&s.Resources.Requests, source.Resources.Requests) } @@ -147,9 +191,5 @@ func (s ServerGroupSpec) ResetImmutableFields(group ServerGroup, fieldPrefix str resetFields = append(resetFields, fieldPrefix+".count") } } - if s.GetStorageClassName() != target.GetStorageClassName() { - target.StorageClassName = util.NewStringOrNil(s.StorageClassName) - resetFields = append(resetFields, fieldPrefix+".storageClassName") - } return resetFields } diff --git a/pkg/apis/deployment/v1alpha/server_group_spec_test.go b/pkg/apis/deployment/v1alpha/server_group_spec_test.go index 53134094e..00ac564b7 100644 --- a/pkg/apis/deployment/v1alpha/server_group_spec_test.go +++ b/pkg/apis/deployment/v1alpha/server_group_spec_test.go @@ -35,14 +35,14 @@ func TestServerGroupSpecValidateCount(t *testing.T) { assert.Nil(t, ServerGroupSpec{Count: util.NewInt(0)}.Validate(ServerGroupSingle, false, DeploymentModeCluster, EnvironmentDevelopment)) assert.Nil(t, ServerGroupSpec{Count: util.NewInt(1)}.Validate(ServerGroupAgents, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Nil(t, ServerGroupSpec{Count: util.NewInt(3)}.Validate(ServerGroupAgents, true, DeploymentModeCluster, EnvironmentDevelopment)) - assert.Nil(t, ServerGroupSpec{Count: util.NewInt(1)}.Validate(ServerGroupAgents, true, DeploymentModeResilientSingle, EnvironmentDevelopment)) - assert.Nil(t, ServerGroupSpec{Count: util.NewInt(3)}.Validate(ServerGroupAgents, true, DeploymentModeResilientSingle, EnvironmentDevelopment)) - assert.Nil(t, ServerGroupSpec{Count: util.NewInt(1)}.Validate(ServerGroupDBServers, true, DeploymentModeCluster, EnvironmentDevelopment)) + assert.Nil(t, ServerGroupSpec{Count: util.NewInt(1)}.Validate(ServerGroupAgents, true, DeploymentModeActiveFailover, EnvironmentDevelopment)) + assert.Nil(t, ServerGroupSpec{Count: util.NewInt(3)}.Validate(ServerGroupAgents, true, DeploymentModeActiveFailover, EnvironmentDevelopment)) + assert.Nil(t, ServerGroupSpec{Count: util.NewInt(2)}.Validate(ServerGroupDBServers, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Nil(t, ServerGroupSpec{Count: util.NewInt(6)}.Validate(ServerGroupDBServers, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Nil(t, ServerGroupSpec{Count: util.NewInt(1)}.Validate(ServerGroupCoordinators, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Nil(t, ServerGroupSpec{Count: util.NewInt(2)}.Validate(ServerGroupCoordinators, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Nil(t, ServerGroupSpec{Count: util.NewInt(3)}.Validate(ServerGroupAgents, true, DeploymentModeCluster, EnvironmentProduction)) - assert.Nil(t, ServerGroupSpec{Count: util.NewInt(3)}.Validate(ServerGroupAgents, true, DeploymentModeResilientSingle, EnvironmentProduction)) + assert.Nil(t, ServerGroupSpec{Count: util.NewInt(3)}.Validate(ServerGroupAgents, true, DeploymentModeActiveFailover, EnvironmentProduction)) assert.Nil(t, ServerGroupSpec{Count: util.NewInt(2)}.Validate(ServerGroupDBServers, true, DeploymentModeCluster, EnvironmentProduction)) assert.Nil(t, ServerGroupSpec{Count: util.NewInt(2)}.Validate(ServerGroupCoordinators, true, DeploymentModeCluster, EnvironmentProduction)) assert.Nil(t, ServerGroupSpec{Count: util.NewInt(2)}.Validate(ServerGroupSyncMasters, true, DeploymentModeCluster, EnvironmentProduction)) @@ -51,21 +51,21 @@ func TestServerGroupSpecValidateCount(t *testing.T) { // Invalid assert.Error(t, ServerGroupSpec{Count: util.NewInt(1)}.Validate(ServerGroupSingle, false, DeploymentModeCluster, EnvironmentDevelopment)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(2)}.Validate(ServerGroupSingle, true, DeploymentModeSingle, EnvironmentDevelopment)) - assert.Error(t, ServerGroupSpec{Count: util.NewInt(1)}.Validate(ServerGroupSingle, true, DeploymentModeResilientSingle, EnvironmentProduction)) + assert.Error(t, ServerGroupSpec{Count: util.NewInt(1)}.Validate(ServerGroupSingle, true, DeploymentModeActiveFailover, EnvironmentProduction)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(0)}.Validate(ServerGroupAgents, true, DeploymentModeCluster, EnvironmentDevelopment)) - assert.Error(t, ServerGroupSpec{Count: util.NewInt(0)}.Validate(ServerGroupAgents, true, DeploymentModeResilientSingle, EnvironmentDevelopment)) + assert.Error(t, ServerGroupSpec{Count: util.NewInt(0)}.Validate(ServerGroupAgents, true, DeploymentModeActiveFailover, EnvironmentDevelopment)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(0)}.Validate(ServerGroupDBServers, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(0)}.Validate(ServerGroupCoordinators, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(0)}.Validate(ServerGroupSyncMasters, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(0)}.Validate(ServerGroupSyncWorkers, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(-1)}.Validate(ServerGroupAgents, true, DeploymentModeCluster, EnvironmentDevelopment)) - assert.Error(t, ServerGroupSpec{Count: util.NewInt(-1)}.Validate(ServerGroupAgents, true, DeploymentModeResilientSingle, EnvironmentDevelopment)) + assert.Error(t, ServerGroupSpec{Count: util.NewInt(-1)}.Validate(ServerGroupAgents, true, DeploymentModeActiveFailover, EnvironmentDevelopment)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(-1)}.Validate(ServerGroupDBServers, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(-1)}.Validate(ServerGroupCoordinators, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(-1)}.Validate(ServerGroupSyncMasters, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(-1)}.Validate(ServerGroupSyncWorkers, true, DeploymentModeCluster, EnvironmentDevelopment)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(2)}.Validate(ServerGroupAgents, true, DeploymentModeCluster, EnvironmentProduction)) - assert.Error(t, ServerGroupSpec{Count: util.NewInt(2)}.Validate(ServerGroupAgents, true, DeploymentModeResilientSingle, EnvironmentProduction)) + assert.Error(t, ServerGroupSpec{Count: util.NewInt(2)}.Validate(ServerGroupAgents, true, DeploymentModeActiveFailover, EnvironmentProduction)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(1)}.Validate(ServerGroupDBServers, true, DeploymentModeCluster, EnvironmentProduction)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(1)}.Validate(ServerGroupCoordinators, true, DeploymentModeCluster, EnvironmentProduction)) assert.Error(t, ServerGroupSpec{Count: util.NewInt(1)}.Validate(ServerGroupSyncMasters, true, DeploymentModeCluster, EnvironmentProduction)) @@ -79,27 +79,27 @@ func TestServerGroupSpecDefault(t *testing.T) { } assert.Equal(t, 1, def(ServerGroupSpec{}, ServerGroupSingle, true, DeploymentModeSingle).GetCount()) - assert.Equal(t, 2, def(ServerGroupSpec{}, ServerGroupSingle, true, DeploymentModeResilientSingle).GetCount()) + assert.Equal(t, 2, def(ServerGroupSpec{}, ServerGroupSingle, true, DeploymentModeActiveFailover).GetCount()) assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupSingle, false, DeploymentModeCluster).GetCount()) assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupAgents, false, DeploymentModeSingle).GetCount()) - assert.Equal(t, 3, def(ServerGroupSpec{}, ServerGroupAgents, true, DeploymentModeResilientSingle).GetCount()) + assert.Equal(t, 3, def(ServerGroupSpec{}, ServerGroupAgents, true, DeploymentModeActiveFailover).GetCount()) assert.Equal(t, 3, def(ServerGroupSpec{}, ServerGroupAgents, true, DeploymentModeCluster).GetCount()) assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupDBServers, false, DeploymentModeSingle).GetCount()) - assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupDBServers, false, DeploymentModeResilientSingle).GetCount()) + assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupDBServers, false, DeploymentModeActiveFailover).GetCount()) assert.Equal(t, 3, def(ServerGroupSpec{}, ServerGroupDBServers, true, DeploymentModeCluster).GetCount()) assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupCoordinators, false, DeploymentModeSingle).GetCount()) - assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupCoordinators, false, DeploymentModeResilientSingle).GetCount()) + assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupCoordinators, false, DeploymentModeActiveFailover).GetCount()) assert.Equal(t, 3, def(ServerGroupSpec{}, ServerGroupCoordinators, true, DeploymentModeCluster).GetCount()) assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupSyncMasters, false, DeploymentModeSingle).GetCount()) - assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupSyncMasters, false, DeploymentModeResilientSingle).GetCount()) + assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupSyncMasters, false, DeploymentModeActiveFailover).GetCount()) assert.Equal(t, 3, def(ServerGroupSpec{}, ServerGroupSyncMasters, true, DeploymentModeCluster).GetCount()) assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupSyncWorkers, false, DeploymentModeSingle).GetCount()) - assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupSyncWorkers, false, DeploymentModeResilientSingle).GetCount()) + assert.Equal(t, 0, def(ServerGroupSpec{}, ServerGroupSyncWorkers, false, DeploymentModeActiveFailover).GetCount()) assert.Equal(t, 3, def(ServerGroupSpec{}, ServerGroupSyncWorkers, true, DeploymentModeCluster).GetCount()) for _, g := range AllServerGroups { diff --git a/pkg/apis/deployment/v1alpha/sync_authentication_spec.go b/pkg/apis/deployment/v1alpha/sync_authentication_spec.go new file mode 100644 index 000000000..e8c9e3242 --- /dev/null +++ b/pkg/apis/deployment/v1alpha/sync_authentication_spec.go @@ -0,0 +1,87 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// SyncAuthenticationSpec holds dc2dc sync authentication specific configuration settings +type SyncAuthenticationSpec struct { + JWTSecretName *string `json:"jwtSecretName,omitempty"` // JWT secret for sync masters + ClientCASecretName *string `json:"clientCASecretName,omitempty"` // Secret containing client authentication CA +} + +// GetJWTSecretName returns the value of jwtSecretName. +func (s SyncAuthenticationSpec) GetJWTSecretName() string { + return util.StringOrDefault(s.JWTSecretName) +} + +// GetClientCASecretName returns the value of clientCASecretName. +func (s SyncAuthenticationSpec) GetClientCASecretName() string { + return util.StringOrDefault(s.ClientCASecretName) +} + +// Validate the given spec +func (s SyncAuthenticationSpec) Validate() error { + if err := k8sutil.ValidateResourceName(s.GetJWTSecretName()); err != nil { + return maskAny(err) + } + if err := k8sutil.ValidateResourceName(s.GetClientCASecretName()); err != nil { + return maskAny(err) + } + return nil +} + +// SetDefaults fills in missing defaults +func (s *SyncAuthenticationSpec) SetDefaults(defaultJWTSecretName, defaultClientCASecretName string) { + if s.GetJWTSecretName() == "" { + // Note that we don't check for nil here, since even a specified, but empty + // string should result in the default value. + s.JWTSecretName = util.NewString(defaultJWTSecretName) + } + if s.GetClientCASecretName() == "" { + // Note that we don't check for nil here, since even a specified, but empty + // string should result in the default value. + s.ClientCASecretName = util.NewString(defaultClientCASecretName) + } +} + +// SetDefaultsFrom fills unspecified fields with a value from given source spec. +func (s *SyncAuthenticationSpec) SetDefaultsFrom(source SyncAuthenticationSpec) { + if s.JWTSecretName == nil { + s.JWTSecretName = util.NewStringOrNil(source.JWTSecretName) + } + if s.ClientCASecretName == nil { + s.ClientCASecretName = util.NewStringOrNil(source.ClientCASecretName) + } +} + +// ResetImmutableFields replaces all immutable fields in the given target with values from the source spec. +// It returns a list of fields that have been reset. +// Field names are relative to given field prefix. +func (s SyncAuthenticationSpec) ResetImmutableFields(fieldPrefix string, target *SyncAuthenticationSpec) []string { + var resetFields []string + return resetFields +} diff --git a/pkg/apis/deployment/v1alpha/sync_external_access_spec.go b/pkg/apis/deployment/v1alpha/sync_external_access_spec.go new file mode 100644 index 000000000..a50794aaf --- /dev/null +++ b/pkg/apis/deployment/v1alpha/sync_external_access_spec.go @@ -0,0 +1,102 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + "fmt" + "net" + "net/url" + "strconv" + + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// SyncExternalAccessSpec holds configuration for the external access provided for the sync deployment. +type SyncExternalAccessSpec struct { + ExternalAccessSpec + MasterEndpoint []string `json:"masterEndpoint,omitempty"` + AccessPackageSecretNames []string `json:"accessPackageSecretNames,omitempty"` +} + +// GetMasterEndpoint returns the value of masterEndpoint. +func (s SyncExternalAccessSpec) GetMasterEndpoint() []string { + return s.MasterEndpoint +} + +// GetAccessPackageSecretNames returns the value of accessPackageSecretNames. +func (s SyncExternalAccessSpec) GetAccessPackageSecretNames() []string { + return s.AccessPackageSecretNames +} + +// ResolveMasterEndpoint returns the value of `--master.endpoint` option passed to arangosync. +func (s SyncExternalAccessSpec) ResolveMasterEndpoint(syncServiceHostName string, syncServicePort int) []string { + if len(s.MasterEndpoint) > 0 { + return s.MasterEndpoint + } + if ip := s.GetLoadBalancerIP(); ip != "" { + syncServiceHostName = ip + } + return []string{"https://" + net.JoinHostPort(syncServiceHostName, strconv.Itoa(syncServicePort))} +} + +// Validate the given spec +func (s SyncExternalAccessSpec) Validate() error { + if err := s.ExternalAccessSpec.Validate(); err != nil { + return maskAny(err) + } + for _, ep := range s.MasterEndpoint { + if _, err := url.Parse(ep); err != nil { + return maskAny(fmt.Errorf("Failed to parse master endpoint '%s': %s", ep, err)) + } + } + for _, name := range s.AccessPackageSecretNames { + if err := k8sutil.ValidateResourceName(name); err != nil { + return maskAny(fmt.Errorf("Invalid name '%s' in accessPackageSecretNames: %s", name, err)) + } + } + return nil +} + +// SetDefaults fills in missing defaults +func (s *SyncExternalAccessSpec) SetDefaults() { + s.ExternalAccessSpec.SetDefaults() +} + +// SetDefaultsFrom fills unspecified fields with a value from given source spec. +func (s *SyncExternalAccessSpec) SetDefaultsFrom(source SyncExternalAccessSpec) { + s.ExternalAccessSpec.SetDefaultsFrom(source.ExternalAccessSpec) + if s.MasterEndpoint == nil && source.MasterEndpoint != nil { + s.MasterEndpoint = append([]string{}, source.MasterEndpoint...) + } + if s.AccessPackageSecretNames == nil && source.AccessPackageSecretNames != nil { + s.AccessPackageSecretNames = append([]string{}, source.AccessPackageSecretNames...) + } +} + +// ResetImmutableFields replaces all immutable fields in the given target with values from the source spec. +// It returns a list of fields that have been reset. +// Field names are relative to given field prefix. +func (s SyncExternalAccessSpec) ResetImmutableFields(fieldPrefix string, target *SyncExternalAccessSpec) []string { + result := s.ExternalAccessSpec.ResetImmutableFields(fieldPrefix, &s.ExternalAccessSpec) + return result +} diff --git a/pkg/apis/deployment/v1alpha/sync_spec.go b/pkg/apis/deployment/v1alpha/sync_spec.go index 56aad0f64..3b0473e2f 100644 --- a/pkg/apis/deployment/v1alpha/sync_spec.go +++ b/pkg/apis/deployment/v1alpha/sync_spec.go @@ -24,20 +24,18 @@ package v1alpha import ( "github.com/pkg/errors" - "k8s.io/api/core/v1" "github.com/arangodb/kube-arangodb/pkg/util" ) // SyncSpec holds dc2dc replication specific configuration settings type SyncSpec struct { - Enabled *bool `json:"enabled,omitempty"` - Image *string `json:"image,omitempty"` - ImagePullPolicy *v1.PullPolicy `json:"imagePullPolicy,omitempty"` + Enabled *bool `json:"enabled,omitempty"` - Authentication AuthenticationSpec `json:"auth"` - TLS TLSSpec `json:"tls"` - Monitoring MonitoringSpec `json:"monitoring"` + ExternalAccess SyncExternalAccessSpec `json:"externalAccess"` + Authentication SyncAuthenticationSpec `json:"auth"` + TLS TLSSpec `json:"tls"` + Monitoring MonitoringSpec `json:"monitoring"` } // IsEnabled returns the value of enabled. @@ -45,28 +43,18 @@ func (s SyncSpec) IsEnabled() bool { return util.BoolOrDefault(s.Enabled) } -// GetImage returns the value of image. -func (s SyncSpec) GetImage() string { - return util.StringOrDefault(s.Image) -} - -// GetImagePullPolicy returns the value of imagePullPolicy. -func (s SyncSpec) GetImagePullPolicy() v1.PullPolicy { - return util.PullPolicyOrDefault(s.ImagePullPolicy) -} - // Validate the given spec func (s SyncSpec) Validate(mode DeploymentMode) error { if s.IsEnabled() && !mode.SupportsSync() { return maskAny(errors.Wrapf(ValidationError, "Cannot enable sync with mode: '%s'", mode)) } - if s.GetImage() == "" { - return maskAny(errors.Wrapf(ValidationError, "image must be set")) - } - if err := s.Authentication.Validate(s.IsEnabled()); err != nil { - return maskAny(err) - } if s.IsEnabled() { + if err := s.ExternalAccess.Validate(); err != nil { + return maskAny(err) + } + if err := s.Authentication.Validate(); err != nil { + return maskAny(err) + } if err := s.TLS.Validate(); err != nil { return maskAny(err) } @@ -78,16 +66,11 @@ func (s SyncSpec) Validate(mode DeploymentMode) error { } // SetDefaults fills in missing defaults -func (s *SyncSpec) SetDefaults(defaultImage string, defaulPullPolicy v1.PullPolicy, defaultJWTSecretName, defaultCASecretName string) { - if s.GetImage() == "" { - s.Image = util.NewString(defaultImage) - } - if s.GetImagePullPolicy() == "" { - s.ImagePullPolicy = util.NewPullPolicy(defaulPullPolicy) - } - s.Authentication.SetDefaults(defaultJWTSecretName) - s.TLS.SetDefaults(defaultCASecretName) - s.Monitoring.SetDefaults() +func (s *SyncSpec) SetDefaults(defaultJWTSecretName, defaultClientAuthCASecretName, defaultTLSCASecretName, defaultMonitoringSecretName string) { + s.ExternalAccess.SetDefaults() + s.Authentication.SetDefaults(defaultJWTSecretName, defaultClientAuthCASecretName) + s.TLS.SetDefaults(defaultTLSCASecretName) + s.Monitoring.SetDefaults(defaultMonitoringSecretName) } // SetDefaultsFrom fills unspecified fields with a value from given source spec. @@ -95,12 +78,7 @@ func (s *SyncSpec) SetDefaultsFrom(source SyncSpec) { if s.Enabled == nil { s.Enabled = util.NewBoolOrNil(source.Enabled) } - if s.Image == nil { - s.Image = util.NewStringOrNil(source.Image) - } - if s.ImagePullPolicy == nil { - s.ImagePullPolicy = util.NewPullPolicyOrNil(source.ImagePullPolicy) - } + s.ExternalAccess.SetDefaultsFrom(source.ExternalAccess) s.Authentication.SetDefaultsFrom(source.Authentication) s.TLS.SetDefaultsFrom(source.TLS) s.Monitoring.SetDefaultsFrom(source.Monitoring) @@ -111,6 +89,9 @@ func (s *SyncSpec) SetDefaultsFrom(source SyncSpec) { // Field names are relative to given field prefix. func (s SyncSpec) ResetImmutableFields(fieldPrefix string, target *SyncSpec) []string { var resetFields []string + if list := s.ExternalAccess.ResetImmutableFields(fieldPrefix+".externalAccess", &target.ExternalAccess); len(list) > 0 { + resetFields = append(resetFields, list...) + } if list := s.Authentication.ResetImmutableFields(fieldPrefix+".auth", &target.Authentication); len(list) > 0 { resetFields = append(resetFields, list...) } diff --git a/pkg/apis/deployment/v1alpha/sync_spec_test.go b/pkg/apis/deployment/v1alpha/sync_spec_test.go index 567edf19c..6d0b18f08 100644 --- a/pkg/apis/deployment/v1alpha/sync_spec_test.go +++ b/pkg/apis/deployment/v1alpha/sync_spec_test.go @@ -27,41 +27,34 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util" "github.com/stretchr/testify/assert" - "k8s.io/api/core/v1" ) func TestSyncSpecValidate(t *testing.T) { // Valid - auth := AuthenticationSpec{JWTSecretName: util.NewString("foo")} + auth := SyncAuthenticationSpec{JWTSecretName: util.NewString("foo"), ClientCASecretName: util.NewString("foo-client")} tls := TLSSpec{CASecretName: util.NewString("None")} - assert.Nil(t, SyncSpec{Image: util.NewString("foo"), Authentication: auth}.Validate(DeploymentModeSingle)) - assert.Nil(t, SyncSpec{Image: util.NewString("foo"), Authentication: auth}.Validate(DeploymentModeResilientSingle)) - assert.Nil(t, SyncSpec{Image: util.NewString("foo"), Authentication: auth}.Validate(DeploymentModeCluster)) - assert.Nil(t, SyncSpec{Image: util.NewString("foo"), Authentication: auth, TLS: tls, Enabled: util.NewBool(true)}.Validate(DeploymentModeCluster)) + assert.Nil(t, SyncSpec{Authentication: auth}.Validate(DeploymentModeSingle)) + assert.Nil(t, SyncSpec{Authentication: auth}.Validate(DeploymentModeActiveFailover)) + assert.Nil(t, SyncSpec{Authentication: auth}.Validate(DeploymentModeCluster)) + assert.Nil(t, SyncSpec{Authentication: auth, TLS: tls, Enabled: util.NewBool(true)}.Validate(DeploymentModeCluster)) // Not valid - assert.Error(t, SyncSpec{Image: util.NewString(""), Authentication: auth}.Validate(DeploymentModeSingle)) - assert.Error(t, SyncSpec{Image: util.NewString(""), Authentication: auth}.Validate(DeploymentModeResilientSingle)) - assert.Error(t, SyncSpec{Image: util.NewString(""), Authentication: auth}.Validate(DeploymentModeCluster)) - assert.Error(t, SyncSpec{Image: util.NewString("foo"), Authentication: auth, TLS: tls, Enabled: util.NewBool(true)}.Validate(DeploymentModeSingle)) - assert.Error(t, SyncSpec{Image: util.NewString("foo"), Authentication: auth, TLS: tls, Enabled: util.NewBool(true)}.Validate(DeploymentModeResilientSingle)) + assert.Error(t, SyncSpec{Authentication: auth, TLS: tls, Enabled: util.NewBool(true)}.Validate(DeploymentModeSingle)) + assert.Error(t, SyncSpec{Authentication: auth, TLS: tls, Enabled: util.NewBool(true)}.Validate(DeploymentModeActiveFailover)) } func TestSyncSpecSetDefaults(t *testing.T) { def := func(spec SyncSpec) SyncSpec { - spec.SetDefaults("test-image", v1.PullAlways, "test-jwt", "test-ca") + spec.SetDefaults("test-jwt", "test-client-auth-ca", "test-tls-ca", "test-mon") return spec } assert.False(t, def(SyncSpec{}).IsEnabled()) assert.False(t, def(SyncSpec{Enabled: util.NewBool(false)}).IsEnabled()) assert.True(t, def(SyncSpec{Enabled: util.NewBool(true)}).IsEnabled()) - assert.Equal(t, "test-image", def(SyncSpec{}).GetImage()) - assert.Equal(t, "foo", def(SyncSpec{Image: util.NewString("foo")}).GetImage()) - assert.Equal(t, v1.PullAlways, def(SyncSpec{}).GetImagePullPolicy()) - assert.Equal(t, v1.PullNever, def(SyncSpec{ImagePullPolicy: util.NewPullPolicy(v1.PullNever)}).GetImagePullPolicy()) assert.Equal(t, "test-jwt", def(SyncSpec{}).Authentication.GetJWTSecretName()) - assert.Equal(t, "foo", def(SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("foo")}}).Authentication.GetJWTSecretName()) + assert.Equal(t, "test-mon", def(SyncSpec{}).Monitoring.GetTokenSecretName()) + assert.Equal(t, "foo", def(SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("foo")}}).Authentication.GetJWTSecretName()) } func TestSyncSpecResetImmutableFields(t *testing.T) { @@ -85,49 +78,23 @@ func TestSyncSpecResetImmutableFields(t *testing.T) { nil, }, { - SyncSpec{Image: util.NewString("foo")}, - SyncSpec{Image: util.NewString("foo2")}, - SyncSpec{Image: util.NewString("foo2")}, + SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("None"), ClientCASecretName: util.NewString("some")}}, + SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("None"), ClientCASecretName: util.NewString("some")}}, + SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("None"), ClientCASecretName: util.NewString("some")}}, nil, }, { - SyncSpec{ImagePullPolicy: util.NewPullPolicy(v1.PullAlways)}, - SyncSpec{ImagePullPolicy: util.NewPullPolicy(v1.PullNever)}, - SyncSpec{ImagePullPolicy: util.NewPullPolicy(v1.PullNever)}, + SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("foo"), ClientCASecretName: util.NewString("some")}}, + SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("foo"), ClientCASecretName: util.NewString("some")}}, + SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("foo"), ClientCASecretName: util.NewString("some")}}, nil, }, { - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("None")}}, - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("None")}}, - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("None")}}, + SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("foo"), ClientCASecretName: util.NewString("some")}}, + SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("foo2"), ClientCASecretName: util.NewString("some")}}, + SyncSpec{Authentication: SyncAuthenticationSpec{JWTSecretName: util.NewString("foo2"), ClientCASecretName: util.NewString("some")}}, nil, }, - { - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("foo")}}, - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("foo")}}, - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("foo")}}, - nil, - }, - { - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("foo")}}, - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("foo2")}}, - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("foo2")}}, - nil, - }, - - // Invalid changes - { - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("foo")}}, - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("None")}}, - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("foo")}}, - []string{"test.auth.jwtSecretName"}, - }, - { - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("None")}}, - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("foo")}}, - SyncSpec{Authentication: AuthenticationSpec{JWTSecretName: util.NewString("None")}}, - []string{"test.auth.jwtSecretName"}, - }, } for _, test := range tests { diff --git a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go index 030f9f26c..c8d083f60 100644 --- a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go @@ -253,6 +253,16 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { **out = **in } } + if in.DowntimeAllowed != nil { + in, out := &in.DowntimeAllowed, &out.DowntimeAllowed + if *in == nil { + *out = nil + } else { + *out = new(bool) + **out = **in + } + } + in.ExternalAccess.DeepCopyInto(&out.ExternalAccess) in.RocksDB.DeepCopyInto(&out.RocksDB) in.Authentication.DeepCopyInto(&out.Authentication) in.TLS.DeepCopyInto(&out.TLS) @@ -389,6 +399,49 @@ func (in *DeploymentStatusMembers) DeepCopy() *DeploymentStatusMembers { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalAccessSpec) DeepCopyInto(out *ExternalAccessSpec) { + *out = *in + if in.Type != nil { + in, out := &in.Type, &out.Type + if *in == nil { + *out = nil + } else { + *out = new(ExternalAccessType) + **out = **in + } + } + if in.NodePort != nil { + in, out := &in.NodePort, &out.NodePort + if *in == nil { + *out = nil + } else { + *out = new(int) + **out = **in + } + } + if in.LoadBalancerIP != nil { + in, out := &in.LoadBalancerIP, &out.LoadBalancerIP + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalAccessSpec. +func (in *ExternalAccessSpec) DeepCopy() *ExternalAccessSpec { + if in == nil { + return nil + } + out := new(ExternalAccessSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ImageInfo) DeepCopyInto(out *ImageInfo) { *out = *in @@ -546,6 +599,22 @@ func (in *ServerGroupSpec) DeepCopyInto(out *ServerGroupSpec) { } } in.Resources.DeepCopyInto(&out.Resources) + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]core_v1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ServiceAccountName != nil { + in, out := &in.ServiceAccountName, &out.ServiceAccountName + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } return } @@ -560,19 +629,19 @@ func (in *ServerGroupSpec) DeepCopy() *ServerGroupSpec { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *SyncSpec) DeepCopyInto(out *SyncSpec) { +func (in *SyncAuthenticationSpec) DeepCopyInto(out *SyncAuthenticationSpec) { *out = *in - if in.Enabled != nil { - in, out := &in.Enabled, &out.Enabled + if in.JWTSecretName != nil { + in, out := &in.JWTSecretName, &out.JWTSecretName if *in == nil { *out = nil } else { - *out = new(bool) + *out = new(string) **out = **in } } - if in.Image != nil { - in, out := &in.Image, &out.Image + if in.ClientCASecretName != nil { + in, out := &in.ClientCASecretName, &out.ClientCASecretName if *in == nil { *out = nil } else { @@ -580,15 +649,59 @@ func (in *SyncSpec) DeepCopyInto(out *SyncSpec) { **out = **in } } - if in.ImagePullPolicy != nil { - in, out := &in.ImagePullPolicy, &out.ImagePullPolicy + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SyncAuthenticationSpec. +func (in *SyncAuthenticationSpec) DeepCopy() *SyncAuthenticationSpec { + if in == nil { + return nil + } + out := new(SyncAuthenticationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SyncExternalAccessSpec) DeepCopyInto(out *SyncExternalAccessSpec) { + *out = *in + in.ExternalAccessSpec.DeepCopyInto(&out.ExternalAccessSpec) + if in.MasterEndpoint != nil { + in, out := &in.MasterEndpoint, &out.MasterEndpoint + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.AccessPackageSecretNames != nil { + in, out := &in.AccessPackageSecretNames, &out.AccessPackageSecretNames + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SyncExternalAccessSpec. +func (in *SyncExternalAccessSpec) DeepCopy() *SyncExternalAccessSpec { + if in == nil { + return nil + } + out := new(SyncExternalAccessSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *SyncSpec) DeepCopyInto(out *SyncSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled if *in == nil { *out = nil } else { - *out = new(core_v1.PullPolicy) + *out = new(bool) **out = **in } } + in.ExternalAccess.DeepCopyInto(&out.ExternalAccess) in.Authentication.DeepCopyInto(&out.Authentication) in.TLS.DeepCopyInto(&out.TLS) in.Monitoring.DeepCopyInto(&out.Monitoring) diff --git a/pkg/apis/replication/v1alpha/collection_status.go b/pkg/apis/replication/v1alpha/collection_status.go new file mode 100644 index 000000000..8e471266b --- /dev/null +++ b/pkg/apis/replication/v1alpha/collection_status.go @@ -0,0 +1,32 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +// CollectionStatus contains the status of a single collection. +type CollectionStatus struct { + // Name of the collection + Name string `json:"name"` + // Replication status per shard. + // The list is ordered by shard index (0..noShards-1) + Shards []ShardStatus `json:"shards,omitempty"` +} diff --git a/pkg/apis/replication/v1alpha/conditions.go b/pkg/apis/replication/v1alpha/conditions.go new file mode 100644 index 000000000..86eeea935 --- /dev/null +++ b/pkg/apis/replication/v1alpha/conditions.go @@ -0,0 +1,131 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// ConditionType is a strongly typed condition name +type ConditionType string + +const ( + // ConditionTypeConfigured indicates that the replication has been configured. + ConditionTypeConfigured ConditionType = "Configured" +) + +// Condition represents one current condition of a deployment or deployment member. +// A condition might not show up if it is not happening. +// For example, if a cluster is not upgrading, the Upgrading condition would not show up. +type Condition struct { + // Type of condition. + Type ConditionType `json:"type"` + // Status of the condition, one of True, False, Unknown. + Status v1.ConditionStatus `json:"status"` + // The last time this condition was updated. + LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` + // Last time the condition transitioned from one status to another. + LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"` + // The reason for the condition's last transition. + Reason string `json:"reason,omitempty"` + // A human readable message indicating details about the transition. + Message string `json:"message,omitempty"` +} + +// ConditionList is a list of conditions. +// Each type is allowed only once. +type ConditionList []Condition + +// IsTrue return true when a condition with given type exists and its status is `True`. +func (list ConditionList) IsTrue(conditionType ConditionType) bool { + c, found := list.Get(conditionType) + return found && c.Status == v1.ConditionTrue +} + +// Get a condition by type. +// Returns true if found, false if not found. +func (list ConditionList) Get(conditionType ConditionType) (Condition, bool) { + for _, x := range list { + if x.Type == conditionType { + return x, true + } + } + // Not found + return Condition{}, false +} + +// Update the condition, replacing an old condition with same type (if any) +// Returns true when changes were made, false otherwise. +func (list *ConditionList) Update(conditionType ConditionType, status bool, reason, message string) bool { + src := *list + statusX := v1.ConditionFalse + if status { + statusX = v1.ConditionTrue + } + for i, x := range src { + if x.Type == conditionType { + if x.Status != statusX { + // Transition to another status + src[i].Status = statusX + now := metav1.Now() + src[i].LastTransitionTime = now + src[i].LastUpdateTime = now + src[i].Reason = reason + src[i].Message = message + } else if x.Reason != reason || x.Message != message { + src[i].LastUpdateTime = metav1.Now() + src[i].Reason = reason + src[i].Message = message + } else { + return false + } + return true + } + } + // Not found + now := metav1.Now() + *list = append(src, Condition{ + Type: conditionType, + LastUpdateTime: now, + LastTransitionTime: now, + Status: statusX, + Reason: reason, + Message: message, + }) + return true +} + +// Remove the condition with given type. +// Returns true if removed, or false if not found. +func (list *ConditionList) Remove(conditionType ConditionType) bool { + src := *list + for i, x := range src { + if x.Type == conditionType { + *list = append(src[:i], src[i+1:]...) + return true + } + } + // Not found + return false +} diff --git a/pkg/apis/replication/v1alpha/conditions_test.go b/pkg/apis/replication/v1alpha/conditions_test.go new file mode 100644 index 000000000..26dcbdbc0 --- /dev/null +++ b/pkg/apis/replication/v1alpha/conditions_test.go @@ -0,0 +1,95 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConditionListIsTrue(t *testing.T) { + assert.False(t, ConditionList{}.IsTrue(ConditionTypeConfigured)) + + cl := ConditionList{} + cl.Update(ConditionTypeConfigured, true, "test", "msg") + assert.True(t, cl.IsTrue(ConditionTypeConfigured)) + //assert.False(t, cl.IsTrue(ConditionTypeTerminated)) + + cl.Update(ConditionTypeConfigured, false, "test", "msg") + assert.False(t, cl.IsTrue(ConditionTypeConfigured)) + + cl.Remove(ConditionTypeConfigured) + assert.False(t, cl.IsTrue(ConditionTypeConfigured)) + assert.Equal(t, 0, len(cl)) +} + +func TestConditionListGet(t *testing.T) { + conv := func(c Condition, b bool) []interface{} { + return []interface{}{c, b} + } + + cl := ConditionList{} + assert.EqualValues(t, conv(Condition{}, false), conv(cl.Get(ConditionTypeConfigured))) + cl.Update(ConditionTypeConfigured, false, "test", "msg") + assert.EqualValues(t, conv(cl[0], true), conv(cl.Get(ConditionTypeConfigured))) +} + +func TestConditionListUpdate(t *testing.T) { + cl := ConditionList{} + assert.Equal(t, 0, len(cl)) + + assert.True(t, cl.Update(ConditionTypeConfigured, true, "test", "msg")) + assert.True(t, cl.IsTrue(ConditionTypeConfigured)) + assert.Equal(t, 1, len(cl)) + + assert.False(t, cl.Update(ConditionTypeConfigured, true, "test", "msg")) + assert.True(t, cl.IsTrue(ConditionTypeConfigured)) + assert.Equal(t, 1, len(cl)) + + assert.True(t, cl.Update(ConditionTypeConfigured, false, "test", "msg")) + assert.False(t, cl.IsTrue(ConditionTypeConfigured)) + assert.Equal(t, 1, len(cl)) + + assert.True(t, cl.Update(ConditionTypeConfigured, false, "test2", "msg")) + assert.False(t, cl.IsTrue(ConditionTypeConfigured)) + assert.Equal(t, 1, len(cl)) + + assert.True(t, cl.Update(ConditionTypeConfigured, false, "test2", "msg2")) + assert.False(t, cl.IsTrue(ConditionTypeConfigured)) + assert.Equal(t, 1, len(cl)) +} + +func TestConditionListRemove(t *testing.T) { + cl := ConditionList{} + assert.Equal(t, 0, len(cl)) + + cl.Update(ConditionTypeConfigured, true, "test", "msg") + assert.Equal(t, 1, len(cl)) + + assert.True(t, cl.Remove(ConditionTypeConfigured)) + assert.Equal(t, 0, len(cl)) + + assert.False(t, cl.Remove(ConditionTypeConfigured)) + assert.Equal(t, 0, len(cl)) +} diff --git a/pkg/apis/replication/v1alpha/database_status.go b/pkg/apis/replication/v1alpha/database_status.go new file mode 100644 index 000000000..b80e0695e --- /dev/null +++ b/pkg/apis/replication/v1alpha/database_status.go @@ -0,0 +1,32 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +// DatabaseStatus contains the status of a single database. +type DatabaseStatus struct { + // Name of the database + Name string `json:"name"` + // Collections holds the replication status of each collection in the database. + // List is ordered by name of the collection. + Collections []CollectionStatus `json:"collections,omitempty"` +} diff --git a/pkg/apis/replication/v1alpha/doc.go b/pkg/apis/replication/v1alpha/doc.go new file mode 100644 index 000000000..a33182847 --- /dev/null +++ b/pkg/apis/replication/v1alpha/doc.go @@ -0,0 +1,25 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +// +k8s:deepcopy-gen=package +// +groupName=replication.database.arangodb.com +package v1alpha diff --git a/pkg/apis/replication/v1alpha/endpoint_authentication_spec.go b/pkg/apis/replication/v1alpha/endpoint_authentication_spec.go new file mode 100644 index 000000000..f2c537ee6 --- /dev/null +++ b/pkg/apis/replication/v1alpha/endpoint_authentication_spec.go @@ -0,0 +1,89 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/pkg/errors" +) + +// EndpointAuthenticationSpec contains the specification to authentication with the syncmasters +// in either source or destination endpoint. +type EndpointAuthenticationSpec struct { + // KeyfileSecretName holds the name of a Secret containing a client authentication + // certificate formatted at keyfile in a `tls.keyfile` field. + KeyfileSecretName *string `json:"keyfileSecretName,omitempty"` + // UserSecretName holds the name of a Secret containing a `username` & `password` + // field used for basic authentication. + // The user identified by the username must have write access in the `_system` database + // of the ArangoDB cluster at the endpoint. + UserSecretName *string `json:"userSecretName,omitempty"` +} + +// GetKeyfileSecretName returns the value of keyfileSecretName. +func (s EndpointAuthenticationSpec) GetKeyfileSecretName() string { + return util.StringOrDefault(s.KeyfileSecretName) +} + +// GetUserSecretName returns the value of userSecretName. +func (s EndpointAuthenticationSpec) GetUserSecretName() string { + return util.StringOrDefault(s.UserSecretName) +} + +// Validate the given spec, returning an error on validation +// problems or nil if all ok. +func (s EndpointAuthenticationSpec) Validate(keyfileSecretNameRequired bool) error { + if err := k8sutil.ValidateOptionalResourceName(s.GetKeyfileSecretName()); err != nil { + return maskAny(err) + } + if err := k8sutil.ValidateOptionalResourceName(s.GetUserSecretName()); err != nil { + return maskAny(err) + } + if keyfileSecretNameRequired && s.GetKeyfileSecretName() == "" { + return maskAny(errors.Wrapf(ValidationError, "Provide a keyfileSecretName")) + } + return nil +} + +// SetDefaults fills empty field with default values. +func (s *EndpointAuthenticationSpec) SetDefaults() { +} + +// SetDefaultsFrom fills empty field with default values from the given source. +func (s *EndpointAuthenticationSpec) SetDefaultsFrom(source EndpointAuthenticationSpec) { + if s.KeyfileSecretName == nil { + s.KeyfileSecretName = util.NewStringOrNil(source.KeyfileSecretName) + } + if s.UserSecretName == nil { + s.UserSecretName = util.NewStringOrNil(source.UserSecretName) + } +} + +// ResetImmutableFields replaces all immutable fields in the given target with values from the source spec. +// It returns a list of fields that have been reset. +// Field names are relative to `spec.`. +func (s EndpointAuthenticationSpec) ResetImmutableFields(target *EndpointAuthenticationSpec, fieldPrefix string) []string { + var result []string + return result +} diff --git a/pkg/apis/replication/v1alpha/endpoint_spec.go b/pkg/apis/replication/v1alpha/endpoint_spec.go new file mode 100644 index 000000000..8a72e2151 --- /dev/null +++ b/pkg/apis/replication/v1alpha/endpoint_spec.go @@ -0,0 +1,111 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + "net/url" + + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/pkg/errors" +) + +// EndpointSpec contains the specification used to reach the syncmasters +// in either source or destination mode. +type EndpointSpec struct { + // DeploymentName holds the name of an ArangoDeployment resource. + // If set this provides default values for masterEndpoint, auth & tls. + DeploymentName *string `json:"deploymentName,omitempty"` + // MasterEndpoint holds a list of URLs used to reach the syncmaster(s). + MasterEndpoint []string `json:"masterEndpoint,omitempty"` + // Authentication holds settings needed to authentication at the syncmaster. + Authentication EndpointAuthenticationSpec `json:"auth"` + // TLS holds settings needed to verify the TLS connection to the syncmaster. + TLS EndpointTLSSpec `json:"tls"` +} + +// GetDeploymentName returns the value of deploymentName. +func (s EndpointSpec) GetDeploymentName() string { + return util.StringOrDefault(s.DeploymentName) +} + +// HasDeploymentName returns the true when a non-empty deployment name it set. +func (s EndpointSpec) HasDeploymentName() bool { + return s.GetDeploymentName() != "" +} + +// Validate the given spec, returning an error on validation +// problems or nil if all ok. +func (s EndpointSpec) Validate(isSourceEndpoint bool) error { + if err := k8sutil.ValidateOptionalResourceName(s.GetDeploymentName()); err != nil { + return maskAny(err) + } + for _, ep := range s.MasterEndpoint { + if _, err := url.Parse(ep); err != nil { + return maskAny(errors.Wrapf(ValidationError, "Invalid master endpoint '%s': %s", ep, err)) + } + } + hasDeploymentName := s.HasDeploymentName() + if !hasDeploymentName && len(s.MasterEndpoint) == 0 { + return maskAny(errors.Wrapf(ValidationError, "Provide a deploy name or at least one master endpoint")) + } + if err := s.Authentication.Validate(isSourceEndpoint || !hasDeploymentName); err != nil { + return maskAny(err) + } + if err := s.TLS.Validate(!hasDeploymentName); err != nil { + return maskAny(err) + } + return nil +} + +// SetDefaults fills empty field with default values. +func (s *EndpointSpec) SetDefaults() { + s.Authentication.SetDefaults() + s.TLS.SetDefaults() +} + +// SetDefaultsFrom fills empty field with default values from the given source. +func (s *EndpointSpec) SetDefaultsFrom(source EndpointSpec) { + if s.DeploymentName == nil { + s.DeploymentName = util.NewStringOrNil(source.DeploymentName) + } + s.Authentication.SetDefaultsFrom(source.Authentication) + s.TLS.SetDefaultsFrom(source.TLS) +} + +// ResetImmutableFields replaces all immutable fields in the given target with values from the source spec. +// It returns a list of fields that have been reset. +// Field names are relative to `spec.`. +func (s EndpointSpec) ResetImmutableFields(target *EndpointSpec, fieldPrefix string) []string { + var result []string + if s.GetDeploymentName() != target.GetDeploymentName() { + result = append(result, fieldPrefix+"deploymentName") + } + if list := s.Authentication.ResetImmutableFields(&target.Authentication, fieldPrefix+"auth."); len(list) > 0 { + result = append(result, list...) + } + if list := s.TLS.ResetImmutableFields(&target.TLS, fieldPrefix+"tls."); len(list) > 0 { + result = append(result, list...) + } + return result +} diff --git a/pkg/apis/replication/v1alpha/endpoint_status.go b/pkg/apis/replication/v1alpha/endpoint_status.go new file mode 100644 index 000000000..91e51d213 --- /dev/null +++ b/pkg/apis/replication/v1alpha/endpoint_status.go @@ -0,0 +1,30 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +// EndpointStatus contains the status of either the source or destination endpoint. +type EndpointStatus struct { + // Databases holds the replication status of all databases from the point of view of this endpoint. + // List is ordered by name of the database. + Databases []DatabaseStatus `json:"databases,omitempty"` +} diff --git a/pkg/apis/replication/v1alpha/endpoint_tls_spec.go b/pkg/apis/replication/v1alpha/endpoint_tls_spec.go new file mode 100644 index 000000000..2ae74d3ca --- /dev/null +++ b/pkg/apis/replication/v1alpha/endpoint_tls_spec.go @@ -0,0 +1,72 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/pkg/errors" +) + +// EndpointTLSSpec contains the specification regarding the TLS connection to the syncmasters +// in either source or destination endpoint. +type EndpointTLSSpec struct { + // CASecretName holds the name of a Secret containing a ca.crt public key for TLS validation. + CASecretName *string `json:"caSecretName,omitempty"` +} + +// GetCASecretName returns the value of caSecretName. +func (s EndpointTLSSpec) GetCASecretName() string { + return util.StringOrDefault(s.CASecretName) +} + +// Validate the given spec, returning an error on validation +// problems or nil if all ok. +func (s EndpointTLSSpec) Validate(caSecretNameRequired bool) error { + if err := k8sutil.ValidateOptionalResourceName(s.GetCASecretName()); err != nil { + return maskAny(err) + } + if caSecretNameRequired && s.GetCASecretName() == "" { + return maskAny(errors.Wrapf(ValidationError, "Provide a caSecretName")) + } + return nil +} + +// SetDefaults fills empty field with default values. +func (s *EndpointTLSSpec) SetDefaults() { +} + +// SetDefaultsFrom fills empty field with default values from the given source. +func (s *EndpointTLSSpec) SetDefaultsFrom(source EndpointTLSSpec) { + if s.CASecretName == nil { + s.CASecretName = util.NewStringOrNil(source.CASecretName) + } +} + +// ResetImmutableFields replaces all immutable fields in the given target with values from the source spec. +// It returns a list of fields that have been reset. +// Field names are relative to `spec.`. +func (s EndpointTLSSpec) ResetImmutableFields(target *EndpointTLSSpec, fieldPrefix string) []string { + var result []string + return result +} diff --git a/pkg/apis/replication/v1alpha/errors.go b/pkg/apis/replication/v1alpha/errors.go new file mode 100644 index 000000000..64f72e81c --- /dev/null +++ b/pkg/apis/replication/v1alpha/errors.go @@ -0,0 +1,37 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import "github.com/pkg/errors" + +var ( + // ValidationError indicates a validation failure + ValidationError = errors.New("validation failed") + + maskAny = errors.WithStack +) + +// IsValidation return true when the given error is or is caused by a ValidationError. +func IsValidation(err error) bool { + return errors.Cause(err) == ValidationError +} diff --git a/pkg/apis/replication/v1alpha/register.go b/pkg/apis/replication/v1alpha/register.go new file mode 100644 index 000000000..ccab2f54d --- /dev/null +++ b/pkg/apis/replication/v1alpha/register.go @@ -0,0 +1,59 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +const ( + ArangoDeploymentReplicationResourceKind = "ArangoDeploymentReplication" + ArangoDeploymentReplicationResourcePlural = "arangodeploymentreplications" + groupName = "replication.database.arangodb.com" +) + +var ( + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = SchemeBuilder.AddToScheme + + SchemeGroupVersion = schema.GroupVersion{Group: groupName, Version: "v1alpha"} + ArangoDeploymentReplicationCRDName = ArangoDeploymentReplicationResourcePlural + "." + groupName + ArangoDeploymentReplicationShortNames = []string{"arangorepl"} +) + +// Resource gets an ArangoCluster GroupResource for a specified resource +func Resource(resource string) schema.GroupResource { + return SchemeGroupVersion.WithResource(resource).GroupResource() +} + +// addKnownTypes adds the set of types defined in this package to the supplied scheme. +func addKnownTypes(s *runtime.Scheme) error { + s.AddKnownTypes(SchemeGroupVersion, + &ArangoDeploymentReplication{}, + &ArangoDeploymentReplicationList{}, + ) + metav1.AddToGroupVersion(s, SchemeGroupVersion) + return nil +} diff --git a/pkg/apis/replication/v1alpha/replication.go b/pkg/apis/replication/v1alpha/replication.go new file mode 100644 index 000000000..4bb3be55f --- /dev/null +++ b/pkg/apis/replication/v1alpha/replication.go @@ -0,0 +1,63 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ArangoDeploymentReplicationList is a list of ArangoDB deployment replications. +type ArangoDeploymentReplicationList struct { + metav1.TypeMeta `json:",inline"` + // Standard list metadata + // More info: http://releases.k8s.io/HEAD/docs/devel/api-conventions.md#metadata + metav1.ListMeta `json:"metadata,omitempty"` + Items []ArangoDeploymentReplication `json:"items"` +} + +// +genclient +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// ArangoDeploymentReplication contains the entire Kubernetes info for an ArangoDB +// local storage provider. +type ArangoDeploymentReplication struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + Spec DeploymentReplicationSpec `json:"spec"` + Status DeploymentReplicationStatus `json:"status"` +} + +// AsOwner creates an OwnerReference for the given replication +func (d *ArangoDeploymentReplication) AsOwner() metav1.OwnerReference { + trueVar := true + return metav1.OwnerReference{ + APIVersion: SchemeGroupVersion.String(), + Kind: ArangoDeploymentReplicationResourceKind, + Name: d.Name, + UID: d.UID, + Controller: &trueVar, + BlockOwnerDeletion: &trueVar, + } +} diff --git a/pkg/apis/replication/v1alpha/replication_phase.go b/pkg/apis/replication/v1alpha/replication_phase.go new file mode 100644 index 000000000..78fcc459c --- /dev/null +++ b/pkg/apis/replication/v1alpha/replication_phase.go @@ -0,0 +1,39 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +// DeploymentReplicationPhase is a strongly typed lifetime phase of a deployment replication +type DeploymentReplicationPhase string + +const ( + // DeploymentReplicationPhaseNone indicates that the phase is not set yet + DeploymentReplicationPhaseNone DeploymentReplicationPhase = "" + // DeploymentReplicationPhaseFailed indicates that a deployment replication is in a failed state + // from which automatic recovery is impossible. Inspect `Reason` for more info. + DeploymentReplicationPhaseFailed DeploymentReplicationPhase = "Failed" +) + +// IsFailed returns true if given state is DeploymentStateFailed +func (cs DeploymentReplicationPhase) IsFailed() bool { + return cs == DeploymentReplicationPhaseFailed +} diff --git a/pkg/apis/replication/v1alpha/replication_spec.go b/pkg/apis/replication/v1alpha/replication_spec.go new file mode 100644 index 000000000..8d6fd403f --- /dev/null +++ b/pkg/apis/replication/v1alpha/replication_spec.go @@ -0,0 +1,68 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +// DeploymentReplicationSpec contains the specification part of +// an ArangoDeploymentReplication. +type DeploymentReplicationSpec struct { + Source EndpointSpec `json:"source"` + Destination EndpointSpec `json:"destination"` +} + +// Validate the given spec, returning an error on validation +// problems or nil if all ok. +func (s DeploymentReplicationSpec) Validate() error { + if err := s.Source.Validate(true); err != nil { + return maskAny(err) + } + if err := s.Destination.Validate(false); err != nil { + return maskAny(err) + } + return nil +} + +// SetDefaults fills empty field with default values. +func (s *DeploymentReplicationSpec) SetDefaults() { + s.Source.SetDefaults() + s.Destination.SetDefaults() +} + +// SetDefaultsFrom fills empty field with default values from the given source. +func (s *DeploymentReplicationSpec) SetDefaultsFrom(source DeploymentReplicationSpec) { + s.Source.SetDefaultsFrom(source.Source) + s.Destination.SetDefaultsFrom(source.Destination) +} + +// ResetImmutableFields replaces all immutable fields in the given target with values from the source spec. +// It returns a list of fields that have been reset. +// Field names are relative to `spec.`. +func (s DeploymentReplicationSpec) ResetImmutableFields(target *DeploymentReplicationSpec) []string { + var result []string + if list := s.Source.ResetImmutableFields(&target.Source, "source."); len(list) > 0 { + result = append(result, list...) + } + if list := s.Destination.ResetImmutableFields(&target.Destination, "destination."); len(list) > 0 { + result = append(result, list...) + } + return result +} diff --git a/pkg/apis/replication/v1alpha/replication_status.go b/pkg/apis/replication/v1alpha/replication_status.go new file mode 100644 index 000000000..10693216c --- /dev/null +++ b/pkg/apis/replication/v1alpha/replication_status.go @@ -0,0 +1,44 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +// DeploymentReplicationStatus contains the status part of +// an ArangoDeploymentReplication. +type DeploymentReplicationStatus struct { + // Phase holds the current lifetime phase of the deployment replication + Phase DeploymentReplicationPhase `json:"phase,omitempty"` + // Reason contains a human readable reason for reaching the current phase (can be empty) + Reason string `json:"reason,omitempty"` // Reason for current phase + + // Conditions specific to the entire deployment replication + Conditions ConditionList `json:"conditions,omitempty"` + + // Source contains the detailed status of the source endpoint + Source EndpointStatus `json:"source"` + // Destination contains the detailed status of the destination endpoint + Destination EndpointStatus `json:"destination"` + + // CancelFailures records the number of times that the configuration was canceled + // which resulted in an error. + CancelFailures int `json:"cancel-failures,omitempty"` +} diff --git a/pkg/apis/replication/v1alpha/shard_status.go b/pkg/apis/replication/v1alpha/shard_status.go new file mode 100644 index 000000000..65d275f29 --- /dev/null +++ b/pkg/apis/replication/v1alpha/shard_status.go @@ -0,0 +1,28 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1alpha + +// ShardStatus contains the status of a single shard. +type ShardStatus struct { + Status string `json:"status"` +} diff --git a/pkg/apis/replication/v1alpha/zz_generated.deepcopy.go b/pkg/apis/replication/v1alpha/zz_generated.deepcopy.go new file mode 100644 index 000000000..ccfa8d871 --- /dev/null +++ b/pkg/apis/replication/v1alpha/zz_generated.deepcopy.go @@ -0,0 +1,325 @@ +// +build !ignore_autogenerated + +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +// This file was autogenerated by deepcopy-gen. Do not edit it manually! + +package v1alpha + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArangoDeploymentReplication) DeepCopyInto(out *ArangoDeploymentReplication) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoDeploymentReplication. +func (in *ArangoDeploymentReplication) DeepCopy() *ArangoDeploymentReplication { + if in == nil { + return nil + } + out := new(ArangoDeploymentReplication) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ArangoDeploymentReplication) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArangoDeploymentReplicationList) DeepCopyInto(out *ArangoDeploymentReplicationList) { + *out = *in + out.TypeMeta = in.TypeMeta + out.ListMeta = in.ListMeta + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]ArangoDeploymentReplication, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArangoDeploymentReplicationList. +func (in *ArangoDeploymentReplicationList) DeepCopy() *ArangoDeploymentReplicationList { + if in == nil { + return nil + } + out := new(ArangoDeploymentReplicationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *ArangoDeploymentReplicationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CollectionStatus) DeepCopyInto(out *CollectionStatus) { + *out = *in + if in.Shards != nil { + in, out := &in.Shards, &out.Shards + *out = make([]ShardStatus, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CollectionStatus. +func (in *CollectionStatus) DeepCopy() *CollectionStatus { + if in == nil { + return nil + } + out := new(CollectionStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Condition) DeepCopyInto(out *Condition) { + *out = *in + in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime) + in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Condition. +func (in *Condition) DeepCopy() *Condition { + if in == nil { + return nil + } + out := new(Condition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DatabaseStatus) DeepCopyInto(out *DatabaseStatus) { + *out = *in + if in.Collections != nil { + in, out := &in.Collections, &out.Collections + *out = make([]CollectionStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DatabaseStatus. +func (in *DatabaseStatus) DeepCopy() *DatabaseStatus { + if in == nil { + return nil + } + out := new(DatabaseStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentReplicationSpec) DeepCopyInto(out *DeploymentReplicationSpec) { + *out = *in + in.Source.DeepCopyInto(&out.Source) + in.Destination.DeepCopyInto(&out.Destination) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentReplicationSpec. +func (in *DeploymentReplicationSpec) DeepCopy() *DeploymentReplicationSpec { + if in == nil { + return nil + } + out := new(DeploymentReplicationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DeploymentReplicationStatus) DeepCopyInto(out *DeploymentReplicationStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make(ConditionList, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + in.Source.DeepCopyInto(&out.Source) + in.Destination.DeepCopyInto(&out.Destination) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DeploymentReplicationStatus. +func (in *DeploymentReplicationStatus) DeepCopy() *DeploymentReplicationStatus { + if in == nil { + return nil + } + out := new(DeploymentReplicationStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EndpointAuthenticationSpec) DeepCopyInto(out *EndpointAuthenticationSpec) { + *out = *in + if in.KeyfileSecretName != nil { + in, out := &in.KeyfileSecretName, &out.KeyfileSecretName + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } + if in.UserSecretName != nil { + in, out := &in.UserSecretName, &out.UserSecretName + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointAuthenticationSpec. +func (in *EndpointAuthenticationSpec) DeepCopy() *EndpointAuthenticationSpec { + if in == nil { + return nil + } + out := new(EndpointAuthenticationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EndpointSpec) DeepCopyInto(out *EndpointSpec) { + *out = *in + if in.DeploymentName != nil { + in, out := &in.DeploymentName, &out.DeploymentName + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } + if in.MasterEndpoint != nil { + in, out := &in.MasterEndpoint, &out.MasterEndpoint + *out = make([]string, len(*in)) + copy(*out, *in) + } + in.Authentication.DeepCopyInto(&out.Authentication) + in.TLS.DeepCopyInto(&out.TLS) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointSpec. +func (in *EndpointSpec) DeepCopy() *EndpointSpec { + if in == nil { + return nil + } + out := new(EndpointSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EndpointStatus) DeepCopyInto(out *EndpointStatus) { + *out = *in + if in.Databases != nil { + in, out := &in.Databases, &out.Databases + *out = make([]DatabaseStatus, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointStatus. +func (in *EndpointStatus) DeepCopy() *EndpointStatus { + if in == nil { + return nil + } + out := new(EndpointStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *EndpointTLSSpec) DeepCopyInto(out *EndpointTLSSpec) { + *out = *in + if in.CASecretName != nil { + in, out := &in.CASecretName, &out.CASecretName + if *in == nil { + *out = nil + } else { + *out = new(string) + **out = **in + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new EndpointTLSSpec. +func (in *EndpointTLSSpec) DeepCopy() *EndpointTLSSpec { + if in == nil { + return nil + } + out := new(EndpointTLSSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ShardStatus) DeepCopyInto(out *ShardStatus) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ShardStatus. +func (in *ShardStatus) DeepCopy() *ShardStatus { + if in == nil { + return nil + } + out := new(ShardStatus) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/deployment/access_package.go b/pkg/deployment/access_package.go new file mode 100644 index 000000000..b9b12af77 --- /dev/null +++ b/pkg/deployment/access_package.go @@ -0,0 +1,214 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package deployment + +import ( + "strings" + "time" + + certificates "github.com/arangodb-helper/go-certificates" + "github.com/ghodss/yaml" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +const ( + clientAuthValidFor = time.Hour * 24 * 365 // 1yr + clientAuthCurve = "P256" + labelKeyOriginalDeployment = "original-deployment-name" +) + +// createAccessPackages creates a arangosync access packages specified +// in spec.sync.externalAccess.accessPackageSecretNames. +func (d *Deployment) createAccessPackages() error { + log := d.deps.Log + spec := d.apiObject.Spec + secrets := d.deps.KubeCli.CoreV1().Secrets(d.GetNamespace()) + + if !spec.Sync.IsEnabled() { + // We're only relevant when sync is enabled + return nil + } + + // Create all access packages that we're asked to build + apNameMap := make(map[string]struct{}) + for _, apSecretName := range spec.Sync.ExternalAccess.AccessPackageSecretNames { + apNameMap[apSecretName] = struct{}{} + if err := d.ensureAccessPackage(apSecretName); err != nil { + return maskAny(err) + } + } + + // Remove all access packages that we did build, but are no longer needed + secretList, err := secrets.List(metav1.ListOptions{}) + if err != nil { + log.Debug().Err(err).Msg("Failed to list secrets") + return maskAny(err) + } + for _, secret := range secretList.Items { + if d.isOwnerOf(&secret) { + if _, found := secret.Data[constants.SecretAccessPackageYaml]; found { + // Secret is an access package + if _, wanted := apNameMap[secret.GetName()]; !wanted { + // We found an obsolete access package secret. Remove it. + if err := secrets.Delete(secret.GetName(), &metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &secret.UID}, + }); err != nil && !k8sutil.IsNotFound(err) { + // Not serious enough to stop everything now, just log and create an event + log.Warn().Err(err).Msg("Failed to remove obsolete access package secret") + d.CreateEvent(k8sutil.NewErrorEvent("Access Package cleanup failed", err, d.apiObject)) + } else { + // Access package removed, notify user + log.Info().Str("secret-name", secret.GetName()).Msg("Removed access package Secret") + d.CreateEvent(k8sutil.NewAccessPackageDeletedEvent(d.apiObject, secret.GetName())) + } + } + } + } + } + + return nil +} + +// ensureAccessPackage creates an arangosync access package with given name +// it is does not already exist. +func (d *Deployment) ensureAccessPackage(apSecretName string) error { + log := d.deps.Log + ns := d.GetNamespace() + secrets := d.deps.KubeCli.CoreV1().Secrets(ns) + spec := d.apiObject.Spec + + if _, err := secrets.Get(apSecretName, metav1.GetOptions{}); err == nil { + // Secret already exists + return nil + } + + // Fetch client authentication CA + clientAuthSecretName := spec.Sync.Authentication.GetClientCASecretName() + clientAuthCert, clientAuthKey, _, err := k8sutil.GetCASecret(d.deps.KubeCli.CoreV1(), clientAuthSecretName, ns, nil) + if err != nil { + log.Debug().Err(err).Msg("Failed to get client-auth CA secret") + return maskAny(err) + } + + // Fetch TLS CA public key + tlsCASecretName := spec.Sync.TLS.GetCASecretName() + tlsCACert, err := k8sutil.GetCACertficateSecret(d.deps.KubeCli.CoreV1(), tlsCASecretName, ns) + if err != nil { + log.Debug().Err(err).Msg("Failed to get TLS CA secret") + return maskAny(err) + } + + // Create keyfile + ca, err := certificates.LoadCAFromPEM(clientAuthCert, clientAuthKey) + if err != nil { + log.Debug().Err(err).Msg("Failed to parse client-auth CA") + return maskAny(err) + } + + // Create certificate + options := certificates.CreateCertificateOptions{ + ValidFor: clientAuthValidFor, + ECDSACurve: clientAuthCurve, + IsClientAuth: true, + } + cert, key, err := certificates.CreateCertificate(options, &ca) + if err != nil { + log.Debug().Err(err).Msg("Failed to create client-auth keyfile") + return maskAny(err) + } + keyfile := strings.TrimSpace(cert) + "\n" + strings.TrimSpace(key) + + // Create secrets (in memory) + keyfileSecret := v1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: apSecretName + "-auth", + Labels: map[string]string{ + labelKeyOriginalDeployment: d.apiObject.GetName(), + }, + }, + Data: map[string][]byte{ + constants.SecretTLSKeyfile: []byte(keyfile), + }, + Type: "Opaque", + } + tlsCASecret := v1.Secret{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Secret", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: apSecretName + "-ca", + Labels: map[string]string{ + labelKeyOriginalDeployment: d.apiObject.GetName(), + }, + }, + Data: map[string][]byte{ + constants.SecretCACertificate: []byte(tlsCACert), + }, + Type: "Opaque", + } + + // Serialize secrets + keyfileYaml, err := yaml.Marshal(keyfileSecret) + if err != nil { + log.Debug().Err(err).Msg("Failed to encode client-auth keyfile Secret") + return maskAny(err) + } + tlsCAYaml, err := yaml.Marshal(tlsCASecret) + if err != nil { + log.Debug().Err(err).Msg("Failed to encode TLS CA Secret") + return maskAny(err) + } + allYaml := strings.TrimSpace(string(keyfileYaml)) + "\n---\n" + strings.TrimSpace(string(tlsCAYaml)) + + // Create secret containing access package + secret := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: apSecretName, + }, + Data: map[string][]byte{ + constants.SecretAccessPackageYaml: []byte(allYaml), + }, + } + // Attach secret to owner + secret.SetOwnerReferences(append(secret.GetOwnerReferences(), d.apiObject.AsOwner())) + if _, err := secrets.Create(secret); err != nil { + // Failed to create secret + log.Debug().Err(err).Str("secret-name", apSecretName).Msg("Failed to create access package Secret") + return maskAny(err) + } + + // Write log entry & create event + log.Info().Str("secret-name", apSecretName).Msg("Created access package Secret") + d.CreateEvent(k8sutil.NewAccessPackageCreatedEvent(d.apiObject, apSecretName)) + + return nil +} diff --git a/pkg/deployment/cleanup.go b/pkg/deployment/cleanup.go new file mode 100644 index 000000000..d285e6186 --- /dev/null +++ b/pkg/deployment/cleanup.go @@ -0,0 +1,61 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package deployment + +import ( + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// removePodFinalizers removes all finalizers from all pods owned by us. +func (d *Deployment) removePodFinalizers() error { + log := d.deps.Log + kubecli := d.GetKubeCli() + pods, err := d.GetOwnedPods() + if err != nil { + return maskAny(err) + } + for _, p := range pods { + ignoreNotFound := true + if err := k8sutil.RemovePodFinalizers(log, kubecli, &p, p.GetFinalizers(), ignoreNotFound); err != nil { + log.Warn().Err(err).Msg("Failed to remove pod finalizers") + } + } + return nil +} + +// removePVCFinalizers removes all finalizers from all PVCs owned by us. +func (d *Deployment) removePVCFinalizers() error { + log := d.deps.Log + kubecli := d.GetKubeCli() + pvcs, err := d.GetOwnedPVCs() + if err != nil { + return maskAny(err) + } + for _, p := range pvcs { + ignoreNotFound := true + if err := k8sutil.RemovePVCFinalizers(log, kubecli, &p, p.GetFinalizers(), ignoreNotFound); err != nil { + log.Warn().Err(err).Msg("Failed to remove PVC finalizers") + } + } + return nil +} diff --git a/pkg/deployment/cluster_scaling_integration.go b/pkg/deployment/cluster_scaling_integration.go index d10c7f9e7..962eee48d 100644 --- a/pkg/deployment/cluster_scaling_integration.go +++ b/pkg/deployment/cluster_scaling_integration.go @@ -33,6 +33,7 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/arangod" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) // clusterScalingIntegration is a helper to communicate with the clusters @@ -50,6 +51,10 @@ type clusterScalingIntegration struct { } } +const ( + maxClusterBootstrapTime = time.Minute * 2 // Time we allow a cluster bootstrap to take, before we can do cluster inspections. +) + // newClusterScalingIntegration creates a new clusterScalingIntegration. func newClusterScalingIntegration(depl *Deployment) *clusterScalingIntegration { return &clusterScalingIntegration{ @@ -67,20 +72,29 @@ func (ci *clusterScalingIntegration) SendUpdateToCluster(spec api.DeploymentSpec // listenForClusterEvents keep listening for changes entered in the UI of the cluster. func (ci *clusterScalingIntegration) ListenForClusterEvents(stopCh <-chan struct{}) { + start := time.Now() + goodInspections := 0 for { delay := time.Second * 2 // Is deployment in running state - if ci.depl.status.Phase == api.DeploymentPhaseRunning { + if ci.depl.GetPhase() == api.DeploymentPhaseRunning { // Update cluster with our state ctx := context.Background() - safeToAskCluster, err := ci.updateClusterServerCount(ctx) + expectSuccess := goodInspections > 0 || time.Since(start) > maxClusterBootstrapTime + safeToAskCluster, err := ci.updateClusterServerCount(ctx, expectSuccess) if err != nil { - ci.log.Debug().Err(err).Msg("Cluster update failed") + if expectSuccess { + ci.log.Debug().Err(err).Msg("Cluster update failed") + } } else if safeToAskCluster { // Inspect once - if err := ci.inspectCluster(ctx); err != nil { - ci.log.Debug().Err(err).Msg("Cluster inspection failed") + if err := ci.inspectCluster(ctx, expectSuccess); err != nil { + if expectSuccess { + ci.log.Debug().Err(err).Msg("Cluster inspection failed") + } + } else { + goodInspections++ } } } @@ -96,7 +110,7 @@ func (ci *clusterScalingIntegration) ListenForClusterEvents(stopCh <-chan struct } // Perform a single inspection of the cluster -func (ci *clusterScalingIntegration) inspectCluster(ctx context.Context) error { +func (ci *clusterScalingIntegration) inspectCluster(ctx context.Context, expectSuccess bool) error { log := ci.log c, err := ci.depl.clientCache.GetDatabase(ctx) if err != nil { @@ -104,7 +118,9 @@ func (ci *clusterScalingIntegration) inspectCluster(ctx context.Context) error { } req, err := arangod.GetNumberOfServers(ctx, c.Connection()) if err != nil { - log.Debug().Err(err).Msg("Failed to get number of servers") + if expectSuccess { + log.Debug().Err(err).Msg("Failed to get number of servers") + } return maskAny(err) } if req.Coordinators == nil && req.DBServers == nil { @@ -135,22 +151,31 @@ func (ci *clusterScalingIntegration) inspectCluster(ctx context.Context) error { log.Debug().Err(err).Msg("Failed to get current deployment") return maskAny(err) } + newSpec := current.Spec.DeepCopy() if coordinatorsChanged { - current.Spec.Coordinators.Count = util.NewInt(req.GetCoordinators()) + newSpec.Coordinators.Count = util.NewInt(req.GetCoordinators()) } if dbserversChanged { - current.Spec.DBServers.Count = util.NewInt(req.GetDBServers()) - } - if err := ci.depl.updateCRSpec(current.Spec); err != nil { - log.Warn().Err(err).Msg("Failed to update current deployment") - return maskAny(err) + newSpec.DBServers.Count = util.NewInt(req.GetDBServers()) + } + if err := newSpec.Validate(); err != nil { + // Log failure & create event + log.Warn().Err(err).Msg("Validation of updated spec has failed") + ci.depl.CreateEvent(k8sutil.NewErrorEvent("Validation failed", err, apiObject)) + // Restore original spec in cluster + ci.SendUpdateToCluster(current.Spec) + } else { + if err := ci.depl.updateCRSpec(*newSpec); err != nil { + log.Warn().Err(err).Msg("Failed to update current deployment") + return maskAny(err) + } } return nil } // updateClusterServerCount updates the intended number of servers of the cluster. // Returns true when it is safe to ask the cluster for updates. -func (ci *clusterScalingIntegration) updateClusterServerCount(ctx context.Context) (bool, error) { +func (ci *clusterScalingIntegration) updateClusterServerCount(ctx context.Context, expectSuccess bool) (bool, error) { // Any update needed? ci.pendingUpdate.mutex.Lock() spec := ci.pendingUpdate.spec @@ -168,7 +193,9 @@ func (ci *clusterScalingIntegration) updateClusterServerCount(ctx context.Contex coordinatorCount := spec.Coordinators.GetCount() dbserverCount := spec.DBServers.GetCount() if err := arangod.SetNumberOfServers(ctx, c.Connection(), coordinatorCount, dbserverCount); err != nil { - log.Debug().Err(err).Msg("Failed to set number of servers") + if expectSuccess { + log.Debug().Err(err).Msg("Failed to set number of servers") + } return false, maskAny(err) } diff --git a/pkg/deployment/context_impl.go b/pkg/deployment/context_impl.go index ce6c9d64a..e4958a119 100644 --- a/pkg/deployment/context_impl.go +++ b/pkg/deployment/context_impl.go @@ -24,8 +24,15 @@ package deployment import ( "context" + "fmt" + "net" + "strconv" + "github.com/arangodb/arangosync/client" + "github.com/arangodb/arangosync/tasks" driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/agency" + "github.com/rs/zerolog/log" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -51,26 +58,55 @@ func (d *Deployment) GetKubeCli() kubernetes.Interface { return d.deps.KubeCli } +// GetLifecycleImage returns the image name containing the lifecycle helper (== name of operator image) +func (d *Deployment) GetLifecycleImage() string { + return d.config.LifecycleImage +} + // GetNamespace returns the kubernetes namespace that contains // this deployment. func (d *Deployment) GetNamespace() string { return d.apiObject.GetNamespace() } +// GetPhase returns the current phase of the deployment +func (d *Deployment) GetPhase() api.DeploymentPhase { + return d.status.last.Phase +} + // GetSpec returns the current specification func (d *Deployment) GetSpec() api.DeploymentSpec { return d.apiObject.Spec } // GetStatus returns the current status of the deployment -func (d *Deployment) GetStatus() api.DeploymentStatus { - return d.status +// together with the current version of that status. +func (d *Deployment) GetStatus() (api.DeploymentStatus, int32) { + d.status.mutex.Lock() + defer d.status.mutex.Unlock() + + version := d.status.version + return *d.status.last.DeepCopy(), version } // UpdateStatus replaces the status of the deployment with the given status and // updates the resources in k8s. -func (d *Deployment) UpdateStatus(status api.DeploymentStatus, force ...bool) error { - d.status = status +// If the given last version does not match the actual last version of the status object, +// an error is returned. +func (d *Deployment) UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error { + d.status.mutex.Lock() + defer d.status.mutex.Unlock() + + if d.status.version != lastVersion { + // Status is obsolete + d.deps.Log.Error(). + Int32("expected-version", lastVersion). + Int32("actual-version", d.status.version). + Msg("UpdateStatus version conflict error.") + return maskAny(fmt.Errorf("Status conflict error. Expected version %d, got %d", lastVersion, d.status.version)) + } + d.status.version++ + d.status.last = *status.DeepCopy() if err := d.updateCRStatus(force...); err != nil { return maskAny(err) } @@ -98,9 +134,9 @@ func (d *Deployment) GetServerClient(ctx context.Context, group api.ServerGroup, // GetAgencyClients returns a client connection for every agency member. // If the given predicate is not nil, only agents are included where the given predicate returns true. -func (d *Deployment) GetAgencyClients(ctx context.Context, predicate func(id string) bool) ([]arangod.Agency, error) { - agencyMembers := d.status.Members.Agents - result := make([]arangod.Agency, 0, len(agencyMembers)) +func (d *Deployment) GetAgencyClients(ctx context.Context, predicate func(id string) bool) ([]driver.Connection, error) { + agencyMembers := d.status.last.Members.Agents + result := make([]driver.Connection, 0, len(agencyMembers)) for _, m := range agencyMembers { if predicate != nil && !predicate(m.ID) { continue @@ -109,33 +145,76 @@ func (d *Deployment) GetAgencyClients(ctx context.Context, predicate func(id str if err != nil { return nil, maskAny(err) } - aClient, err := arangod.NewAgencyClient(client) - if err != nil { - return nil, maskAny(err) - } - result = append(result, aClient) + conn := client.Connection() + result = append(result, conn) } return result, nil } +// GetAgency returns a connection to the entire agency. +func (d *Deployment) GetAgency(ctx context.Context) (agency.Agency, error) { + result, err := arangod.CreateArangodAgencyClient(ctx, d.deps.KubeCli.CoreV1(), d.apiObject) + if err != nil { + return nil, maskAny(err) + } + return result, nil +} + +// GetSyncServerClient returns a cached client for a specific arangosync server. +func (d *Deployment) GetSyncServerClient(ctx context.Context, group api.ServerGroup, id string) (client.API, error) { + // Fetch monitoring token + log := d.deps.Log + kubecli := d.deps.KubeCli + ns := d.apiObject.GetNamespace() + secretName := d.apiObject.Spec.Sync.Monitoring.GetTokenSecretName() + monitoringToken, err := k8sutil.GetTokenSecret(kubecli.CoreV1(), secretName, ns) + if err != nil { + log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get sync monitoring secret") + return nil, maskAny(err) + } + + // Fetch server DNS name + dnsName := k8sutil.CreatePodDNSName(d.apiObject, group.AsRole(), id) + + // Build client + port := k8sutil.ArangoSyncMasterPort + if group == api.ServerGroupSyncWorkers { + port = k8sutil.ArangoSyncWorkerPort + } + source := client.Endpoint{"https://" + net.JoinHostPort(dnsName, strconv.Itoa(port))} + tlsAuth := tasks.TLSAuthentication{ + TLSClientAuthentication: tasks.TLSClientAuthentication{ + ClientToken: monitoringToken, + }, + } + auth := client.NewAuthentication(tlsAuth, "") + insecureSkipVerify := true + c, err := d.syncClientCache.GetClient(d.deps.Log, source, auth, insecureSkipVerify) + if err != nil { + return nil, maskAny(err) + } + return c, nil +} + // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. -func (d *Deployment) CreateMember(group api.ServerGroup, id string) error { +func (d *Deployment) CreateMember(group api.ServerGroup, id string) (string, error) { log := d.deps.Log - id, err := d.createMember(group, id, d.apiObject) + status, lastVersion := d.GetStatus() + id, err := createMember(log, &status, group, id, d.apiObject) if err != nil { log.Debug().Err(err).Str("group", group.AsRole()).Msg("Failed to create member") - return maskAny(err) + return "", maskAny(err) } // Save added member - if err := d.updateCRStatus(); err != nil { + if err := d.UpdateStatus(status, lastVersion); err != nil { log.Debug().Err(err).Msg("Updating CR status failed") - return maskAny(err) + return "", maskAny(err) } // Create event about it d.CreateEvent(k8sutil.NewMemberAddEvent(id, group.AsRole(), d.apiObject)) - return nil + return id, nil } // DeletePod deletes a pod with given name in the namespace @@ -165,6 +244,25 @@ func (d *Deployment) CleanupPod(p v1.Pod) error { return nil } +// RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace +// of the deployment. If the pod does not exist, the error is ignored. +func (d *Deployment) RemovePodFinalizers(podName string) error { + log := d.deps.Log + ns := d.GetNamespace() + kubecli := d.deps.KubeCli + p, err := kubecli.CoreV1().Pods(ns).Get(podName, metav1.GetOptions{}) + if err != nil { + if k8sutil.IsNotFound(err) { + return nil + } + return maskAny(err) + } + if err := k8sutil.RemovePodFinalizers(log, d.deps.KubeCli, p, p.GetFinalizers(), true); err != nil { + return maskAny(err) + } + return nil +} + // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. func (d *Deployment) DeletePvc(pvcName string) error { @@ -195,6 +293,34 @@ func (d *Deployment) GetOwnedPods() ([]v1.Pod, error) { return myPods, nil } +// GetOwnedPVCs returns a list of all PVCs owned by the deployment. +func (d *Deployment) GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) { + // Get all current PVCs + log := d.deps.Log + pvcs, err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(d.apiObject.GetNamespace()).List(k8sutil.DeploymentListOpt(d.apiObject.GetName())) + if err != nil { + log.Debug().Err(err).Msg("Failed to list PVCs") + return nil, maskAny(err) + } + myPVCs := make([]v1.PersistentVolumeClaim, 0, len(pvcs.Items)) + for _, p := range pvcs.Items { + if d.isOwnerOf(&p) { + myPVCs = append(myPVCs, p) + } + } + return myPVCs, nil +} + +// GetPvc gets a PVC by the given name, in the samespace of the deployment. +func (d *Deployment) GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error) { + pvc, err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(d.apiObject.GetNamespace()).Get(pvcName, metav1.GetOptions{}) + if err != nil { + log.Debug().Err(err).Str("pvc-name", pvcName).Msg("Failed to get PVC") + return nil, maskAny(err) + } + return pvc, nil +} + // GetTLSKeyfile returns the keyfile encoded TLS certificate+key for // the given member. func (d *Deployment) GetTLSKeyfile(group api.ServerGroup, member api.MemberStatus) (string, error) { @@ -217,3 +343,26 @@ func (d *Deployment) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberSt } return nil } + +// GetTLSCA returns the TLS CA certificate in the secret with given name. +// Returns: publicKey, privateKey, ownerByDeployment, error +func (d *Deployment) GetTLSCA(secretName string) (string, string, bool, error) { + ns := d.apiObject.GetNamespace() + owner := d.apiObject.AsOwner() + cert, priv, isOwned, err := k8sutil.GetCASecret(d.deps.KubeCli.CoreV1(), secretName, ns, &owner) + if err != nil { + return "", "", false, maskAny(err) + } + return cert, priv, isOwned, nil + +} + +// DeleteSecret removes the Secret with given name. +// If the secret does not exist, the error is ignored. +func (d *Deployment) DeleteSecret(secretName string) error { + ns := d.apiObject.GetNamespace() + if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(secretName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + return maskAny(err) + } + return nil +} diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index 902dc60df..918c97a32 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -25,15 +25,16 @@ package deployment import ( "fmt" "reflect" + "sync" "sync/atomic" "time" + "github.com/arangodb/arangosync/client" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" - corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/deployment/chaos" @@ -50,6 +51,7 @@ import ( type Config struct { ServiceAccount string AllowChaos bool + LifecycleImage string } // Dependencies holds dependent services for a Deployment @@ -57,6 +59,7 @@ type Dependencies struct { Log zerolog.Logger KubeCli kubernetes.Interface DatabaseCRCli versioned.Interface + EventRecorder record.EventRecorder } // deploymentEventType strongly typed type of event @@ -81,16 +84,18 @@ const ( // Deployment is the in process state of an ArangoDeployment. type Deployment struct { apiObject *api.ArangoDeployment // API object - status api.DeploymentStatus // Internal status of the CR - config Config - deps Dependencies + status struct { + mutex sync.Mutex + version int32 + last api.DeploymentStatus // Internal status copy of the CR + } + config Config + deps Dependencies eventCh chan *deploymentEvent stopCh chan struct{} stopped int32 - eventsCli corev1.EventInterface - inspectTrigger trigger.Trigger updateDeploymentTrigger trigger.Trigger clientCache *clientCache @@ -100,6 +105,7 @@ type Deployment struct { resilience *resilience.Resilience resources *resources.Resources chaosMonkey *chaos.Monkey + syncClientCache client.ClientCache } // New creates a new Deployment from the given API object. @@ -109,20 +115,19 @@ func New(config Config, deps Dependencies, apiObject *api.ArangoDeployment) (*De } d := &Deployment{ apiObject: apiObject, - status: *(apiObject.Status.DeepCopy()), config: config, deps: deps, eventCh: make(chan *deploymentEvent, deploymentEventQueueSize), stopCh: make(chan struct{}), - eventsCli: deps.KubeCli.Core().Events(apiObject.GetNamespace()), clientCache: newClientCache(deps.KubeCli, apiObject), } + d.status.last = *(apiObject.Status.DeepCopy()) d.reconciler = reconcile.NewReconciler(deps.Log, d) d.resilience = resilience.NewResilience(deps.Log, d) d.resources = resources.NewResources(deps.Log, d) - if d.status.AcceptedSpec == nil { + if d.status.last.AcceptedSpec == nil { // We've validated the spec, so let's use it from now. - d.status.AcceptedSpec = apiObject.Spec.DeepCopy() + d.status.last.AcceptedSpec = apiObject.Spec.DeepCopy() } go d.run() @@ -182,7 +187,7 @@ func (d *Deployment) send(ev *deploymentEvent) { func (d *Deployment) run() { log := d.deps.Log - if d.status.Phase == api.DeploymentPhaseNone { + if d.GetPhase() == api.DeploymentPhaseNone { // Create secrets if err := d.resources.EnsureSecrets(); err != nil { d.CreateEvent(k8sutil.NewErrorEvent("Failed to create secrets", err, d.GetAPIObject())) @@ -208,8 +213,9 @@ func (d *Deployment) run() { d.CreateEvent(k8sutil.NewErrorEvent("Failed to create pods", err, d.GetAPIObject())) } - d.status.Phase = api.DeploymentPhaseRunning - if err := d.updateCRStatus(); err != nil { + status, lastVersion := d.GetStatus() + status.Phase = api.DeploymentPhaseRunning + if err := d.UpdateStatus(status, lastVersion); err != nil { log.Warn().Err(err).Msg("update initial CR status failed") } log.Info().Msg("start running...") @@ -219,6 +225,14 @@ func (d *Deployment) run() { for { select { case <-d.stopCh: + // Remove finalizers from created resources + log.Info().Msg("Deployment removed, removing finalizers to prevent orphaned resources") + if err := d.removePodFinalizers(); err != nil { + log.Warn().Err(err).Msg("Failed to remove Pod finalizers") + } + if err := d.removePVCFinalizers(); err != nil { + log.Warn().Err(err).Msg("Failed to remove PVC finalizers") + } // We're being stopped. return @@ -266,12 +280,14 @@ func (d *Deployment) handleArangoDeploymentUpdatedEvent() error { } specBefore := d.apiObject.Spec - if d.status.AcceptedSpec != nil { - specBefore = *d.status.AcceptedSpec + status := d.status.last + if d.status.last.AcceptedSpec != nil { + specBefore = *status.AcceptedSpec.DeepCopy() } newAPIObject := current.DeepCopy() newAPIObject.Spec.SetDefaultsFrom(specBefore) - newAPIObject.Status = d.status + newAPIObject.Spec.SetDefaults(d.apiObject.GetName()) + newAPIObject.Status = status resetFields := specBefore.ResetImmutableFields(&newAPIObject.Spec) if len(resetFields) > 0 { log.Debug().Strs("fields", resetFields).Msg("Found modified immutable fields") @@ -297,9 +313,12 @@ func (d *Deployment) handleArangoDeploymentUpdatedEvent() error { return maskAny(fmt.Errorf("failed to update ArangoDeployment spec: %v", err)) } // Save updated accepted spec - d.status.AcceptedSpec = newAPIObject.Spec.DeepCopy() - if err := d.updateCRStatus(); err != nil { - return maskAny(fmt.Errorf("failed to update ArangoDeployment status: %v", err)) + { + status, lastVersion := d.GetStatus() + status.AcceptedSpec = newAPIObject.Spec.DeepCopy() + if err := d.UpdateStatus(status, lastVersion); err != nil { + return maskAny(fmt.Errorf("failed to update ArangoDeployment status: %v", err)) + } } // Notify cluster of desired server count @@ -315,11 +334,8 @@ func (d *Deployment) handleArangoDeploymentUpdatedEvent() error { // CreateEvent creates a given event. // On error, the error is logged. -func (d *Deployment) CreateEvent(evt *v1.Event) { - _, err := d.eventsCli.Create(evt) - if err != nil { - d.deps.Log.Error().Err(err).Interface("event", *evt).Msg("Failed to record event") - } +func (d *Deployment) CreateEvent(evt *k8sutil.Event) { + d.deps.EventRecorder.Event(evt.InvolvedObject, evt.Type, evt.Reason, evt.Message) } // Update the status of the API object from the internal status @@ -333,13 +349,17 @@ func (d *Deployment) updateCRStatus(force ...bool) error { } // Send update to API server + ns := d.apiObject.GetNamespace() + depls := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(ns) update := d.apiObject.DeepCopy() attempt := 0 for { attempt++ - update.Status = d.status - ns := d.apiObject.GetNamespace() - newAPIObject, err := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(ns).Update(update) + update.Status = d.status.last + if update.GetDeletionTimestamp() == nil { + ensureFinalizers(update) + } + newAPIObject, err := depls.Update(update) if err == nil { // Update internal object d.apiObject = newAPIObject @@ -349,7 +369,7 @@ func (d *Deployment) updateCRStatus(force ...bool) error { // API object may have been changed already, // Reload api object and try again var current *api.ArangoDeployment - current, err = d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(ns).Get(update.GetName(), metav1.GetOptions{}) + current, err = depls.Get(update.GetName(), metav1.GetOptions{}) if err == nil { update = current.DeepCopy() continue @@ -372,7 +392,7 @@ func (d *Deployment) updateCRSpec(newSpec api.DeploymentSpec) error { for { attempt++ update.Spec = newSpec - update.Status = d.status + update.Status = d.status.last ns := d.apiObject.GetNamespace() newAPIObject, err := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(ns).Update(update) if err == nil { @@ -401,7 +421,7 @@ func (d *Deployment) updateCRSpec(newSpec api.DeploymentSpec) error { // Since there is no recovery from a failed deployment, use with care! func (d *Deployment) failOnError(err error, msg string) { log.Error().Err(err).Msg(msg) - d.status.Reason = err.Error() + d.status.last.Reason = err.Error() d.reportFailedStatus() } @@ -412,7 +432,7 @@ func (d *Deployment) reportFailedStatus() { log.Info().Msg("deployment failed. Reporting failed reason...") op := func() error { - d.status.Phase = api.DeploymentPhaseFailed + d.status.last.Phase = api.DeploymentPhaseFailed err := d.updateCRStatus() if err == nil || k8sutil.IsNotFound(err) { // Status has been updated diff --git a/pkg/deployment/deployment_finalizers.go b/pkg/deployment/deployment_finalizers.go new file mode 100644 index 000000000..e57e1e7ab --- /dev/null +++ b/pkg/deployment/deployment_finalizers.go @@ -0,0 +1,117 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package deployment + +import ( + "context" + + "github.com/rs/zerolog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// ensureFinalizers adds all required finalizers to the given deployment (in memory). +func ensureFinalizers(depl *api.ArangoDeployment) { + for _, f := range depl.GetFinalizers() { + if f == constants.FinalizerDeplRemoveChildFinalizers { + // Finalizer already set + return + } + } + // Set finalizers + depl.SetFinalizers(append(depl.GetFinalizers(), constants.FinalizerDeplRemoveChildFinalizers)) +} + +// runDeploymentFinalizers goes through the list of ArangoDeployoment finalizers to see if they can be removed. +func (d *Deployment) runDeploymentFinalizers(ctx context.Context) error { + log := d.deps.Log + var removalList []string + + depls := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(d.GetNamespace()) + updated, err := depls.Get(d.apiObject.GetName(), metav1.GetOptions{}) + if err != nil { + return maskAny(err) + } + for _, f := range updated.ObjectMeta.GetFinalizers() { + switch f { + case constants.FinalizerDeplRemoveChildFinalizers: + log.Debug().Msg("Inspecting 'remove child finalizers' finalizer") + if err := d.inspectRemoveChildFinalizers(ctx, log, updated); err == nil { + removalList = append(removalList, f) + } else { + log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet") + } + } + } + // Remove finalizers (if needed) + if len(removalList) > 0 { + if err := removeDeploymentFinalizers(log, d.deps.DatabaseCRCli, updated, removalList); err != nil { + log.Debug().Err(err).Msg("Failed to update ArangoDeployment (to remove finalizers)") + return maskAny(err) + } + } + return nil +} + +// inspectRemoveChildFinalizers checks the finalizer condition for remove-child-finalizers. +// It returns nil if the finalizer can be removed. +func (d *Deployment) inspectRemoveChildFinalizers(ctx context.Context, log zerolog.Logger, depl *api.ArangoDeployment) error { + if err := d.removePodFinalizers(); err != nil { + return maskAny(err) + } + if err := d.removePVCFinalizers(); err != nil { + return maskAny(err) + } + + return nil +} + +// removeDeploymentFinalizers removes the given finalizers from the given PVC. +func removeDeploymentFinalizers(log zerolog.Logger, cli versioned.Interface, depl *api.ArangoDeployment, finalizers []string) error { + depls := cli.DatabaseV1alpha().ArangoDeployments(depl.GetNamespace()) + getFunc := func() (metav1.Object, error) { + result, err := depls.Get(depl.GetName(), metav1.GetOptions{}) + if err != nil { + return nil, maskAny(err) + } + return result, nil + } + updateFunc := func(updated metav1.Object) error { + updatedDepl := updated.(*api.ArangoDeployment) + result, err := depls.Update(updatedDepl) + if err != nil { + return maskAny(err) + } + *depl = *result + return nil + } + ignoreNotFound := false + if err := k8sutil.RemoveFinalizers(log, finalizers, getFunc, updateFunc, ignoreNotFound); err != nil { + return maskAny(err) + } + return nil +} diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index c4c9f3063..13557009f 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -46,85 +46,113 @@ func (d *Deployment) inspectDeployment(lastInterval time.Duration) time.Duration ctx := context.Background() // Check deployment still exists - if _, err := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(d.apiObject.GetNamespace()).Get(d.apiObject.GetName(), metav1.GetOptions{}); k8sutil.IsNotFound(err) { + updated, err := d.deps.DatabaseCRCli.DatabaseV1alpha().ArangoDeployments(d.apiObject.GetNamespace()).Get(d.apiObject.GetName(), metav1.GetOptions{}) + if k8sutil.IsNotFound(err) { // Deployment is gone log.Info().Msg("Deployment is gone") d.Delete() return nextInterval - } + } else if updated != nil && updated.GetDeletionTimestamp() != nil { + // Deployment is marked for deletion + if err := d.runDeploymentFinalizers(ctx); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("ArangoDeployment finalizer inspection failed", err, d.apiObject)) + } + } else { + // Is the deployment in failed state, if so, give up. + if d.GetPhase() == api.DeploymentPhaseFailed { + log.Debug().Msg("Deployment is in Failed state.") + return nextInterval + } - // Is the deployment in failed state, if so, give up. - if d.status.Phase == api.DeploymentPhaseFailed { - log.Debug().Msg("Deployment is in Failed state.") - return nextInterval - } + // Inspect secret hashes + if err := d.resources.ValidateSecretHashes(); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Secret hash validation failed", err, d.apiObject)) + } - // Inspect secret hashes - if err := d.resources.ValidateSecretHashes(); err != nil { - hasError = true - d.CreateEvent(k8sutil.NewErrorEvent("Secret hash validation failed", err, d.apiObject)) - } + // Is the deployment in a good state? + status, _ := d.GetStatus() + if status.Conditions.IsTrue(api.ConditionTypeSecretsChanged) { + log.Debug().Msg("Condition SecretsChanged is true. Revert secrets before we can continue") + return nextInterval + } - // Is the deployment in a good state? - if d.status.Conditions.IsTrue(api.ConditionTypeSecretsChanged) { - log.Debug().Msg("Condition SecretsChanged is true. Revert secrets before we can continue") - return nextInterval - } + // Ensure we have image info + if retrySoon, err := d.ensureImages(d.apiObject); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Image detection failed", err, d.apiObject)) + } else if retrySoon { + nextInterval = minInspectionInterval + } - // Ensure we have image info - if retrySoon, err := d.ensureImages(d.apiObject); err != nil { - hasError = true - d.CreateEvent(k8sutil.NewErrorEvent("Image detection failed", err, d.apiObject)) - } else if retrySoon { - nextInterval = minInspectionInterval - } + // Inspection of generated resources needed + if err := d.resources.InspectPods(ctx); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Pod inspection failed", err, d.apiObject)) + } + if err := d.resources.InspectPVCs(ctx); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("PVC inspection failed", err, d.apiObject)) + } - // Inspection of generated resources needed - if err := d.resources.InspectPods(); err != nil { - hasError = true - d.CreateEvent(k8sutil.NewErrorEvent("Pod inspection failed", err, d.apiObject)) - } + // Check members for resilience + if err := d.resilience.CheckMemberFailure(); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Member failure detection failed", err, d.apiObject)) + } - // Check members for resilience - if err := d.resilience.CheckMemberFailure(); err != nil { - hasError = true - d.CreateEvent(k8sutil.NewErrorEvent("Member failure detection failed", err, d.apiObject)) - } + // Create scale/update plan + if err := d.reconciler.CreatePlan(); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Plan creation failed", err, d.apiObject)) + } - // Create scale/update plan - if err := d.reconciler.CreatePlan(); err != nil { - hasError = true - d.CreateEvent(k8sutil.NewErrorEvent("Plan creation failed", err, d.apiObject)) - } + // Execute current step of scale/update plan + retrySoon, err := d.reconciler.ExecutePlan(ctx) + if err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Plan execution failed", err, d.apiObject)) + } + if retrySoon { + nextInterval = minInspectionInterval + } - // Execute current step of scale/update plan - retrySoon, err := d.reconciler.ExecutePlan(ctx) - if err != nil { - hasError = true - d.CreateEvent(k8sutil.NewErrorEvent("Plan execution failed", err, d.apiObject)) - } - if retrySoon { - nextInterval = minInspectionInterval - } + // Ensure all resources are created + if err := d.resources.EnsureSecrets(); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Secret creation failed", err, d.apiObject)) + } + if err := d.resources.EnsureServices(); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Service creation failed", err, d.apiObject)) + } + if err := d.resources.EnsurePVCs(); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("PVC creation failed", err, d.apiObject)) + } + if err := d.resources.EnsurePods(); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Pod creation failed", err, d.apiObject)) + } - // Ensure all resources are created - if err := d.resources.EnsureServices(); err != nil { - hasError = true - d.CreateEvent(k8sutil.NewErrorEvent("Service creation failed", err, d.apiObject)) - } - if err := d.resources.EnsurePVCs(); err != nil { - hasError = true - d.CreateEvent(k8sutil.NewErrorEvent("PVC creation failed", err, d.apiObject)) - } - if err := d.resources.EnsurePods(); err != nil { - hasError = true - d.CreateEvent(k8sutil.NewErrorEvent("Pod creation failed", err, d.apiObject)) - } + // Create access packages + if err := d.createAccessPackages(); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("AccessPackage creation failed", err, d.apiObject)) + } + + // Inspect deployment for obsolete members + if err := d.resources.CleanupRemovedMembers(); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Removed member cleanup failed", err, d.apiObject)) + } - // At the end of the inspect, we cleanup terminated pods. - if d.resources.CleanupTerminatedPods(); err != nil { - hasError = true - d.CreateEvent(k8sutil.NewErrorEvent("Pod cleanup failed", err, d.apiObject)) + // At the end of the inspect, we cleanup terminated pods. + if err := d.resources.CleanupTerminatedPods(); err != nil { + hasError = true + d.CreateEvent(k8sutil.NewErrorEvent("Pod cleanup failed", err, d.apiObject)) + } } // Update next interval (on errors) diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index 504a4ca7b..7c76bc3b7 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -27,8 +27,10 @@ import ( "crypto/sha1" "fmt" "strings" + "time" "github.com/rs/zerolog" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -38,7 +40,7 @@ import ( ) const ( - dockerPullableImageIDPrefix = "docker-pullable://" + dockerPullableImageIDPrefix_ = "docker-pullable://" ) type imagesBuilder struct { @@ -53,15 +55,15 @@ type imagesBuilder struct { // ensureImages creates pods needed to detect ImageID for specified images. // Returns: retrySoon, error func (d *Deployment) ensureImages(apiObject *api.ArangoDeployment) (bool, error) { + status, lastVersion := d.GetStatus() ib := imagesBuilder{ APIObject: apiObject, Spec: apiObject.Spec, - Status: d.status, + Status: status, Log: d.deps.Log, KubeCli: d.deps.KubeCli, UpdateCRStatus: func(status api.DeploymentStatus) error { - d.status = status - if err := d.updateCRStatus(); err != nil { + if err := d.UpdateStatus(status, lastVersion); err != nil { return maskAny(err) } return nil @@ -117,10 +119,8 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima log.Warn().Msg("Empty list of ContainerStatuses") return true, nil } - imageID := pod.Status.ContainerStatuses[0].ImageID - if strings.HasPrefix(imageID, dockerPullableImageIDPrefix) { - imageID = imageID[len(dockerPullableImageIDPrefix):] - } else if imageID == "" { + imageID := k8sutil.ConvertImageID2Image(pod.Status.ContainerStatuses[0].ImageID) + if imageID == "" { // Fall back to specified image imageID = image } @@ -137,6 +137,7 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima return true, nil } version := v.Version + enterprise := strings.ToLower(v.License) == "enterprise" // We have all the info we need now, kill the pod and store the image info. if err := ib.KubeCli.CoreV1().Pods(ns).Delete(podName, nil); err != nil && !k8sutil.IsNotFound(err) { @@ -148,6 +149,7 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima Image: image, ImageID: imageID, ArangoDBVersion: version, + Enterprise: enterprise, } ib.Status.Images.AddOrUpdate(info) if err := ib.UpdateCRStatus(ib.Status); err != nil { @@ -165,8 +167,19 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima args := []string{ "--server.authentication=false", fmt.Sprintf("--server.endpoint=tcp://[::]:%d", k8sutil.ArangoPort), + "--database.directory=" + k8sutil.ArangodVolumeMountDir, + "--log.output=+", } - if err := k8sutil.CreateArangodPod(ib.KubeCli, true, ib.APIObject, role, id, podName, "", image, ib.Spec.GetImagePullPolicy(), "", false, args, nil, nil, nil, "", ""); err != nil { + terminationGracePeriod := time.Second * 30 + tolerations := make([]v1.Toleration, 0, 2) + shortDur := k8sutil.TolerationDuration{Forever: false, TimeSpan: time.Second * 5} + tolerations = k8sutil.AddTolerationIfNotFound(tolerations, k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeNotReady, shortDur)) + tolerations = k8sutil.AddTolerationIfNotFound(tolerations, k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeUnreachable, shortDur)) + tolerations = k8sutil.AddTolerationIfNotFound(tolerations, k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeAlphaUnreachable, shortDur)) + serviceAccountName := "" + + if err := k8sutil.CreateArangodPod(ib.KubeCli, true, ib.APIObject, role, id, podName, "", image, "", ib.Spec.GetImagePullPolicy(), "", false, terminationGracePeriod, args, nil, nil, nil, nil, + tolerations, serviceAccountName, "", ""); err != nil { log.Debug().Err(err).Msg("Failed to create image ID pod") return true, maskAny(err) } diff --git a/pkg/deployment/members.go b/pkg/deployment/members.go index 27435e052..37ed06a98 100644 --- a/pkg/deployment/members.go +++ b/pkg/deployment/members.go @@ -27,7 +27,7 @@ import ( "strings" "github.com/dchest/uniuri" - "k8s.io/api/core/v1" + "github.com/rs/zerolog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" @@ -41,23 +41,24 @@ func (d *Deployment) createInitialMembers(apiObject *api.ArangoDeployment) error log.Debug().Msg("creating initial members...") // Go over all groups and create members - var events []*v1.Event - if err := apiObject.ForeachServerGroup(func(group api.ServerGroup, spec api.ServerGroupSpec, status *api.MemberStatusList) error { - for len(*status) < spec.GetCount() { - id, err := d.createMember(group, "", apiObject) + var events []*k8sutil.Event + status, lastVersion := d.GetStatus() + if err := apiObject.ForeachServerGroup(func(group api.ServerGroup, spec api.ServerGroupSpec, members *api.MemberStatusList) error { + for len(*members) < spec.GetCount() { + id, err := createMember(log, &status, group, "", apiObject) if err != nil { return maskAny(err) } events = append(events, k8sutil.NewMemberAddEvent(id, group.AsRole(), apiObject)) } return nil - }, &d.status); err != nil { + }, &status); err != nil { return maskAny(err) } // Save status log.Debug().Msg("saving initial members...") - if err := d.updateCRStatus(); err != nil { + if err := d.UpdateStatus(status, lastVersion); err != nil { return maskAny(err) } // Save events @@ -71,13 +72,12 @@ func (d *Deployment) createInitialMembers(apiObject *api.ArangoDeployment) error // createMember creates member and adds it to the applicable member list. // Note: This does not create any pods of PVCs // Note: The updated status is not yet written to the apiserver. -func (d *Deployment) createMember(group api.ServerGroup, id string, apiObject *api.ArangoDeployment) (string, error) { - log := d.deps.Log +func createMember(log zerolog.Logger, status *api.DeploymentStatus, group api.ServerGroup, id string, apiObject *api.ArangoDeployment) (string, error) { if id == "" { idPrefix := getArangodIDPrefix(group) for { id = idPrefix + strings.ToLower(uniuri.NewLen(8)) // K8s accepts only lowercase, so we use it here as well - if !d.status.Members.ContainsID(id) { + if !status.Members.ContainsID(id) { break } // Duplicate, try again @@ -89,68 +89,68 @@ func (d *Deployment) createMember(group api.ServerGroup, id string, apiObject *a switch group { case api.ServerGroupSingle: log.Debug().Str("id", id).Msg("Adding single server") - if err := d.status.Members.Single.Add(api.MemberStatus{ + if err := status.Members.Add(api.MemberStatus{ ID: id, CreatedAt: metav1.Now(), Phase: api.MemberPhaseNone, PersistentVolumeClaimName: k8sutil.CreatePersistentVolumeClaimName(deploymentName, role, id), PodName: "", - }); err != nil { + }, group); err != nil { return "", maskAny(err) } case api.ServerGroupAgents: log.Debug().Str("id", id).Msg("Adding agent") - if err := d.status.Members.Agents.Add(api.MemberStatus{ + if err := status.Members.Add(api.MemberStatus{ ID: id, CreatedAt: metav1.Now(), Phase: api.MemberPhaseNone, PersistentVolumeClaimName: k8sutil.CreatePersistentVolumeClaimName(deploymentName, role, id), PodName: "", - }); err != nil { + }, group); err != nil { return "", maskAny(err) } case api.ServerGroupDBServers: log.Debug().Str("id", id).Msg("Adding dbserver") - if err := d.status.Members.DBServers.Add(api.MemberStatus{ + if err := status.Members.Add(api.MemberStatus{ ID: id, CreatedAt: metav1.Now(), Phase: api.MemberPhaseNone, PersistentVolumeClaimName: k8sutil.CreatePersistentVolumeClaimName(deploymentName, role, id), PodName: "", - }); err != nil { + }, group); err != nil { return "", maskAny(err) } case api.ServerGroupCoordinators: log.Debug().Str("id", id).Msg("Adding coordinator") - if err := d.status.Members.Coordinators.Add(api.MemberStatus{ + if err := status.Members.Add(api.MemberStatus{ ID: id, CreatedAt: metav1.Now(), Phase: api.MemberPhaseNone, PersistentVolumeClaimName: "", PodName: "", - }); err != nil { + }, group); err != nil { return "", maskAny(err) } case api.ServerGroupSyncMasters: log.Debug().Str("id", id).Msg("Adding syncmaster") - if err := d.status.Members.SyncMasters.Add(api.MemberStatus{ + if err := status.Members.Add(api.MemberStatus{ ID: id, CreatedAt: metav1.Now(), Phase: api.MemberPhaseNone, PersistentVolumeClaimName: "", PodName: "", - }); err != nil { + }, group); err != nil { return "", maskAny(err) } case api.ServerGroupSyncWorkers: log.Debug().Str("id", id).Msg("Adding syncworker") - if err := d.status.Members.SyncWorkers.Add(api.MemberStatus{ + if err := status.Members.Add(api.MemberStatus{ ID: id, CreatedAt: metav1.Now(), Phase: api.MemberPhaseNone, PersistentVolumeClaimName: "", PodName: "", - }); err != nil { + }, group); err != nil { return "", maskAny(err) } default: @@ -164,6 +164,8 @@ func (d *Deployment) createMember(group api.ServerGroup, id string, apiObject *a // in the given group. func getArangodIDPrefix(group api.ServerGroup) string { switch group { + case api.ServerGroupSingle: + return "SNGL-" case api.ServerGroupCoordinators: return "CRDN-" case api.ServerGroupDBServers: diff --git a/pkg/deployment/reconcile/action.go b/pkg/deployment/reconcile/action.go index e3b0ded79..eecf9176d 100644 --- a/pkg/deployment/reconcile/action.go +++ b/pkg/deployment/reconcile/action.go @@ -24,6 +24,7 @@ package reconcile import ( "context" + "time" ) // Action executes a single Plan item. @@ -33,6 +34,10 @@ type Action interface { // the start time needs to be recorded and a ready condition needs to be checked. Start(ctx context.Context) (bool, error) // CheckProgress checks the progress of the action. - // Returns true if the action is completely finished, false otherwise. - CheckProgress(ctx context.Context) (bool, error) + // Returns: ready, abort, error. + CheckProgress(ctx context.Context) (bool, bool, error) + // Timeout returns the amount of time after which this action will timeout. + Timeout() time.Duration + // Return the MemberID used / created in this action + MemberID() string } diff --git a/pkg/deployment/reconcile/action_add_member.go b/pkg/deployment/reconcile/action_add_member.go index cf96ca726..1e282c67e 100644 --- a/pkg/deployment/reconcile/action_add_member.go +++ b/pkg/deployment/reconcile/action_add_member.go @@ -24,6 +24,7 @@ package reconcile import ( "context" + "time" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/rs/zerolog" @@ -42,25 +43,38 @@ func NewAddMemberAction(log zerolog.Logger, action api.Action, actionCtx ActionC // actionAddMember implements an AddMemberAction. type actionAddMember struct { - log zerolog.Logger - action api.Action - actionCtx ActionContext + log zerolog.Logger + action api.Action + actionCtx ActionContext + newMemberID string } // Start performs the start of the action. // Returns true if the action is completely finished, false in case // the start time needs to be recorded and a ready condition needs to be checked. func (a *actionAddMember) Start(ctx context.Context) (bool, error) { - if err := a.actionCtx.CreateMember(a.action.Group, a.action.MemberID); err != nil { + newID, err := a.actionCtx.CreateMember(a.action.Group, a.action.MemberID) + if err != nil { log.Debug().Err(err).Msg("Failed to create member") return false, maskAny(err) } + a.newMemberID = newID return true, nil } // CheckProgress checks the progress of the action. // Returns true if the action is completely finished, false otherwise. -func (a *actionAddMember) CheckProgress(ctx context.Context) (bool, error) { +func (a *actionAddMember) CheckProgress(ctx context.Context) (bool, bool, error) { // Nothing todo - return true, nil + return true, false, nil +} + +// Timeout returns the amount of time after which this action will timeout. +func (a *actionAddMember) Timeout() time.Duration { + return addMemberTimeout +} + +// Return the MemberID used / created in this action +func (a *actionAddMember) MemberID() string { + return a.newMemberID } diff --git a/pkg/deployment/reconcile/action_cleanout_member.go b/pkg/deployment/reconcile/action_cleanout_member.go index 6292a5c52..263bac205 100644 --- a/pkg/deployment/reconcile/action_cleanout_member.go +++ b/pkg/deployment/reconcile/action_cleanout_member.go @@ -24,9 +24,13 @@ package reconcile import ( "context" + "time" + driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/rs/zerolog" + + "github.com/arangodb/kube-arangodb/pkg/util/arangod" ) // NewCleanOutMemberAction creates a new Action that implements the given @@ -66,12 +70,16 @@ func (a *actionCleanoutMember) Start(ctx context.Context) (bool, error) { log.Debug().Err(err).Msg("Failed to access cluster") return false, maskAny(err) } + var jobID string + ctx = driver.WithJobIDResponse(ctx, &jobID) if err := cluster.CleanOutServer(ctx, a.action.MemberID); err != nil { log.Debug().Err(err).Msg("Failed to cleanout member") return false, maskAny(err) } + log.Debug().Str("job-id", jobID).Msg("Cleanout member started") // Update status m.Phase = api.MemberPhaseCleanOut + m.CleanoutJobID = jobID if a.actionCtx.UpdateMember(m); err != nil { return false, maskAny(err) } @@ -79,27 +87,76 @@ func (a *actionCleanoutMember) Start(ctx context.Context) (bool, error) { } // CheckProgress checks the progress of the action. -// Returns true if the action is completely finished, false otherwise. -func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, error) { +// Returns: ready, abort, error. +func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, error) { log := a.log + m, ok := a.actionCtx.GetMemberStatusByID(a.action.MemberID) + if !ok { + // We wanted to remove and it is already gone. All ok + return true, false, nil + } c, err := a.actionCtx.GetDatabaseClient(ctx) if err != nil { - log.Debug().Err(err).Msg("Failed to create member client") - return false, maskAny(err) + log.Debug().Err(err).Msg("Failed to create database client") + return false, false, maskAny(err) } cluster, err := c.Cluster(ctx) if err != nil { log.Debug().Err(err).Msg("Failed to access cluster") - return false, maskAny(err) + return false, false, maskAny(err) } cleanedOut, err := cluster.IsCleanedOut(ctx, a.action.MemberID) if err != nil { - return false, maskAny(err) + log.Debug().Err(err).Msg("IsCleanedOut failed") + return false, false, maskAny(err) } if !cleanedOut { - // We're not done yet - return false, nil + // We're not done yet, check job status + log.Debug().Msg("IsCleanedOut returned false") + + c, err := a.actionCtx.GetDatabaseClient(ctx) + if err != nil { + log.Debug().Err(err).Msg("Failed to create database client") + return false, false, maskAny(err) + } + agency, err := a.actionCtx.GetAgency(ctx) + if err != nil { + log.Debug().Err(err).Msg("Failed to create agency client") + return false, false, maskAny(err) + } + jobStatus, err := arangod.CleanoutServerJobStatus(ctx, m.CleanoutJobID, c, agency) + if err != nil { + log.Debug().Err(err).Msg("Failed to fetch cleanout job status") + return false, false, maskAny(err) + } + if jobStatus.IsFailed() { + log.Warn().Str("reason", jobStatus.Reason()).Msg("Cleanout Job failed. Aborting plan") + // Revert cleanout state + m.Phase = api.MemberPhaseCreated + m.CleanoutJobID = "" + if a.actionCtx.UpdateMember(m); err != nil { + return false, false, maskAny(err) + } + return false, true, nil + } + return false, false, nil + } + // Cleanout completed + if m.Conditions.Update(api.ConditionTypeCleanedOut, true, "CleanedOut", "") { + if a.actionCtx.UpdateMember(m); err != nil { + return false, false, maskAny(err) + } } // Cleanout completed - return true, nil + return true, false, nil +} + +// Timeout returns the amount of time after which this action will timeout. +func (a *actionCleanoutMember) Timeout() time.Duration { + return cleanoutMemberTimeout +} + +// Return the MemberID used / created in this action +func (a *actionCleanoutMember) MemberID() string { + return a.action.MemberID } diff --git a/pkg/deployment/reconcile/action_context.go b/pkg/deployment/reconcile/action_context.go index 03bfde1e0..1802b3bda 100644 --- a/pkg/deployment/reconcile/action_context.go +++ b/pkg/deployment/reconcile/action_context.go @@ -26,12 +26,14 @@ import ( "context" "fmt" + "github.com/arangodb/go-driver/agency" + + "github.com/arangodb/arangosync/client" driver "github.com/arangodb/go-driver" "github.com/rs/zerolog" "github.com/rs/zerolog/log" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" - "github.com/arangodb/kube-arangodb/pkg/util/arangod" ) // ActionContext provides methods to the Action implementations @@ -45,7 +47,11 @@ type ActionContext interface { // GetServerClient returns a cached client for a specific server. GetServerClient(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error) // GetAgencyClients returns a client connection for every agency member. - GetAgencyClients(ctx context.Context) ([]arangod.Agency, error) + GetAgencyClients(ctx context.Context) ([]driver.Connection, error) + // GetAgency returns a connection to the entire agency. + GetAgency(ctx context.Context) (agency.Agency, error) + // GetSyncServerClient returns a cached client for a specific arangosync server. + GetSyncServerClient(ctx context.Context, group api.ServerGroup, id string) (client.API, error) // GetMemberStatusByID returns the current member status // for the member with given id. // Returns member status, true when found, or false @@ -53,7 +59,7 @@ type ActionContext interface { GetMemberStatusByID(id string) (api.MemberStatus, bool) // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. - CreateMember(group api.ServerGroup, id string) error + CreateMember(group api.ServerGroup, id string) (string, error) // UpdateMember updates the deployment status wrt the given member. UpdateMember(member api.MemberStatus) error // RemoveMemberByID removes a member with given id. @@ -64,9 +70,14 @@ type ActionContext interface { // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. DeletePvc(pvcName string) error + // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace + // of the deployment. If the pod does not exist, the error is ignored. + RemovePodFinalizers(podName string) error // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member. // If the secret does not exist, the error is ignored. DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error + // DeleteTLSCASecret removes the Secret containing the TLS CA certificate. + DeleteTLSCASecret() error } // newActionContext creates a new ActionContext implementation. @@ -108,7 +119,7 @@ func (ac *actionContext) GetServerClient(ctx context.Context, group api.ServerGr } // GetAgencyClients returns a client connection for every agency member. -func (ac *actionContext) GetAgencyClients(ctx context.Context) ([]arangod.Agency, error) { +func (ac *actionContext) GetAgencyClients(ctx context.Context) ([]driver.Connection, error) { c, err := ac.context.GetAgencyClients(ctx, nil) if err != nil { return nil, maskAny(err) @@ -116,33 +127,55 @@ func (ac *actionContext) GetAgencyClients(ctx context.Context) ([]arangod.Agency return c, nil } +// GetAgency returns a connection to the entire agency. +func (ac *actionContext) GetAgency(ctx context.Context) (agency.Agency, error) { + a, err := ac.context.GetAgency(ctx) + if err != nil { + return nil, maskAny(err) + } + return a, nil +} + +// GetSyncServerClient returns a cached client for a specific arangosync server. +func (ac *actionContext) GetSyncServerClient(ctx context.Context, group api.ServerGroup, id string) (client.API, error) { + c, err := ac.context.GetSyncServerClient(ctx, group, id) + if err != nil { + return nil, maskAny(err) + } + return c, nil +} + // GetMemberStatusByID returns the current member status // for the member with given id. // Returns member status, true when found, or false // when no such member is found. func (ac *actionContext) GetMemberStatusByID(id string) (api.MemberStatus, bool) { - m, _, ok := ac.context.GetStatus().Members.ElementByID(id) + status, _ := ac.context.GetStatus() + m, _, ok := status.Members.ElementByID(id) return m, ok } // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. -func (ac *actionContext) CreateMember(group api.ServerGroup, id string) error { - if err := ac.context.CreateMember(group, id); err != nil { - return maskAny(err) +func (ac *actionContext) CreateMember(group api.ServerGroup, id string) (string, error) { + result, err := ac.context.CreateMember(group, id) + if err != nil { + return "", maskAny(err) } - return nil + return result, nil } // UpdateMember updates the deployment status wrt the given member. func (ac *actionContext) UpdateMember(member api.MemberStatus) error { - status := ac.context.GetStatus() + status, lastVersion := ac.context.GetStatus() _, group, found := status.Members.ElementByID(member.ID) if !found { return maskAny(fmt.Errorf("Member %s not found", member.ID)) } - status.Members.UpdateMemberStatus(member, group) - if err := ac.context.UpdateStatus(status); err != nil { + if err := status.Members.Update(member, group); err != nil { + return maskAny(err) + } + if err := ac.context.UpdateStatus(status, lastVersion); err != nil { log.Debug().Err(err).Msg("Updating CR status failed") return maskAny(err) } @@ -151,7 +184,7 @@ func (ac *actionContext) UpdateMember(member api.MemberStatus) error { // RemoveMemberByID removes a member with given id. func (ac *actionContext) RemoveMemberByID(id string) error { - status := ac.context.GetStatus() + status, lastVersion := ac.context.GetStatus() _, group, found := status.Members.ElementByID(id) if !found { return nil @@ -161,7 +194,7 @@ func (ac *actionContext) RemoveMemberByID(id string) error { return maskAny(err) } // Save removed member - if err := ac.context.UpdateStatus(status); err != nil { + if err := ac.context.UpdateStatus(status, lastVersion); err != nil { return maskAny(err) } return nil @@ -185,6 +218,15 @@ func (ac *actionContext) DeletePvc(pvcName string) error { return nil } +// RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace +// of the deployment. If the pod does not exist, the error is ignored. +func (ac *actionContext) RemovePodFinalizers(podName string) error { + if err := ac.context.RemovePodFinalizers(podName); err != nil { + return maskAny(err) + } + return nil +} + // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member. // If the secret does not exist, the error is ignored. func (ac *actionContext) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error { @@ -193,3 +235,28 @@ func (ac *actionContext) DeleteTLSKeyfile(group api.ServerGroup, member api.Memb } return nil } + +// DeleteTLSCASecret removes the Secret containing the TLS CA certificate. +func (ac *actionContext) DeleteTLSCASecret() error { + spec := ac.context.GetSpec().TLS + if !spec.IsSecure() { + return nil + } + secretName := spec.GetCASecretName() + if secretName == "" { + return nil + } + // Remove secret hash, since it is going to change + status, lastVersion := ac.context.GetStatus() + if status.SecretHashes != nil { + status.SecretHashes.TLSCA = "" + if err := ac.context.UpdateStatus(status, lastVersion); err != nil { + return maskAny(err) + } + } + // Do delete the secret + if err := ac.context.DeleteSecret(secretName); err != nil { + return maskAny(err) + } + return nil +} diff --git a/pkg/deployment/reconcile/action_remove_member.go b/pkg/deployment/reconcile/action_remove_member.go index feb5ac8c6..8824d373c 100644 --- a/pkg/deployment/reconcile/action_remove_member.go +++ b/pkg/deployment/reconcile/action_remove_member.go @@ -24,6 +24,8 @@ package reconcile import ( "context" + "fmt" + "time" "github.com/pkg/errors" "github.com/rs/zerolog" @@ -85,12 +87,26 @@ func (a *actionRemoveMember) Start(ctx context.Context) (bool, error) { if err := a.actionCtx.RemoveMemberByID(a.action.MemberID); err != nil { return false, maskAny(err) } + // Check that member has been removed + if _, found := a.actionCtx.GetMemberStatusByID(a.action.MemberID); found { + return false, maskAny(fmt.Errorf("Member %s still exists", a.action.MemberID)) + } return true, nil } // CheckProgress checks the progress of the action. // Returns true if the action is completely finished, false otherwise. -func (a *actionRemoveMember) CheckProgress(ctx context.Context) (bool, error) { +func (a *actionRemoveMember) CheckProgress(ctx context.Context) (bool, bool, error) { // Nothing todo - return true, nil + return true, false, nil +} + +// Timeout returns the amount of time after which this action will timeout. +func (a *actionRemoveMember) Timeout() time.Duration { + return removeMemberTimeout +} + +// Return the MemberID used / created in this action +func (a *actionRemoveMember) MemberID() string { + return a.action.MemberID } diff --git a/pkg/deployment/reconcile/action_renew_tls_ca_certificate.go b/pkg/deployment/reconcile/action_renew_tls_ca_certificate.go new file mode 100644 index 000000000..5a0a3543e --- /dev/null +++ b/pkg/deployment/reconcile/action_renew_tls_ca_certificate.go @@ -0,0 +1,76 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package reconcile + +import ( + "context" + "time" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/rs/zerolog" +) + +// NewRenewTLSCACertificateAction creates a new Action that implements the given +// planned RenewTLSCACertificate action. +func NewRenewTLSCACertificateAction(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + return &renewTLSCACertificateAction{ + log: log, + action: action, + actionCtx: actionCtx, + } +} + +// renewTLSCACertificateAction implements a RenewTLSCACertificate action. +type renewTLSCACertificateAction struct { + log zerolog.Logger + action api.Action + actionCtx ActionContext +} + +// Start performs the start of the action. +// Returns true if the action is completely finished, false in case +// the start time needs to be recorded and a ready condition needs to be checked. +func (a *renewTLSCACertificateAction) Start(ctx context.Context) (bool, error) { + // Just delete the secret. + // It will be re-created. + if err := a.actionCtx.DeleteTLSCASecret(); err != nil { + return false, maskAny(err) + } + return true, nil +} + +// CheckProgress checks the progress of the action. +// Returns true if the action is completely finished, false otherwise. +func (a *renewTLSCACertificateAction) CheckProgress(ctx context.Context) (bool, bool, error) { + return true, false, nil +} + +// Timeout returns the amount of time after which this action will timeout. +func (a *renewTLSCACertificateAction) Timeout() time.Duration { + return renewTLSCACertificateTimeout +} + +// Return the MemberID used / created in this action +func (a *renewTLSCACertificateAction) MemberID() string { + return a.action.MemberID +} diff --git a/pkg/deployment/reconcile/action_renew_tls_certificate.go b/pkg/deployment/reconcile/action_renew_tls_certificate.go index 284394a0f..8ea4c839f 100644 --- a/pkg/deployment/reconcile/action_renew_tls_certificate.go +++ b/pkg/deployment/reconcile/action_renew_tls_certificate.go @@ -24,6 +24,7 @@ package reconcile import ( "context" + "time" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/rs/zerolog" @@ -66,6 +67,16 @@ func (a *renewTLSCertificateAction) Start(ctx context.Context) (bool, error) { // CheckProgress checks the progress of the action. // Returns true if the action is completely finished, false otherwise. -func (a *renewTLSCertificateAction) CheckProgress(ctx context.Context) (bool, error) { - return true, nil +func (a *renewTLSCertificateAction) CheckProgress(ctx context.Context) (bool, bool, error) { + return true, false, nil +} + +// Timeout returns the amount of time after which this action will timeout. +func (a *renewTLSCertificateAction) Timeout() time.Duration { + return renewTLSCertificateTimeout +} + +// Return the MemberID used / created in this action +func (a *renewTLSCertificateAction) MemberID() string { + return a.action.MemberID } diff --git a/pkg/deployment/reconcile/action_rotate_member.go b/pkg/deployment/reconcile/action_rotate_member.go index 84af5c8bf..d6368281e 100644 --- a/pkg/deployment/reconcile/action_rotate_member.go +++ b/pkg/deployment/reconcile/action_rotate_member.go @@ -24,6 +24,7 @@ package reconcile import ( "context" + "time" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/rs/zerolog" @@ -56,6 +57,10 @@ func (a *actionRotateMember) Start(ctx context.Context) (bool, error) { if !ok { log.Error().Msg("No such member") } + // Remove finalizers, so Kubernetes will quickly terminate the pod + if err := a.actionCtx.RemovePodFinalizers(m.PodName); err != nil { + return false, maskAny(err) + } if group.IsArangod() { // Invoke shutdown endpoint c, err := a.actionCtx.GetServerClient(ctx, group, a.action.MemberID) @@ -69,7 +74,7 @@ func (a *actionRotateMember) Start(ctx context.Context) (bool, error) { defer cancel() if err := c.Shutdown(ctx, removeFromCluster); err != nil { // Shutdown failed. Let's check if we're already done - if ready, err := a.CheckProgress(ctx); err == nil && ready { + if ready, _, err := a.CheckProgress(ctx); err == nil && ready { // We're done return true, nil } @@ -91,28 +96,39 @@ func (a *actionRotateMember) Start(ctx context.Context) (bool, error) { } // CheckProgress checks the progress of the action. -// Returns true if the action is completely finished, false otherwise. -func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, error) { +// Returns: ready, abort, error. +func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, bool, error) { // Check that pod is removed log := a.log m, found := a.actionCtx.GetMemberStatusByID(a.action.MemberID) if !found { log.Error().Msg("No such member") - return true, nil + return true, false, nil } if !m.Conditions.IsTrue(api.ConditionTypeTerminated) { // Pod is not yet terminated - return false, nil + return false, false, nil } // Pod is terminated, we can now remove it if err := a.actionCtx.DeletePod(m.PodName); err != nil { - return false, maskAny(err) + return false, false, maskAny(err) } // Pod is now gone, update the member status m.Phase = api.MemberPhaseNone m.RecentTerminations = nil // Since we're rotating, we do not care about old terminations. + m.CleanoutJobID = "" if err := a.actionCtx.UpdateMember(m); err != nil { - return false, maskAny(err) + return false, false, maskAny(err) } - return true, nil + return true, false, nil +} + +// Timeout returns the amount of time after which this action will timeout. +func (a *actionRotateMember) Timeout() time.Duration { + return rotateMemberTimeout +} + +// Return the MemberID used / created in this action +func (a *actionRotateMember) MemberID() string { + return a.action.MemberID } diff --git a/pkg/deployment/reconcile/action_shutdown_member.go b/pkg/deployment/reconcile/action_shutdown_member.go index bb0ec47ca..e4db9ddc2 100644 --- a/pkg/deployment/reconcile/action_shutdown_member.go +++ b/pkg/deployment/reconcile/action_shutdown_member.go @@ -75,7 +75,7 @@ func (a *actionShutdownMember) Start(ctx context.Context) (bool, error) { defer cancel() if err := c.Shutdown(ctx, removeFromCluster); err != nil { // Shutdown failed. Let's check if we're already done - if ready, err := a.CheckProgress(ctx); err == nil && ready { + if ready, _, err := a.CheckProgress(ctx); err == nil && ready { // We're done return true, nil } @@ -97,17 +97,27 @@ func (a *actionShutdownMember) Start(ctx context.Context) (bool, error) { } // CheckProgress checks the progress of the action. -// Returns true if the action is completely finished, false otherwise. -func (a *actionShutdownMember) CheckProgress(ctx context.Context) (bool, error) { +// Returns: ready, abort, error. +func (a *actionShutdownMember) CheckProgress(ctx context.Context) (bool, bool, error) { m, found := a.actionCtx.GetMemberStatusByID(a.action.MemberID) if !found { // Member not long exists - return true, nil + return true, false, nil } if m.Conditions.IsTrue(api.ConditionTypeTerminated) { // Shutdown completed - return true, nil + return true, false, nil } // Member still not shutdown, retry soon - return false, nil + return false, false, nil +} + +// Timeout returns the amount of time after which this action will timeout. +func (a *actionShutdownMember) Timeout() time.Duration { + return shutdownMemberTimeout +} + +// Return the MemberID used / created in this action +func (a *actionShutdownMember) MemberID() string { + return a.action.MemberID } diff --git a/pkg/deployment/reconcile/action_upgrade_member.go b/pkg/deployment/reconcile/action_upgrade_member.go index a9ec6564d..97f82368a 100644 --- a/pkg/deployment/reconcile/action_upgrade_member.go +++ b/pkg/deployment/reconcile/action_upgrade_member.go @@ -24,6 +24,7 @@ package reconcile import ( "context" + "time" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/rs/zerolog" @@ -74,7 +75,7 @@ func (a *actionUpgradeMember) Start(ctx context.Context) (bool, error) { defer cancel() if err := c.Shutdown(ctx, removeFromCluster); err != nil { // Shutdown failed. Let's check if we're already done - if ready, err := a.CheckProgress(ctx); err == nil && ready { + if ready, _, err := a.CheckProgress(ctx); err == nil && ready { // We're done return true, nil } @@ -97,13 +98,13 @@ func (a *actionUpgradeMember) Start(ctx context.Context) (bool, error) { // CheckProgress checks the progress of the action. // Returns true if the action is completely finished, false otherwise. -func (a *actionUpgradeMember) CheckProgress(ctx context.Context) (bool, error) { +func (a *actionUpgradeMember) CheckProgress(ctx context.Context) (bool, bool, error) { // Check that pod is removed log := a.log m, found := a.actionCtx.GetMemberStatusByID(a.action.MemberID) if !found { log.Error().Msg("No such member") - return true, nil + return true, false, nil } isUpgrading := m.Phase == api.MemberPhaseUpgrading log = log.With(). @@ -111,18 +112,29 @@ func (a *actionUpgradeMember) CheckProgress(ctx context.Context) (bool, error) { Bool("is-upgrading", isUpgrading).Logger() if !m.Conditions.IsTrue(api.ConditionTypeTerminated) { // Pod is not yet terminated - return false, nil + return false, false, nil } // Pod is terminated, we can now remove it log.Debug().Msg("Deleting pod") if err := a.actionCtx.DeletePod(m.PodName); err != nil { - return false, maskAny(err) + return false, false, maskAny(err) } // Pod is now gone, update the member status m.Phase = api.MemberPhaseNone m.RecentTerminations = nil // Since we're upgrading, we do not care about old terminations. + m.CleanoutJobID = "" if err := a.actionCtx.UpdateMember(m); err != nil { - return false, maskAny(err) + return false, false, maskAny(err) } - return isUpgrading, nil + return isUpgrading, false, nil +} + +// Timeout returns the amount of time after which this action will timeout. +func (a *actionUpgradeMember) Timeout() time.Duration { + return upgradeMemberTimeout +} + +// Return the MemberID used / created in this action +func (a *actionUpgradeMember) MemberID() string { + return a.action.MemberID } diff --git a/pkg/deployment/reconcile/action_wait_for_member_up.go b/pkg/deployment/reconcile/action_wait_for_member_up.go index fb3462412..8bdd0d1e7 100644 --- a/pkg/deployment/reconcile/action_wait_for_member_up.go +++ b/pkg/deployment/reconcile/action_wait_for_member_up.go @@ -24,12 +24,13 @@ package reconcile import ( "context" + "time" driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/agency" "github.com/rs/zerolog" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" - "github.com/arangodb/kube-arangodb/pkg/util/arangod" ) // NewWaitForMemberUpAction creates a new Action that implements the given @@ -53,7 +54,7 @@ type actionWaitForMemberUp struct { // Returns true if the action is completely finished, false in case // the start time needs to be recorded and a ready condition needs to be checked. func (a *actionWaitForMemberUp) Start(ctx context.Context) (bool, error) { - ready, err := a.CheckProgress(ctx) + ready, _, err := a.CheckProgress(ctx) if err != nil { return false, maskAny(err) } @@ -62,18 +63,18 @@ func (a *actionWaitForMemberUp) Start(ctx context.Context) (bool, error) { // CheckProgress checks the progress of the action. // Returns true if the action is completely finished, false otherwise. -func (a *actionWaitForMemberUp) CheckProgress(ctx context.Context) (bool, error) { +func (a *actionWaitForMemberUp) CheckProgress(ctx context.Context) (bool, bool, error) { if a.action.Group.IsArangosync() { return a.checkProgressArangoSync(ctx) } switch a.actionCtx.GetMode() { case api.DeploymentModeSingle: return a.checkProgressSingle(ctx) - case api.DeploymentModeResilientSingle: + case api.DeploymentModeActiveFailover: if a.action.Group == api.ServerGroupAgents { return a.checkProgressAgent(ctx) } - return a.checkProgressSingle(ctx) + return a.checkProgressSingleInActiveFailover(ctx) default: if a.action.Group == api.ServerGroupAgents { return a.checkProgressAgent(ctx) @@ -84,74 +85,113 @@ func (a *actionWaitForMemberUp) CheckProgress(ctx context.Context) (bool, error) // checkProgressSingle checks the progress of the action in the case // of a single server. -func (a *actionWaitForMemberUp) checkProgressSingle(ctx context.Context) (bool, error) { +func (a *actionWaitForMemberUp) checkProgressSingle(ctx context.Context) (bool, bool, error) { log := a.log c, err := a.actionCtx.GetDatabaseClient(ctx) if err != nil { log.Debug().Err(err).Msg("Failed to create database client") - return false, maskAny(err) + return false, false, maskAny(err) } if _, err := c.Version(ctx); err != nil { log.Debug().Err(err).Msg("Failed to get version") - return false, maskAny(err) + return false, false, maskAny(err) + } + return true, false, nil +} + +// checkProgressSingleInActiveFailover checks the progress of the action in the case +// of a single server as part of an active failover deployment. +func (a *actionWaitForMemberUp) checkProgressSingleInActiveFailover(ctx context.Context) (bool, bool, error) { + log := a.log + c, err := a.actionCtx.GetDatabaseClient(ctx) + if err != nil { + log.Debug().Err(err).Msg("Failed to create database client") + return false, false, maskAny(err) + } + if _, err := c.Version(ctx); err != nil { + log.Debug().Err(err).Msg("Failed to get version") + return false, false, maskAny(err) } - return true, nil + if _, err := c.Databases(ctx); err != nil { + log.Debug().Err(err).Msg("Failed to get databases") + return false, false, maskAny(err) + } + return true, false, nil } // checkProgressAgent checks the progress of the action in the case // of an agent. -func (a *actionWaitForMemberUp) checkProgressAgent(ctx context.Context) (bool, error) { +func (a *actionWaitForMemberUp) checkProgressAgent(ctx context.Context) (bool, bool, error) { log := a.log clients, err := a.actionCtx.GetAgencyClients(ctx) if err != nil { log.Debug().Err(err).Msg("Failed to create agency clients") - return false, maskAny(err) + return false, false, maskAny(err) } - if err := arangod.AreAgentsHealthy(ctx, clients); err != nil { + if err := agency.AreAgentsHealthy(ctx, clients); err != nil { log.Debug().Err(err).Msg("Not all agents are ready") - return false, nil + return false, false, nil } log.Debug().Msg("Agency is happy") - return true, nil + return true, false, nil } // checkProgressCluster checks the progress of the action in the case // of a cluster deployment (coordinator/dbserver). -func (a *actionWaitForMemberUp) checkProgressCluster(ctx context.Context) (bool, error) { +func (a *actionWaitForMemberUp) checkProgressCluster(ctx context.Context) (bool, bool, error) { log := a.log c, err := a.actionCtx.GetDatabaseClient(ctx) if err != nil { log.Debug().Err(err).Msg("Failed to create database client") - return false, maskAny(err) + return false, false, maskAny(err) } cluster, err := c.Cluster(ctx) if err != nil { log.Debug().Err(err).Msg("Failed to access cluster") - return false, maskAny(err) + return false, false, maskAny(err) } h, err := cluster.Health(ctx) if err != nil { log.Debug().Err(err).Msg("Failed to get cluster health") - return false, maskAny(err) + return false, false, maskAny(err) } sh, found := h.Health[driver.ServerID(a.action.MemberID)] if !found { log.Debug().Msg("Member not yet found in cluster health") - return false, nil + return false, false, nil } if sh.Status != driver.ServerStatusGood { log.Debug().Str("status", string(sh.Status)).Msg("Member set status not yet good") - return false, nil + return false, false, nil } - return true, nil + return true, false, nil } // checkProgressArangoSync checks the progress of the action in the case // of a sync master / worker. -func (a *actionWaitForMemberUp) checkProgressArangoSync(ctx context.Context) (bool, error) { - // TODO - return true, nil +func (a *actionWaitForMemberUp) checkProgressArangoSync(ctx context.Context) (bool, bool, error) { + log := a.log + c, err := a.actionCtx.GetSyncServerClient(ctx, a.action.Group, a.action.MemberID) + if err != nil { + log.Debug().Err(err).Msg("Failed to create arangosync client") + return false, false, maskAny(err) + } + if err := c.Health(ctx); err != nil { + log.Debug().Err(err).Msg("Health not ok yet") + return false, false, maskAny(err) + } + return true, false, nil +} + +// Timeout returns the amount of time after which this action will timeout. +func (a *actionWaitForMemberUp) Timeout() time.Duration { + return waitForMemberUpTimeout +} + +// Return the MemberID used / created in this action +func (a *actionWaitForMemberUp) MemberID() string { + return a.action.MemberID } diff --git a/pkg/deployment/reconcile/context.go b/pkg/deployment/reconcile/context.go index fa2ca0795..8d16679a1 100644 --- a/pkg/deployment/reconcile/context.go +++ b/pkg/deployment/reconcile/context.go @@ -25,11 +25,12 @@ package reconcile import ( "context" + "github.com/arangodb/arangosync/client" driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/agency" "k8s.io/api/core/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" - "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) @@ -40,10 +41,10 @@ type Context interface { // GetSpec returns the current specification of the deployment GetSpec() api.DeploymentSpec // GetStatus returns the current status of the deployment - GetStatus() api.DeploymentStatus + GetStatus() (api.DeploymentStatus, int32) // UpdateStatus replaces the status of the deployment with the given status and // updates the resources in k8s. - UpdateStatus(status api.DeploymentStatus, force ...bool) error + UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), // creating one if needed. GetDatabaseClient(ctx context.Context) (driver.Client, error) @@ -51,22 +52,41 @@ type Context interface { GetServerClient(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error) // GetAgencyClients returns a client connection for every agency member. // If the given predicate is not nil, only agents are included where the given predicate returns true. - GetAgencyClients(ctx context.Context, predicate func(id string) bool) ([]arangod.Agency, error) + GetAgencyClients(ctx context.Context, predicate func(id string) bool) ([]driver.Connection, error) + // GetAgency returns a connection to the entire agency. + GetAgency(ctx context.Context) (agency.Agency, error) + // GetSyncServerClient returns a cached client for a specific arangosync server. + GetSyncServerClient(ctx context.Context, group api.ServerGroup, id string) (client.API, error) + // CreateEvent creates a given event. + // On error, the error is logged. + CreateEvent(evt *k8sutil.Event) // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. - CreateMember(group api.ServerGroup, id string) error + // Returns ID, error + CreateMember(group api.ServerGroup, id string) (string, error) // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. DeletePod(podName string) error // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. DeletePvc(pvcName string) error + // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace + // of the deployment. If the pod does not exist, the error is ignored. + RemovePodFinalizers(podName string) error // GetOwnedPods returns a list of all pods owned by the deployment. GetOwnedPods() ([]v1.Pod, error) + // GetPvc gets a PVC by the given name, in the samespace of the deployment. + GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error) // GetTLSKeyfile returns the keyfile encoded TLS certificate+key for // the given member. GetTLSKeyfile(group api.ServerGroup, member api.MemberStatus) (string, error) // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member. // If the secret does not exist, the error is ignored. DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error + // GetTLSCA returns the TLS CA certificate in the secret with given name. + // Returns: publicKey, privateKey, ownerByDeployment, error + GetTLSCA(secretName string) (string, string, bool, error) + // DeleteSecret removes the Secret with given name. + // If the secret does not exist, the error is ignored. + DeleteSecret(secretName string) error } diff --git a/pkg/deployment/reconcile/plan_builder.go b/pkg/deployment/reconcile/plan_builder.go index 29cfd22e7..9057a6d71 100644 --- a/pkg/deployment/reconcile/plan_builder.go +++ b/pkg/deployment/reconcile/plan_builder.go @@ -23,16 +23,13 @@ package reconcile import ( - "crypto/x509" - "encoding/pem" - "time" - "github.com/rs/zerolog" "github.com/rs/zerolog/log" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) // upgradeDecision is the result of an upgrade check. @@ -56,8 +53,8 @@ func (d *Reconciler) CreatePlan() error { // Create plan apiObject := d.context.GetAPIObject() spec := d.context.GetSpec() - status := d.context.GetStatus() - newPlan, changed := createPlan(d.log, apiObject, status.Plan, spec, status, pods, d.context.GetTLSKeyfile) + status, lastVersion := d.context.GetStatus() + newPlan, changed := createPlan(d.log, apiObject, status.Plan, spec, status, pods, d.context.GetTLSKeyfile, d.context.GetTLSCA, d.context.GetPvc, d.context.CreateEvent) // If not change, we're done if !changed { @@ -70,7 +67,7 @@ func (d *Reconciler) CreatePlan() error { return nil } status.Plan = newPlan - if err := d.context.UpdateStatus(status); err != nil { + if err := d.context.UpdateStatus(status, lastVersion); err != nil { return maskAny(err) } return nil @@ -79,10 +76,13 @@ func (d *Reconciler) CreatePlan() error { // createPlan considers the given specification & status and creates a plan to get the status in line with the specification. // If a plan already exists, the given plan is returned with false. // Otherwise the new plan is returned with a boolean true. -func createPlan(log zerolog.Logger, apiObject metav1.Object, +func createPlan(log zerolog.Logger, apiObject k8sutil.APIObject, currentPlan api.Plan, spec api.DeploymentSpec, status api.DeploymentStatus, pods []v1.Pod, - getTLSKeyfile func(group api.ServerGroup, member api.MemberStatus) (string, error)) (api.Plan, bool) { + getTLSKeyfile func(group api.ServerGroup, member api.MemberStatus) (string, error), + getTLSCA func(string) (string, string, bool, error), + getPVC func(pvcName string) (*v1.PersistentVolumeClaim, error), + createEvent func(evt *k8sutil.Event)) (api.Plan, bool) { if len(currentPlan) > 0 { // Plan already exists, complete that first return currentPlan, false @@ -92,8 +92,8 @@ func createPlan(log zerolog.Logger, apiObject metav1.Object, var plan api.Plan // Check for members in failed state - status.Members.ForeachServerGroup(func(group api.ServerGroup, members *api.MemberStatusList) error { - for _, m := range *members { + status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { + for _, m := range members { if m.Phase == api.MemberPhaseFailed && len(plan) == 0 { newID := "" if group == api.ServerGroupAgents { @@ -108,18 +108,31 @@ func createPlan(log zerolog.Logger, apiObject metav1.Object, return nil }) + // Check for cleaned out dbserver in created state + for _, m := range status.Members.DBServers { + if len(plan) == 0 && m.Phase == api.MemberPhaseCreated && m.Conditions.IsTrue(api.ConditionTypeCleanedOut) { + plan = append(plan, + api.NewAction(api.ActionTypeRemoveMember, api.ServerGroupDBServers, m.ID), + api.NewAction(api.ActionTypeAddMember, api.ServerGroupDBServers, ""), + ) + } + } + // Check for scale up/down if len(plan) == 0 { switch spec.GetMode() { case api.DeploymentModeSingle: // Never scale down - case api.DeploymentModeResilientSingle: + case api.DeploymentModeActiveFailover: // Only scale singles plan = append(plan, createScalePlan(log, status.Members.Single, api.ServerGroupSingle, spec.Single.GetCount())...) case api.DeploymentModeCluster: - // Scale dbservers, coordinators, syncmasters & syncworkers + // Scale dbservers, coordinators plan = append(plan, createScalePlan(log, status.Members.DBServers, api.ServerGroupDBServers, spec.DBServers.GetCount())...) plan = append(plan, createScalePlan(log, status.Members.Coordinators, api.ServerGroupCoordinators, spec.Coordinators.GetCount())...) + } + if spec.GetMode().SupportsSync() { + // Scale syncmasters & syncworkers plan = append(plan, createScalePlan(log, status.Members.SyncMasters, api.ServerGroupSyncMasters, spec.SyncMasters.GetCount())...) plan = append(plan, createScalePlan(log, status.Members.SyncWorkers, api.ServerGroupSyncWorkers, spec.SyncWorkers.GetCount())...) } @@ -135,8 +148,8 @@ func createPlan(log zerolog.Logger, apiObject metav1.Object, } return nil } - status.Members.ForeachServerGroup(func(group api.ServerGroup, members *api.MemberStatusList) error { - for _, m := range *members { + status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { + for _, m := range members { if len(plan) > 0 { // Only 1 change at a time continue @@ -165,36 +178,18 @@ func createPlan(log zerolog.Logger, apiObject metav1.Object, } // Check for the need to rotate TLS certificate of a members - if len(plan) == 0 && spec.TLS.IsSecure() { - status.Members.ForeachServerGroup(func(group api.ServerGroup, members *api.MemberStatusList) error { - for _, m := range *members { - if len(plan) > 0 { - // Only 1 change at a time - continue - } - if m.Phase != api.MemberPhaseCreated { - // Only make changes when phase is created - continue - } - // Load keyfile - keyfile, err := getTLSKeyfile(group, m) - if err != nil { - log.Warn().Err(err). - Str("role", group.AsRole()). - Str("id", m.ID). - Msg("Failed to get TLS secret") - continue - } - renewalNeeded := tlsKeyfileNeedsRenewal(log, keyfile) - if renewalNeeded { - plan = append(append(plan, - api.NewAction(api.ActionTypeRenewTLSCertificate, group, m.ID)), - createRotateMemberPlan(log, m, group, "TLS certificate renewal")..., - ) - } - } - return nil - }) + if len(plan) == 0 { + plan = createRotateTLSServerCertificatePlan(log, spec, status, getTLSKeyfile) + } + + // Check for changes storage classes or requirements + if len(plan) == 0 { + plan = createRotateServerStoragePlan(log, apiObject, spec, status, getPVC, createEvent) + } + + // Check for the need to rotate TLS CA certificate and all members + if len(plan) == 0 { + plan = createRotateTLSCAPlan(log, apiObject, spec, status, getTLSCA, createEvent) } // Return plan @@ -204,8 +199,7 @@ func createPlan(log zerolog.Logger, apiObject metav1.Object, // podNeedsUpgrading decides if an upgrade of the pod is needed (to comply with // the given spec) and if that is allowed. func podNeedsUpgrading(p v1.Pod, spec api.DeploymentSpec, images api.ImageInfoList) upgradeDecision { - if len(p.Spec.Containers) == 1 { - c := p.Spec.Containers[0] + if c, found := k8sutil.GetContainerByName(&p, k8sutil.ServerContainerName); found { specImageInfo, found := images.GetByImage(spec.GetImage()) if !found { return upgradeDecision{UpgradeNeeded: false} @@ -249,15 +243,17 @@ func podNeedsUpgrading(p v1.Pod, spec api.DeploymentSpec, images api.ImageInfoLi // When true is returned, a reason for the rotation is already returned. func podNeedsRotation(p v1.Pod, apiObject metav1.Object, spec api.DeploymentSpec, group api.ServerGroup, agents api.MemberStatusList, id string) (bool, string) { - // Check number of containers - if len(p.Spec.Containers) != 1 { - return true, "Number of containers changed" - } + groupSpec := spec.GetServerGroupSpec(group) + // Check image pull policy - c := p.Spec.Containers[0] - if c.ImagePullPolicy != spec.GetImagePullPolicy() { - return true, "Image pull policy changed" + if c, found := k8sutil.GetContainerByName(&p, k8sutil.ServerContainerName); found { + if c.ImagePullPolicy != spec.GetImagePullPolicy() { + return true, "Image pull policy changed" + } + } else { + return true, "Server container not found" } + // Check arguments /*expectedArgs := createArangodArgs(apiObject, spec, group, agents, id) if len(expectedArgs) != len(c.Args) { @@ -269,45 +265,20 @@ func podNeedsRotation(p v1.Pod, apiObject metav1.Object, spec api.DeploymentSpec } }*/ + // Check service account + if normalizeServiceAccountName(p.Spec.ServiceAccountName) != normalizeServiceAccountName(groupSpec.GetServiceAccountName()) { + return true, "ServiceAccountName changed" + } + return false, "" } -// tlsKeyfileNeedsRenewal decides if the certificate in the given keyfile -// should be renewed. -func tlsKeyfileNeedsRenewal(log zerolog.Logger, keyfile string) bool { - raw := []byte(keyfile) - for { - var derBlock *pem.Block - derBlock, raw = pem.Decode(raw) - if derBlock == nil { - break - } - if derBlock.Type == "CERTIFICATE" { - cert, err := x509.ParseCertificate(derBlock.Bytes) - if err != nil { - // We do not understand the certificate, let's renew it - log.Warn().Err(err).Msg("Failed to parse x509 certificate. Renewing it") - return true - } - if cert.IsCA { - // Only look at the server certificate, not CA or intermediate - continue - } - // Check expiration date. Renewal at 2/3 of lifetime. - ttl := cert.NotAfter.Sub(cert.NotBefore) - expirationDate := cert.NotBefore.Add((ttl / 3) * 2) - if expirationDate.Before(time.Now()) { - // We should renew now - log.Debug(). - Str("not-before", cert.NotBefore.String()). - Str("not-after", cert.NotAfter.String()). - Str("expiration-date", expirationDate.String()). - Msg("TLS certificate renewal needed") - return true - } - } +// normalizeServiceAccountName replaces default with empty string, otherwise returns the input. +func normalizeServiceAccountName(name string) string { + if name == "default" { + return "" } - return false + return "" } // createScalePlan creates a scaling plan for a single server group @@ -320,12 +291,16 @@ func createScalePlan(log zerolog.Logger, members api.MemberStatusList, group api plan = append(plan, api.NewAction(api.ActionTypeAddMember, group, "")) } log.Debug(). + Int("count", count). + Int("actual-count", len(members)). Int("delta", toAdd). Str("role", group.AsRole()). Msg("Creating scale-up plan") } else if len(members) > count { - // Note, we scale down 1 member as a time - if m, err := members.SelectMemberToRemove(); err == nil { + // Note, we scale down 1 member at a time + if m, err := members.SelectMemberToRemove(); err != nil { + log.Warn().Err(err).Str("role", group.AsRole()).Msg("Failed to select member to remove") + } else { if group == api.ServerGroupDBServers { plan = append(plan, api.NewAction(api.ActionTypeCleanOutMember, group, m.ID), @@ -336,7 +311,10 @@ func createScalePlan(log zerolog.Logger, members api.MemberStatusList, group api api.NewAction(api.ActionTypeRemoveMember, group, m.ID), ) log.Debug(). + Int("count", count). + Int("actual-count", len(members)). Str("role", group.AsRole()). + Str("member-id", m.ID). Msg("Creating scale-down plan") } } @@ -350,6 +328,7 @@ func createRotateMemberPlan(log zerolog.Logger, member api.MemberStatus, log.Debug(). Str("id", member.ID). Str("role", group.AsRole()). + Str("reason", reason). Msg("Creating rotation plan") plan := api.Plan{ api.NewAction(api.ActionTypeRotateMember, group, member.ID, reason), @@ -365,6 +344,7 @@ func createUpgradeMemberPlan(log zerolog.Logger, member api.MemberStatus, log.Debug(). Str("id", member.ID). Str("role", group.AsRole()). + Str("reason", reason). Msg("Creating upgrade plan") plan := api.Plan{ api.NewAction(api.ActionTypeUpgradeMember, group, member.ID, reason), diff --git a/pkg/deployment/reconcile/plan_builder_storage.go b/pkg/deployment/reconcile/plan_builder_storage.go new file mode 100644 index 000000000..87090785c --- /dev/null +++ b/pkg/deployment/reconcile/plan_builder_storage.go @@ -0,0 +1,115 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package reconcile + +import ( + "github.com/rs/zerolog" + "k8s.io/api/core/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// createRotateServerStoragePlan creates plan to rotate a server and its volume because of a +// different storage class or a difference in storage resource requirements. +func createRotateServerStoragePlan(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, + getPVC func(pvcName string) (*v1.PersistentVolumeClaim, error), + createEvent func(evt *k8sutil.Event)) api.Plan { + if spec.GetMode() == api.DeploymentModeSingle { + // Storage cannot be changed in single server deployments + return nil + } + var plan api.Plan + status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { + for _, m := range members { + if len(plan) > 0 { + // Only 1 change at a time + continue + } + if m.Phase != api.MemberPhaseCreated { + // Only make changes when phase is created + continue + } + if m.PersistentVolumeClaimName == "" { + // Plan is irrelevant without PVC + continue + } + groupSpec := spec.GetServerGroupSpec(group) + storageClassName := groupSpec.GetStorageClassName() + if storageClassName == "" { + // Using default storage class name + continue + } + // Load PVC + pvc, err := getPVC(m.PersistentVolumeClaimName) + if err != nil { + log.Warn().Err(err). + Str("role", group.AsRole()). + Str("id", m.ID). + Msg("Failed to get PVC") + continue + } + replacementNeeded := false + if util.StringOrDefault(pvc.Spec.StorageClassName) != storageClassName { + // Storageclass has changed + replacementNeeded = true + } + if replacementNeeded { + if group != api.ServerGroupAgents && group != api.ServerGroupDBServers { + // Only agents & dbservers are allowed to change their storage class. + createEvent(k8sutil.NewCannotChangeStorageClassEvent(apiObject, m.ID, group.AsRole(), "Not supported")) + continue + } else { + if group != api.ServerGroupAgents { + plan = append(plan, + // Scale up, so we're sure that a new member is available + api.NewAction(api.ActionTypeAddMember, group, ""), + api.NewAction(api.ActionTypeWaitForMemberUp, group, api.MemberIDPreviousAction), + ) + } + if group == api.ServerGroupDBServers { + plan = append(plan, + // Cleanout + api.NewAction(api.ActionTypeCleanOutMember, group, m.ID), + ) + } + plan = append(plan, + // Shutdown & remove the server + api.NewAction(api.ActionTypeShutdownMember, group, m.ID), + api.NewAction(api.ActionTypeRemoveMember, group, m.ID), + ) + if group == api.ServerGroupAgents { + plan = append(plan, + // Scale up, so we're adding the removed agent (note: with the old ID) + api.NewAction(api.ActionTypeAddMember, group, m.ID), + api.NewAction(api.ActionTypeWaitForMemberUp, group, m.ID), + ) + } + } + } + } + return nil + }) + return plan +} diff --git a/pkg/deployment/reconcile/plan_builder_test.go b/pkg/deployment/reconcile/plan_builder_test.go index 20a7640bf..7663d1c40 100644 --- a/pkg/deployment/reconcile/plan_builder_test.go +++ b/pkg/deployment/reconcile/plan_builder_test.go @@ -29,10 +29,12 @@ import ( "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) // TestCreatePlanSingleScale creates a `single` deployment to test the creating of scaling plan. @@ -40,6 +42,13 @@ func TestCreatePlanSingleScale(t *testing.T) { getTLSKeyfile := func(group api.ServerGroup, member api.MemberStatus) (string, error) { return "", maskAny(fmt.Errorf("Not implemented")) } + getTLSCA := func(string) (string, string, bool, error) { + return "", "", false, maskAny(fmt.Errorf("Not implemented")) + } + getPVC := func(pvcName string) (*v1.PersistentVolumeClaim, error) { + return nil, maskAny(fmt.Errorf("Not implemented")) + } + createEvent := func(evt *k8sutil.Event) {} log := zerolog.Nop() spec := api.DeploymentSpec{ Mode: api.NewMode(api.DeploymentModeSingle), @@ -55,7 +64,7 @@ func TestCreatePlanSingleScale(t *testing.T) { // Test with empty status var status api.DeploymentStatus - newPlan, changed := createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile) + newPlan, changed := createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile, getTLSCA, getPVC, createEvent) assert.True(t, changed) assert.Len(t, newPlan, 0) // Single mode does not scale @@ -66,7 +75,7 @@ func TestCreatePlanSingleScale(t *testing.T) { PodName: "something", }, } - newPlan, changed = createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile) + newPlan, changed = createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile, getTLSCA, getPVC, createEvent) assert.True(t, changed) assert.Len(t, newPlan, 0) // Single mode does not scale @@ -81,19 +90,26 @@ func TestCreatePlanSingleScale(t *testing.T) { PodName: "something1", }, } - newPlan, changed = createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile) + newPlan, changed = createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile, getTLSCA, getPVC, createEvent) assert.True(t, changed) assert.Len(t, newPlan, 0) // Single mode does not scale } -// TestCreatePlanResilientSingleScale creates a `resilientsingle` deployment to test the creating of scaling plan. -func TestCreatePlanResilientSingleScale(t *testing.T) { +// TestCreatePlanActiveFailoverScale creates a `ActiveFailover` deployment to test the creating of scaling plan. +func TestCreatePlanActiveFailoverScale(t *testing.T) { getTLSKeyfile := func(group api.ServerGroup, member api.MemberStatus) (string, error) { return "", maskAny(fmt.Errorf("Not implemented")) } + getTLSCA := func(string) (string, string, bool, error) { + return "", "", false, maskAny(fmt.Errorf("Not implemented")) + } + getPVC := func(pvcName string) (*v1.PersistentVolumeClaim, error) { + return nil, maskAny(fmt.Errorf("Not implemented")) + } + createEvent := func(evt *k8sutil.Event) {} log := zerolog.Nop() spec := api.DeploymentSpec{ - Mode: api.NewMode(api.DeploymentModeResilientSingle), + Mode: api.NewMode(api.DeploymentModeActiveFailover), } spec.SetDefaults("test") spec.Single.Count = util.NewInt(2) @@ -107,7 +123,7 @@ func TestCreatePlanResilientSingleScale(t *testing.T) { // Test with empty status var status api.DeploymentStatus - newPlan, changed := createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile) + newPlan, changed := createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile, getTLSCA, getPVC, createEvent) assert.True(t, changed) require.Len(t, newPlan, 2) assert.Equal(t, api.ActionTypeAddMember, newPlan[0].Type) @@ -120,7 +136,7 @@ func TestCreatePlanResilientSingleScale(t *testing.T) { PodName: "something", }, } - newPlan, changed = createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile) + newPlan, changed = createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile, getTLSCA, getPVC, createEvent) assert.True(t, changed) require.Len(t, newPlan, 1) assert.Equal(t, api.ActionTypeAddMember, newPlan[0].Type) @@ -145,7 +161,7 @@ func TestCreatePlanResilientSingleScale(t *testing.T) { PodName: "something4", }, } - newPlan, changed = createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile) + newPlan, changed = createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile, getTLSCA, getPVC, createEvent) assert.True(t, changed) require.Len(t, newPlan, 2) // Note: Downscaling is only down 1 at a time assert.Equal(t, api.ActionTypeShutdownMember, newPlan[0].Type) @@ -159,6 +175,13 @@ func TestCreatePlanClusterScale(t *testing.T) { getTLSKeyfile := func(group api.ServerGroup, member api.MemberStatus) (string, error) { return "", maskAny(fmt.Errorf("Not implemented")) } + getTLSCA := func(string) (string, string, bool, error) { + return "", "", false, maskAny(fmt.Errorf("Not implemented")) + } + getPVC := func(pvcName string) (*v1.PersistentVolumeClaim, error) { + return nil, maskAny(fmt.Errorf("Not implemented")) + } + createEvent := func(evt *k8sutil.Event) {} log := zerolog.Nop() spec := api.DeploymentSpec{ Mode: api.NewMode(api.DeploymentModeCluster), @@ -174,7 +197,7 @@ func TestCreatePlanClusterScale(t *testing.T) { // Test with empty status var status api.DeploymentStatus - newPlan, changed := createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile) + newPlan, changed := createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile, getTLSCA, getPVC, createEvent) assert.True(t, changed) require.Len(t, newPlan, 6) // Adding 3 dbservers & 3 coordinators (note: agents do not scale now) assert.Equal(t, api.ActionTypeAddMember, newPlan[0].Type) @@ -207,7 +230,7 @@ func TestCreatePlanClusterScale(t *testing.T) { PodName: "coordinator1", }, } - newPlan, changed = createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile) + newPlan, changed = createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile, getTLSCA, getPVC, createEvent) assert.True(t, changed) require.Len(t, newPlan, 3) assert.Equal(t, api.ActionTypeAddMember, newPlan[0].Type) @@ -244,7 +267,7 @@ func TestCreatePlanClusterScale(t *testing.T) { } spec.DBServers.Count = util.NewInt(1) spec.Coordinators.Count = util.NewInt(1) - newPlan, changed = createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile) + newPlan, changed = createPlan(log, depl, nil, spec, status, nil, getTLSKeyfile, getTLSCA, getPVC, createEvent) assert.True(t, changed) require.Len(t, newPlan, 5) // Note: Downscaling is done 1 at a time assert.Equal(t, api.ActionTypeCleanOutMember, newPlan[0].Type) diff --git a/pkg/deployment/reconcile/plan_builder_tls.go b/pkg/deployment/reconcile/plan_builder_tls.go new file mode 100644 index 000000000..53086ecf7 --- /dev/null +++ b/pkg/deployment/reconcile/plan_builder_tls.go @@ -0,0 +1,250 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package reconcile + +import ( + "crypto/x509" + "encoding/pem" + "net" + "time" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/rs/zerolog" +) + +// createRotateTLSServerCertificatePlan creates plan to rotate a server because of an (soon to be) expired TLS certificate. +func createRotateTLSServerCertificatePlan(log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, + getTLSKeyfile func(group api.ServerGroup, member api.MemberStatus) (string, error)) api.Plan { + if !spec.TLS.IsSecure() { + return nil + } + var plan api.Plan + status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { + for _, m := range members { + if len(plan) > 0 { + // Only 1 change at a time + continue + } + if m.Phase != api.MemberPhaseCreated { + // Only make changes when phase is created + continue + } + if group == api.ServerGroupSyncWorkers { + // SyncWorkers have no externally created TLS keyfile + continue + } + // Load keyfile + keyfile, err := getTLSKeyfile(group, m) + if err != nil { + log.Warn().Err(err). + Str("role", group.AsRole()). + Str("id", m.ID). + Msg("Failed to get TLS secret") + continue + } + tlsSpec := spec.TLS + if group.IsArangosync() { + tlsSpec = spec.Sync.TLS + } + renewalNeeded, reason := tlsKeyfileNeedsRenewal(log, keyfile, tlsSpec) + if renewalNeeded { + plan = append(append(plan, + api.NewAction(api.ActionTypeRenewTLSCertificate, group, m.ID, reason)), + createRotateMemberPlan(log, m, group, "TLS certificate renewal")..., + ) + } + } + return nil + }) + return plan +} + +// createRotateTLSCAPlan creates plan to replace a TLS CA and rotate all server. +func createRotateTLSCAPlan(log zerolog.Logger, apiObject k8sutil.APIObject, + spec api.DeploymentSpec, status api.DeploymentStatus, + getTLSCA func(string) (string, string, bool, error), + createEvent func(evt *k8sutil.Event)) api.Plan { + if !spec.TLS.IsSecure() { + return nil + } + secretName := spec.TLS.GetCASecretName() + cert, _, isOwned, err := getTLSCA(secretName) + if err != nil { + log.Warn().Err(err).Str("secret-name", secretName).Msg("Failed to fetch TLS CA secret") + return nil + } + if !isOwned { + // TLS CA is not owned by the deployment, we cannot change it + return nil + } + var plan api.Plan + if renewalNeeded, reason := tlsCANeedsRenewal(log, cert, spec.TLS); renewalNeeded { + if spec.IsDowntimeAllowed() { + var planSuffix api.Plan + plan = append(plan, + api.NewAction(api.ActionTypeRenewTLSCACertificate, 0, "", reason), + ) + status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { + for _, m := range members { + if m.Phase != api.MemberPhaseCreated { + // Only make changes when phase is created + continue + } + if !group.IsArangod() { + // Sync master/worker is not applicable here + continue + } + plan = append(plan, + api.NewAction(api.ActionTypeRenewTLSCertificate, group, m.ID), + api.NewAction(api.ActionTypeRotateMember, group, m.ID, "TLS CA certificate changed"), + ) + planSuffix = append(planSuffix, + api.NewAction(api.ActionTypeWaitForMemberUp, group, m.ID, "TLS CA certificate changed"), + ) + } + return nil + }) + plan = append(plan, planSuffix...) + } else { + // Rotating the CA results in downtime. + // That is currently not allowed. + createEvent(k8sutil.NewDowntimeNotAllowedEvent(apiObject, "Rotate TLS CA")) + } + } + return plan +} + +// tlsKeyfileNeedsRenewal decides if the certificate in the given keyfile +// should be renewed. +func tlsKeyfileNeedsRenewal(log zerolog.Logger, keyfile string, spec api.TLSSpec) (bool, string) { + raw := []byte(keyfile) + // containsAll returns true when all elements in the expected list + // are in the actual list. + containsAll := func(actual []string, expected []string) bool { + for _, x := range expected { + found := false + for _, y := range actual { + if x == y { + found = true + break + } + } + if !found { + return false + } + } + return true + } + ipsToStringSlice := func(list []net.IP) []string { + result := make([]string, len(list)) + for i, x := range list { + result[i] = x.String() + } + return result + } + for { + var derBlock *pem.Block + derBlock, raw = pem.Decode(raw) + if derBlock == nil { + break + } + if derBlock.Type == "CERTIFICATE" { + cert, err := x509.ParseCertificate(derBlock.Bytes) + if err != nil { + // We do not understand the certificate, let's renew it + log.Warn().Err(err).Msg("Failed to parse x509 certificate. Renewing it") + return true, "Cannot parse x509 certificate: " + err.Error() + } + if cert.IsCA { + // Only look at the server certificate, not CA or intermediate + continue + } + // Check expiration date. Renewal at 2/3 of lifetime. + ttl := cert.NotAfter.Sub(cert.NotBefore) + expirationDate := cert.NotBefore.Add((ttl / 3) * 2) + if expirationDate.Before(time.Now()) { + // We should renew now + log.Debug(). + Str("not-before", cert.NotBefore.String()). + Str("not-after", cert.NotAfter.String()). + Str("expiration-date", expirationDate.String()). + Msg("TLS certificate renewal needed") + return true, "Server certificate about to expire" + } + // Check alternate names against spec + dnsNames, ipAddresses, emailAddress, err := spec.GetParsedAltNames() + if err == nil { + if !containsAll(cert.DNSNames, dnsNames) { + return true, "Some alternate DNS names are missing" + } + if !containsAll(ipsToStringSlice(cert.IPAddresses), ipAddresses) { + return true, "Some alternate IP addresses are missing" + } + if !containsAll(cert.EmailAddresses, emailAddress) { + return true, "Some alternate email addresses are missing" + } + } + } + } + return false, "" +} + +// tlsCANeedsRenewal decides if the given CA certificate +// should be renewed. +// Returns: shouldRenew, reason +func tlsCANeedsRenewal(log zerolog.Logger, cert string, spec api.TLSSpec) (bool, string) { + raw := []byte(cert) + for { + var derBlock *pem.Block + derBlock, raw = pem.Decode(raw) + if derBlock == nil { + break + } + if derBlock.Type == "CERTIFICATE" { + cert, err := x509.ParseCertificate(derBlock.Bytes) + if err != nil { + // We do not understand the certificate, let's renew it + log.Warn().Err(err).Msg("Failed to parse x509 certificate. Renewing it") + return true, "Cannot parse x509 certificate: " + err.Error() + } + if !cert.IsCA { + // Only look at the CA certificate + continue + } + // Check expiration date. Renewal at 90% of lifetime. + ttl := cert.NotAfter.Sub(cert.NotBefore) + expirationDate := cert.NotBefore.Add((ttl / 10) * 9) + if expirationDate.Before(time.Now()) { + // We should renew now + log.Debug(). + Str("not-before", cert.NotBefore.String()). + Str("not-after", cert.NotAfter.String()). + Str("expiration-date", expirationDate.String()). + Msg("TLS CA certificate renewal needed") + return true, "CA Certificate about to expire" + } + } + } + return false, "" +} diff --git a/pkg/deployment/reconcile/plan_executor.go b/pkg/deployment/reconcile/plan_executor.go index ca0291e38..a423a7b64 100644 --- a/pkg/deployment/reconcile/plan_executor.go +++ b/pkg/deployment/reconcile/plan_executor.go @@ -25,11 +25,13 @@ package reconcile import ( "context" "fmt" + "time" + "github.com/rs/zerolog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" - "github.com/rs/zerolog" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) // ExecutePlan tries to execute the plan as far as possible. @@ -37,19 +39,23 @@ import ( // False otherwise. func (d *Reconciler) ExecutePlan(ctx context.Context) (bool, error) { log := d.log - initialPlanLen := len(d.context.GetStatus().Plan) + firstLoop := true for { - status := d.context.GetStatus() - if len(status.Plan) == 0 { - // No plan exists, nothing to be done - return initialPlanLen > 0, nil + loopStatus, _ := d.context.GetStatus() + if len(loopStatus.Plan) == 0 { + // No plan exists or all action have finished, nothing to be done + if !firstLoop { + log.Debug().Msg("Reconciliation plan has finished") + } + return !firstLoop, nil } + firstLoop = false // Take first action - planAction := status.Plan[0] + planAction := loopStatus.Plan[0] log := log.With(). - Int("plan-len", len(status.Plan)). + Int("plan-len", len(loopStatus.Plan)). Str("action-id", planAction.ID). Str("action-type", string(planAction.Type)). Str("group", planAction.Group.AsRole()). @@ -64,21 +70,26 @@ func (d *Reconciler) ExecutePlan(ctx context.Context) (bool, error) { Msg("Failed to start action") return false, maskAny(err) } - // action.Start may have changed status, so reload it. - status = d.context.GetStatus() - // Update status according to result on action.Start. - if ready { - // Remove action from list - status.Plan = status.Plan[1:] - } else { - // Mark start time - now := metav1.Now() - status.Plan[0].StartTime = &now - } - // Save plan update - if err := d.context.UpdateStatus(status, true); err != nil { - log.Debug().Err(err).Msg("Failed to update CR status") - return false, maskAny(err) + { // action.Start may have changed status, so reload it. + status, lastVersion := d.context.GetStatus() + // Update status according to result on action.Start. + if ready { + // Remove action from list + status.Plan = status.Plan[1:] + if len(status.Plan) > 0 && status.Plan[0].MemberID == api.MemberIDPreviousAction { + // Fill in MemberID from previous action + status.Plan[0].MemberID = action.MemberID() + } + } else { + // Mark start time + now := metav1.Now() + status.Plan[0].StartTime = &now + } + // Save plan update + if err := d.context.UpdateStatus(status, lastVersion, true); err != nil { + log.Debug().Err(err).Msg("Failed to update CR status") + return false, maskAny(err) + } } log.Debug().Bool("ready", ready).Msg("Action Start completed") if !ready { @@ -88,25 +99,57 @@ func (d *Reconciler) ExecutePlan(ctx context.Context) (bool, error) { // Continue with next action } else { // First action of plan has been started, check its progress - ready, err := action.CheckProgress(ctx) + ready, abort, err := action.CheckProgress(ctx) if err != nil { log.Debug().Err(err).Msg("Failed to check action progress") return false, maskAny(err) } if ready { - // action.CheckProgress may have changed status, so reload it. - status = d.context.GetStatus() - // Remove action from list - status.Plan = status.Plan[1:] - // Save plan update - if err := d.context.UpdateStatus(status); err != nil { - log.Debug().Err(err).Msg("Failed to update CR status") - return false, maskAny(err) + { // action.CheckProgress may have changed status, so reload it. + status, lastVersion := d.context.GetStatus() + // Remove action from list + status.Plan = status.Plan[1:] + if len(status.Plan) > 0 && status.Plan[0].MemberID == api.MemberIDPreviousAction { + // Fill in MemberID from previous action + status.Plan[0].MemberID = action.MemberID() + } + // Save plan update + if err := d.context.UpdateStatus(status, lastVersion); err != nil { + log.Debug().Err(err).Msg("Failed to update CR status") + return false, maskAny(err) + } } } - log.Debug().Bool("ready", ready).Msg("Action CheckProgress completed") + log.Debug(). + Bool("abort", abort). + Bool("ready", ready). + Msg("Action CheckProgress completed") if !ready { - // Not ready check, come back soon + deadlineExpired := false + if abort { + log.Warn().Msg("Action aborted. Removing the entire plan") + d.context.CreateEvent(k8sutil.NewPlanAbortedEvent(d.context.GetAPIObject(), string(planAction.Type), planAction.MemberID, planAction.Group.AsRole())) + } else { + // Not ready yet & no abort, check timeout + deadline := planAction.CreationTime.Add(action.Timeout()) + if time.Now().After(deadline) { + // Timeout has expired + deadlineExpired = true + log.Warn().Msg("Action not finished in time. Removing the entire plan") + d.context.CreateEvent(k8sutil.NewPlanTimeoutEvent(d.context.GetAPIObject(), string(planAction.Type), planAction.MemberID, planAction.Group.AsRole())) + } + } + if abort || deadlineExpired { + // Replace plan with empty one and save it. + status, lastVersion := d.context.GetStatus() + status.Plan = api.Plan{} + if err := d.context.UpdateStatus(status, lastVersion); err != nil { + log.Debug().Err(err).Msg("Failed to update CR status") + return false, maskAny(err) + } + return true, nil + } + // Timeout not yet expired, come back soon return true, nil } // Continue with next action @@ -136,6 +179,8 @@ func (d *Reconciler) createAction(ctx context.Context, log zerolog.Logger, actio return NewWaitForMemberUpAction(log, action, actionCtx) case api.ActionTypeRenewTLSCertificate: return NewRenewTLSCertificateAction(log, action, actionCtx) + case api.ActionTypeRenewTLSCACertificate: + return NewRenewTLSCACertificateAction(log, action, actionCtx) default: panic(fmt.Sprintf("Unknown action type '%s'", action.Type)) } diff --git a/pkg/deployment/reconcile/timeouts.go b/pkg/deployment/reconcile/timeouts.go new file mode 100644 index 000000000..a2273d18d --- /dev/null +++ b/pkg/deployment/reconcile/timeouts.go @@ -0,0 +1,37 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package reconcile + +import "time" + +const ( + addMemberTimeout = time.Minute * 5 + cleanoutMemberTimeout = time.Hour * 12 + removeMemberTimeout = time.Minute * 15 + renewTLSCertificateTimeout = time.Minute * 30 + renewTLSCACertificateTimeout = time.Minute * 30 + rotateMemberTimeout = time.Minute * 30 + shutdownMemberTimeout = time.Minute * 30 + upgradeMemberTimeout = time.Hour * 6 + waitForMemberUpTimeout = time.Minute * 15 +) diff --git a/pkg/deployment/resilience/context.go b/pkg/deployment/resilience/context.go index 6dad0d4f2..4a0c4fa0e 100644 --- a/pkg/deployment/resilience/context.go +++ b/pkg/deployment/resilience/context.go @@ -27,7 +27,6 @@ import ( driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" - "github.com/arangodb/kube-arangodb/pkg/util/arangod" ) // Context provides methods to the resilience package. @@ -35,13 +34,13 @@ type Context interface { // GetSpec returns the current specification of the deployment GetSpec() api.DeploymentSpec // GetStatus returns the current status of the deployment - GetStatus() api.DeploymentStatus + GetStatus() (api.DeploymentStatus, int32) // UpdateStatus replaces the status of the deployment with the given status and // updates the resources in k8s. - UpdateStatus(status api.DeploymentStatus, force ...bool) error + UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error // GetAgencyClients returns a client connection for every agency member. // If the given predicate is not nil, only agents are included where the given predicate returns true. - GetAgencyClients(ctx context.Context, predicate func(id string) bool) ([]arangod.Agency, error) + GetAgencyClients(ctx context.Context, predicate func(id string) bool) ([]driver.Connection, error) // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), // creating one if needed. GetDatabaseClient(ctx context.Context) (driver.Client, error) diff --git a/pkg/deployment/resilience/member_failure.go b/pkg/deployment/resilience/member_failure.go index ff161d500..fb67492f4 100644 --- a/pkg/deployment/resilience/member_failure.go +++ b/pkg/deployment/resilience/member_failure.go @@ -26,6 +26,7 @@ import ( "context" "time" + "github.com/arangodb/go-driver/agency" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/util/arangod" ) @@ -40,10 +41,10 @@ const ( // - They are frequently restarted // - They cannot be scheduled for a long time (TODO) func (r *Resilience) CheckMemberFailure() error { - status := r.context.GetStatus() + status, lastVersion := r.context.GetStatus() updateStatusNeeded := false - if err := status.Members.ForeachServerGroup(func(group api.ServerGroup, list *api.MemberStatusList) error { - for _, m := range *list { + if err := status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { + for _, m := range list { log := r.log.With(). Str("id", m.ID). Str("role", group.AsRole()). @@ -69,7 +70,7 @@ func (r *Resilience) CheckMemberFailure() error { } else if failureAcceptable { log.Info().Msg("Member is not ready for long time, marking is failed") m.Phase = api.MemberPhaseFailed - list.Update(m) + status.Members.Update(m, group) updateStatusNeeded = true } else { log.Warn().Msgf("Member is not ready for long time, but it is not safe to mark it a failed because: %s", reason) @@ -88,7 +89,7 @@ func (r *Resilience) CheckMemberFailure() error { } else if failureAcceptable { log.Info().Msg("Member has terminated too often in recent history, marking is failed") m.Phase = api.MemberPhaseFailed - list.Update(m) + status.Members.Update(m, group) updateStatusNeeded = true } else { log.Warn().Msgf("Member has terminated too often in recent history, but it is not safe to mark it a failed because: %s", reason) @@ -102,7 +103,7 @@ func (r *Resilience) CheckMemberFailure() error { return maskAny(err) } if updateStatusNeeded { - if err := r.context.UpdateStatus(status); err != nil { + if err := r.context.UpdateStatus(status, lastVersion); err != nil { return maskAny(err) } } @@ -122,7 +123,7 @@ func (r *Resilience) isMemberFailureAcceptable(status api.DeploymentStatus, grou if err != nil { return false, "", maskAny(err) } - if err := arangod.AreAgentsHealthy(ctx, clients); err != nil { + if err := agency.AreAgentsHealthy(ctx, clients); err != nil { return false, err.Error(), nil } return true, "", nil diff --git a/pkg/deployment/resources/certificates_client_auth.go b/pkg/deployment/resources/certificates_client_auth.go new file mode 100644 index 000000000..d3f3be143 --- /dev/null +++ b/pkg/deployment/resources/certificates_client_auth.go @@ -0,0 +1,113 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package resources + +import ( + "fmt" + "strings" + "time" + + certificates "github.com/arangodb-helper/go-certificates" + "github.com/rs/zerolog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/typed/core/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +const ( + clientAuthECDSACurve = "P256" // This curve is the default that ArangoDB accepts and plenty strong +) + +// createClientAuthCACertificate creates a client authentication CA certificate and stores it in a secret with name +// specified in the given spec. +func createClientAuthCACertificate(log zerolog.Logger, cli v1.CoreV1Interface, spec api.SyncAuthenticationSpec, deploymentName, namespace string, ownerRef *metav1.OwnerReference) error { + log = log.With().Str("secret", spec.GetClientCASecretName()).Logger() + options := certificates.CreateCertificateOptions{ + CommonName: fmt.Sprintf("%s Client Authentication Root Certificate", deploymentName), + ValidFrom: time.Now(), + ValidFor: caTTL, + IsCA: true, + IsClientAuth: true, + ECDSACurve: clientAuthECDSACurve, + } + cert, priv, err := certificates.CreateCertificate(options, nil) + if err != nil { + log.Debug().Err(err).Msg("Failed to create CA certificate") + return maskAny(err) + } + if err := k8sutil.CreateCASecret(cli, spec.GetClientCASecretName(), namespace, cert, priv, ownerRef); err != nil { + if k8sutil.IsAlreadyExists(err) { + log.Debug().Msg("CA Secret already exists") + } else { + log.Debug().Err(err).Msg("Failed to create CA Secret") + } + return maskAny(err) + } + log.Debug().Msg("Created CA Secret") + return nil +} + +// createClientAuthCertificateKeyfile creates a client authentication certificate for a specific user and stores +// it in a secret with the given name. +func createClientAuthCertificateKeyfile(log zerolog.Logger, cli v1.CoreV1Interface, commonName string, ttl time.Duration, spec api.SyncAuthenticationSpec, secretName, namespace string, ownerRef *metav1.OwnerReference) error { + log = log.With().Str("secret", secretName).Logger() + // Load CA certificate + caCert, caKey, _, err := k8sutil.GetCASecret(cli, spec.GetClientCASecretName(), namespace, nil) + if err != nil { + log.Debug().Err(err).Msg("Failed to load CA certificate") + return maskAny(err) + } + ca, err := certificates.LoadCAFromPEM(caCert, caKey) + if err != nil { + log.Debug().Err(err).Msg("Failed to decode CA certificate") + return maskAny(err) + } + + options := certificates.CreateCertificateOptions{ + CommonName: commonName, + ValidFrom: time.Now(), + ValidFor: ttl, + IsCA: false, + IsClientAuth: true, + ECDSACurve: clientAuthECDSACurve, + } + cert, priv, err := certificates.CreateCertificate(options, &ca) + if err != nil { + log.Debug().Err(err).Msg("Failed to create server certificate") + return maskAny(err) + } + keyfile := strings.TrimSpace(cert) + "\n" + + strings.TrimSpace(priv) + if err := k8sutil.CreateTLSKeyfileSecret(cli, secretName, namespace, keyfile, ownerRef); err != nil { + if k8sutil.IsAlreadyExists(err) { + log.Debug().Msg("Server Secret already exists") + } else { + log.Debug().Err(err).Msg("Failed to create server Secret") + } + return maskAny(err) + } + log.Debug().Msg("Created server Secret") + return nil +} diff --git a/pkg/deployment/resources/tls.go b/pkg/deployment/resources/certificates_tls.go similarity index 76% rename from pkg/deployment/resources/tls.go rename to pkg/deployment/resources/certificates_tls.go index 725fa0c40..d2b295c08 100644 --- a/pkg/deployment/resources/tls.go +++ b/pkg/deployment/resources/certificates_tls.go @@ -41,24 +41,17 @@ const ( tlsECDSACurve = "P256" // This curve is the default that ArangoDB accepts and plenty strong ) -// createCACertificate creates a CA certificate and stores it in a secret with name +// createTLSCACertificate creates a CA certificate and stores it in a secret with name // specified in the given spec. -func createCACertificate(log zerolog.Logger, cli v1.CoreV1Interface, spec api.TLSSpec, deploymentName, namespace string, ownerRef *metav1.OwnerReference) error { +func createTLSCACertificate(log zerolog.Logger, cli v1.CoreV1Interface, spec api.TLSSpec, deploymentName, namespace string, ownerRef *metav1.OwnerReference) error { log = log.With().Str("secret", spec.GetCASecretName()).Logger() - dnsNames, ipAddresses, emailAddress, err := spec.GetParsedAltNames() - if err != nil { - log.Debug().Err(err).Msg("Failed to get alternate names") - return maskAny(err) - } options := certificates.CreateCertificateOptions{ - CommonName: fmt.Sprintf("%s Root Certificate", deploymentName), - Hosts: append(dnsNames, ipAddresses...), - EmailAddresses: emailAddress, - ValidFrom: time.Now(), - ValidFor: caTTL, - IsCA: true, - ECDSACurve: tlsECDSACurve, + CommonName: fmt.Sprintf("%s Root Certificate", deploymentName), + ValidFrom: time.Now(), + ValidFor: caTTL, + IsCA: true, + ECDSACurve: tlsECDSACurve, } cert, priv, err := certificates.CreateCertificate(options, nil) if err != nil { @@ -77,9 +70,9 @@ func createCACertificate(log zerolog.Logger, cli v1.CoreV1Interface, spec api.TL return nil } -// createServerCertificate creates a TLS certificate for a specific server and stores +// createTLSServerCertificate creates a TLS certificate for a specific server and stores // it in a secret with the given name. -func createServerCertificate(log zerolog.Logger, cli v1.CoreV1Interface, serverNames []string, spec api.TLSSpec, secretName, namespace string, ownerRef *metav1.OwnerReference) error { +func createTLSServerCertificate(log zerolog.Logger, cli v1.CoreV1Interface, serverNames []string, spec api.TLSSpec, secretName, namespace string, ownerRef *metav1.OwnerReference) error { log = log.With().Str("secret", secretName).Logger() // Load alt names dnsNames, ipAddresses, emailAddress, err := spec.GetParsedAltNames() @@ -89,7 +82,7 @@ func createServerCertificate(log zerolog.Logger, cli v1.CoreV1Interface, serverN } // Load CA certificate - caCert, caKey, err := k8sutil.GetCASecret(cli, spec.GetCASecretName(), namespace) + caCert, caKey, _, err := k8sutil.GetCASecret(cli, spec.GetCASecretName(), namespace, nil) if err != nil { log.Debug().Err(err).Msg("Failed to load CA certificate") return maskAny(err) diff --git a/pkg/deployment/resources/context.go b/pkg/deployment/resources/context.go index e7a5d6683..9e4db969f 100644 --- a/pkg/deployment/resources/context.go +++ b/pkg/deployment/resources/context.go @@ -23,6 +23,9 @@ package resources import ( + "context" + + driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "k8s.io/api/core/v1" @@ -49,20 +52,35 @@ type Context interface { // GetSpec returns the current specification of the deployment GetSpec() api.DeploymentSpec // GetStatus returns the current status of the deployment - GetStatus() api.DeploymentStatus + GetStatus() (api.DeploymentStatus, int32) // UpdateStatus replaces the status of the deployment with the given status and // updates the resources in k8s. - UpdateStatus(status api.DeploymentStatus, force ...bool) error + UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error // GetKubeCli returns the kubernetes client GetKubeCli() kubernetes.Interface + // GetLifecycleImage returns the image name containing the lifecycle helper (== name of operator image) + GetLifecycleImage() string // GetNamespace returns the namespace that contains the deployment GetNamespace() string - // createEvent creates a given event. + // CreateEvent creates a given event. // On error, the error is logged. - CreateEvent(evt *v1.Event) + CreateEvent(evt *k8sutil.Event) // GetOwnedPods returns a list of all pods owned by the deployment. GetOwnedPods() ([]v1.Pod, error) + // GetOwnedPVCs returns a list of all PVCs owned by the deployment. + GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) // CleanupPod deletes a given pod with force and explicit UID. // If the pod does not exist, the error is ignored. CleanupPod(p v1.Pod) error + // DeletePod deletes a pod with given name in the namespace + // of the deployment. If the pod does not exist, the error is ignored. + DeletePod(podName string) error + // DeletePvc deletes a persistent volume claim with given name in the namespace + // of the deployment. If the pvc does not exist, the error is ignored. + DeletePvc(pvcName string) error + // GetAgencyClients returns a client connection for every agency member. + GetAgencyClients(ctx context.Context, predicate func(memberID string) bool) ([]driver.Connection, error) + // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), + // creating one if needed. + GetDatabaseClient(ctx context.Context) (driver.Client, error) } diff --git a/pkg/deployment/resources/member_cleanup.go b/pkg/deployment/resources/member_cleanup.go new file mode 100644 index 000000000..87b4ef34a --- /dev/null +++ b/pkg/deployment/resources/member_cleanup.go @@ -0,0 +1,137 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package resources + +import ( + "context" + "time" + + driver "github.com/arangodb/go-driver" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +const ( + // minMemberAge is the minimum duration we expect a member to be created before we remove it because + // it is not part of a deployment. + minMemberAge = time.Minute * 10 +) + +// CleanupRemovedMembers removes all arangod members that are no longer part of ArangoDB deployment. +func (r *Resources) CleanupRemovedMembers() error { + // Decide what to do depending on cluster mode + switch r.context.GetSpec().GetMode() { + case api.DeploymentModeCluster: + if err := r.cleanupRemovedClusterMembers(); err != nil { + return maskAny(err) + } + return nil + default: + // Other mode have no concept of cluster in which members can be removed + return nil + } +} + +// cleanupRemovedClusterMembers removes all arangod members that are no longer part of the cluster. +func (r *Resources) cleanupRemovedClusterMembers() error { + log := r.log + ctx := context.Background() + + // Ask cluster for its health + client, err := r.context.GetDatabaseClient(ctx) + if err != nil { + return maskAny(err) + } + c, err := client.Cluster(ctx) + if err != nil { + return maskAny(err) + } + h, err := c.Health(ctx) + if err != nil { + return maskAny(err) + } + + serverFound := func(id string) bool { + _, found := h.Health[driver.ServerID(id)] + return found + } + + // For over all members that can be removed + status, lastVersion := r.context.GetStatus() + updateStatusNeeded := false + var podNamesToRemove, pvcNamesToRemove []string + status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { + if group != api.ServerGroupCoordinators && group != api.ServerGroupDBServers { + // We're not interested in these other groups + return nil + } + for _, m := range list { + if serverFound(m.ID) { + // Member is (still) found, skip it + if m.Conditions.Update(api.ConditionTypeMemberOfCluster, true, "", "") { + status.Members.Update(m, group) + updateStatusNeeded = true + } + continue + } else if !m.Conditions.IsTrue(api.ConditionTypeMemberOfCluster) { + // Member is not yet recorded as member of cluster + if m.Age() < minMemberAge { + continue + } + log.Info().Str("member", m.ID).Str("role", group.AsRole()).Msg("Member has never been part of the cluster for a long time. Removing it.") + } else { + // Member no longer part of cluster, remove it + log.Info().Str("member", m.ID).Str("role", group.AsRole()).Msg("Member is no longer part of the ArangoDB cluster. Removing it.") + } + status.Members.RemoveByID(m.ID, group) + updateStatusNeeded = true + // Remove Pod & PVC (if any) + if m.PodName != "" { + podNamesToRemove = append(podNamesToRemove, m.PodName) + } + if m.PersistentVolumeClaimName != "" { + pvcNamesToRemove = append(pvcNamesToRemove, m.PersistentVolumeClaimName) + } + } + return nil + }) + + if updateStatusNeeded { + if err := r.context.UpdateStatus(status, lastVersion); err != nil { + return maskAny(err) + } + } + + for _, podName := range podNamesToRemove { + if err := r.context.DeletePod(podName); err != nil && !k8sutil.IsNotFound(err) { + log.Warn().Err(err).Str("pod", podName).Msg("Failed to remove obsolete pod") + } + } + for _, pvcName := range pvcNamesToRemove { + if err := r.context.DeletePvc(pvcName); err != nil && !k8sutil.IsNotFound(err) { + log.Warn().Err(err).Str("pvc", pvcName).Msg("Failed to remove obsolete PVC") + } + } + + return nil +} diff --git a/pkg/deployment/resources/pod_cleanup.go b/pkg/deployment/resources/pod_cleanup.go index a625c0cf8..e507c4a7c 100644 --- a/pkg/deployment/resources/pod_cleanup.go +++ b/pkg/deployment/resources/pod_cleanup.go @@ -23,11 +23,17 @@ package resources import ( + "time" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" ) +const ( + statelessTerminationPeriod = time.Minute // We wait this long for a stateless server to terminate on it's own. Afterwards we kill it. +) + // CleanupTerminatedPods removes all pods in Terminated state that belong to a member in Created state. func (r *Resources) CleanupTerminatedPods() error { log := r.log @@ -39,7 +45,7 @@ func (r *Resources) CleanupTerminatedPods() error { } // Update member status from all pods found - status := r.context.GetStatus() + status, _ := r.context.GetStatus() for _, p := range pods { if k8sutil.IsArangoDBImageIDAndVersionPod(p) { // Image ID pods are not relevant to inspect here @@ -47,20 +53,29 @@ func (r *Resources) CleanupTerminatedPods() error { } // Check pod state - if !(k8sutil.IsPodSucceeded(&p) || k8sutil.IsPodFailed(&p)) { + if !(k8sutil.IsPodSucceeded(&p) || k8sutil.IsPodFailed(&p) || k8sutil.IsPodTerminating(&p)) { continue } // Find member status - memberStatus, _, found := status.Members.MemberStatusByPodName(p.GetName()) + memberStatus, group, found := status.Members.MemberStatusByPodName(p.GetName()) if !found { - log.Debug().Str("pod", p.GetName()).Msg("no memberstatus found for pod") - continue - } - - // Check member termination condition - if !memberStatus.Conditions.IsTrue(api.ConditionTypeTerminated) { - continue + log.Debug().Str("pod", p.GetName()).Msg("no memberstatus found for pod. Performing cleanup") + } else { + // Check member termination condition + if !memberStatus.Conditions.IsTrue(api.ConditionTypeTerminated) { + if !group.IsStateless() { + // For statefull members, we have to wait for confirmed termination + continue + } else { + // If a stateless server does not terminate within a reasonable amount or time, we kill it. + t := p.GetDeletionTimestamp() + if t == nil || t.Add(statelessTerminationPeriod).After(time.Now()) { + // Either delete timestamp is not set, or not yet waiting long enough + continue + } + } + } } // Ok, we can delete the pod diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 8c3a2a44d..5a6bd2df7 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -27,16 +27,20 @@ import ( "encoding/json" "fmt" "net" + "net/url" "path/filepath" "sort" "strconv" "strings" + "sync" + "time" + "github.com/arangodb/go-driver/jwt" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" - "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "github.com/pkg/errors" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -185,7 +189,7 @@ func createArangodArgs(apiObject metav1.Object, deplSpec api.DeploymentSpec, gro optionPair{"--foxx.queues", "true"}, optionPair{"--server.statistics", "true"}, ) - if deplSpec.GetMode() == api.DeploymentModeResilientSingle { + if deplSpec.GetMode() == api.DeploymentModeActiveFailover { addAgentEndpoints = true options = append(options, optionPair{"--replication.automatic-failover", "true"}, @@ -217,9 +221,77 @@ func createArangodArgs(apiObject metav1.Object, deplSpec api.DeploymentSpec, gro } // createArangoSyncArgs creates command line arguments for an arangosync server in the given group. -func createArangoSyncArgs(spec api.DeploymentSpec, group api.ServerGroup, groupSpec api.ServerGroupSpec, agents api.MemberStatusList, id string) []string { - // TODO - return nil +func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, group api.ServerGroup, groupSpec api.ServerGroupSpec, agents api.MemberStatusList, id string) []string { + options := make([]optionPair, 0, 64) + var runCmd string + var port int + + /*if config.DebugCluster { + options = append(options, + optionPair{"--log.level", "debug"}) + }*/ + if spec.Sync.Monitoring.GetTokenSecretName() != "" { + options = append(options, + optionPair{"--monitoring.token", "$(" + constants.EnvArangoSyncMonitoringToken + ")"}, + ) + } + masterSecretPath := filepath.Join(k8sutil.MasterJWTSecretVolumeMountDir, constants.SecretKeyToken) + options = append(options, + optionPair{"--master.jwt-secret", masterSecretPath}, + ) + var masterEndpoint []string + switch group { + case api.ServerGroupSyncMasters: + runCmd = "master" + port = k8sutil.ArangoSyncMasterPort + masterEndpoint = spec.Sync.ExternalAccess.ResolveMasterEndpoint(k8sutil.CreateSyncMasterClientServiceDNSName(apiObject), port) + keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile) + clientCAPath := filepath.Join(k8sutil.ClientAuthCAVolumeMountDir, constants.SecretCACertificate) + options = append(options, + optionPair{"--server.keyfile", keyPath}, + optionPair{"--server.client-cafile", clientCAPath}, + optionPair{"--mq.type", "direct"}, + ) + if spec.IsAuthenticated() { + clusterSecretPath := filepath.Join(k8sutil.ClusterJWTSecretVolumeMountDir, constants.SecretKeyToken) + options = append(options, + optionPair{"--cluster.jwt-secret", clusterSecretPath}, + ) + } + dbServiceName := k8sutil.CreateDatabaseClientServiceName(apiObject.GetName()) + scheme := "http" + if spec.IsSecure() { + scheme = "https" + } + options = append(options, + optionPair{"--cluster.endpoint", fmt.Sprintf("%s://%s:%d", scheme, dbServiceName, k8sutil.ArangoPort)}) + case api.ServerGroupSyncWorkers: + runCmd = "worker" + port = k8sutil.ArangoSyncWorkerPort + masterEndpointHost := k8sutil.CreateSyncMasterClientServiceName(apiObject.GetName()) + masterEndpoint = []string{"https://" + net.JoinHostPort(masterEndpointHost, strconv.Itoa(k8sutil.ArangoSyncMasterPort))} + } + for _, ep := range masterEndpoint { + options = append(options, + optionPair{"--master.endpoint", ep}) + } + serverEndpoint := "https://" + net.JoinHostPort(k8sutil.CreatePodDNSName(apiObject, group.AsRole(), id), strconv.Itoa(port)) + options = append(options, + optionPair{"--server.endpoint", serverEndpoint}, + optionPair{"--server.port", strconv.Itoa(port)}, + ) + + args := make([]string, 0, 2+len(options)+len(groupSpec.Args)) + sort.Slice(options, func(i, j int) bool { + return options[i].CompareTo(options[j]) < 0 + }) + args = append(args, "run", runCmd) + for _, o := range options { + args = append(args, o.Key+"="+o.Value) + } + args = append(args, groupSpec.Args...) + + return args } // createLivenessProbe creates configuration for a liveness probe of a server in the given group. @@ -232,7 +304,7 @@ func (r *Resources) createLivenessProbe(spec api.DeploymentSpec, group api.Serve if err != nil { return nil, maskAny(err) } - authorization, err = arangod.CreateArangodJwtAuthorizationHeader(secretData) + authorization, err = jwt.CreateArangodJwtAuthorizationHeader(secretData, "kube-arangodb") if err != nil { return nil, maskAny(err) } @@ -246,13 +318,17 @@ func (r *Resources) createLivenessProbe(spec api.DeploymentSpec, group api.Serve return nil, nil case api.ServerGroupSyncMasters, api.ServerGroupSyncWorkers: authorization := "" + port := k8sutil.ArangoSyncMasterPort + if group == api.ServerGroupSyncWorkers { + port = k8sutil.ArangoSyncWorkerPort + } if spec.Sync.Monitoring.GetTokenSecretName() != "" { // Use monitoring token token, err := r.getSyncMonitoringToken(spec) if err != nil { return nil, maskAny(err) } - authorization = "bearer: " + token + authorization = "bearer " + token if err != nil { return nil, maskAny(err) } @@ -262,7 +338,7 @@ func (r *Resources) createLivenessProbe(spec api.DeploymentSpec, group api.Serve if err != nil { return nil, maskAny(err) } - authorization, err = arangod.CreateArangodJwtAuthorizationHeader(secretData) + authorization, err = jwt.CreateArangodJwtAuthorizationHeader(secretData, "kube-arangodb") if err != nil { return nil, maskAny(err) } @@ -274,6 +350,7 @@ func (r *Resources) createLivenessProbe(spec api.DeploymentSpec, group api.Serve LocalPath: "/_api/version", Secure: spec.IsSecure(), Authorization: authorization, + Port: port, }, nil default: return nil, nil @@ -282,7 +359,7 @@ func (r *Resources) createLivenessProbe(spec api.DeploymentSpec, group api.Serve // createReadinessProbe creates configuration for a readiness probe of a server in the given group. func (r *Resources) createReadinessProbe(spec api.DeploymentSpec, group api.ServerGroup) (*k8sutil.HTTPProbeConfig, error) { - if group != api.ServerGroupCoordinators { + if group != api.ServerGroupSingle && group != api.ServerGroupCoordinators { return nil, nil } authorization := "" @@ -291,26 +368,89 @@ func (r *Resources) createReadinessProbe(spec api.DeploymentSpec, group api.Serv if err != nil { return nil, maskAny(err) } - authorization, err = arangod.CreateArangodJwtAuthorizationHeader(secretData) + authorization, err = jwt.CreateArangodJwtAuthorizationHeader(secretData, "kube-arangodb") if err != nil { return nil, maskAny(err) } } - return &k8sutil.HTTPProbeConfig{ - LocalPath: "/_api/version", - Secure: spec.IsSecure(), - Authorization: authorization, - }, nil + probeCfg := &k8sutil.HTTPProbeConfig{ + LocalPath: "/_api/version", + Secure: spec.IsSecure(), + Authorization: authorization, + InitialDelaySeconds: 2, + PeriodSeconds: 2, + } + switch spec.GetMode() { + case api.DeploymentModeActiveFailover: + probeCfg.LocalPath = "/_admin/echo" + } + return probeCfg, nil +} + +// createPodFinalizers creates a list of finalizers for a pod created for the given group. +func (r *Resources) createPodFinalizers(group api.ServerGroup) []string { + switch group { + case api.ServerGroupAgents: + return []string{constants.FinalizerPodAgencyServing} + case api.ServerGroupDBServers: + return []string{constants.FinalizerPodDrainDBServer} + default: + return nil + } +} + +// createPodTolerations creates a list of tolerations for a pod created for the given group. +func (r *Resources) createPodTolerations(group api.ServerGroup, groupSpec api.ServerGroupSpec) []v1.Toleration { + notReadyDur := k8sutil.TolerationDuration{Forever: false, TimeSpan: time.Minute} + unreachableDur := k8sutil.TolerationDuration{Forever: false, TimeSpan: time.Minute} + switch group { + case api.ServerGroupAgents: + notReadyDur.Forever = true + unreachableDur.Forever = true + case api.ServerGroupCoordinators: + notReadyDur.TimeSpan = 15 * time.Second + unreachableDur.TimeSpan = 15 * time.Second + case api.ServerGroupDBServers: + notReadyDur.TimeSpan = 5 * time.Minute + unreachableDur.TimeSpan = 5 * time.Minute + case api.ServerGroupSingle: + if r.context.GetSpec().GetMode() == api.DeploymentModeSingle { + notReadyDur.Forever = true + unreachableDur.Forever = true + } else { + notReadyDur.TimeSpan = 5 * time.Minute + unreachableDur.TimeSpan = 5 * time.Minute + } + case api.ServerGroupSyncMasters: + notReadyDur.TimeSpan = 15 * time.Second + unreachableDur.TimeSpan = 15 * time.Second + case api.ServerGroupSyncWorkers: + notReadyDur.TimeSpan = 1 * time.Minute + unreachableDur.TimeSpan = 1 * time.Minute + } + tolerations := groupSpec.GetTolerations() + tolerations = k8sutil.AddTolerationIfNotFound(tolerations, k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeNotReady, notReadyDur)) + tolerations = k8sutil.AddTolerationIfNotFound(tolerations, k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeUnreachable, unreachableDur)) + tolerations = k8sutil.AddTolerationIfNotFound(tolerations, k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeAlphaUnreachable, unreachableDur)) + return tolerations } // createPodForMember creates all Pods listed in member status -func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.ServerGroup, - groupSpec api.ServerGroupSpec, m api.MemberStatus, memberStatusList *api.MemberStatusList) error { +func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, imageNotFoundOnce *sync.Once) error { kubecli := r.context.GetKubeCli() log := r.log apiObject := r.context.GetAPIObject() ns := r.context.GetNamespace() - status := r.context.GetStatus() + status, lastVersion := r.context.GetStatus() + m, group, found := status.Members.ElementByID(memberID) + if !found { + return maskAny(fmt.Errorf("Member '%s' not found", memberID)) + } + groupSpec := spec.GetServerGroupSpec(group) + lifecycleImage := r.context.GetLifecycleImage() + terminationGracePeriod := group.DefaultTerminationGracePeriod() + tolerations := r.createPodTolerations(group, groupSpec) + serviceAccountName := groupSpec.GetServiceAccountName() // Update pod name role := group.AsRole() @@ -318,14 +458,16 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server podSuffix := createPodSuffix(spec) m.PodName = k8sutil.CreatePodName(apiObject.GetName(), roleAbbr, m.ID, podSuffix) newPhase := api.MemberPhaseCreated + // Find image ID + imageInfo, imageFound := status.Images.GetByImage(spec.GetImage()) + if !imageFound { + imageNotFoundOnce.Do(func() { + log.Debug().Str("image", spec.GetImage()).Msg("Image ID is not known yet for image") + }) + return nil + } // Create pod if group.IsArangod() { - // Find image ID - info, found := status.Images.GetByImage(spec.GetImage()) - if !found { - log.Debug().Str("image", spec.GetImage()).Msg("Image ID is not known yet for image") - return nil - } // Prepare arguments autoUpgrade := m.Conditions.IsTrue(api.ConditionTypeAutoUpgrade) if autoUpgrade { @@ -348,8 +490,11 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server k8sutil.CreateDatabaseClientServiceDNSName(apiObject), k8sutil.CreatePodDNSName(apiObject, role, m.ID), } + if ip := spec.ExternalAccess.GetLoadBalancerIP(); ip != "" { + serverNames = append(serverNames, ip) + } owner := apiObject.AsOwner() - if err := createServerCertificate(log, kubecli.CoreV1(), serverNames, spec.TLS, tlsKeyfileSecretName, ns, &owner); err != nil && !k8sutil.IsAlreadyExists(err) { + if err := createTLSServerCertificate(log, kubecli.CoreV1(), serverNames, spec.TLS, tlsKeyfileSecretName, ns, &owner); err != nil && !k8sutil.IsAlreadyExists(err) { return maskAny(errors.Wrapf(err, "Failed to create TLS keyfile secret")) } } @@ -363,26 +508,75 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server if spec.IsAuthenticated() { env[constants.EnvArangodJWTSecret] = k8sutil.EnvValue{ SecretName: spec.Authentication.GetJWTSecretName(), - SecretKey: constants.SecretKeyJWT, + SecretKey: constants.SecretKeyToken, } } engine := spec.GetStorageEngine().AsArangoArgument() requireUUID := group == api.ServerGroupDBServers && m.IsInitialized - if err := k8sutil.CreateArangodPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, m.PersistentVolumeClaimName, info.ImageID, spec.GetImagePullPolicy(), - engine, requireUUID, args, env, livenessProbe, readinessProbe, tlsKeyfileSecretName, rocksdbEncryptionSecretName); err != nil { + finalizers := r.createPodFinalizers(group) + if err := k8sutil.CreateArangodPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, m.PersistentVolumeClaimName, imageInfo.ImageID, lifecycleImage, spec.GetImagePullPolicy(), + engine, requireUUID, terminationGracePeriod, args, env, finalizers, livenessProbe, readinessProbe, tolerations, serviceAccountName, tlsKeyfileSecretName, rocksdbEncryptionSecretName); err != nil { return maskAny(err) } log.Debug().Str("pod-name", m.PodName).Msg("Created pod") } else if group.IsArangosync() { - // Find image ID - info, found := status.Images.GetByImage(spec.Sync.GetImage()) - if !found { - log.Debug().Str("image", spec.Sync.GetImage()).Msg("Image ID is not known yet for image") - return nil + // Check image + if !imageInfo.Enterprise { + log.Debug().Str("image", spec.GetImage()).Msg("Image is not an enterprise image") + return maskAny(fmt.Errorf("Image '%s' does not contain an Enterprise version of ArangoDB", spec.GetImage())) + } + var tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName string + // Check master JWT secret + masterJWTSecretName = spec.Sync.Authentication.GetJWTSecretName() + if err := k8sutil.ValidateTokenSecret(kubecli.CoreV1(), masterJWTSecretName, ns); err != nil { + return maskAny(errors.Wrapf(err, "Master JWT secret validation failed")) } + // Check monitoring token secret + monitoringTokenSecretName := spec.Sync.Monitoring.GetTokenSecretName() + if err := k8sutil.ValidateTokenSecret(kubecli.CoreV1(), monitoringTokenSecretName, ns); err != nil { + return maskAny(errors.Wrapf(err, "Monitoring token secret validation failed")) + } + if group == api.ServerGroupSyncMasters { + // Create TLS secret + tlsKeyfileSecretName = k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), role, m.ID) + serverNames := []string{ + k8sutil.CreateSyncMasterClientServiceName(apiObject.GetName()), + k8sutil.CreateSyncMasterClientServiceDNSName(apiObject), + k8sutil.CreatePodDNSName(apiObject, role, m.ID), + } + masterEndpoint := spec.Sync.ExternalAccess.ResolveMasterEndpoint(k8sutil.CreateSyncMasterClientServiceDNSName(apiObject), k8sutil.ArangoSyncMasterPort) + for _, ep := range masterEndpoint { + if u, err := url.Parse(ep); err == nil { + serverNames = append(serverNames, u.Hostname()) + } + } + owner := apiObject.AsOwner() + if err := createTLSServerCertificate(log, kubecli.CoreV1(), serverNames, spec.Sync.TLS, tlsKeyfileSecretName, ns, &owner); err != nil && !k8sutil.IsAlreadyExists(err) { + return maskAny(errors.Wrapf(err, "Failed to create TLS keyfile secret")) + } + // Check cluster JWT secret + if spec.IsAuthenticated() { + clusterJWTSecretName = spec.Authentication.GetJWTSecretName() + if err := k8sutil.ValidateTokenSecret(kubecli.CoreV1(), clusterJWTSecretName, ns); err != nil { + return maskAny(errors.Wrapf(err, "Cluster JWT secret validation failed")) + } + } + // Check client-auth CA certificate secret + clientAuthCASecretName = spec.Sync.Authentication.GetClientCASecretName() + if err := k8sutil.ValidateCACertificateSecret(kubecli.CoreV1(), clientAuthCASecretName, ns); err != nil { + return maskAny(errors.Wrapf(err, "Client authentication CA certificate secret validation failed")) + } + } + // Prepare arguments - args := createArangoSyncArgs(spec, group, groupSpec, status.Members.Agents, m.ID) + args := createArangoSyncArgs(apiObject, spec, group, groupSpec, status.Members.Agents, m.ID) env := make(map[string]k8sutil.EnvValue) + if spec.Sync.Monitoring.GetTokenSecretName() != "" { + env[constants.EnvArangoSyncMonitoringToken] = k8sutil.EnvValue{ + SecretName: spec.Sync.Monitoring.GetTokenSecretName(), + SecretKey: constants.SecretKeyToken, + } + } livenessProbe, err := r.createLivenessProbe(spec, group) if err != nil { return maskAny(err) @@ -391,7 +585,8 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server if group == api.ServerGroupSyncWorkers { affinityWithRole = api.ServerGroupDBServers.AsRole() } - if err := k8sutil.CreateArangoSyncPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, info.ImageID, spec.Sync.GetImagePullPolicy(), args, env, livenessProbe, affinityWithRole); err != nil { + if err := k8sutil.CreateArangoSyncPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, imageInfo.ImageID, lifecycleImage, spec.GetImagePullPolicy(), terminationGracePeriod, args, env, + livenessProbe, tolerations, serviceAccountName, tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName, affinityWithRole); err != nil { return maskAny(err) } log.Debug().Str("pod-name", m.PodName).Msg("Created pod") @@ -401,10 +596,10 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server m.Conditions.Remove(api.ConditionTypeReady) m.Conditions.Remove(api.ConditionTypeTerminated) m.Conditions.Remove(api.ConditionTypeAutoUpgrade) - if err := memberStatusList.Update(m); err != nil { + if err := status.Members.Update(m, group); err != nil { return maskAny(err) } - if err := r.context.UpdateStatus(status); err != nil { + if err := r.context.UpdateStatus(status, lastVersion); err != nil { return maskAny(err) } // Create event @@ -416,14 +611,18 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, group api.Server // EnsurePods creates all Pods listed in member status func (r *Resources) EnsurePods() error { iterator := r.context.GetServerGroupIterator() - status := r.context.GetStatus() + status, _ := r.context.GetStatus() + imageNotFoundOnce := &sync.Once{} if err := iterator.ForeachServerGroup(func(group api.ServerGroup, groupSpec api.ServerGroupSpec, status *api.MemberStatusList) error { for _, m := range *status { if m.Phase != api.MemberPhaseNone { continue } + if m.Conditions.IsTrue(api.ConditionTypeCleanedOut) { + continue + } spec := r.context.GetSpec() - if err := r.createPodForMember(spec, group, groupSpec, m, status); err != nil { + if err := r.createPodForMember(spec, m.ID, imageNotFoundOnce); err != nil { return maskAny(err) } } diff --git a/pkg/deployment/resources/pod_creator_single_args_test.go b/pkg/deployment/resources/pod_creator_single_args_test.go index 9916f394b..6920481cb 100644 --- a/pkg/deployment/resources/pod_creator_single_args_test.go +++ b/pkg/deployment/resources/pod_creator_single_args_test.go @@ -202,7 +202,7 @@ func TestCreateArangodArgsSingle(t *testing.T) { ) } - // Resilient single + // ActiveFailover { apiObject := &api.ArangoDeployment{ ObjectMeta: metav1.ObjectMeta{ @@ -210,7 +210,7 @@ func TestCreateArangodArgsSingle(t *testing.T) { Namespace: "ns", }, Spec: api.DeploymentSpec{ - Mode: api.NewMode(api.DeploymentModeResilientSingle), + Mode: api.NewMode(api.DeploymentModeActiveFailover), }, } apiObject.Spec.SetDefaults("test") diff --git a/pkg/deployment/resources/pod_finalizers.go b/pkg/deployment/resources/pod_finalizers.go new file mode 100644 index 000000000..4cc8f90e5 --- /dev/null +++ b/pkg/deployment/resources/pod_finalizers.go @@ -0,0 +1,236 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package resources + +import ( + "context" + "fmt" + "time" + + "github.com/rs/zerolog" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/arangodb/go-driver/agency" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// runPodFinalizers goes through the list of pod finalizers to see if they can be removed. +func (r *Resources) runPodFinalizers(ctx context.Context, p *v1.Pod, memberStatus api.MemberStatus, updateMember func(api.MemberStatus) error) error { + log := r.log.With().Str("pod-name", p.GetName()).Logger() + var removalList []string + for _, f := range p.ObjectMeta.GetFinalizers() { + switch f { + case constants.FinalizerPodAgencyServing: + log.Debug().Msg("Inspecting agency-serving finalizer") + if err := r.inspectFinalizerPodAgencyServing(ctx, log, p, memberStatus); err == nil { + removalList = append(removalList, f) + } else { + log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet") + } + case constants.FinalizerPodDrainDBServer: + log.Debug().Msg("Inspecting drain dbserver finalizer") + if err := r.inspectFinalizerPodDrainDBServer(ctx, log, p, memberStatus, updateMember); err == nil { + removalList = append(removalList, f) + } else { + log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet") + } + } + } + // Remove finalizers (if needed) + if len(removalList) > 0 { + kubecli := r.context.GetKubeCli() + ignoreNotFound := false + if err := k8sutil.RemovePodFinalizers(log, kubecli, p, removalList, ignoreNotFound); err != nil { + log.Debug().Err(err).Msg("Failed to update pod (to remove finalizers)") + return maskAny(err) + } + } + return nil +} + +// inspectFinalizerPodAgencyServing checks the finalizer condition for agency-serving. +// It returns nil if the finalizer can be removed. +func (r *Resources) inspectFinalizerPodAgencyServing(ctx context.Context, log zerolog.Logger, p *v1.Pod, memberStatus api.MemberStatus) error { + // Inspect member phase + if memberStatus.Phase.IsFailed() { + log.Debug().Msg("Pod is already failed, safe to remove agency serving finalizer") + return nil + } + // Inspect deployment deletion state + apiObject := r.context.GetAPIObject() + if apiObject.GetDeletionTimestamp() != nil { + log.Debug().Msg("Entire deployment is being deleted, safe to remove agency serving finalizer") + return nil + } + + // Check node the pod is scheduled on + agentDataWillBeGone := false + if p.Spec.NodeName != "" { + node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(p.Spec.NodeName, metav1.GetOptions{}) + if err != nil { + log.Warn().Err(err).Msg("Failed to get node for member") + return maskAny(err) + } + if node.Spec.Unschedulable { + agentDataWillBeGone = true + } + } + + // Check PVC + pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(apiObject.GetNamespace()) + pvc, err := pvcs.Get(memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) + if err != nil { + log.Warn().Err(err).Msg("Failed to get PVC for member") + return maskAny(err) + } + if k8sutil.IsPersistentVolumeClaimMarkedForDeletion(pvc) { + agentDataWillBeGone = true + } + + // Is this a simple pod restart? + if !agentDataWillBeGone { + log.Debug().Msg("Pod is just being restarted, safe to remove agency serving finalizer") + return nil + } + + // Inspect agency state + log.Debug().Msg("Agent data will be gone, so we will check agency serving status first") + ctx = agency.WithAllowNoLeader(ctx) // The ID we're checking may be the leader, so ignore situations where all other agents are followers + ctx, cancel := context.WithTimeout(ctx, time.Second*15) // Force a quick check + defer cancel() + agencyConns, err := r.context.GetAgencyClients(ctx, func(id string) bool { return id != memberStatus.ID }) + if err != nil { + log.Debug().Err(err).Msg("Failed to create member client") + return maskAny(err) + } + if len(agencyConns) == 0 { + log.Debug().Err(err).Msg("No more remaining agents, we cannot delete this one") + return maskAny(fmt.Errorf("No more remaining agents")) + } + if err := agency.AreAgentsHealthy(ctx, agencyConns); err != nil { + log.Debug().Err(err).Msg("Remaining agents are not health") + return maskAny(err) + } + + // Remaining agents are healthy, we can remove this one and trigger a delete of the PVC + if err := pvcs.Delete(memberStatus.PersistentVolumeClaimName, &metav1.DeleteOptions{}); err != nil { + log.Warn().Err(err).Msg("Failed to delete PVC for member") + return maskAny(err) + } + + return nil +} + +// inspectFinalizerPodDrainDBServer checks the finalizer condition for drain-dbserver. +// It returns nil if the finalizer can be removed. +func (r *Resources) inspectFinalizerPodDrainDBServer(ctx context.Context, log zerolog.Logger, p *v1.Pod, memberStatus api.MemberStatus, updateMember func(api.MemberStatus) error) error { + // Inspect member phase + if memberStatus.Phase.IsFailed() { + log.Debug().Msg("Pod is already failed, safe to remove drain dbserver finalizer") + return nil + } + // Inspect deployment deletion state + apiObject := r.context.GetAPIObject() + if apiObject.GetDeletionTimestamp() != nil { + log.Debug().Msg("Entire deployment is being deleted, safe to remove drain dbserver finalizer") + return nil + } + + // Check node the pod is scheduled on + dbserverDataWillBeGone := false + if p.Spec.NodeName != "" { + node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(p.Spec.NodeName, metav1.GetOptions{}) + if err != nil { + log.Warn().Err(err).Msg("Failed to get node for member") + return maskAny(err) + } + if node.Spec.Unschedulable { + dbserverDataWillBeGone = true + } + } + + // Check PVC + pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(apiObject.GetNamespace()) + pvc, err := pvcs.Get(memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) + if err != nil { + log.Warn().Err(err).Msg("Failed to get PVC for member") + return maskAny(err) + } + if k8sutil.IsPersistentVolumeClaimMarkedForDeletion(pvc) { + dbserverDataWillBeGone = true + } + + // Is this a simple pod restart? + if !dbserverDataWillBeGone { + log.Debug().Msg("Pod is just being restarted, safe to remove drain dbserver finalizer") + return nil + } + + // Inspect cleaned out state + log.Debug().Msg("DBServer data is being deleted, so we will cleanout the dbserver first") + c, err := r.context.GetDatabaseClient(ctx) + if err != nil { + log.Debug().Err(err).Msg("Failed to create member client") + return maskAny(err) + } + cluster, err := c.Cluster(ctx) + if err != nil { + log.Debug().Err(err).Msg("Failed to access cluster") + return maskAny(err) + } + cleanedOut, err := cluster.IsCleanedOut(ctx, memberStatus.ID) + if err != nil { + return maskAny(err) + } + if cleanedOut { + // Cleanout completed + if memberStatus.Conditions.Update(api.ConditionTypeCleanedOut, true, "CleanedOut", "") { + if err := updateMember(memberStatus); err != nil { + return maskAny(err) + } + } + // Trigger PVC removal + if err := pvcs.Delete(memberStatus.PersistentVolumeClaimName, &metav1.DeleteOptions{}); err != nil { + log.Warn().Err(err).Msg("Failed to delete PVC for member") + return maskAny(err) + } + + log.Debug().Msg("Server is cleaned out. Save to remove drain dbserver finalizer") + return nil + } + // Not cleaned out yet, check member status + if memberStatus.Conditions.IsTrue(api.ConditionTypeTerminated) { + log.Warn().Msg("Member is already terminated before it could be cleaned out. Not good, but removing drain dbserver finalizer because we cannot do anything further") + return nil + } + // Ensure the cleanout is triggered + log.Debug().Msg("Server is not yet clean out. Triggering a clean out now") + if err := cluster.CleanOutServer(ctx, memberStatus.ID); err != nil { + log.Debug().Err(err).Msg("Failed to clean out server") + return maskAny(err) + } + return maskAny(fmt.Errorf("Server is not yet cleaned out")) +} diff --git a/pkg/deployment/resources/pod_inspector.go b/pkg/deployment/resources/pod_inspector.go index 5594878e0..ebad40ec1 100644 --- a/pkg/deployment/resources/pod_inspector.go +++ b/pkg/deployment/resources/pod_inspector.go @@ -23,10 +23,10 @@ package resources import ( + "context" "fmt" "time" - "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" @@ -44,9 +44,9 @@ const ( // InspectPods lists all pods that belong to the given deployment and updates // the member status of the deployment accordingly. -func (r *Resources) InspectPods() error { +func (r *Resources) InspectPods(ctx context.Context) error { log := r.log - var events []*v1.Event + var events []*k8sutil.Event pods, err := r.context.GetOwnedPods() if err != nil { @@ -55,7 +55,7 @@ func (r *Resources) InspectPods() error { } // Update member status from all pods found - status := r.context.GetStatus() + status, lastVersion := r.context.GetStatus() apiObject := r.context.GetAPIObject() var podNamesWithScheduleTimeout []string var unscheduledPodNames []string @@ -72,6 +72,17 @@ func (r *Resources) InspectPods() error { memberStatus, group, found := status.Members.MemberStatusByPodName(p.GetName()) if !found { log.Debug().Str("pod", p.GetName()).Msg("no memberstatus found for pod") + if k8sutil.IsPodMarkedForDeletion(&p) && len(p.GetFinalizers()) > 0 { + // Strange, pod belongs to us, but we have no member for it. + // Remove all finalizers, so it can be removed. + log.Warn().Msg("Pod belongs to this deployment, but we don't know the member. Removing all finalizers") + kubecli := r.context.GetKubeCli() + ignoreNotFound := false + if err := k8sutil.RemovePodFinalizers(log, kubecli, &p, p.GetFinalizers(), ignoreNotFound); err != nil { + log.Debug().Err(err).Msg("Failed to update pod (to remove all finalizers)") + return maskAny(err) + } + } continue } @@ -123,8 +134,19 @@ func (r *Resources) InspectPods() error { } else if !k8sutil.IsPodScheduled(&p) { unscheduledPodNames = append(unscheduledPodNames, p.GetName()) } + if k8sutil.IsPodMarkedForDeletion(&p) { + // Process finalizers + if err := r.runPodFinalizers(ctx, &p, memberStatus, func(m api.MemberStatus) error { + updateMemberStatusNeeded = true + memberStatus = m + return nil + }); err != nil { + // Only log here, since we'll be called to try again. + log.Warn().Err(err).Msg("Failed to run pod finalizers") + } + } if updateMemberStatusNeeded { - if err := status.Members.UpdateMemberStatus(memberStatus, group); err != nil { + if err := status.Members.Update(memberStatus, group); err != nil { return maskAny(err) } } @@ -140,8 +162,8 @@ func (r *Resources) InspectPods() error { } // Go over all members, check for missing pods - status.Members.ForeachServerGroup(func(group api.ServerGroup, members *api.MemberStatusList) error { - for _, m := range *members { + status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { + for _, m := range members { if podName := m.PodName; podName != "" { if !podExists(podName) { switch m.Phase { @@ -158,7 +180,7 @@ func (r *Resources) InspectPods() error { m.RecentTerminations = append(m.RecentTerminations, now) } // Save it - if err := status.Members.UpdateMemberStatus(m, group); err != nil { + if err := status.Members.Update(m, group); err != nil { return maskAny(err) } } @@ -182,7 +204,7 @@ func (r *Resources) InspectPods() error { } if updateMemberNeeded { // Save it - if err := status.Members.UpdateMemberStatus(m, group); err != nil { + if err := status.Members.Update(m, group); err != nil { return maskAny(err) } } @@ -194,7 +216,8 @@ func (r *Resources) InspectPods() error { }) // Update overall conditions - allMembersReady := status.Members.AllMembersReady() + spec := r.context.GetSpec() + allMembersReady := status.Members.AllMembersReady(spec.GetMode(), spec.Sync.IsEnabled()) status.Conditions.Update(api.ConditionTypeReady, allMembersReady, "", "") // Update conditions @@ -214,7 +237,7 @@ func (r *Resources) InspectPods() error { } // Save status - if err := r.context.UpdateStatus(status); err != nil { + if err := r.context.UpdateStatus(status, lastVersion); err != nil { return maskAny(err) } diff --git a/pkg/deployment/resources/pvc_finalizers.go b/pkg/deployment/resources/pvc_finalizers.go new file mode 100644 index 000000000..fd01bbe6a --- /dev/null +++ b/pkg/deployment/resources/pvc_finalizers.go @@ -0,0 +1,105 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package resources + +import ( + "context" + "fmt" + + "github.com/rs/zerolog" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// runPVCFinalizers goes through the list of PVC finalizers to see if they can be removed. +func (r *Resources) runPVCFinalizers(ctx context.Context, p *v1.PersistentVolumeClaim, group api.ServerGroup, memberStatus api.MemberStatus) error { + log := r.log.With().Str("pvc-name", p.GetName()).Logger() + var removalList []string + for _, f := range p.ObjectMeta.GetFinalizers() { + switch f { + case constants.FinalizerPVCMemberExists: + log.Debug().Msg("Inspecting member exists finalizer") + if err := r.inspectFinalizerPVCMemberExists(ctx, log, p, group, memberStatus); err == nil { + removalList = append(removalList, f) + } else { + log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet") + } + } + } + // Remove finalizers (if needed) + if len(removalList) > 0 { + kubecli := r.context.GetKubeCli() + ignoreNotFound := false + if err := k8sutil.RemovePVCFinalizers(log, kubecli, p, removalList, ignoreNotFound); err != nil { + log.Debug().Err(err).Msg("Failed to update PVC (to remove finalizers)") + return maskAny(err) + } + } + return nil +} + +// inspectFinalizerPVCMemberExists checks the finalizer condition for member-exists. +// It returns nil if the finalizer can be removed. +func (r *Resources) inspectFinalizerPVCMemberExists(ctx context.Context, log zerolog.Logger, p *v1.PersistentVolumeClaim, group api.ServerGroup, memberStatus api.MemberStatus) error { + // Inspect member phase + if memberStatus.Phase.IsFailed() { + log.Debug().Msg("Member is already failed, safe to remove member-exists finalizer") + return nil + } + // Inspect deployment deletion state + apiObject := r.context.GetAPIObject() + if apiObject.GetDeletionTimestamp() != nil { + log.Debug().Msg("Entire deployment is being deleted, safe to remove member-exists finalizer") + return nil + } + + // We do allow to rebuild agents & replace dbservers + switch group { + case api.ServerGroupAgents: + if memberStatus.Conditions.IsTrue(api.ConditionTypeTerminated) { + log.Debug().Msg("Rebuilding terminated agents is allowed, safe to remove member-exists finalizer") + return nil + } + case api.ServerGroupDBServers: + if memberStatus.Conditions.IsTrue(api.ConditionTypeCleanedOut) { + log.Debug().Msg("Removing cleanedout dbservers is allowed, safe to remove member-exists finalizer") + return nil + } + } + + // Member still exists, let's trigger a delete of it + if memberStatus.PodName != "" { + log.Info().Msg("Removing Pod of member, because PVC is being removed") + pods := r.context.GetKubeCli().CoreV1().Pods(apiObject.GetNamespace()) + if err := pods.Delete(memberStatus.PodName, &metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + log.Debug().Err(err).Msg("Failed to delete pod") + return maskAny(err) + } + } + + return maskAny(fmt.Errorf("Member still exists")) +} diff --git a/pkg/deployment/resources/pvc_inspector.go b/pkg/deployment/resources/pvc_inspector.go new file mode 100644 index 000000000..b7c2d0e8c --- /dev/null +++ b/pkg/deployment/resources/pvc_inspector.go @@ -0,0 +1,81 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package resources + +import ( + "context" + + "github.com/arangodb/kube-arangodb/pkg/metrics" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +var ( + inspectedPVCCounter = metrics.MustRegisterCounter("deployment", "inspected_ppvcs", "Number of PVCs inspections") +) + +// InspectPVCs lists all PVCs that belong to the given deployment and updates +// the member status of the deployment accordingly. +func (r *Resources) InspectPVCs(ctx context.Context) error { + log := r.log + + pvcs, err := r.context.GetOwnedPVCs() + if err != nil { + log.Debug().Err(err).Msg("Failed to get owned PVCs") + return maskAny(err) + } + + // Update member status from all pods found + status, _ := r.context.GetStatus() + for _, p := range pvcs { + // PVC belongs to this deployment, update metric + inspectedPVCCounter.Inc() + + // Find member status + memberStatus, group, found := status.Members.MemberStatusByPVCName(p.GetName()) + if !found { + log.Debug().Str("pvc", p.GetName()).Msg("no memberstatus found for PVC") + if k8sutil.IsPersistentVolumeClaimMarkedForDeletion(&p) && len(p.GetFinalizers()) > 0 { + // Strange, pvc belongs to us, but we have no member for it. + // Remove all finalizers, so it can be removed. + log.Warn().Msg("PVC belongs to this deployment, but we don't know the member. Removing all finalizers") + kubecli := r.context.GetKubeCli() + ignoreNotFound := false + if err := k8sutil.RemovePVCFinalizers(log, kubecli, &p, p.GetFinalizers(), ignoreNotFound); err != nil { + log.Debug().Err(err).Msg("Failed to update PVC (to remove all finalizers)") + return maskAny(err) + } + } + continue + } + + if k8sutil.IsPersistentVolumeClaimMarkedForDeletion(&p) { + // Process finalizers + if err := r.runPVCFinalizers(ctx, &p, group, memberStatus); err != nil { + // Only log here, since we'll be called to try again. + log.Warn().Err(err).Msg("Failed to run PVC finalizers") + } + } + } + + return nil +} diff --git a/pkg/deployment/resources/pvcs.go b/pkg/deployment/resources/pvcs.go index abb2b135b..86ac90374 100644 --- a/pkg/deployment/resources/pvcs.go +++ b/pkg/deployment/resources/pvcs.go @@ -25,9 +25,15 @@ package resources import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) +// createPVCFinalizers creates a list of finalizers for a PVC created for the given group. +func (r *Resources) createPVCFinalizers(group api.ServerGroup) []string { + return []string{constants.FinalizerPVCMemberExists} +} + // EnsurePVCs creates all PVC's listed in member status func (r *Resources) EnsurePVCs() error { kubecli := r.context.GetKubeCli() @@ -36,7 +42,8 @@ func (r *Resources) EnsurePVCs() error { ns := apiObject.GetNamespace() owner := apiObject.AsOwner() iterator := r.context.GetServerGroupIterator() - status := r.context.GetStatus() + status, _ := r.context.GetStatus() + enforceAntiAffinity := r.context.GetSpec().GetEnvironment().IsProduction() if err := iterator.ForeachServerGroup(func(group api.ServerGroup, spec api.ServerGroupSpec, status *api.MemberStatusList) error { for _, m := range *status { @@ -44,7 +51,8 @@ func (r *Resources) EnsurePVCs() error { storageClassName := spec.GetStorageClassName() role := group.AsRole() resources := spec.Resources - if err := k8sutil.CreatePersistentVolumeClaim(kubecli, m.PersistentVolumeClaimName, deploymentName, ns, storageClassName, role, resources, owner); err != nil { + finalizers := r.createPVCFinalizers(group) + if err := k8sutil.CreatePersistentVolumeClaim(kubecli, m.PersistentVolumeClaimName, deploymentName, ns, storageClassName, role, enforceAntiAffinity, resources, finalizers, owner); err != nil { return maskAny(err) } } diff --git a/pkg/deployment/resources/secret_hashes.go b/pkg/deployment/resources/secret_hashes.go index 425ad1486..64e58c812 100644 --- a/pkg/deployment/resources/secret_hashes.go +++ b/pkg/deployment/resources/secret_hashes.go @@ -44,9 +44,9 @@ func (r *Resources) ValidateSecretHashes() error { // validate performs a secret hash comparison for a single secret. // Return true if all is good, false when the SecretChanged condition // must be set. - validate := func(secretName string, expectedHashRef *string, status *api.DeploymentStatus) (bool, error) { + validate := func(secretName string, getExpectedHash func() string, setExpectedHash func(string) error) (bool, error) { log := r.log.With().Str("secret-name", secretName).Logger() - expectedHash := *expectedHashRef + expectedHash := getExpectedHash() hash, err := r.getSecretHash(secretName) if expectedHash == "" { // No hash set yet, try to fill it @@ -59,8 +59,7 @@ func (r *Resources) ValidateSecretHashes() error { return true, nil // Since we do not yet have a hash, we let this go with only a warning. } // Hash fetched succesfully, store it - *expectedHashRef = hash - if r.context.UpdateStatus(*status); err != nil { + if err := setExpectedHash(hash); err != nil { log.Debug().Msg("Failed to save secret hash") return true, maskAny(err) } @@ -89,14 +88,33 @@ func (r *Resources) ValidateSecretHashes() error { spec := r.context.GetSpec() log := r.log var badSecretNames []string - status := r.context.GetStatus() - if status.SecretHashes == nil { - status.SecretHashes = &api.SecretHashes{} + status, lastVersion := r.context.GetStatus() + getHashes := func() *api.SecretHashes { + if status.SecretHashes == nil { + status.SecretHashes = &api.SecretHashes{} + } + return status.SecretHashes + } + updateHashes := func(updater func(*api.SecretHashes)) error { + if status.SecretHashes == nil { + status.SecretHashes = &api.SecretHashes{} + } + updater(status.SecretHashes) + if err := r.context.UpdateStatus(status, lastVersion); err != nil { + return maskAny(err) + } + // Reload status + status, lastVersion = r.context.GetStatus() + return nil } - hashes := status.SecretHashes + if spec.IsAuthenticated() { secretName := spec.Authentication.GetJWTSecretName() - if hashOK, err := validate(secretName, &hashes.AuthJWT, &status); err != nil { + getExpectedHash := func() string { return getHashes().AuthJWT } + setExpectedHash := func(h string) error { + return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.AuthJWT = h })) + } + if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash); err != nil { return maskAny(err) } else if !hashOK { badSecretNames = append(badSecretNames, secretName) @@ -104,7 +122,11 @@ func (r *Resources) ValidateSecretHashes() error { } if spec.RocksDB.IsEncrypted() { secretName := spec.RocksDB.Encryption.GetKeySecretName() - if hashOK, err := validate(secretName, &hashes.RocksDBEncryptionKey, &status); err != nil { + getExpectedHash := func() string { return getHashes().RocksDBEncryptionKey } + setExpectedHash := func(h string) error { + return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.RocksDBEncryptionKey = h })) + } + if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash); err != nil { return maskAny(err) } else if !hashOK { badSecretNames = append(badSecretNames, secretName) @@ -112,7 +134,11 @@ func (r *Resources) ValidateSecretHashes() error { } if spec.IsSecure() { secretName := spec.TLS.GetCASecretName() - if hashOK, err := validate(secretName, &hashes.TLSCA, &status); err != nil { + getExpectedHash := func() string { return getHashes().TLSCA } + setExpectedHash := func(h string) error { + return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.TLSCA = h })) + } + if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash); err != nil { return maskAny(err) } else if !hashOK { badSecretNames = append(badSecretNames, secretName) @@ -120,7 +146,11 @@ func (r *Resources) ValidateSecretHashes() error { } if spec.Sync.IsEnabled() { secretName := spec.Sync.TLS.GetCASecretName() - if hashOK, err := validate(secretName, &hashes.SyncTLSCA, &status); err != nil { + getExpectedHash := func() string { return getHashes().SyncTLSCA } + setExpectedHash := func(h string) error { + return maskAny(updateHashes(func(dst *api.SecretHashes) { dst.SyncTLSCA = h })) + } + if hashOK, err := validate(secretName, getExpectedHash, setExpectedHash); err != nil { return maskAny(err) } else if !hashOK { badSecretNames = append(badSecretNames, secretName) @@ -132,7 +162,7 @@ func (r *Resources) ValidateSecretHashes() error { if status.Conditions.Update(api.ConditionTypeSecretsChanged, true, "Secrets have changed", fmt.Sprintf("Found %d changed secrets", len(badSecretNames))) { log.Warn().Msgf("Found %d changed secrets. Settings SecretsChanged condition", len(badSecretNames)) - if err := r.context.UpdateStatus(status); err != nil { + if err := r.context.UpdateStatus(status, lastVersion); err != nil { log.Error().Err(err).Msg("Failed to save SecretsChanged condition") return maskAny(err) } @@ -143,7 +173,7 @@ func (r *Resources) ValidateSecretHashes() error { // All good, we van remove the SecretsChanged condition if status.Conditions.Remove(api.ConditionTypeSecretsChanged) { log.Info().Msg("Resetting SecretsChanged condition") - if err := r.context.UpdateStatus(status); err != nil { + if err := r.context.UpdateStatus(status, lastVersion); err != nil { log.Error().Err(err).Msg("Failed to save SecretsChanged condition") return maskAny(err) } diff --git a/pkg/deployment/resources/secrets.go b/pkg/deployment/resources/secrets.go index 44aa75c4a..205736a43 100644 --- a/pkg/deployment/resources/secrets.go +++ b/pkg/deployment/resources/secrets.go @@ -25,7 +25,6 @@ package resources import ( "crypto/rand" "encoding/hex" - "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -37,27 +36,36 @@ import ( func (r *Resources) EnsureSecrets() error { spec := r.context.GetSpec() if spec.IsAuthenticated() { - if err := r.ensureJWTSecret(spec.Authentication.GetJWTSecretName()); err != nil { + if err := r.ensureTokenSecret(spec.Authentication.GetJWTSecretName()); err != nil { return maskAny(err) } } if spec.IsSecure() { - if err := r.ensureCACertificateSecret(spec.TLS); err != nil { + if err := r.ensureTLSCACertificateSecret(spec.TLS); err != nil { return maskAny(err) } } if spec.Sync.IsEnabled() { - if err := r.ensureCACertificateSecret(spec.Sync.TLS); err != nil { + if err := r.ensureTokenSecret(spec.Sync.Authentication.GetJWTSecretName()); err != nil { + return maskAny(err) + } + if err := r.ensureTokenSecret(spec.Sync.Monitoring.GetTokenSecretName()); err != nil { + return maskAny(err) + } + if err := r.ensureTLSCACertificateSecret(spec.Sync.TLS); err != nil { + return maskAny(err) + } + if err := r.ensureClientAuthCACertificateSecret(spec.Sync.Authentication); err != nil { return maskAny(err) } } return nil } -// ensureJWTSecret checks if a secret with given name exists in the namespace +// ensureTokenSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with a random -// JWT token. -func (r *Resources) ensureJWTSecret(secretName string) error { +// token. +func (r *Resources) ensureTokenSecret(secretName string) error { kubecli := r.context.GetKubeCli() ns := r.context.GetNamespace() if _, err := kubecli.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{}); k8sutil.IsNotFound(err) { @@ -69,7 +77,7 @@ func (r *Resources) ensureJWTSecret(secretName string) error { // Create secret owner := r.context.GetAPIObject().AsOwner() - if err := k8sutil.CreateJWTSecret(kubecli.CoreV1(), secretName, ns, token, &owner); k8sutil.IsAlreadyExists(err) { + if err := k8sutil.CreateTokenSecret(kubecli.CoreV1(), secretName, ns, token, &owner); k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { @@ -83,10 +91,9 @@ func (r *Resources) ensureJWTSecret(secretName string) error { return nil } -// ensureCACertificateSecret checks if a secret with given name exists in the namespace +// ensureTLSCACertificateSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with a generated CA certificate. -// JWT token. -func (r *Resources) ensureCACertificateSecret(spec api.TLSSpec) error { +func (r *Resources) ensureTLSCACertificateSecret(spec api.TLSSpec) error { kubecli := r.context.GetKubeCli() ns := r.context.GetNamespace() if _, err := kubecli.CoreV1().Secrets(ns).Get(spec.GetCASecretName(), metav1.GetOptions{}); k8sutil.IsNotFound(err) { @@ -94,7 +101,31 @@ func (r *Resources) ensureCACertificateSecret(spec api.TLSSpec) error { apiObject := r.context.GetAPIObject() owner := apiObject.AsOwner() deploymentName := apiObject.GetName() - if err := createCACertificate(r.log, kubecli.CoreV1(), spec, deploymentName, ns, &owner); k8sutil.IsAlreadyExists(err) { + if err := createTLSCACertificate(r.log, kubecli.CoreV1(), spec, deploymentName, ns, &owner); k8sutil.IsAlreadyExists(err) { + // Secret added while we tried it also + return nil + } else if err != nil { + // Failed to create secret + return maskAny(err) + } + } else if err != nil { + // Failed to get secret for other reasons + return maskAny(err) + } + return nil +} + +// ensureClientAuthCACertificateSecret checks if a secret with given name exists in the namespace +// of the deployment. If not, it will add such a secret with a generated CA certificate. +func (r *Resources) ensureClientAuthCACertificateSecret(spec api.SyncAuthenticationSpec) error { + kubecli := r.context.GetKubeCli() + ns := r.context.GetNamespace() + if _, err := kubecli.CoreV1().Secrets(ns).Get(spec.GetClientCASecretName(), metav1.GetOptions{}); k8sutil.IsNotFound(err) { + // Secret not found, create it + apiObject := r.context.GetAPIObject() + owner := apiObject.AsOwner() + deploymentName := apiObject.GetName() + if err := createClientAuthCACertificate(r.log, kubecli.CoreV1(), spec, deploymentName, ns, &owner); k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { @@ -116,7 +147,7 @@ func (r *Resources) getJWTSecret(spec api.DeploymentSpec) (string, error) { kubecli := r.context.GetKubeCli() ns := r.context.GetNamespace() secretName := spec.Authentication.GetJWTSecretName() - s, err := k8sutil.GetJWTSecret(kubecli.CoreV1(), secretName, ns) + s, err := k8sutil.GetTokenSecret(kubecli.CoreV1(), secretName, ns) if err != nil { r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get JWT secret") return "", maskAny(err) @@ -129,7 +160,7 @@ func (r *Resources) getSyncJWTSecret(spec api.DeploymentSpec) (string, error) { kubecli := r.context.GetKubeCli() ns := r.context.GetNamespace() secretName := spec.Sync.Authentication.GetJWTSecretName() - s, err := k8sutil.GetJWTSecret(kubecli.CoreV1(), secretName, ns) + s, err := k8sutil.GetTokenSecret(kubecli.CoreV1(), secretName, ns) if err != nil { r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get sync JWT secret") return "", maskAny(err) @@ -142,13 +173,10 @@ func (r *Resources) getSyncMonitoringToken(spec api.DeploymentSpec) (string, err kubecli := r.context.GetKubeCli() ns := r.context.GetNamespace() secretName := spec.Sync.Monitoring.GetTokenSecretName() - s, err := kubecli.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{}) + s, err := k8sutil.GetTokenSecret(kubecli.CoreV1(), secretName, ns) if err != nil { - r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get monitoring token secret") - } - // Take the first data - for _, v := range s.Data { - return string(v), nil + r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get sync monitoring secret") + return "", maskAny(err) } - return "", maskAny(fmt.Errorf("No data found in secret '%s'", secretName)) + return s, nil } diff --git a/pkg/deployment/resources/services.go b/pkg/deployment/resources/services.go index 49ec7558f..009b99b5c 100644 --- a/pkg/deployment/resources/services.go +++ b/pkg/deployment/resources/services.go @@ -23,7 +23,16 @@ package resources import ( + "time" + + "k8s.io/client-go/kubernetes" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/rs/zerolog" ) // EnsureServices creates all services needed to service the deployment @@ -31,9 +40,11 @@ func (r *Resources) EnsureServices() error { log := r.log kubecli := r.context.GetKubeCli() apiObject := r.context.GetAPIObject() + ns := apiObject.GetNamespace() owner := apiObject.AsOwner() spec := r.context.GetSpec() + // Headless service svcName, newlyCreated, err := k8sutil.CreateHeadlessService(kubecli, apiObject, owner) if err != nil { log.Debug().Err(err).Msg("Failed to create headless service") @@ -42,6 +53,8 @@ func (r *Resources) EnsureServices() error { if newlyCreated { log.Debug().Str("service", svcName).Msg("Created headless service") } + + // Internal database client service single := spec.GetMode().HasSingleServers() svcName, newlyCreated, err = k8sutil.CreateDatabaseClientService(kubecli, apiObject, single, owner) if err != nil { @@ -51,30 +64,120 @@ func (r *Resources) EnsureServices() error { if newlyCreated { log.Debug().Str("service", svcName).Msg("Created database client service") } - status := r.context.GetStatus() - if status.ServiceName != svcName { - status.ServiceName = svcName - if err := r.context.UpdateStatus(status); err != nil { - return maskAny(err) + { + status, lastVersion := r.context.GetStatus() + if status.ServiceName != svcName { + status.ServiceName = svcName + if err := r.context.UpdateStatus(status, lastVersion); err != nil { + return maskAny(err) + } } } + // Database external access service + eaServiceName := k8sutil.CreateDatabaseExternalAccessServiceName(apiObject.GetName()) + role := "coordinator" + if single { + role = "single" + } + if err := r.ensureExternalAccessServices(eaServiceName, ns, role, "database", k8sutil.ArangoPort, false, spec.ExternalAccess, apiObject, log, kubecli); err != nil { + return maskAny(err) + } + if spec.Sync.IsEnabled() { - svcName, newlyCreated, err := k8sutil.CreateSyncMasterClientService(kubecli, apiObject, owner) - if err != nil { - log.Debug().Err(err).Msg("Failed to create syncmaster client service") + // External (and internal) Sync master service + eaServiceName := k8sutil.CreateSyncMasterClientServiceName(apiObject.GetName()) + role := "syncmaster" + if err := r.ensureExternalAccessServices(eaServiceName, ns, role, "sync", k8sutil.ArangoSyncMasterPort, true, spec.Sync.ExternalAccess.ExternalAccessSpec, apiObject, log, kubecli); err != nil { return maskAny(err) } - if newlyCreated { - log.Debug().Str("service", svcName).Msg("Created syncmasters service") - } - status := r.context.GetStatus() - if status.SyncServiceName != svcName { - status.SyncServiceName = svcName - if err := r.context.UpdateStatus(status); err != nil { + status, lastVersion := r.context.GetStatus() + if status.SyncServiceName != eaServiceName { + status.SyncServiceName = eaServiceName + if err := r.context.UpdateStatus(status, lastVersion); err != nil { return maskAny(err) } } } return nil } + +// EnsureServices creates all services needed to service the deployment +func (r *Resources) ensureExternalAccessServices(eaServiceName, ns, svcRole, title string, port int, noneIsClusterIP bool, spec api.ExternalAccessSpec, apiObject k8sutil.APIObject, log zerolog.Logger, kubecli kubernetes.Interface) error { + // Database external access service + createExternalAccessService := false + deleteExternalAccessService := false + eaServiceType := spec.GetType().AsServiceType() // Note: Type auto defaults to ServiceTypeLoadBalancer + svcCli := kubecli.CoreV1().Services(ns) + if existing, err := svcCli.Get(eaServiceName, metav1.GetOptions{}); err == nil { + // External access service exists + loadBalancerIP := spec.GetLoadBalancerIP() + nodePort := spec.GetNodePort() + if spec.GetType().IsNone() { + if noneIsClusterIP { + eaServiceType = v1.ServiceTypeClusterIP + if existing.Spec.Type != v1.ServiceTypeClusterIP { + deleteExternalAccessService = true // Remove the current and replace with proper one + createExternalAccessService = true + } + } else { + // Should not be there, remove it + deleteExternalAccessService = true + } + } else if spec.GetType().IsAuto() { + // Inspect existing service. + if existing.Spec.Type == v1.ServiceTypeLoadBalancer { + // See if LoadBalancer has been configured & the service is "old enough" + oldEnoughTimestamp := time.Now().Add(-1 * time.Minute) // How long does the load-balancer provisioner have to act. + if len(existing.Status.LoadBalancer.Ingress) == 0 && existing.GetObjectMeta().GetCreationTimestamp().Time.Before(oldEnoughTimestamp) { + log.Info().Str("service", eaServiceName).Msgf("LoadBalancerIP of %s external access service is not set, switching to NodePort", title) + createExternalAccessService = true + eaServiceType = v1.ServiceTypeNodePort + deleteExternalAccessService = true // Remove the LoadBalancer ex service, then add the NodePort one + } else if existing.Spec.Type == v1.ServiceTypeLoadBalancer && (loadBalancerIP != "" && existing.Spec.LoadBalancerIP != loadBalancerIP) { + deleteExternalAccessService = true // LoadBalancerIP is wrong, remove the current and replace with proper one + createExternalAccessService = true + } else if existing.Spec.Type == v1.ServiceTypeNodePort && len(existing.Spec.Ports) == 1 && (nodePort != 0 && existing.Spec.Ports[0].NodePort != int32(nodePort)) { + deleteExternalAccessService = true // NodePort is wrong, remove the current and replace with proper one + createExternalAccessService = true + } + } + } else if spec.GetType().IsLoadBalancer() { + if existing.Spec.Type != v1.ServiceTypeLoadBalancer || (loadBalancerIP != "" && existing.Spec.LoadBalancerIP != loadBalancerIP) { + deleteExternalAccessService = true // Remove the current and replace with proper one + createExternalAccessService = true + } + } else if spec.GetType().IsNodePort() { + if existing.Spec.Type != v1.ServiceTypeNodePort || len(existing.Spec.Ports) != 1 || (nodePort != 0 && existing.Spec.Ports[0].NodePort != int32(nodePort)) { + deleteExternalAccessService = true // Remove the current and replace with proper one + createExternalAccessService = true + } + } + } else if k8sutil.IsNotFound(err) { + // External access service does not exist + if !spec.GetType().IsNone() || noneIsClusterIP { + createExternalAccessService = true + } + } + if deleteExternalAccessService { + log.Info().Str("service", eaServiceName).Msgf("Removing obsolete %s external access service", title) + if err := svcCli.Delete(eaServiceName, &metav1.DeleteOptions{}); err != nil { + log.Debug().Err(err).Msgf("Failed to remove %s external access service", title) + return maskAny(err) + } + } + if createExternalAccessService { + // Let's create or update the database external access service + nodePort := spec.GetNodePort() + loadBalancerIP := spec.GetLoadBalancerIP() + _, newlyCreated, err := k8sutil.CreateExternalAccessService(kubecli, eaServiceName, svcRole, apiObject, eaServiceType, port, nodePort, loadBalancerIP, apiObject.AsOwner()) + if err != nil { + log.Debug().Err(err).Msgf("Failed to create %s external access service", title) + return maskAny(err) + } + if newlyCreated { + log.Debug().Str("service", eaServiceName).Msgf("Created %s external access service", title) + } + } + return nil +} diff --git a/pkg/generated/clientset/versioned/clientset.go b/pkg/generated/clientset/versioned/clientset.go index 6d76c9bde..44936e362 100644 --- a/pkg/generated/clientset/versioned/clientset.go +++ b/pkg/generated/clientset/versioned/clientset.go @@ -21,6 +21,7 @@ package versioned import ( databasev1alpha "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/typed/deployment/v1alpha" + replicationv1alpha "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/typed/replication/v1alpha" storagev1alpha "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/typed/storage/v1alpha" glog "github.com/golang/glog" discovery "k8s.io/client-go/discovery" @@ -33,6 +34,9 @@ type Interface interface { DatabaseV1alpha() databasev1alpha.DatabaseV1alphaInterface // Deprecated: please explicitly pick a version if possible. Database() databasev1alpha.DatabaseV1alphaInterface + ReplicationV1alpha() replicationv1alpha.ReplicationV1alphaInterface + // Deprecated: please explicitly pick a version if possible. + Replication() replicationv1alpha.ReplicationV1alphaInterface StorageV1alpha() storagev1alpha.StorageV1alphaInterface // Deprecated: please explicitly pick a version if possible. Storage() storagev1alpha.StorageV1alphaInterface @@ -42,8 +46,9 @@ type Interface interface { // version included in a Clientset. type Clientset struct { *discovery.DiscoveryClient - databaseV1alpha *databasev1alpha.DatabaseV1alphaClient - storageV1alpha *storagev1alpha.StorageV1alphaClient + databaseV1alpha *databasev1alpha.DatabaseV1alphaClient + replicationV1alpha *replicationv1alpha.ReplicationV1alphaClient + storageV1alpha *storagev1alpha.StorageV1alphaClient } // DatabaseV1alpha retrieves the DatabaseV1alphaClient @@ -57,6 +62,17 @@ func (c *Clientset) Database() databasev1alpha.DatabaseV1alphaInterface { return c.databaseV1alpha } +// ReplicationV1alpha retrieves the ReplicationV1alphaClient +func (c *Clientset) ReplicationV1alpha() replicationv1alpha.ReplicationV1alphaInterface { + return c.replicationV1alpha +} + +// Deprecated: Replication retrieves the default version of ReplicationClient. +// Please explicitly pick a version. +func (c *Clientset) Replication() replicationv1alpha.ReplicationV1alphaInterface { + return c.replicationV1alpha +} + // StorageV1alpha retrieves the StorageV1alphaClient func (c *Clientset) StorageV1alpha() storagev1alpha.StorageV1alphaInterface { return c.storageV1alpha @@ -88,6 +104,10 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { if err != nil { return nil, err } + cs.replicationV1alpha, err = replicationv1alpha.NewForConfig(&configShallowCopy) + if err != nil { + return nil, err + } cs.storageV1alpha, err = storagev1alpha.NewForConfig(&configShallowCopy) if err != nil { return nil, err @@ -106,6 +126,7 @@ func NewForConfig(c *rest.Config) (*Clientset, error) { func NewForConfigOrDie(c *rest.Config) *Clientset { var cs Clientset cs.databaseV1alpha = databasev1alpha.NewForConfigOrDie(c) + cs.replicationV1alpha = replicationv1alpha.NewForConfigOrDie(c) cs.storageV1alpha = storagev1alpha.NewForConfigOrDie(c) cs.DiscoveryClient = discovery.NewDiscoveryClientForConfigOrDie(c) @@ -116,6 +137,7 @@ func NewForConfigOrDie(c *rest.Config) *Clientset { func New(c rest.Interface) *Clientset { var cs Clientset cs.databaseV1alpha = databasev1alpha.New(c) + cs.replicationV1alpha = replicationv1alpha.New(c) cs.storageV1alpha = storagev1alpha.New(c) cs.DiscoveryClient = discovery.NewDiscoveryClient(c) diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go index 1ca192f22..1e3cefd46 100644 --- a/pkg/generated/clientset/versioned/fake/clientset_generated.go +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -23,6 +23,8 @@ import ( clientset "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" databasev1alpha "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/typed/deployment/v1alpha" fakedatabasev1alpha "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/typed/deployment/v1alpha/fake" + replicationv1alpha "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/typed/replication/v1alpha" + fakereplicationv1alpha "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/typed/replication/v1alpha/fake" storagev1alpha "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/typed/storage/v1alpha" fakestoragev1alpha "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/typed/storage/v1alpha/fake" "k8s.io/apimachinery/pkg/runtime" @@ -83,6 +85,16 @@ func (c *Clientset) Database() databasev1alpha.DatabaseV1alphaInterface { return &fakedatabasev1alpha.FakeDatabaseV1alpha{Fake: &c.Fake} } +// ReplicationV1alpha retrieves the ReplicationV1alphaClient +func (c *Clientset) ReplicationV1alpha() replicationv1alpha.ReplicationV1alphaInterface { + return &fakereplicationv1alpha.FakeReplicationV1alpha{Fake: &c.Fake} +} + +// Replication retrieves the ReplicationV1alphaClient +func (c *Clientset) Replication() replicationv1alpha.ReplicationV1alphaInterface { + return &fakereplicationv1alpha.FakeReplicationV1alpha{Fake: &c.Fake} +} + // StorageV1alpha retrieves the StorageV1alphaClient func (c *Clientset) StorageV1alpha() storagev1alpha.StorageV1alphaInterface { return &fakestoragev1alpha.FakeStorageV1alpha{Fake: &c.Fake} diff --git a/pkg/generated/clientset/versioned/fake/register.go b/pkg/generated/clientset/versioned/fake/register.go index 33cdee33e..70c37b32b 100644 --- a/pkg/generated/clientset/versioned/fake/register.go +++ b/pkg/generated/clientset/versioned/fake/register.go @@ -21,6 +21,7 @@ package fake import ( databasev1alpha "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + replicationv1alpha "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" storagev1alpha "github.com/arangodb/kube-arangodb/pkg/apis/storage/v1alpha" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" @@ -53,5 +54,6 @@ func init() { // correctly. func AddToScheme(scheme *runtime.Scheme) { databasev1alpha.AddToScheme(scheme) + replicationv1alpha.AddToScheme(scheme) storagev1alpha.AddToScheme(scheme) } diff --git a/pkg/generated/clientset/versioned/scheme/register.go b/pkg/generated/clientset/versioned/scheme/register.go index 63ca2d1cd..de3b9f360 100644 --- a/pkg/generated/clientset/versioned/scheme/register.go +++ b/pkg/generated/clientset/versioned/scheme/register.go @@ -21,6 +21,7 @@ package scheme import ( databasev1alpha "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + replicationv1alpha "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" storagev1alpha "github.com/arangodb/kube-arangodb/pkg/apis/storage/v1alpha" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" @@ -53,5 +54,6 @@ func init() { // correctly. func AddToScheme(scheme *runtime.Scheme) { databasev1alpha.AddToScheme(scheme) + replicationv1alpha.AddToScheme(scheme) storagev1alpha.AddToScheme(scheme) } diff --git a/pkg/generated/clientset/versioned/typed/replication/v1alpha/arangodeploymentreplication.go b/pkg/generated/clientset/versioned/typed/replication/v1alpha/arangodeploymentreplication.go new file mode 100644 index 000000000..b548a7665 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/replication/v1alpha/arangodeploymentreplication.go @@ -0,0 +1,175 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +package v1alpha + +import ( + v1alpha "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" + scheme "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/scheme" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + rest "k8s.io/client-go/rest" +) + +// ArangoDeploymentReplicationsGetter has a method to return a ArangoDeploymentReplicationInterface. +// A group's client should implement this interface. +type ArangoDeploymentReplicationsGetter interface { + ArangoDeploymentReplications(namespace string) ArangoDeploymentReplicationInterface +} + +// ArangoDeploymentReplicationInterface has methods to work with ArangoDeploymentReplication resources. +type ArangoDeploymentReplicationInterface interface { + Create(*v1alpha.ArangoDeploymentReplication) (*v1alpha.ArangoDeploymentReplication, error) + Update(*v1alpha.ArangoDeploymentReplication) (*v1alpha.ArangoDeploymentReplication, error) + UpdateStatus(*v1alpha.ArangoDeploymentReplication) (*v1alpha.ArangoDeploymentReplication, error) + Delete(name string, options *v1.DeleteOptions) error + DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error + Get(name string, options v1.GetOptions) (*v1alpha.ArangoDeploymentReplication, error) + List(opts v1.ListOptions) (*v1alpha.ArangoDeploymentReplicationList, error) + Watch(opts v1.ListOptions) (watch.Interface, error) + Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha.ArangoDeploymentReplication, err error) + ArangoDeploymentReplicationExpansion +} + +// arangoDeploymentReplications implements ArangoDeploymentReplicationInterface +type arangoDeploymentReplications struct { + client rest.Interface + ns string +} + +// newArangoDeploymentReplications returns a ArangoDeploymentReplications +func newArangoDeploymentReplications(c *ReplicationV1alphaClient, namespace string) *arangoDeploymentReplications { + return &arangoDeploymentReplications{ + client: c.RESTClient(), + ns: namespace, + } +} + +// Get takes name of the arangoDeploymentReplication, and returns the corresponding arangoDeploymentReplication object, and an error if there is any. +func (c *arangoDeploymentReplications) Get(name string, options v1.GetOptions) (result *v1alpha.ArangoDeploymentReplication, err error) { + result = &v1alpha.ArangoDeploymentReplication{} + err = c.client.Get(). + Namespace(c.ns). + Resource("arangodeploymentreplications"). + Name(name). + VersionedParams(&options, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// List takes label and field selectors, and returns the list of ArangoDeploymentReplications that match those selectors. +func (c *arangoDeploymentReplications) List(opts v1.ListOptions) (result *v1alpha.ArangoDeploymentReplicationList, err error) { + result = &v1alpha.ArangoDeploymentReplicationList{} + err = c.client.Get(). + Namespace(c.ns). + Resource("arangodeploymentreplications"). + VersionedParams(&opts, scheme.ParameterCodec). + Do(). + Into(result) + return +} + +// Watch returns a watch.Interface that watches the requested arangoDeploymentReplications. +func (c *arangoDeploymentReplications) Watch(opts v1.ListOptions) (watch.Interface, error) { + opts.Watch = true + return c.client.Get(). + Namespace(c.ns). + Resource("arangodeploymentreplications"). + VersionedParams(&opts, scheme.ParameterCodec). + Watch() +} + +// Create takes the representation of a arangoDeploymentReplication and creates it. Returns the server's representation of the arangoDeploymentReplication, and an error, if there is any. +func (c *arangoDeploymentReplications) Create(arangoDeploymentReplication *v1alpha.ArangoDeploymentReplication) (result *v1alpha.ArangoDeploymentReplication, err error) { + result = &v1alpha.ArangoDeploymentReplication{} + err = c.client.Post(). + Namespace(c.ns). + Resource("arangodeploymentreplications"). + Body(arangoDeploymentReplication). + Do(). + Into(result) + return +} + +// Update takes the representation of a arangoDeploymentReplication and updates it. Returns the server's representation of the arangoDeploymentReplication, and an error, if there is any. +func (c *arangoDeploymentReplications) Update(arangoDeploymentReplication *v1alpha.ArangoDeploymentReplication) (result *v1alpha.ArangoDeploymentReplication, err error) { + result = &v1alpha.ArangoDeploymentReplication{} + err = c.client.Put(). + Namespace(c.ns). + Resource("arangodeploymentreplications"). + Name(arangoDeploymentReplication.Name). + Body(arangoDeploymentReplication). + Do(). + Into(result) + return +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). + +func (c *arangoDeploymentReplications) UpdateStatus(arangoDeploymentReplication *v1alpha.ArangoDeploymentReplication) (result *v1alpha.ArangoDeploymentReplication, err error) { + result = &v1alpha.ArangoDeploymentReplication{} + err = c.client.Put(). + Namespace(c.ns). + Resource("arangodeploymentreplications"). + Name(arangoDeploymentReplication.Name). + SubResource("status"). + Body(arangoDeploymentReplication). + Do(). + Into(result) + return +} + +// Delete takes name of the arangoDeploymentReplication and deletes it. Returns an error if one occurs. +func (c *arangoDeploymentReplications) Delete(name string, options *v1.DeleteOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("arangodeploymentreplications"). + Name(name). + Body(options). + Do(). + Error() +} + +// DeleteCollection deletes a collection of objects. +func (c *arangoDeploymentReplications) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + return c.client.Delete(). + Namespace(c.ns). + Resource("arangodeploymentreplications"). + VersionedParams(&listOptions, scheme.ParameterCodec). + Body(options). + Do(). + Error() +} + +// Patch applies the patch and returns the patched arangoDeploymentReplication. +func (c *arangoDeploymentReplications) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha.ArangoDeploymentReplication, err error) { + result = &v1alpha.ArangoDeploymentReplication{} + err = c.client.Patch(pt). + Namespace(c.ns). + Resource("arangodeploymentreplications"). + SubResource(subresources...). + Name(name). + Body(data). + Do(). + Into(result) + return +} diff --git a/pkg/generated/clientset/versioned/typed/replication/v1alpha/doc.go b/pkg/generated/clientset/versioned/typed/replication/v1alpha/doc.go new file mode 100644 index 000000000..f48feba5c --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/replication/v1alpha/doc.go @@ -0,0 +1,21 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// This package has the automatically generated typed clients. +package v1alpha diff --git a/pkg/generated/clientset/versioned/typed/replication/v1alpha/fake/doc.go b/pkg/generated/clientset/versioned/typed/replication/v1alpha/fake/doc.go new file mode 100644 index 000000000..6055f5176 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/replication/v1alpha/fake/doc.go @@ -0,0 +1,21 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Package fake has the automatically generated clients. +package fake diff --git a/pkg/generated/clientset/versioned/typed/replication/v1alpha/fake/fake_arangodeploymentreplication.go b/pkg/generated/clientset/versioned/typed/replication/v1alpha/fake/fake_arangodeploymentreplication.go new file mode 100644 index 000000000..96c937aed --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/replication/v1alpha/fake/fake_arangodeploymentreplication.go @@ -0,0 +1,141 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +package fake + +import ( + v1alpha "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + labels "k8s.io/apimachinery/pkg/labels" + schema "k8s.io/apimachinery/pkg/runtime/schema" + types "k8s.io/apimachinery/pkg/types" + watch "k8s.io/apimachinery/pkg/watch" + testing "k8s.io/client-go/testing" +) + +// FakeArangoDeploymentReplications implements ArangoDeploymentReplicationInterface +type FakeArangoDeploymentReplications struct { + Fake *FakeReplicationV1alpha + ns string +} + +var arangodeploymentreplicationsResource = schema.GroupVersionResource{Group: "replication.database.arangodb.com", Version: "v1alpha", Resource: "arangodeploymentreplications"} + +var arangodeploymentreplicationsKind = schema.GroupVersionKind{Group: "replication.database.arangodb.com", Version: "v1alpha", Kind: "ArangoDeploymentReplication"} + +// Get takes name of the arangoDeploymentReplication, and returns the corresponding arangoDeploymentReplication object, and an error if there is any. +func (c *FakeArangoDeploymentReplications) Get(name string, options v1.GetOptions) (result *v1alpha.ArangoDeploymentReplication, err error) { + obj, err := c.Fake. + Invokes(testing.NewGetAction(arangodeploymentreplicationsResource, c.ns, name), &v1alpha.ArangoDeploymentReplication{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha.ArangoDeploymentReplication), err +} + +// List takes label and field selectors, and returns the list of ArangoDeploymentReplications that match those selectors. +func (c *FakeArangoDeploymentReplications) List(opts v1.ListOptions) (result *v1alpha.ArangoDeploymentReplicationList, err error) { + obj, err := c.Fake. + Invokes(testing.NewListAction(arangodeploymentreplicationsResource, arangodeploymentreplicationsKind, c.ns, opts), &v1alpha.ArangoDeploymentReplicationList{}) + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + list := &v1alpha.ArangoDeploymentReplicationList{} + for _, item := range obj.(*v1alpha.ArangoDeploymentReplicationList).Items { + if label.Matches(labels.Set(item.Labels)) { + list.Items = append(list.Items, item) + } + } + return list, err +} + +// Watch returns a watch.Interface that watches the requested arangoDeploymentReplications. +func (c *FakeArangoDeploymentReplications) Watch(opts v1.ListOptions) (watch.Interface, error) { + return c.Fake. + InvokesWatch(testing.NewWatchAction(arangodeploymentreplicationsResource, c.ns, opts)) + +} + +// Create takes the representation of a arangoDeploymentReplication and creates it. Returns the server's representation of the arangoDeploymentReplication, and an error, if there is any. +func (c *FakeArangoDeploymentReplications) Create(arangoDeploymentReplication *v1alpha.ArangoDeploymentReplication) (result *v1alpha.ArangoDeploymentReplication, err error) { + obj, err := c.Fake. + Invokes(testing.NewCreateAction(arangodeploymentreplicationsResource, c.ns, arangoDeploymentReplication), &v1alpha.ArangoDeploymentReplication{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha.ArangoDeploymentReplication), err +} + +// Update takes the representation of a arangoDeploymentReplication and updates it. Returns the server's representation of the arangoDeploymentReplication, and an error, if there is any. +func (c *FakeArangoDeploymentReplications) Update(arangoDeploymentReplication *v1alpha.ArangoDeploymentReplication) (result *v1alpha.ArangoDeploymentReplication, err error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateAction(arangodeploymentreplicationsResource, c.ns, arangoDeploymentReplication), &v1alpha.ArangoDeploymentReplication{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha.ArangoDeploymentReplication), err +} + +// UpdateStatus was generated because the type contains a Status member. +// Add a +genclient:noStatus comment above the type to avoid generating UpdateStatus(). +func (c *FakeArangoDeploymentReplications) UpdateStatus(arangoDeploymentReplication *v1alpha.ArangoDeploymentReplication) (*v1alpha.ArangoDeploymentReplication, error) { + obj, err := c.Fake. + Invokes(testing.NewUpdateSubresourceAction(arangodeploymentreplicationsResource, "status", c.ns, arangoDeploymentReplication), &v1alpha.ArangoDeploymentReplication{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha.ArangoDeploymentReplication), err +} + +// Delete takes name of the arangoDeploymentReplication and deletes it. Returns an error if one occurs. +func (c *FakeArangoDeploymentReplications) Delete(name string, options *v1.DeleteOptions) error { + _, err := c.Fake. + Invokes(testing.NewDeleteAction(arangodeploymentreplicationsResource, c.ns, name), &v1alpha.ArangoDeploymentReplication{}) + + return err +} + +// DeleteCollection deletes a collection of objects. +func (c *FakeArangoDeploymentReplications) DeleteCollection(options *v1.DeleteOptions, listOptions v1.ListOptions) error { + action := testing.NewDeleteCollectionAction(arangodeploymentreplicationsResource, c.ns, listOptions) + + _, err := c.Fake.Invokes(action, &v1alpha.ArangoDeploymentReplicationList{}) + return err +} + +// Patch applies the patch and returns the patched arangoDeploymentReplication. +func (c *FakeArangoDeploymentReplications) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (result *v1alpha.ArangoDeploymentReplication, err error) { + obj, err := c.Fake. + Invokes(testing.NewPatchSubresourceAction(arangodeploymentreplicationsResource, c.ns, name, data, subresources...), &v1alpha.ArangoDeploymentReplication{}) + + if obj == nil { + return nil, err + } + return obj.(*v1alpha.ArangoDeploymentReplication), err +} diff --git a/pkg/generated/clientset/versioned/typed/replication/v1alpha/fake/fake_replication_client.go b/pkg/generated/clientset/versioned/typed/replication/v1alpha/fake/fake_replication_client.go new file mode 100644 index 000000000..bea672c63 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/replication/v1alpha/fake/fake_replication_client.go @@ -0,0 +1,41 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +package fake + +import ( + v1alpha "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/typed/replication/v1alpha" + rest "k8s.io/client-go/rest" + testing "k8s.io/client-go/testing" +) + +type FakeReplicationV1alpha struct { + *testing.Fake +} + +func (c *FakeReplicationV1alpha) ArangoDeploymentReplications(namespace string) v1alpha.ArangoDeploymentReplicationInterface { + return &FakeArangoDeploymentReplications{c, namespace} +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *FakeReplicationV1alpha) RESTClient() rest.Interface { + var ret *rest.RESTClient + return ret +} diff --git a/pkg/generated/clientset/versioned/typed/replication/v1alpha/generated_expansion.go b/pkg/generated/clientset/versioned/typed/replication/v1alpha/generated_expansion.go new file mode 100644 index 000000000..c102dfab5 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/replication/v1alpha/generated_expansion.go @@ -0,0 +1,22 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +package v1alpha + +type ArangoDeploymentReplicationExpansion interface{} diff --git a/pkg/generated/clientset/versioned/typed/replication/v1alpha/replication_client.go b/pkg/generated/clientset/versioned/typed/replication/v1alpha/replication_client.go new file mode 100644 index 000000000..a2ae5c1c2 --- /dev/null +++ b/pkg/generated/clientset/versioned/typed/replication/v1alpha/replication_client.go @@ -0,0 +1,91 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +package v1alpha + +import ( + v1alpha "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/scheme" + serializer "k8s.io/apimachinery/pkg/runtime/serializer" + rest "k8s.io/client-go/rest" +) + +type ReplicationV1alphaInterface interface { + RESTClient() rest.Interface + ArangoDeploymentReplicationsGetter +} + +// ReplicationV1alphaClient is used to interact with features provided by the replication.database.arangodb.com group. +type ReplicationV1alphaClient struct { + restClient rest.Interface +} + +func (c *ReplicationV1alphaClient) ArangoDeploymentReplications(namespace string) ArangoDeploymentReplicationInterface { + return newArangoDeploymentReplications(c, namespace) +} + +// NewForConfig creates a new ReplicationV1alphaClient for the given config. +func NewForConfig(c *rest.Config) (*ReplicationV1alphaClient, error) { + config := *c + if err := setConfigDefaults(&config); err != nil { + return nil, err + } + client, err := rest.RESTClientFor(&config) + if err != nil { + return nil, err + } + return &ReplicationV1alphaClient{client}, nil +} + +// NewForConfigOrDie creates a new ReplicationV1alphaClient for the given config and +// panics if there is an error in the config. +func NewForConfigOrDie(c *rest.Config) *ReplicationV1alphaClient { + client, err := NewForConfig(c) + if err != nil { + panic(err) + } + return client +} + +// New creates a new ReplicationV1alphaClient for the given RESTClient. +func New(c rest.Interface) *ReplicationV1alphaClient { + return &ReplicationV1alphaClient{c} +} + +func setConfigDefaults(config *rest.Config) error { + gv := v1alpha.SchemeGroupVersion + config.GroupVersion = &gv + config.APIPath = "/apis" + config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs} + + if config.UserAgent == "" { + config.UserAgent = rest.DefaultKubernetesUserAgent() + } + + return nil +} + +// RESTClient returns a RESTClient that is used to communicate +// with API server by this client implementation. +func (c *ReplicationV1alphaClient) RESTClient() rest.Interface { + if c == nil { + return nil + } + return c.restClient +} diff --git a/pkg/generated/informers/externalversions/factory.go b/pkg/generated/informers/externalversions/factory.go index bb541551e..c88fc2a3f 100644 --- a/pkg/generated/informers/externalversions/factory.go +++ b/pkg/generated/informers/externalversions/factory.go @@ -30,6 +30,7 @@ import ( versioned "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" deployment "github.com/arangodb/kube-arangodb/pkg/generated/informers/externalversions/deployment" internalinterfaces "github.com/arangodb/kube-arangodb/pkg/generated/informers/externalversions/internalinterfaces" + replication "github.com/arangodb/kube-arangodb/pkg/generated/informers/externalversions/replication" storage "github.com/arangodb/kube-arangodb/pkg/generated/informers/externalversions/storage" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" @@ -129,6 +130,7 @@ type SharedInformerFactory interface { WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool Database() deployment.Interface + Replication() replication.Interface Storage() storage.Interface } @@ -136,6 +138,10 @@ func (f *sharedInformerFactory) Database() deployment.Interface { return deployment.New(f, f.namespace, f.tweakListOptions) } +func (f *sharedInformerFactory) Replication() replication.Interface { + return replication.New(f, f.namespace, f.tweakListOptions) +} + func (f *sharedInformerFactory) Storage() storage.Interface { return storage.New(f, f.namespace, f.tweakListOptions) } diff --git a/pkg/generated/informers/externalversions/generic.go b/pkg/generated/informers/externalversions/generic.go index e076d52f0..b169aa0ef 100644 --- a/pkg/generated/informers/externalversions/generic.go +++ b/pkg/generated/informers/externalversions/generic.go @@ -26,6 +26,7 @@ import ( "fmt" v1alpha "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + replication_v1alpha "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" storage_v1alpha "github.com/arangodb/kube-arangodb/pkg/apis/storage/v1alpha" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" @@ -61,6 +62,10 @@ func (f *sharedInformerFactory) ForResource(resource schema.GroupVersionResource case v1alpha.SchemeGroupVersion.WithResource("arangodeployments"): return &genericInformer{resource: resource.GroupResource(), informer: f.Database().V1alpha().ArangoDeployments().Informer()}, nil + // Group=replication.database.arangodb.com, Version=v1alpha + case replication_v1alpha.SchemeGroupVersion.WithResource("arangodeploymentreplications"): + return &genericInformer{resource: resource.GroupResource(), informer: f.Replication().V1alpha().ArangoDeploymentReplications().Informer()}, nil + // Group=storage.arangodb.com, Version=v1alpha case storage_v1alpha.SchemeGroupVersion.WithResource("arangolocalstorages"): return &genericInformer{resource: resource.GroupResource(), informer: f.Storage().V1alpha().ArangoLocalStorages().Informer()}, nil diff --git a/pkg/generated/informers/externalversions/replication/interface.go b/pkg/generated/informers/externalversions/replication/interface.go new file mode 100644 index 000000000..6c7a3bcfb --- /dev/null +++ b/pkg/generated/informers/externalversions/replication/interface.go @@ -0,0 +1,50 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +// This file was automatically generated by informer-gen + +package replication + +import ( + internalinterfaces "github.com/arangodb/kube-arangodb/pkg/generated/informers/externalversions/internalinterfaces" + v1alpha "github.com/arangodb/kube-arangodb/pkg/generated/informers/externalversions/replication/v1alpha" +) + +// Interface provides access to each of this group's versions. +type Interface interface { + // V1alpha provides access to shared informers for resources in V1alpha. + V1alpha() v1alpha.Interface +} + +type group struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &group{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// V1alpha returns a new v1alpha.Interface. +func (g *group) V1alpha() v1alpha.Interface { + return v1alpha.New(g.factory, g.namespace, g.tweakListOptions) +} diff --git a/pkg/generated/informers/externalversions/replication/v1alpha/arangodeploymentreplication.go b/pkg/generated/informers/externalversions/replication/v1alpha/arangodeploymentreplication.go new file mode 100644 index 000000000..2c45a1651 --- /dev/null +++ b/pkg/generated/informers/externalversions/replication/v1alpha/arangodeploymentreplication.go @@ -0,0 +1,93 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +// This file was automatically generated by informer-gen + +package v1alpha + +import ( + time "time" + + replication_v1alpha "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" + versioned "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" + internalinterfaces "github.com/arangodb/kube-arangodb/pkg/generated/informers/externalversions/internalinterfaces" + v1alpha "github.com/arangodb/kube-arangodb/pkg/generated/listers/replication/v1alpha" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + runtime "k8s.io/apimachinery/pkg/runtime" + watch "k8s.io/apimachinery/pkg/watch" + cache "k8s.io/client-go/tools/cache" +) + +// ArangoDeploymentReplicationInformer provides access to a shared informer and lister for +// ArangoDeploymentReplications. +type ArangoDeploymentReplicationInformer interface { + Informer() cache.SharedIndexInformer + Lister() v1alpha.ArangoDeploymentReplicationLister +} + +type arangoDeploymentReplicationInformer struct { + factory internalinterfaces.SharedInformerFactory + tweakListOptions internalinterfaces.TweakListOptionsFunc + namespace string +} + +// NewArangoDeploymentReplicationInformer constructs a new informer for ArangoDeploymentReplication type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewArangoDeploymentReplicationInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers) cache.SharedIndexInformer { + return NewFilteredArangoDeploymentReplicationInformer(client, namespace, resyncPeriod, indexers, nil) +} + +// NewFilteredArangoDeploymentReplicationInformer constructs a new informer for ArangoDeploymentReplication type. +// Always prefer using an informer factory to get a shared informer instead of getting an independent +// one. This reduces memory footprint and number of connections to the server. +func NewFilteredArangoDeploymentReplicationInformer(client versioned.Interface, namespace string, resyncPeriod time.Duration, indexers cache.Indexers, tweakListOptions internalinterfaces.TweakListOptionsFunc) cache.SharedIndexInformer { + return cache.NewSharedIndexInformer( + &cache.ListWatch{ + ListFunc: func(options v1.ListOptions) (runtime.Object, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ReplicationV1alpha().ArangoDeploymentReplications(namespace).List(options) + }, + WatchFunc: func(options v1.ListOptions) (watch.Interface, error) { + if tweakListOptions != nil { + tweakListOptions(&options) + } + return client.ReplicationV1alpha().ArangoDeploymentReplications(namespace).Watch(options) + }, + }, + &replication_v1alpha.ArangoDeploymentReplication{}, + resyncPeriod, + indexers, + ) +} + +func (f *arangoDeploymentReplicationInformer) defaultInformer(client versioned.Interface, resyncPeriod time.Duration) cache.SharedIndexInformer { + return NewFilteredArangoDeploymentReplicationInformer(client, f.namespace, resyncPeriod, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, f.tweakListOptions) +} + +func (f *arangoDeploymentReplicationInformer) Informer() cache.SharedIndexInformer { + return f.factory.InformerFor(&replication_v1alpha.ArangoDeploymentReplication{}, f.defaultInformer) +} + +func (f *arangoDeploymentReplicationInformer) Lister() v1alpha.ArangoDeploymentReplicationLister { + return v1alpha.NewArangoDeploymentReplicationLister(f.Informer().GetIndexer()) +} diff --git a/pkg/generated/informers/externalversions/replication/v1alpha/interface.go b/pkg/generated/informers/externalversions/replication/v1alpha/interface.go new file mode 100644 index 000000000..c5f378749 --- /dev/null +++ b/pkg/generated/informers/externalversions/replication/v1alpha/interface.go @@ -0,0 +1,49 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +// This file was automatically generated by informer-gen + +package v1alpha + +import ( + internalinterfaces "github.com/arangodb/kube-arangodb/pkg/generated/informers/externalversions/internalinterfaces" +) + +// Interface provides access to all the informers in this group version. +type Interface interface { + // ArangoDeploymentReplications returns a ArangoDeploymentReplicationInformer. + ArangoDeploymentReplications() ArangoDeploymentReplicationInformer +} + +type version struct { + factory internalinterfaces.SharedInformerFactory + namespace string + tweakListOptions internalinterfaces.TweakListOptionsFunc +} + +// New returns a new Interface. +func New(f internalinterfaces.SharedInformerFactory, namespace string, tweakListOptions internalinterfaces.TweakListOptionsFunc) Interface { + return &version{factory: f, namespace: namespace, tweakListOptions: tweakListOptions} +} + +// ArangoDeploymentReplications returns a ArangoDeploymentReplicationInformer. +func (v *version) ArangoDeploymentReplications() ArangoDeploymentReplicationInformer { + return &arangoDeploymentReplicationInformer{factory: v.factory, namespace: v.namespace, tweakListOptions: v.tweakListOptions} +} diff --git a/pkg/generated/listers/replication/v1alpha/arangodeploymentreplication.go b/pkg/generated/listers/replication/v1alpha/arangodeploymentreplication.go new file mode 100644 index 000000000..09bdea528 --- /dev/null +++ b/pkg/generated/listers/replication/v1alpha/arangodeploymentreplication.go @@ -0,0 +1,98 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +// This file was automatically generated by lister-gen + +package v1alpha + +import ( + v1alpha "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/tools/cache" +) + +// ArangoDeploymentReplicationLister helps list ArangoDeploymentReplications. +type ArangoDeploymentReplicationLister interface { + // List lists all ArangoDeploymentReplications in the indexer. + List(selector labels.Selector) (ret []*v1alpha.ArangoDeploymentReplication, err error) + // ArangoDeploymentReplications returns an object that can list and get ArangoDeploymentReplications. + ArangoDeploymentReplications(namespace string) ArangoDeploymentReplicationNamespaceLister + ArangoDeploymentReplicationListerExpansion +} + +// arangoDeploymentReplicationLister implements the ArangoDeploymentReplicationLister interface. +type arangoDeploymentReplicationLister struct { + indexer cache.Indexer +} + +// NewArangoDeploymentReplicationLister returns a new ArangoDeploymentReplicationLister. +func NewArangoDeploymentReplicationLister(indexer cache.Indexer) ArangoDeploymentReplicationLister { + return &arangoDeploymentReplicationLister{indexer: indexer} +} + +// List lists all ArangoDeploymentReplications in the indexer. +func (s *arangoDeploymentReplicationLister) List(selector labels.Selector) (ret []*v1alpha.ArangoDeploymentReplication, err error) { + err = cache.ListAll(s.indexer, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha.ArangoDeploymentReplication)) + }) + return ret, err +} + +// ArangoDeploymentReplications returns an object that can list and get ArangoDeploymentReplications. +func (s *arangoDeploymentReplicationLister) ArangoDeploymentReplications(namespace string) ArangoDeploymentReplicationNamespaceLister { + return arangoDeploymentReplicationNamespaceLister{indexer: s.indexer, namespace: namespace} +} + +// ArangoDeploymentReplicationNamespaceLister helps list and get ArangoDeploymentReplications. +type ArangoDeploymentReplicationNamespaceLister interface { + // List lists all ArangoDeploymentReplications in the indexer for a given namespace. + List(selector labels.Selector) (ret []*v1alpha.ArangoDeploymentReplication, err error) + // Get retrieves the ArangoDeploymentReplication from the indexer for a given namespace and name. + Get(name string) (*v1alpha.ArangoDeploymentReplication, error) + ArangoDeploymentReplicationNamespaceListerExpansion +} + +// arangoDeploymentReplicationNamespaceLister implements the ArangoDeploymentReplicationNamespaceLister +// interface. +type arangoDeploymentReplicationNamespaceLister struct { + indexer cache.Indexer + namespace string +} + +// List lists all ArangoDeploymentReplications in the indexer for a given namespace. +func (s arangoDeploymentReplicationNamespaceLister) List(selector labels.Selector) (ret []*v1alpha.ArangoDeploymentReplication, err error) { + err = cache.ListAllByNamespace(s.indexer, s.namespace, selector, func(m interface{}) { + ret = append(ret, m.(*v1alpha.ArangoDeploymentReplication)) + }) + return ret, err +} + +// Get retrieves the ArangoDeploymentReplication from the indexer for a given namespace and name. +func (s arangoDeploymentReplicationNamespaceLister) Get(name string) (*v1alpha.ArangoDeploymentReplication, error) { + obj, exists, err := s.indexer.GetByKey(s.namespace + "/" + name) + if err != nil { + return nil, err + } + if !exists { + return nil, errors.NewNotFound(v1alpha.Resource("arangodeploymentreplication"), name) + } + return obj.(*v1alpha.ArangoDeploymentReplication), nil +} diff --git a/pkg/generated/listers/replication/v1alpha/expansion_generated.go b/pkg/generated/listers/replication/v1alpha/expansion_generated.go new file mode 100644 index 000000000..51d8b72b8 --- /dev/null +++ b/pkg/generated/listers/replication/v1alpha/expansion_generated.go @@ -0,0 +1,31 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +// This file was automatically generated by lister-gen + +package v1alpha + +// ArangoDeploymentReplicationListerExpansion allows custom methods to be added to +// ArangoDeploymentReplicationLister. +type ArangoDeploymentReplicationListerExpansion interface{} + +// ArangoDeploymentReplicationNamespaceListerExpansion allows custom methods to be added to +// ArangoDeploymentReplicationNamespaceLister. +type ArangoDeploymentReplicationNamespaceListerExpansion interface{} diff --git a/pkg/operator/crd.go b/pkg/operator/crd.go index 0b20f3cac..826a02405 100644 --- a/pkg/operator/crd.go +++ b/pkg/operator/crd.go @@ -24,13 +24,14 @@ package operator import ( deplapi "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + replapi "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" lsapi "github.com/arangodb/kube-arangodb/pkg/apis/storage/v1alpha" "github.com/arangodb/kube-arangodb/pkg/util/crd" ) // waitForCRD waits for the CustomResourceDefinition (created externally) // to be ready. -func (o *Operator) waitForCRD(enableDeployment, enableStorage bool) error { +func (o *Operator) waitForCRD(enableDeployment, enableDeploymentReplication, enableStorage bool) error { log := o.log if enableDeployment { @@ -40,6 +41,13 @@ func (o *Operator) waitForCRD(enableDeployment, enableStorage bool) error { } } + if enableDeploymentReplication { + log.Debug().Msg("Waiting for ArangoDeploymentReplication CRD to be ready") + if err := crd.WaitCRDReady(o.KubeExtCli, replapi.ArangoDeploymentReplicationCRDName); err != nil { + return maskAny(err) + } + } + if enableStorage { log.Debug().Msg("Waiting for ArangoLocalStorage CRD to be ready") if err := crd.WaitCRDReady(o.KubeExtCli, lsapi.ArangoLocalStorageCRDName); err != nil { diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 5fdeb85a9..ba43136b4 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -34,10 +34,12 @@ import ( "k8s.io/client-go/tools/record" deplapi "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + replapi "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" lsapi "github.com/arangodb/kube-arangodb/pkg/apis/storage/v1alpha" "github.com/arangodb/kube-arangodb/pkg/deployment" "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" "github.com/arangodb/kube-arangodb/pkg/logging" + "github.com/arangodb/kube-arangodb/pkg/replication" "github.com/arangodb/kube-arangodb/pkg/storage" "github.com/arangodb/kube-arangodb/pkg/util/probe" ) @@ -47,49 +49,55 @@ const ( ) type Event struct { - Type kwatch.EventType - Deployment *deplapi.ArangoDeployment - LocalStorage *lsapi.ArangoLocalStorage + Type kwatch.EventType + Deployment *deplapi.ArangoDeployment + DeploymentReplication *replapi.ArangoDeploymentReplication + LocalStorage *lsapi.ArangoLocalStorage } type Operator struct { Config Dependencies - log zerolog.Logger - deployments map[string]*deployment.Deployment - localStorages map[string]*storage.LocalStorage + log zerolog.Logger + deployments map[string]*deployment.Deployment + deploymentReplications map[string]*replication.DeploymentReplication + localStorages map[string]*storage.LocalStorage } type Config struct { - ID string - Namespace string - PodName string - ServiceAccount string - EnableDeployment bool - EnableStorage bool - AllowChaos bool + ID string + Namespace string + PodName string + ServiceAccount string + LifecycleImage string + EnableDeployment bool + EnableDeploymentReplication bool + EnableStorage bool + AllowChaos bool } type Dependencies struct { - LogService logging.Service - KubeCli kubernetes.Interface - KubeExtCli apiextensionsclient.Interface - CRCli versioned.Interface - EventRecorder record.EventRecorder - LivenessProbe *probe.LivenessProbe - DeploymentProbe *probe.ReadyProbe - StorageProbe *probe.ReadyProbe + LogService logging.Service + KubeCli kubernetes.Interface + KubeExtCli apiextensionsclient.Interface + CRCli versioned.Interface + EventRecorder record.EventRecorder + LivenessProbe *probe.LivenessProbe + DeploymentProbe *probe.ReadyProbe + DeploymentReplicationProbe *probe.ReadyProbe + StorageProbe *probe.ReadyProbe } // NewOperator instantiates a new operator from given config & dependencies. func NewOperator(config Config, deps Dependencies) (*Operator, error) { o := &Operator{ - Config: config, - Dependencies: deps, - log: deps.LogService.MustGetLogger("operator"), - deployments: make(map[string]*deployment.Deployment), - localStorages: make(map[string]*storage.LocalStorage), + Config: config, + Dependencies: deps, + log: deps.LogService.MustGetLogger("operator"), + deployments: make(map[string]*deployment.Deployment), + deploymentReplications: make(map[string]*replication.DeploymentReplication), + localStorages: make(map[string]*storage.LocalStorage), } return o, nil } @@ -99,6 +107,9 @@ func (o *Operator) Run() { if o.Config.EnableDeployment { go o.runLeaderElection("arango-deployment-operator", o.onStartDeployment) } + if o.Config.EnableDeploymentReplication { + go o.runLeaderElection("arango-deployment-replication-operator", o.onStartDeploymentReplication) + } if o.Config.EnableStorage { go o.runLeaderElection("arango-storage-operator", o.onStartStorage) } @@ -109,7 +120,7 @@ func (o *Operator) Run() { // onStartDeployment starts the deployment operator and run till given channel is closed. func (o *Operator) onStartDeployment(stop <-chan struct{}) { for { - if err := o.waitForCRD(true, false); err == nil { + if err := o.waitForCRD(true, false, false); err == nil { break } else { log.Error().Err(err).Msg("Resource initialization failed") @@ -120,10 +131,24 @@ func (o *Operator) onStartDeployment(stop <-chan struct{}) { o.runDeployments(stop) } +// onStartDeploymentReplication starts the deployment replication operator and run till given channel is closed. +func (o *Operator) onStartDeploymentReplication(stop <-chan struct{}) { + for { + if err := o.waitForCRD(false, true, false); err == nil { + break + } else { + log.Error().Err(err).Msg("Resource initialization failed") + log.Info().Msgf("Retrying in %s...", initRetryWaitTime) + time.Sleep(initRetryWaitTime) + } + } + o.runDeploymentReplications(stop) +} + // onStartStorage starts the storage operator and run till given channel is closed. func (o *Operator) onStartStorage(stop <-chan struct{}) { for { - if err := o.waitForCRD(false, true); err == nil { + if err := o.waitForCRD(false, false, true); err == nil { break } else { log.Error().Err(err).Msg("Resource initialization failed") diff --git a/pkg/operator/operator_deployment.go b/pkg/operator/operator_deployment.go index d62d32090..63524c7f4 100644 --- a/pkg/operator/operator_deployment.go +++ b/pkg/operator/operator_deployment.go @@ -203,6 +203,7 @@ func (o *Operator) handleDeploymentEvent(event *Event) error { func (o *Operator) makeDeploymentConfigAndDeps(apiObject *api.ArangoDeployment) (deployment.Config, deployment.Dependencies) { cfg := deployment.Config{ ServiceAccount: o.Config.ServiceAccount, + LifecycleImage: o.Config.LifecycleImage, AllowChaos: o.Config.AllowChaos, } deps := deployment.Dependencies{ @@ -211,6 +212,7 @@ func (o *Operator) makeDeploymentConfigAndDeps(apiObject *api.ArangoDeployment) Logger(), KubeCli: o.Dependencies.KubeCli, DatabaseCRCli: o.Dependencies.CRCli, + EventRecorder: o.Dependencies.EventRecorder, } return cfg, deps } diff --git a/pkg/operator/operator_deployment_relication.go b/pkg/operator/operator_deployment_relication.go new file mode 100644 index 000000000..013e7ce34 --- /dev/null +++ b/pkg/operator/operator_deployment_relication.go @@ -0,0 +1,216 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package operator + +import ( + "fmt" + + "github.com/pkg/errors" + kwatch "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/tools/cache" + + api "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/metrics" + "github.com/arangodb/kube-arangodb/pkg/replication" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +var ( + deploymentReplicationsCreated = metrics.MustRegisterCounter("controller", "deployment_replications_created", "Number of deployment replications that have been created") + deploymentReplicationsDeleted = metrics.MustRegisterCounter("controller", "deployment_replications_deleted", "Number of deployment replications that have been deleted") + deploymentReplicationsFailed = metrics.MustRegisterCounter("controller", "deployment_replications_failed", "Number of deployment replications that have failed") + deploymentReplicationsModified = metrics.MustRegisterCounter("controller", "deployment_replications_modified", "Number of deployment replication modifications") + deploymentReplicationsCurrent = metrics.MustRegisterGauge("controller", "deployment_replications", "Number of deployment replications currently being managed") +) + +// run the deployment replications part of the operator. +// This registers a listener and waits until the process stops. +func (o *Operator) runDeploymentReplications(stop <-chan struct{}) { + rw := k8sutil.NewResourceWatcher( + o.log, + o.Dependencies.CRCli.ReplicationV1alpha().RESTClient(), + api.ArangoDeploymentReplicationResourcePlural, + o.Config.Namespace, + &api.ArangoDeploymentReplication{}, + cache.ResourceEventHandlerFuncs{ + AddFunc: o.onAddArangoDeploymentReplication, + UpdateFunc: o.onUpdateArangoDeploymentReplication, + DeleteFunc: o.onDeleteArangoDeploymentReplication, + }) + + o.Dependencies.DeploymentReplicationProbe.SetReady() + rw.Run(stop) +} + +// onAddArangoDeploymentReplication deployment replication addition callback +func (o *Operator) onAddArangoDeploymentReplication(obj interface{}) { + o.Dependencies.LivenessProbe.Lock() + defer o.Dependencies.LivenessProbe.Unlock() + + apiObject := obj.(*api.ArangoDeploymentReplication) + o.log.Debug(). + Str("name", apiObject.GetObjectMeta().GetName()). + Msg("ArangoDeploymentReplication added") + o.syncArangoDeploymentReplication(apiObject) +} + +// onUpdateArangoDeploymentReplication deployment replication update callback +func (o *Operator) onUpdateArangoDeploymentReplication(oldObj, newObj interface{}) { + o.Dependencies.LivenessProbe.Lock() + defer o.Dependencies.LivenessProbe.Unlock() + + apiObject := newObj.(*api.ArangoDeploymentReplication) + o.log.Debug(). + Str("name", apiObject.GetObjectMeta().GetName()). + Msg("ArangoDeploymentReplication updated") + o.syncArangoDeploymentReplication(apiObject) +} + +// onDeleteArangoDeploymentReplication deployment replication delete callback +func (o *Operator) onDeleteArangoDeploymentReplication(obj interface{}) { + o.Dependencies.LivenessProbe.Lock() + defer o.Dependencies.LivenessProbe.Unlock() + + log := o.log + apiObject, ok := obj.(*api.ArangoDeploymentReplication) + if !ok { + tombstone, ok := obj.(cache.DeletedFinalStateUnknown) + if !ok { + log.Error().Interface("event-object", obj).Msg("unknown object from ArangoDeploymentReplication delete event") + return + } + apiObject, ok = tombstone.Obj.(*api.ArangoDeploymentReplication) + if !ok { + log.Error().Interface("event-object", obj).Msg("Tombstone contained object that is not an ArangoDeploymentReplication") + return + } + } + log.Debug(). + Str("name", apiObject.GetObjectMeta().GetName()). + Msg("ArangoDeploymentReplication deleted") + ev := &Event{ + Type: kwatch.Deleted, + DeploymentReplication: apiObject, + } + + // pt.start() + err := o.handleDeploymentReplicationEvent(ev) + if err != nil { + log.Warn().Err(err).Msg("Failed to handle event") + } + //pt.stop() +} + +// syncArangoDeploymentReplication synchronized the given deployment replication. +func (o *Operator) syncArangoDeploymentReplication(apiObject *api.ArangoDeploymentReplication) { + ev := &Event{ + Type: kwatch.Added, + DeploymentReplication: apiObject, + } + // re-watch or restart could give ADD event. + // If for an ADD event the cluster spec is invalid then it is not added to the local cache + // so modifying that deployment will result in another ADD event + if _, ok := o.deploymentReplications[apiObject.Name]; ok { + ev.Type = kwatch.Modified + } + + //pt.start() + err := o.handleDeploymentReplicationEvent(ev) + if err != nil { + o.log.Warn().Err(err).Msg("Failed to handle event") + } + //pt.stop() +} + +// handleDeploymentReplicationEvent processed the given event. +func (o *Operator) handleDeploymentReplicationEvent(event *Event) error { + apiObject := event.DeploymentReplication + + if apiObject.Status.Phase.IsFailed() { + deploymentReplicationsFailed.Inc() + if event.Type == kwatch.Deleted { + delete(o.deploymentReplications, apiObject.Name) + return nil + } + return maskAny(fmt.Errorf("ignore failed deployment replication (%s). Please delete its CR", apiObject.Name)) + } + + switch event.Type { + case kwatch.Added: + if _, ok := o.deploymentReplications[apiObject.Name]; ok { + return maskAny(fmt.Errorf("unsafe state. deployment replication (%s) was created before but we received event (%s)", apiObject.Name, event.Type)) + } + + // Fill in defaults + apiObject.Spec.SetDefaults() + // Validate deployment spec + if err := apiObject.Spec.Validate(); err != nil { + return maskAny(errors.Wrapf(err, "invalid deployment replication spec. please fix the following problem with the deployment replication spec: %v", err)) + } + + cfg, deps := o.makeDeploymentReplicationConfigAndDeps(apiObject) + nc, err := replication.New(cfg, deps, apiObject) + if err != nil { + return maskAny(fmt.Errorf("failed to create deployment: %s", err)) + } + o.deploymentReplications[apiObject.Name] = nc + + deploymentReplicationsCreated.Inc() + deploymentReplicationsCurrent.Set(float64(len(o.deploymentReplications))) + + case kwatch.Modified: + repl, ok := o.deploymentReplications[apiObject.Name] + if !ok { + return maskAny(fmt.Errorf("unsafe state. deployment replication (%s) was never created but we received event (%s)", apiObject.Name, event.Type)) + } + repl.Update(apiObject) + deploymentReplicationsModified.Inc() + + case kwatch.Deleted: + repl, ok := o.deploymentReplications[apiObject.Name] + if !ok { + return maskAny(fmt.Errorf("unsafe state. deployment replication (%s) was never created but we received event (%s)", apiObject.Name, event.Type)) + } + repl.Delete() + delete(o.deploymentReplications, apiObject.Name) + deploymentReplicationsDeleted.Inc() + deploymentReplicationsCurrent.Set(float64(len(o.deploymentReplications))) + } + return nil +} + +// makeDeploymentReplicationConfigAndDeps creates a Config & Dependencies object for a new DeploymentReplication. +func (o *Operator) makeDeploymentReplicationConfigAndDeps(apiObject *api.ArangoDeploymentReplication) (replication.Config, replication.Dependencies) { + cfg := replication.Config{ + Namespace: o.Config.Namespace, + } + deps := replication.Dependencies{ + Log: o.Dependencies.LogService.MustGetLogger("deployment-replication").With(). + Str("deployment-replication", apiObject.GetName()). + Logger(), + KubeCli: o.Dependencies.KubeCli, + CRCli: o.Dependencies.CRCli, + EventRecorder: o.Dependencies.EventRecorder, + } + return cfg, deps +} diff --git a/pkg/operator/operator_local_storage.go b/pkg/operator/operator_local_storage.go index 008df5098..5d0b3b5ac 100644 --- a/pkg/operator/operator_local_storage.go +++ b/pkg/operator/operator_local_storage.go @@ -210,8 +210,9 @@ func (o *Operator) makeLocalStorageConfigAndDeps(apiObject *api.ArangoLocalStora Log: o.Dependencies.LogService.MustGetLogger("storage").With(). Str("localStorage", apiObject.GetName()). Logger(), - KubeCli: o.Dependencies.KubeCli, - StorageCRCli: o.Dependencies.CRCli, + KubeCli: o.Dependencies.KubeCli, + StorageCRCli: o.Dependencies.CRCli, + EventRecorder: o.Dependencies.EventRecorder, } return cfg, deps } diff --git a/pkg/replication/deployment_replication.go b/pkg/replication/deployment_replication.go new file mode 100644 index 000000000..0275dc8c7 --- /dev/null +++ b/pkg/replication/deployment_replication.go @@ -0,0 +1,373 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package replication + +import ( + "fmt" + "reflect" + "sync/atomic" + "time" + + "github.com/rs/zerolog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/record" + + "github.com/arangodb/arangosync/client" + api "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/retry" + "github.com/arangodb/kube-arangodb/pkg/util/trigger" +) + +// Config holds configuration settings for a DeploymentReplication +type Config struct { + Namespace string +} + +// Dependencies holds dependent services for a DeploymentReplication +type Dependencies struct { + Log zerolog.Logger + KubeCli kubernetes.Interface + CRCli versioned.Interface + EventRecorder record.EventRecorder +} + +// deploymentReplicationEvent strongly typed type of event +type deploymentReplicationEventType string + +const ( + eventArangoDeploymentReplicationUpdated deploymentReplicationEventType = "DeploymentReplicationUpdated" +) + +// seploymentReplicationEvent holds an event passed from the controller to the deployment replication. +type deploymentReplicationEvent struct { + Type deploymentReplicationEventType + DeploymentReplication *api.ArangoDeploymentReplication +} + +const ( + deploymentReplicationEventQueueSize = 100 + minInspectionInterval = time.Second // Ensure we inspect the generated resources no less than with this interval + maxInspectionInterval = time.Minute // Ensure we inspect the generated resources no less than with this interval +) + +// DeploymentReplication is the in process state of an ArangoDeploymentReplication. +type DeploymentReplication struct { + apiObject *api.ArangoDeploymentReplication // API object + status api.DeploymentReplicationStatus // Internal status of the CR + config Config + deps Dependencies + + eventCh chan *deploymentReplicationEvent + stopCh chan struct{} + stopped int32 + + inspectTrigger trigger.Trigger + recentInspectionErrors int + clientCache client.ClientCache +} + +// New creates a new DeploymentReplication from the given API object. +func New(config Config, deps Dependencies, apiObject *api.ArangoDeploymentReplication) (*DeploymentReplication, error) { + if err := apiObject.Spec.Validate(); err != nil { + return nil, maskAny(err) + } + dr := &DeploymentReplication{ + apiObject: apiObject, + status: *(apiObject.Status.DeepCopy()), + config: config, + deps: deps, + eventCh: make(chan *deploymentReplicationEvent, deploymentReplicationEventQueueSize), + stopCh: make(chan struct{}), + } + + go dr.run() + + return dr, nil +} + +// Update the deployment replication. +// This sends an update event in the event queue. +func (dr *DeploymentReplication) Update(apiObject *api.ArangoDeploymentReplication) { + dr.send(&deploymentReplicationEvent{ + Type: eventArangoDeploymentReplicationUpdated, + DeploymentReplication: apiObject, + }) +} + +// Delete the deployment replication. +// Called when the local storage was deleted by the user. +func (dr *DeploymentReplication) Delete() { + dr.deps.Log.Info().Msg("deployment replication is deleted by user") + if atomic.CompareAndSwapInt32(&dr.stopped, 0, 1) { + close(dr.stopCh) + } +} + +// send given event into the deployment replication event queue. +func (dr *DeploymentReplication) send(ev *deploymentReplicationEvent) { + select { + case dr.eventCh <- ev: + l, ecap := len(dr.eventCh), cap(dr.eventCh) + if l > int(float64(ecap)*0.8) { + dr.deps.Log.Warn(). + Int("used", l). + Int("capacity", ecap). + Msg("event queue buffer is almost full") + } + case <-dr.stopCh: + } +} + +// run is the core the core worker. +// It processes the event queue and polls the state of generated +// resource on a regular basis. +func (dr *DeploymentReplication) run() { + log := dr.deps.Log + + // Add finalizers + if err := dr.addFinalizers(); err != nil { + log.Warn().Err(err).Msg("Failed to add finalizers") + } + + inspectionInterval := maxInspectionInterval + dr.inspectTrigger.Trigger() + for { + select { + case <-dr.stopCh: + // We're being stopped. + return + + case event := <-dr.eventCh: + // Got event from event queue + switch event.Type { + case eventArangoDeploymentReplicationUpdated: + if err := dr.handleArangoDeploymentReplicationUpdatedEvent(event); err != nil { + dr.failOnError(err, "Failed to handle deployment replication update") + return + } + default: + panic("unknown event type" + event.Type) + } + + case <-dr.inspectTrigger.Done(): + inspectionInterval = dr.inspectDeploymentReplication(inspectionInterval) + + case <-time.After(inspectionInterval): + // Trigger inspection + dr.inspectTrigger.Trigger() + // Backoff with next interval + inspectionInterval = time.Duration(float64(inspectionInterval) * 1.5) + if inspectionInterval > maxInspectionInterval { + inspectionInterval = maxInspectionInterval + } + } + } +} + +// handleArangoDeploymentReplicationUpdatedEvent is called when the deployment replication is updated by the user. +func (dr *DeploymentReplication) handleArangoDeploymentReplicationUpdatedEvent(event *deploymentReplicationEvent) error { + log := dr.deps.Log.With().Str("deployoment-replication", event.DeploymentReplication.GetName()).Logger() + repls := dr.deps.CRCli.ReplicationV1alpha().ArangoDeploymentReplications(dr.apiObject.GetNamespace()) + + // Get the most recent version of the deployment replication from the API server + current, err := repls.Get(dr.apiObject.GetName(), metav1.GetOptions{}) + if err != nil { + log.Debug().Err(err).Msg("Failed to get current version of deployment replication from API server") + if k8sutil.IsNotFound(err) { + return nil + } + return maskAny(err) + } + + newAPIObject := current.DeepCopy() + newAPIObject.Spec.SetDefaults() + newAPIObject.Status = dr.status + resetFields := dr.apiObject.Spec.ResetImmutableFields(&newAPIObject.Spec) + if len(resetFields) > 0 { + log.Debug().Strs("fields", resetFields).Msg("Found modified immutable fields") + } + if err := newAPIObject.Spec.Validate(); err != nil { + dr.createEvent(k8sutil.NewErrorEvent("Validation failed", err, dr.apiObject)) + // Try to reset object + if err := dr.updateCRSpec(dr.apiObject.Spec); err != nil { + log.Error().Err(err).Msg("Restore original spec failed") + dr.createEvent(k8sutil.NewErrorEvent("Restore original failed", err, dr.apiObject)) + } + return nil + } + if len(resetFields) > 0 { + for _, fieldName := range resetFields { + log.Debug().Str("field", fieldName).Msg("Reset immutable field") + dr.createEvent(k8sutil.NewImmutableFieldEvent(fieldName, dr.apiObject)) + } + } + + // Save updated spec + if err := dr.updateCRSpec(newAPIObject.Spec); err != nil { + return maskAny(fmt.Errorf("failed to update ArangoDeploymentReplication spec: %v", err)) + } + + // Trigger inspect + dr.inspectTrigger.Trigger() + + return nil +} + +// createEvent creates a given event. +// On error, the error is logged. +func (dr *DeploymentReplication) createEvent(evt *k8sutil.Event) { + dr.deps.EventRecorder.Event(evt.InvolvedObject, evt.Type, evt.Reason, evt.Message) +} + +// Update the status of the API object from the internal status +func (dr *DeploymentReplication) updateCRStatus() error { + if reflect.DeepEqual(dr.apiObject.Status, dr.status) { + // Nothing has changed + return nil + } + + // Send update to API server + log := dr.deps.Log + repls := dr.deps.CRCli.ReplicationV1alpha().ArangoDeploymentReplications(dr.apiObject.GetNamespace()) + update := dr.apiObject.DeepCopy() + attempt := 0 + for { + attempt++ + update.Status = dr.status + newAPIObject, err := repls.Update(update) + if err == nil { + // Update internal object + dr.apiObject = newAPIObject + return nil + } + if attempt < 10 && k8sutil.IsConflict(err) { + // API object may have been changed already, + // Reload api object and try again + var current *api.ArangoDeploymentReplication + current, err = repls.Get(update.GetName(), metav1.GetOptions{}) + if err == nil { + update = current.DeepCopy() + continue + } + } + if err != nil { + log.Debug().Err(err).Msg("failed to patch ArangoDeploymentReplication status") + return maskAny(fmt.Errorf("failed to patch ArangoDeploymentReplication status: %v", err)) + } + } +} + +// Update the spec part of the API object (d.apiObject) +// to the given object, while preserving the status. +// On success, d.apiObject is updated. +func (dr *DeploymentReplication) updateCRSpec(newSpec api.DeploymentReplicationSpec) error { + log := dr.deps.Log + repls := dr.deps.CRCli.ReplicationV1alpha().ArangoDeploymentReplications(dr.apiObject.GetNamespace()) + + // Send update to API server + update := dr.apiObject.DeepCopy() + attempt := 0 + for { + attempt++ + update.Spec = newSpec + update.Status = dr.status + newAPIObject, err := repls.Update(update) + if err == nil { + // Update internal object + dr.apiObject = newAPIObject + return nil + } + if attempt < 10 && k8sutil.IsConflict(err) { + // API object may have been changed already, + // Reload api object and try again + var current *api.ArangoDeploymentReplication + current, err = repls.Get(update.GetName(), metav1.GetOptions{}) + if err == nil { + update = current.DeepCopy() + continue + } + } + if err != nil { + log.Debug().Err(err).Msg("failed to patch ArangoDeploymentReplication spec") + return maskAny(fmt.Errorf("failed to patch ArangoDeploymentReplication spec: %v", err)) + } + } +} + +// failOnError reports the given error and sets the deployment replication status to failed. +func (dr *DeploymentReplication) failOnError(err error, msg string) { + log := dr.deps.Log + log.Error().Err(err).Msg(msg) + dr.status.Reason = err.Error() + dr.reportFailedStatus() +} + +// reportFailedStatus sets the status of the deployment replication to Failed and keeps trying to forward +// that to the API server. +func (dr *DeploymentReplication) reportFailedStatus() { + log := dr.deps.Log + log.Info().Msg("local storage failed. Reporting failed reason...") + repls := dr.deps.CRCli.ReplicationV1alpha().ArangoDeploymentReplications(dr.apiObject.GetNamespace()) + + op := func() error { + dr.status.Phase = api.DeploymentReplicationPhaseFailed + err := dr.updateCRStatus() + if err == nil || k8sutil.IsNotFound(err) { + // Status has been updated + return nil + } + + if !k8sutil.IsConflict(err) { + log.Warn().Err(err).Msg("retry report status: fail to update") + return maskAny(err) + } + + depl, err := repls.Get(dr.apiObject.Name, metav1.GetOptions{}) + if err != nil { + // Update (PUT) will return conflict even if object is deleted since we have UID set in object. + // Because it will check UID first and return something like: + // "Precondition failed: UID in precondition: 0xc42712c0f0, UID in object meta: ". + if k8sutil.IsNotFound(err) { + return nil + } + log.Warn().Err(err).Msg("retry report status: fail to get latest version") + return maskAny(err) + } + dr.apiObject = depl + return maskAny(fmt.Errorf("retry needed")) + } + + retry.Retry(op, time.Hour*24*365) +} + +// isOwnerOf returns true if the given object belong to this local storage. +func (dr *DeploymentReplication) isOwnerOf(obj metav1.Object) bool { + ownerRefs := obj.GetOwnerReferences() + if len(ownerRefs) < 1 { + return false + } + return ownerRefs[0].UID == dr.apiObject.UID +} diff --git a/pkg/replication/errors.go b/pkg/replication/errors.go new file mode 100644 index 000000000..e48fb52d3 --- /dev/null +++ b/pkg/replication/errors.go @@ -0,0 +1,29 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package replication + +import "github.com/pkg/errors" + +var ( + maskAny = errors.WithStack +) diff --git a/pkg/replication/finalizers.go b/pkg/replication/finalizers.go new file mode 100644 index 000000000..112782124 --- /dev/null +++ b/pkg/replication/finalizers.go @@ -0,0 +1,191 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package replication + +import ( + "context" + "fmt" + "time" + + "github.com/rs/zerolog" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/arangodb/arangosync/client" + api "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +const ( + maxCancelFailures = 5 // After this amount of failed cancel-synchronization attempts, the operator switch to abort-sychronization. +) + +// addFinalizers adds a stop-sync finalizer to the api object when needed. +func (dr *DeploymentReplication) addFinalizers() error { + apiObject := dr.apiObject + if apiObject.GetDeletionTimestamp() != nil { + // Delete already triggered, cannot add. + return nil + } + for _, f := range apiObject.GetFinalizers() { + if f == constants.FinalizerDeplReplStopSync { + // Finalizer already added + return nil + } + } + apiObject.SetFinalizers(append(apiObject.GetFinalizers(), constants.FinalizerDeplReplStopSync)) + if err := dr.updateCRSpec(apiObject.Spec); err != nil { + return maskAny(err) + } + return nil +} + +// runFinalizers goes through the list of ArangoDeploymentReplication finalizers to see if they can be removed. +func (dr *DeploymentReplication) runFinalizers(ctx context.Context, p *api.ArangoDeploymentReplication) error { + log := dr.deps.Log.With().Str("replication-name", p.GetName()).Logger() + var removalList []string + for _, f := range p.ObjectMeta.GetFinalizers() { + switch f { + case constants.FinalizerDeplReplStopSync: + log.Debug().Msg("Inspecting stop-sync finalizer") + if err := dr.inspectFinalizerDeplReplStopSync(ctx, log, p); err == nil { + removalList = append(removalList, f) + } else { + log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet") + } + } + } + // Remove finalizers (if needed) + if len(removalList) > 0 { + ignoreNotFound := false + if err := removeDeploymentReplicationFinalizers(log, dr.deps.CRCli, p, removalList, ignoreNotFound); err != nil { + log.Debug().Err(err).Msg("Failed to update deployment replication (to remove finalizers)") + return maskAny(err) + } + } + return nil +} + +// inspectFinalizerDeplReplStopSync checks the finalizer condition for stop-sync. +// It returns nil if the finalizer can be removed. +func (dr *DeploymentReplication) inspectFinalizerDeplReplStopSync(ctx context.Context, log zerolog.Logger, p *api.ArangoDeploymentReplication) error { + // Inspect phase + if p.Status.Phase.IsFailed() { + log.Debug().Msg("Deployment replication is already failed, safe to remove stop-sync finalizer") + return nil + } + + // Inspect deployment deletion state in source + abort := dr.status.CancelFailures > maxCancelFailures + depls := dr.deps.CRCli.DatabaseV1alpha().ArangoDeployments(p.GetNamespace()) + if name := p.Spec.Source.GetDeploymentName(); name != "" { + depl, err := depls.Get(name, metav1.GetOptions{}) + if k8sutil.IsNotFound(err) { + log.Debug().Msg("Source deployment is gone. Abort enabled") + abort = true + } else if err != nil { + log.Warn().Err(err).Msg("Failed to get source deployment") + return maskAny(err) + } else if depl.GetDeletionTimestamp() != nil { + log.Debug().Msg("Source deployment is being deleted. Abort enabled") + abort = true + } + } + + // Inspect deployment deletion state in destination + cleanupSource := false + if name := p.Spec.Destination.GetDeploymentName(); name != "" { + depl, err := depls.Get(name, metav1.GetOptions{}) + if k8sutil.IsNotFound(err) { + log.Debug().Msg("Destination deployment is gone. Source cleanup enabled") + cleanupSource = true + } else if err != nil { + log.Warn().Err(err).Msg("Failed to get destinaton deployment") + return maskAny(err) + } else if depl.GetDeletionTimestamp() != nil { + log.Debug().Msg("Destination deployment is being deleted. Source cleanup enabled") + cleanupSource = true + } + } + + // Cleanup source or stop sync + if cleanupSource { + // Destination is gone, cleanup source + /*sourceClient, err := dr.createSyncMasterClient(p.Spec.Source) + if err != nil { + log.Warn().Err(err).Msg("Failed to create source client") + return maskAny(err) + }*/ + //sourceClient.Master().C + return maskAny(fmt.Errorf("TODO")) + } else { + // Destination still exists, stop/abort sync + destClient, err := dr.createSyncMasterClient(p.Spec.Destination) + if err != nil { + log.Warn().Err(err).Msg("Failed to create destination client") + return maskAny(err) + } + req := client.CancelSynchronizationRequest{ + WaitTimeout: time.Minute * 3, + Force: abort, + ForceTimeout: time.Minute * 2, + } + log.Debug().Bool("abort", abort).Msg("Stopping synchronization...") + _, err = destClient.Master().CancelSynchronization(ctx, req) + if err != nil && !client.IsPreconditionFailed(err) { + log.Warn().Err(err).Bool("abort", abort).Msg("Failed to stop synchronization") + dr.status.CancelFailures++ + if err := dr.updateCRStatus(); err != nil { + log.Warn().Err(err).Msg("Failed to update status to reflect cancel-failures increment") + } + return maskAny(err) + } + return nil + } +} + +// removeDeploymentReplicationFinalizers removes the given finalizers from the given DeploymentReplication. +func removeDeploymentReplicationFinalizers(log zerolog.Logger, crcli versioned.Interface, p *api.ArangoDeploymentReplication, finalizers []string, ignoreNotFound bool) error { + repls := crcli.ReplicationV1alpha().ArangoDeploymentReplications(p.GetNamespace()) + getFunc := func() (metav1.Object, error) { + result, err := repls.Get(p.GetName(), metav1.GetOptions{}) + if err != nil { + return nil, maskAny(err) + } + return result, nil + } + updateFunc := func(updated metav1.Object) error { + updatedRepl := updated.(*api.ArangoDeploymentReplication) + result, err := repls.Update(updatedRepl) + if err != nil { + return maskAny(err) + } + *p = *result + return nil + } + if err := k8sutil.RemoveFinalizers(log, finalizers, getFunc, updateFunc, ignoreNotFound); err != nil { + return maskAny(err) + } + return nil +} diff --git a/pkg/replication/sync_client.go b/pkg/replication/sync_client.go new file mode 100644 index 000000000..df787266e --- /dev/null +++ b/pkg/replication/sync_client.go @@ -0,0 +1,174 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package replication + +import ( + "net" + "strconv" + + certificates "github.com/arangodb-helper/go-certificates" + "github.com/arangodb/arangosync/client" + "github.com/arangodb/arangosync/tasks" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// createSyncMasterClient creates an arangosync client for the given endpoint. +func (dr *DeploymentReplication) createSyncMasterClient(epSpec api.EndpointSpec) (client.API, error) { + log := dr.deps.Log + + // Endpoint + source, err := dr.createArangoSyncEndpoint(epSpec) + if err != nil { + return nil, maskAny(err) + } + + // Authentication + insecureSkipVerify := true + tlsAuth := tasks.TLSAuthentication{} + clientAuthKeyfileSecretName, userSecretName, authJWTSecretName, tlsCASecretName, err := dr.getEndpointSecretNames(epSpec) + if err != nil { + return nil, maskAny(err) + } + username := "" + password := "" + jwtSecret := "" + if userSecretName != "" { + var err error + username, password, err = k8sutil.GetBasicAuthSecret(dr.deps.KubeCli.CoreV1(), userSecretName, dr.apiObject.GetNamespace()) + if err != nil { + return nil, maskAny(err) + } + } else if authJWTSecretName != "" { + var err error + jwtSecret, err = k8sutil.GetTokenSecret(dr.deps.KubeCli.CoreV1(), authJWTSecretName, dr.apiObject.GetNamespace()) + if err != nil { + return nil, maskAny(err) + } + } else if clientAuthKeyfileSecretName != "" { + keyFileContent, err := k8sutil.GetTLSKeyfileSecret(dr.deps.KubeCli.CoreV1(), clientAuthKeyfileSecretName, dr.apiObject.GetNamespace()) + if err != nil { + return nil, maskAny(err) + } + kf, err := certificates.NewKeyfile(keyFileContent) + if err != nil { + return nil, maskAny(err) + } + tlsAuth.TLSClientAuthentication = tasks.TLSClientAuthentication{ + ClientCertificate: kf.EncodeCertificates(), + ClientKey: kf.EncodePrivateKey(), + } + } + if tlsCASecretName != "" { + caCert, err := k8sutil.GetCACertficateSecret(dr.deps.KubeCli.CoreV1(), tlsCASecretName, dr.apiObject.GetNamespace()) + if err != nil { + return nil, maskAny(err) + } + tlsAuth.CACertificate = caCert + } + auth := client.NewAuthentication(tlsAuth, jwtSecret) + auth.Username = username + auth.Password = password + + // Create client + c, err := dr.clientCache.GetClient(log, source, auth, insecureSkipVerify) + if err != nil { + return nil, maskAny(err) + } + return c, nil +} + +// createArangoSyncEndpoint creates the endpoints for the given spec. +func (dr *DeploymentReplication) createArangoSyncEndpoint(epSpec api.EndpointSpec) (client.Endpoint, error) { + if epSpec.HasDeploymentName() { + deploymentName := epSpec.GetDeploymentName() + depls := dr.deps.CRCli.DatabaseV1alpha().ArangoDeployments(dr.apiObject.GetNamespace()) + depl, err := depls.Get(deploymentName, metav1.GetOptions{}) + if err != nil { + dr.deps.Log.Debug().Err(err).Str("deployment", deploymentName).Msg("Failed to get deployment") + return nil, maskAny(err) + } + dnsName := k8sutil.CreateSyncMasterClientServiceDNSName(depl) + return client.Endpoint{"https://" + net.JoinHostPort(dnsName, strconv.Itoa(k8sutil.ArangoSyncMasterPort))}, nil + } + return client.Endpoint(epSpec.MasterEndpoint), nil +} + +// createArangoSyncTLSAuthentication creates the authentication needed to authenticate +// the destination syncmaster at the source syncmaster. +func (dr *DeploymentReplication) createArangoSyncTLSAuthentication(spec api.DeploymentReplicationSpec) (client.TLSAuthentication, error) { + // Fetch secret names of source + clientAuthKeyfileSecretName, _, _, tlsCASecretName, err := dr.getEndpointSecretNames(spec.Source) + if err != nil { + return client.TLSAuthentication{}, maskAny(err) + } + + // Fetch keyfile + keyFileContent, err := k8sutil.GetTLSKeyfileSecret(dr.deps.KubeCli.CoreV1(), clientAuthKeyfileSecretName, dr.apiObject.GetNamespace()) + if err != nil { + return client.TLSAuthentication{}, maskAny(err) + } + kf, err := certificates.NewKeyfile(keyFileContent) + if err != nil { + return client.TLSAuthentication{}, maskAny(err) + } + + // Fetch TLS CA certificate for source + caCert, err := k8sutil.GetCACertficateSecret(dr.deps.KubeCli.CoreV1(), tlsCASecretName, dr.apiObject.GetNamespace()) + if err != nil { + return client.TLSAuthentication{}, maskAny(err) + } + + // Create authentication + result := client.TLSAuthentication{ + TLSClientAuthentication: tasks.TLSClientAuthentication{ + ClientCertificate: kf.EncodeCertificates(), + ClientKey: kf.EncodePrivateKey(), + }, + CACertificate: caCert, + } + return result, nil +} + +// getEndpointSecretNames returns the names of secrets that hold the: +// - client authentication certificate keyfile, +// - user (basic auth) secret, +// - JWT secret name, +// - TLS ca.crt +func (dr *DeploymentReplication) getEndpointSecretNames(epSpec api.EndpointSpec) (clientAuthCertKeyfileSecretName, userSecretName, jwtSecretName, tlsCASecretName string, err error) { + clientAuthCertKeyfileSecretName = epSpec.Authentication.GetKeyfileSecretName() + userSecretName = epSpec.Authentication.GetUserSecretName() + if epSpec.HasDeploymentName() { + deploymentName := epSpec.GetDeploymentName() + depls := dr.deps.CRCli.DatabaseV1alpha().ArangoDeployments(dr.apiObject.GetNamespace()) + depl, err := depls.Get(deploymentName, metav1.GetOptions{}) + if err != nil { + dr.deps.Log.Debug().Err(err).Str("deployment", deploymentName).Msg("Failed to get deployment") + return "", "", "", "", maskAny(err) + } + return clientAuthCertKeyfileSecretName, userSecretName, depl.Spec.Sync.Authentication.GetJWTSecretName(), depl.Spec.Sync.TLS.GetCASecretName(), nil + } + return clientAuthCertKeyfileSecretName, userSecretName, "", epSpec.TLS.GetCASecretName(), nil +} diff --git a/pkg/replication/sync_inspector.go b/pkg/replication/sync_inspector.go new file mode 100644 index 000000000..352542d1d --- /dev/null +++ b/pkg/replication/sync_inspector.go @@ -0,0 +1,306 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package replication + +import ( + "context" + "sort" + "time" + + "github.com/arangodb/arangosync/client" + api "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1alpha" +) + +// inspectDeploymentReplication inspects the entire deployment replication +// and configures the replication when needed. +// This function should be called when: +// - the deployment replication has changed +// - any of the underlying resources has changed +// - once in a while +// Returns the delay until this function should be called again. +func (dr *DeploymentReplication) inspectDeploymentReplication(lastInterval time.Duration) time.Duration { + log := dr.deps.Log + + spec := dr.apiObject.Spec + nextInterval := lastInterval + hasError := false + ctx := context.Background() + + // Add finalizers + if err := dr.addFinalizers(); err != nil { + log.Warn().Err(err).Msg("Failed to add finalizers") + } + + // Is the deployment in failed state, if so, give up. + if dr.status.Phase == api.DeploymentReplicationPhaseFailed { + log.Debug().Msg("Deployment replication is in Failed state.") + return nextInterval + } + + // Is delete triggered? + if dr.apiObject.GetDeletionTimestamp() != nil { + // Deployment replication is triggered for deletion. + if err := dr.runFinalizers(ctx, dr.apiObject); err != nil { + log.Warn().Err(err).Msg("Failed to run finalizers") + hasError = true + } + } else { + // Inspect configuration status + destClient, err := dr.createSyncMasterClient(spec.Destination) + if err != nil { + log.Warn().Err(err).Msg("Failed to create destination syncmaster client") + } else { + // Fetch status of destination + updateStatusNeeded := false + configureSyncNeeded := false + cancelSyncNeeded := false + destEndpoint, err := destClient.Master().GetEndpoints(ctx) + if err != nil { + log.Warn().Err(err).Msg("Failed to fetch endpoints from destination syncmaster") + } + destStatus, err := destClient.Master().Status(ctx) + if err != nil { + log.Warn().Err(err).Msg("Failed to fetch status from destination syncmaster") + } else { + // Inspect destination status + if destStatus.Status.IsActive() { + isIncomingEndpoint, err := dr.isIncomingEndpoint(destStatus, spec.Source) + if err != nil { + log.Warn().Err(err).Msg("Failed to check is-incoming-endpoint") + } else { + if isIncomingEndpoint { + // Destination is correctly configured + if dr.status.Conditions.Update(api.ConditionTypeConfigured, true, "Active", "Destination syncmaster is configured correctly and active") { + updateStatusNeeded = true + } + // Fetch shard status + dr.status.Destination = createEndpointStatus(destStatus, "") + updateStatusNeeded = true + } else { + // Sync is active, but from different source + log.Warn().Msg("Destination syncmaster is configured for different source") + cancelSyncNeeded = true + if dr.status.Conditions.Update(api.ConditionTypeConfigured, false, "Invalid", "Destination syncmaster is configured for different source") { + updateStatusNeeded = true + } + } + } + } else { + // Destination has correct source, but is inactive + configureSyncNeeded = true + if dr.status.Conditions.Update(api.ConditionTypeConfigured, false, "Inactive", "Destination syncmaster is configured correctly but in-active") { + updateStatusNeeded = true + } + } + } + + // Inspect source + sourceClient, err := dr.createSyncMasterClient(spec.Source) + if err != nil { + log.Warn().Err(err).Msg("Failed to create source syncmaster client") + } else { + sourceStatus, err := sourceClient.Master().Status(ctx) + if err != nil { + log.Warn().Err(err).Msg("Failed to fetch status from source syncmaster") + } + + //if sourceStatus.Status.IsActive() { + outgoingID, hasOutgoingEndpoint, err := dr.hasOutgoingEndpoint(sourceStatus, spec.Destination, destEndpoint) + if err != nil { + log.Warn().Err(err).Msg("Failed to check has-outgoing-endpoint") + } else if hasOutgoingEndpoint { + // Destination is know in source + // Fetch shard status + dr.status.Source = createEndpointStatus(sourceStatus, outgoingID) + updateStatusNeeded = true + } else { + // We cannot find the destination in the source status + log.Info().Err(err).Msg("Destination not yet known in source syncmasters") + } + } + + // Update status if needed + if updateStatusNeeded { + if err := dr.updateCRStatus(); err != nil { + log.Warn().Err(err).Msg("Failed to update status") + hasError = true + } + } + + // Cancel sync if needed + if cancelSyncNeeded { + req := client.CancelSynchronizationRequest{} + log.Info().Msg("Canceling synchronization") + if _, err := destClient.Master().CancelSynchronization(ctx, req); err != nil { + log.Warn().Err(err).Msg("Failed to cancel synchronization") + hasError = true + } else { + log.Info().Msg("Canceled synchronization") + nextInterval = time.Second * 10 + } + } + + // Configure sync if needed + if configureSyncNeeded { + source, err := dr.createArangoSyncEndpoint(spec.Source) + if err != nil { + log.Warn().Err(err).Msg("Failed to create syncmaster endpoint") + hasError = true + } else { + auth, err := dr.createArangoSyncTLSAuthentication(spec) + if err != nil { + log.Warn().Err(err).Msg("Failed to configure synchronization authentication") + hasError = true + } else { + req := client.SynchronizationRequest{ + Source: source, + Authentication: auth, + } + log.Info().Msg("Configuring synchronization") + if err := destClient.Master().Synchronize(ctx, req); err != nil { + log.Warn().Err(err).Msg("Failed to configure synchronization") + hasError = true + } else { + log.Info().Msg("Configured synchronization") + nextInterval = time.Second * 10 + } + } + } + } + } + } + + // Update next interval (on errors) + if hasError { + if dr.recentInspectionErrors == 0 { + nextInterval = minInspectionInterval + dr.recentInspectionErrors++ + } + } else { + dr.recentInspectionErrors = 0 + } + if nextInterval > maxInspectionInterval { + nextInterval = maxInspectionInterval + } + return nextInterval +} + +// triggerInspection ensures that an inspection is run soon. +func (dr *DeploymentReplication) triggerInspection() { + dr.inspectTrigger.Trigger() +} + +// isIncomingEndpoint returns true when given sync status's endpoint +// intersects with the given endpoint spec. +func (dr *DeploymentReplication) isIncomingEndpoint(status client.SyncInfo, epSpec api.EndpointSpec) (bool, error) { + ep, err := dr.createArangoSyncEndpoint(epSpec) + if err != nil { + return false, maskAny(err) + } + return !status.Source.Intersection(ep).IsEmpty(), nil +} + +// hasOutgoingEndpoint returns true when given sync status has an outgoing +// item that intersects with the given endpoint spec. +// Returns: outgoing-ID, outgoing-found, error +func (dr *DeploymentReplication) hasOutgoingEndpoint(status client.SyncInfo, epSpec api.EndpointSpec, reportedEndpoint client.Endpoint) (string, bool, error) { + ep, err := dr.createArangoSyncEndpoint(epSpec) + if err != nil { + return "", false, maskAny(err) + } + ep = ep.Merge(reportedEndpoint...) + for _, o := range status.Outgoing { + if !o.Endpoint.Intersection(ep).IsEmpty() { + return o.ID, true, nil + } + } + return "", false, nil +} + +// createEndpointStatus creates an api EndpointStatus from the given sync status. +func createEndpointStatus(status client.SyncInfo, outgoingID string) api.EndpointStatus { + result := api.EndpointStatus{} + if outgoingID == "" { + return createEndpointStatusFromShards(status.Shards) + } + for _, o := range status.Outgoing { + if o.ID != outgoingID { + continue + } + return createEndpointStatusFromShards(o.Shards) + } + + return result +} + +// createEndpointStatusFromShards creates an api EndpointStatus from the given list of shard statuses. +func createEndpointStatusFromShards(shards []client.ShardSyncInfo) api.EndpointStatus { + result := api.EndpointStatus{} + + getDatabase := func(name string) *api.DatabaseStatus { + for i, d := range result.Databases { + if d.Name == name { + return &result.Databases[i] + } + } + // Not found, add it + result.Databases = append(result.Databases, api.DatabaseStatus{Name: name}) + return &result.Databases[len(result.Databases)-1] + } + + getCollection := func(db *api.DatabaseStatus, name string) *api.CollectionStatus { + for i, c := range db.Collections { + if c.Name == name { + return &db.Collections[i] + } + } + // Not found, add it + db.Collections = append(db.Collections, api.CollectionStatus{Name: name}) + return &db.Collections[len(db.Collections)-1] + } + + // Sort shard by index + sort.Slice(shards, func(i, j int) bool { + return shards[i].ShardIndex < shards[j].ShardIndex + }) + for _, s := range shards { + db := getDatabase(s.Database) + col := getCollection(db, s.Collection) + + // Add "missing" shards if needed + for len(col.Shards) < s.ShardIndex { + col.Shards = append(col.Shards, api.ShardStatus{Status: ""}) + } + + // Add current shard + col.Shards = append(col.Shards, api.ShardStatus{Status: string(s.Status)}) + } + + // Sort result + sort.Slice(result.Databases, func(i, j int) bool { return result.Databases[i].Name < result.Databases[j].Name }) + for i, db := range result.Databases { + sort.Slice(db.Collections, func(i, j int) bool { return db.Collections[i].Name < db.Collections[j].Name }) + result.Databases[i] = db + } + return result +} diff --git a/pkg/storage/local_storage.go b/pkg/storage/local_storage.go index 2591bda6d..b1dfeca9c 100644 --- a/pkg/storage/local_storage.go +++ b/pkg/storage/local_storage.go @@ -35,6 +35,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/client-go/tools/record" api "github.com/arangodb/kube-arangodb/pkg/apis/storage/v1alpha" "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" @@ -52,9 +53,10 @@ type Config struct { // Dependencies holds dependent services for a LocalStorage type Dependencies struct { - Log zerolog.Logger - KubeCli kubernetes.Interface - StorageCRCli versioned.Interface + Log zerolog.Logger + KubeCli kubernetes.Interface + StorageCRCli versioned.Interface + EventRecorder record.EventRecorder } // localStorageEvent strongly typed type of event @@ -112,7 +114,6 @@ func New(config Config, deps Dependencies, apiObject *api.ArangoLocalStorage) (* deps: deps, eventCh: make(chan *localStorageEvent, localStorageEventQueueSize), stopCh: make(chan struct{}), - eventsCli: deps.KubeCli.Core().Events(apiObject.GetNamespace()), } ls.pvCleaner = newPVCleaner(deps.Log, deps.KubeCli, ls.GetClientByNodeName) @@ -324,11 +325,8 @@ func (ls *LocalStorage) handleArangoLocalStorageUpdatedEvent(event *localStorage // createEvent creates a given event. // On error, the error is logged. -func (ls *LocalStorage) createEvent(evt *v1.Event) { - _, err := ls.eventsCli.Create(evt) - if err != nil { - ls.deps.Log.Error().Err(err).Interface("event", *evt).Msg("Failed to record event") - } +func (ls *LocalStorage) createEvent(evt *k8sutil.Event) { + ls.deps.EventRecorder.Event(evt.InvolvedObject, evt.Type, evt.Reason, evt.Message) } // Update the status of the API object from the internal status diff --git a/pkg/storage/provisioner/mocks/mocks.go b/pkg/storage/provisioner/mocks/mocks.go new file mode 100644 index 000000000..a3feb1728 --- /dev/null +++ b/pkg/storage/provisioner/mocks/mocks.go @@ -0,0 +1,36 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package mocks + +import ( + "github.com/stretchr/testify/mock" +) + +type MockGetter interface { + AsMock() *mock.Mock +} + +// AsMock performs a typeconversion to *Mock. +func AsMock(obj interface{}) *mock.Mock { + return obj.(MockGetter).AsMock() +} diff --git a/pkg/storage/provisioner/mocks/provisioner.go b/pkg/storage/provisioner/mocks/provisioner.go new file mode 100644 index 000000000..be3918036 --- /dev/null +++ b/pkg/storage/provisioner/mocks/provisioner.go @@ -0,0 +1,95 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package mocks + +import ( + "context" + "fmt" + + "github.com/stretchr/testify/mock" + + "github.com/arangodb/kube-arangodb/pkg/storage/provisioner" +) + +type Provisioner interface { + provisioner.API + MockGetter +} + +type provisionerMock struct { + mock.Mock + nodeName string + available, capacity int64 + localPaths map[string]struct{} +} + +// NewProvisioner returns a new mocked provisioner +func NewProvisioner(nodeName string, available, capacity int64) Provisioner { + return &provisionerMock{ + nodeName: nodeName, + available: available, + capacity: capacity, + localPaths: make(map[string]struct{}), + } +} + +func (m *provisionerMock) AsMock() *mock.Mock { + return &m.Mock +} + +// GetNodeInfo fetches information from the current node. +func (m *provisionerMock) GetNodeInfo(ctx context.Context) (provisioner.NodeInfo, error) { + return provisioner.NodeInfo{ + NodeName: m.nodeName, + }, nil +} + +// GetInfo fetches information from the filesystem containing +// the given local path on the current node. +func (m *provisionerMock) GetInfo(ctx context.Context, localPath string) (provisioner.Info, error) { + return provisioner.Info{ + NodeInfo: provisioner.NodeInfo{ + NodeName: m.nodeName, + }, + Available: m.available, + Capacity: m.capacity, + }, nil +} + +// Prepare a volume at the given local path +func (m *provisionerMock) Prepare(ctx context.Context, localPath string) error { + if _, found := m.localPaths[localPath]; found { + return fmt.Errorf("Path already exists: %s", localPath) + } + m.localPaths[localPath] = struct{}{} + return nil +} + +// Remove a volume with the given local path +func (m *provisionerMock) Remove(ctx context.Context, localPath string) error { + if _, found := m.localPaths[localPath]; !found { + return fmt.Errorf("Path not found: %s", localPath) + } + delete(m.localPaths, localPath) + return nil +} diff --git a/pkg/storage/pv_creator.go b/pkg/storage/pv_creator.go index b79d26b6b..55be5d3d6 100644 --- a/pkg/storage/pv_creator.go +++ b/pkg/storage/pv_creator.go @@ -24,6 +24,7 @@ package storage import ( "context" + "crypto/sha1" "encoding/json" "fmt" "math/rand" @@ -32,6 +33,7 @@ import ( "sort" "strconv" "strings" + "time" "k8s.io/apimachinery/pkg/api/resource" @@ -41,6 +43,8 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/storage/v1alpha" "github.com/arangodb/kube-arangodb/pkg/storage/provisioner" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) const ( @@ -72,7 +76,29 @@ func (ls *LocalStorage) createPVs(ctx context.Context, apiObject *api.ArangoLoca clients[i], clients[j] = clients[j], clients[i] }) + var nodeClientMap map[string]provisioner.API for i, claim := range unboundClaims { + // Find deployment name & role in the claim (if any) + deplName, role, enforceAniAffinity := getDeploymentInfo(claim) + allowedClients := clients + if deplName != "" { + // Select nodes to choose from such that no volume in group lands on the same node + if nodeClientMap == nil { + nodeClientMap = createNodeClientMap(ctx, clients) + } + var err error + allowedClients, err = ls.filterAllowedNodes(nodeClientMap, deplName, role) + if err != nil { + log.Warn().Err(err).Msg("Failed to filter allowed nodes") + continue // We'll try this claim again later + } + if !enforceAniAffinity && len(allowedClients) == 0 { + // No possible nodes found that have no other volume (in same group) on it. + // We don't have to enforce separate nodes, so use all clients. + allowedClients = clients + } + } + // Find size of PVC volSize := defaultVolumeSize if reqStorage := claim.Spec.Resources.Requests.StorageEphemeral(); reqStorage != nil { @@ -81,7 +107,7 @@ func (ls *LocalStorage) createPVs(ctx context.Context, apiObject *api.ArangoLoca } } // Create PV - if err := ls.createPV(ctx, apiObject, clients, i, volSize); err != nil { + if err := ls.createPV(ctx, apiObject, allowedClients, i, volSize, claim, deplName, role); err != nil { log.Error().Err(err).Msg("Failed to create PersistentVolume") } } @@ -90,7 +116,7 @@ func (ls *LocalStorage) createPVs(ctx context.Context, apiObject *api.ArangoLoca } // createPV creates a PersistentVolume. -func (ls *LocalStorage) createPV(ctx context.Context, apiObject *api.ArangoLocalStorage, clients []provisioner.API, clientsOffset int, volSize int64) error { +func (ls *LocalStorage) createPV(ctx context.Context, apiObject *api.ArangoLocalStorage, clients []provisioner.API, clientsOffset int, volSize int64, claim v1.PersistentVolumeClaim, deploymentName, role string) error { log := ls.deps.Log // Try clients for clientIdx := 0; clientIdx < len(clients); clientIdx++ { @@ -117,7 +143,7 @@ func (ls *LocalStorage) createPV(ctx context.Context, apiObject *api.ArangoLocal continue } // Create a volume - pvName := apiObject.GetName() + "-" + name + pvName := strings.ToLower(apiObject.GetName() + "-" + shortHash(info.NodeName) + "-" + name) volumeMode := v1.PersistentVolumeFilesystem nodeAff, err := createNodeAffinity(info.NodeName) if err != nil { @@ -131,6 +157,10 @@ func (ls *LocalStorage) createPV(ctx context.Context, apiObject *api.ArangoLocal v1.AlphaStorageNodeAffinityAnnotation: nodeAff, nodeNameAnnotation: info.NodeName, }, + Labels: map[string]string{ + k8sutil.LabelKeyArangoDeployment: deploymentName, + k8sutil.LabelKeyRole: role, + }, }, Spec: v1.PersistentVolumeSpec{ Capacity: v1.ResourceList{ @@ -147,6 +177,13 @@ func (ls *LocalStorage) createPV(ctx context.Context, apiObject *api.ArangoLocal }, StorageClassName: apiObject.Spec.StorageClass.Name, VolumeMode: &volumeMode, + ClaimRef: &v1.ObjectReference{ + Kind: "PersistentVolumeClaim", + APIVersion: "", + Name: claim.GetName(), + Namespace: claim.GetNamespace(), + UID: claim.GetUID(), + }, }, } // Attach PV to ArangoLocalStorage @@ -159,6 +196,16 @@ func (ls *LocalStorage) createPV(ctx context.Context, apiObject *api.ArangoLocal Str("name", pvName). Str("node-name", info.NodeName). Msg("Created PersistentVolume") + + // Bind claim to volume + if err := ls.bindClaimToVolume(claim, pv.GetName()); err != nil { + // Try to delete the PV now + if err := ls.deps.KubeCli.CoreV1().PersistentVolumes().Delete(pv.GetName(), &metav1.DeleteOptions{}); err != nil { + log.Error().Err(err).Msg("Failed to delete PV after binding PVC failed") + } + return maskAny(err) + } + return nil } } @@ -204,3 +251,100 @@ func createNodeAffinity(nodeName string) (string, error) { } return string(encoded), nil } + +// createNodeClientMap creates a map from node name to API. +// Clients that do not respond properly on a GetNodeInfo request are +// ignored. +func createNodeClientMap(ctx context.Context, clients []provisioner.API) map[string]provisioner.API { + result := make(map[string]provisioner.API) + for _, c := range clients { + if info, err := c.GetNodeInfo(ctx); err == nil { + result[info.NodeName] = c + } + } + return result +} + +// getDeploymentInfo returns the name of the deployment that created the given claim, +// the role of the server that the claim is used for and the value for `enforceAntiAffinity`. +// If not found, empty strings are returned. +// Returns deploymentName, role, enforceAntiAffinity. +func getDeploymentInfo(pvc v1.PersistentVolumeClaim) (string, string, bool) { + deploymentName := pvc.GetLabels()[k8sutil.LabelKeyArangoDeployment] + role := pvc.GetLabels()[k8sutil.LabelKeyRole] + enforceAntiAffinity, _ := strconv.ParseBool(pvc.GetAnnotations()[constants.AnnotationEnforceAntiAffinity]) // If annotation empty, this will yield false. + return deploymentName, role, enforceAntiAffinity +} + +// filterAllowedNodes returns those clients that do not yet have a volume for the given deployment name & role. +func (ls *LocalStorage) filterAllowedNodes(clients map[string]provisioner.API, deploymentName, role string) ([]provisioner.API, error) { + // Find all PVs for given deployment & role + list, err := ls.deps.KubeCli.CoreV1().PersistentVolumes().List(metav1.ListOptions{ + LabelSelector: fmt.Sprintf("%s=%s,%s=%s", k8sutil.LabelKeyArangoDeployment, deploymentName, k8sutil.LabelKeyRole, role), + }) + if err != nil { + return nil, maskAny(err) + } + excludedNodes := make(map[string]struct{}) + for _, pv := range list.Items { + nodeName := pv.GetAnnotations()[nodeNameAnnotation] + excludedNodes[nodeName] = struct{}{} + } + result := make([]provisioner.API, 0, len(clients)) + for nodeName, c := range clients { + if _, found := excludedNodes[nodeName]; !found { + result = append(result, c) + } + } + return result, nil +} + +// bindClaimToVolume tries to bind the given claim to the volume with given name. +// If the claim has been updated, the function retries several times. +func (ls *LocalStorage) bindClaimToVolume(claim v1.PersistentVolumeClaim, volumeName string) error { + log := ls.deps.Log.With().Str("pvc-name", claim.GetName()).Str("volume-name", volumeName).Logger() + pvcs := ls.deps.KubeCli.CoreV1().PersistentVolumeClaims(claim.GetNamespace()) + + for attempt := 0; attempt < 10; attempt++ { + // Backoff if needed + time.Sleep(time.Millisecond * time.Duration(10*attempt)) + + // Fetch latest version of claim + updated, err := pvcs.Get(claim.GetName(), metav1.GetOptions{}) + if k8sutil.IsNotFound(err) { + return maskAny(err) + } else if err != nil { + log.Warn().Err(err).Msg("Failed to load updated PersistentVolumeClaim") + continue + } + + // Check claim. If already bound, bail out + if !pvcNeedsVolume(*updated) { + if updated.Spec.VolumeName == volumeName { + log.Info().Msg("PersistentVolumeClaim already bound to PersistentVolume") + return nil + } + return maskAny(fmt.Errorf("PersistentVolumeClaim '%s' no longer needs a volume", claim.GetName())) + } + + // Try to bind + updated.Spec.VolumeName = volumeName + if _, err := pvcs.Update(updated); k8sutil.IsConflict(err) { + // Claim modified already, retry + log.Debug().Err(err).Msg("PersistentVolumeClaim has been modified. Retrying.") + } else if err != nil { + log.Error().Err(err).Msg("Failed to bind PVC to volume") + return maskAny(err) + } + log.Debug().Msg("Bound volume to PersistentVolumeClaim") + return nil + } + log.Error().Msg("All attempts to bind PVC to volume failed") + return maskAny(fmt.Errorf("All attempts to bind PVC to volume failed")) +} + +// shortHash creates a 6 letter hash of the given name. +func shortHash(name string) string { + h := sha1.Sum([]byte(name)) + return fmt.Sprintf("%0x", h)[:6] +} diff --git a/pkg/storage/pv_creator_test.go b/pkg/storage/pv_creator_test.go new file mode 100644 index 000000000..bbc4311a2 --- /dev/null +++ b/pkg/storage/pv_creator_test.go @@ -0,0 +1,209 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package storage + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/arangodb/kube-arangodb/pkg/storage/provisioner" + "github.com/arangodb/kube-arangodb/pkg/storage/provisioner/mocks" +) + +// TestCreateValidEndpointList tests createValidEndpointList. +func TestCreateValidEndpointList(t *testing.T) { + tests := []struct { + Input *v1.EndpointsList + Expected []string + }{ + { + Input: &v1.EndpointsList{}, + Expected: []string{}, + }, + { + Input: &v1.EndpointsList{ + Items: []v1.Endpoints{ + v1.Endpoints{ + Subsets: []v1.EndpointSubset{ + v1.EndpointSubset{ + Addresses: []v1.EndpointAddress{ + v1.EndpointAddress{ + IP: "1.2.3.4", + }, + }, + }, + v1.EndpointSubset{ + Addresses: []v1.EndpointAddress{ + v1.EndpointAddress{ + IP: "5.6.7.8", + }, + v1.EndpointAddress{ + IP: "9.10.11.12", + }, + }, + }, + }, + }, + }, + }, + Expected: []string{ + "1.2.3.4:8929", + "5.6.7.8:8929", + "9.10.11.12:8929", + }, + }, + } + for _, test := range tests { + output := createValidEndpointList(test.Input) + assert.Equal(t, test.Expected, output) + } +} + +// TestCreateNodeAffinity tests createNodeAffinity. +func TestCreateNodeAffinity(t *testing.T) { + tests := map[string]string{ + "foo": "{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":[{\"matchExpressions\":[{\"key\":\"kubernetes.io/hostname\",\"operator\":\"In\",\"values\":[\"foo\"]}]}]}}", + "bar": "{\"requiredDuringSchedulingIgnoredDuringExecution\":{\"nodeSelectorTerms\":[{\"matchExpressions\":[{\"key\":\"kubernetes.io/hostname\",\"operator\":\"In\",\"values\":[\"bar\"]}]}]}}", + } + for input, expected := range tests { + output, err := createNodeAffinity(input) + assert.NoError(t, err) + assert.Equal(t, expected, output, "Input: '%s'", input) + } +} + +// TestCreateNodeClientMap tests createNodeClientMap. +func TestCreateNodeClientMap(t *testing.T) { + GB := int64(1024 * 1024 * 1024) + foo := mocks.NewProvisioner("foo", 100*GB, 200*GB) + bar := mocks.NewProvisioner("bar", 300*GB, 400*GB) + tests := []struct { + Input []provisioner.API + Expected map[string]provisioner.API + }{ + { + Input: nil, + Expected: map[string]provisioner.API{}, + }, + { + Input: []provisioner.API{foo, bar}, + Expected: map[string]provisioner.API{ + "bar": bar, + "foo": foo, + }, + }, + } + ctx := context.Background() + for _, test := range tests { + output := createNodeClientMap(ctx, test.Input) + assert.Equal(t, test.Expected, output) + } +} + +// TestGetDeploymentInfo tests getDeploymentInfo. +func TestGetDeploymentInfo(t *testing.T) { + tests := []struct { + Input v1.PersistentVolumeClaim + ExpectedDeploymentName string + ExpectedRole string + ExpectedEnforceAntiAffinity bool + }{ + { + Input: v1.PersistentVolumeClaim{}, + ExpectedDeploymentName: "", + ExpectedRole: "", + ExpectedEnforceAntiAffinity: false, + }, + { + Input: v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "database.arangodb.com/enforce-anti-affinity": "true", + }, + Labels: map[string]string{ + "arango_deployment": "foo", + "role": "r1", + }, + }, + }, + ExpectedDeploymentName: "foo", + ExpectedRole: "r1", + ExpectedEnforceAntiAffinity: true, + }, + { + Input: v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "database.arangodb.com/enforce-anti-affinity": "false", + }, + Labels: map[string]string{ + "arango_deployment": "foo", + "role": "r1", + }, + }, + }, + ExpectedDeploymentName: "foo", + ExpectedRole: "r1", + ExpectedEnforceAntiAffinity: false, + }, + { + Input: v1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + "database.arangodb.com/enforce-anti-affinity": "wrong", + }, + Labels: map[string]string{ + "arango_deployment": "bar", + "role": "r77", + }, + }, + }, + ExpectedDeploymentName: "bar", + ExpectedRole: "r77", + ExpectedEnforceAntiAffinity: false, + }, + } + for _, test := range tests { + deploymentName, role, enforceAntiAffinity := getDeploymentInfo(test.Input) + assert.Equal(t, test.ExpectedDeploymentName, deploymentName) + assert.Equal(t, test.ExpectedRole, role) + assert.Equal(t, test.ExpectedEnforceAntiAffinity, enforceAntiAffinity) + } +} + +// TestShortHash tests shortHash. +func TestShortHash(t *testing.T) { + tests := map[string]string{ + "foo": "0beec7", + "": "da39a3", + "something very very very very very looooooooooooooooooooooooooooooooong": "68ff76", + } + for input, expected := range tests { + output := shortHash(input) + assert.Equal(t, expected, output, "Input: '%s'", input) + } +} diff --git a/pkg/util/arangod/agency.go b/pkg/util/arangod/agency.go deleted file mode 100644 index 0e1f71148..000000000 --- a/pkg/util/arangod/agency.go +++ /dev/null @@ -1,147 +0,0 @@ -// -// DISCLAIMER -// -// Copyright 2018 ArangoDB GmbH, Cologne, Germany -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Copyright holder is ArangoDB GmbH, Cologne, Germany -// -// Author Ewout Prangsma -// - -package arangod - -import ( - "context" - "encoding/json" - "fmt" - "strings" - - driver "github.com/arangodb/go-driver" - "github.com/pkg/errors" -) - -// Agency provides API implemented by the ArangoDB agency. -type Agency interface { - // ReadKey reads the value of a given key in the agency. - ReadKey(ctx context.Context, key []string, value interface{}) error - // Endpoint returns the endpoint of this agent connection - Endpoint() string -} - -// NewAgencyClient creates a new Agency connection from the given client -// connection. -// The number of endpoints of the client must be exactly 1. -func NewAgencyClient(c driver.Client) (Agency, error) { - if len(c.Connection().Endpoints()) > 1 { - return nil, maskAny(fmt.Errorf("Got multiple endpoints")) - } - return &agency{ - conn: c.Connection(), - }, nil -} - -type agency struct { - conn driver.Connection -} - -// ReadKey reads the value of a given key in the agency. -func (a *agency) ReadKey(ctx context.Context, key []string, value interface{}) error { - conn := a.conn - req, err := conn.NewRequest("POST", "_api/agency/read") - if err != nil { - return maskAny(err) - } - fullKey := createFullKey(key) - input := [][]string{{fullKey}} - req, err = req.SetBody(input) - if err != nil { - return maskAny(err) - } - //var raw []byte - //ctx = driver.WithRawResponse(ctx, &raw) - resp, err := conn.Do(ctx, req) - if err != nil { - fmt.Printf("conn.Do failed, err=%v, resp=%#v\n", err, resp) - return maskAny(err) - } - if resp.StatusCode() == 307 { - // Not leader - location := resp.Header("Location") - return NotLeaderError{Leader: location} - } - if err := resp.CheckStatus(200, 201, 202); err != nil { - return maskAny(err) - } - //fmt.Printf("Agent response: %s\n", string(raw)) - elems, err := resp.ParseArrayBody() - if err != nil { - return maskAny(err) - } - if len(elems) != 1 { - return maskAny(fmt.Errorf("Expected 1 element, got %d", len(elems))) - } - // If empty key parse directly - if len(key) == 0 { - if err := elems[0].ParseBody("", &value); err != nil { - return maskAny(err) - } - } else { - // Now remove all wrapping objects for each key element - var rawObject map[string]interface{} - if err := elems[0].ParseBody("", &rawObject); err != nil { - return maskAny(err) - } - var rawMsg interface{} - for keyIndex := 0; keyIndex < len(key); keyIndex++ { - if keyIndex > 0 { - var ok bool - rawObject, ok = rawMsg.(map[string]interface{}) - if !ok { - return maskAny(fmt.Errorf("Data is not an object at key %s", key[:keyIndex+1])) - } - } - var found bool - rawMsg, found = rawObject[key[keyIndex]] - if !found { - return errors.Wrapf(KeyNotFoundError, "Missing data at key %s", key[:keyIndex+1]) - } - } - // Encode to json ... - encoded, err := json.Marshal(rawMsg) - if err != nil { - return maskAny(err) - } - // and decode back into result - if err := json.Unmarshal(encoded, &value); err != nil { - return maskAny(err) - } - } - - // fmt.Printf("result as JSON: %s\n", rawResult) - return nil -} - -// Endpoint returns the endpoint of this agent connection -func (a *agency) Endpoint() string { - ep := a.conn.Endpoints() - if len(ep) == 0 { - return "" - } - return ep[0] -} - -func createFullKey(key []string) string { - return "/" + strings.Join(key, "/") -} diff --git a/pkg/util/arangod/agency_health.go b/pkg/util/arangod/agency_health.go deleted file mode 100644 index f76dcf94c..000000000 --- a/pkg/util/arangod/agency_health.go +++ /dev/null @@ -1,98 +0,0 @@ -// -// DISCLAIMER -// -// Copyright 2018 ArangoDB GmbH, Cologne, Germany -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Copyright holder is ArangoDB GmbH, Cologne, Germany -// -// Author Ewout Prangsma -// - -package arangod - -import ( - "context" - "fmt" - "sync" - "time" -) - -const ( - maxAgentResponseTime = time.Second * 10 -) - -// agentStatus is a helper structure used in AreAgentsHealthy. -type agentStatus struct { - IsLeader bool - LeaderEndpoint string - IsResponding bool -} - -// AreAgentsHealthy performs a health check on all given agents. -// Of the given agents, 1 must respond as leader and all others must redirect to the leader. -// The function returns nil when all agents are healthy or an error when something is wrong. -func AreAgentsHealthy(ctx context.Context, clients []Agency) error { - wg := sync.WaitGroup{} - invalidKey := []string{"does-not-exist-149e97e8-4b81-5664-a8a8-9ba93881d64c"} - statuses := make([]agentStatus, len(clients)) - for i, c := range clients { - wg.Add(1) - go func(i int, c Agency) { - defer wg.Done() - var trash interface{} - lctx, cancel := context.WithTimeout(ctx, maxAgentResponseTime) - defer cancel() - if err := c.ReadKey(lctx, invalidKey, &trash); err == nil || IsKeyNotFound(err) { - // We got a valid read from the leader - statuses[i].IsLeader = true - statuses[i].LeaderEndpoint = c.Endpoint() - statuses[i].IsResponding = true - } else { - if location, ok := IsNotLeader(err); ok { - // Valid response from a follower - statuses[i].IsLeader = false - statuses[i].LeaderEndpoint = location - statuses[i].IsResponding = true - } else { - // Unexpected / invalid response - statuses[i].IsResponding = false - } - } - }(i, c) - } - wg.Wait() - - // Check the results - noLeaders := 0 - for i, status := range statuses { - if !status.IsResponding { - return maskAny(fmt.Errorf("Agent %s is not responding", clients[i].Endpoint())) - } - if status.IsLeader { - noLeaders++ - } - if i > 0 { - // Compare leader endpoint with previous - prev := statuses[i-1].LeaderEndpoint - if !IsSameEndpoint(prev, status.LeaderEndpoint) { - return maskAny(fmt.Errorf("Not all agents report the same leader endpoint")) - } - } - } - if noLeaders != 1 { - return maskAny(fmt.Errorf("Unexpected number of agency leaders: %d", noLeaders)) - } - return nil -} diff --git a/pkg/util/arangod/cleanout_server.go b/pkg/util/arangod/cleanout_server.go new file mode 100644 index 000000000..b464efbb7 --- /dev/null +++ b/pkg/util/arangod/cleanout_server.go @@ -0,0 +1,99 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package arangod + +import ( + "context" + "fmt" + + driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/agency" +) + +// CleanoutJobStatus is a strongly typed status of an agency cleanout-server-job. +type CleanoutJobStatus struct { + state string + reason string +} + +// IsFailed returns true when the job is failed +func (s CleanoutJobStatus) IsFailed() bool { + return s.state == "Failed" +} + +// IsFinished returns true when the job is finished +func (s CleanoutJobStatus) IsFinished() bool { + return s.state == "Finished" +} + +// Reason returns the reason for the current state. +func (s CleanoutJobStatus) Reason() string { + return s.reason +} + +// String returns a string representation of the given state. +func (s CleanoutJobStatus) String() string { + return fmt.Sprintf("state: '%s', reason: '%s'", s.state, s.reason) +} + +var ( + agencyJobStateKeyPrefixes = [][]string{ + {"arango", "Target", "ToDo"}, + {"arango", "Target", "Pending"}, + {"arango", "Target", "Finished"}, + {"arango", "Target", "Failed"}, + } +) + +type agencyJob struct { + Reason string `json:"reason,omitempty"` + Server string `json:"server,omitempty"` + JobID string `json:"jobId,omitempty"` + Type string `json:"type,omitempty"` +} + +const ( + agencyJobTypeCleanOutServer = "cleanOutServer" +) + +// CleanoutServerJobStatus checks the status of a cleanout-server job with given ID. +func CleanoutServerJobStatus(ctx context.Context, jobID string, client driver.Client, agencyClient agency.Agency) (CleanoutJobStatus, error) { + for _, keyPrefix := range agencyJobStateKeyPrefixes { + key := append(keyPrefix, jobID) + var job agencyJob + if err := agencyClient.ReadKey(ctx, key, &job); err == nil { + return CleanoutJobStatus{ + state: keyPrefix[len(keyPrefix)-1], + reason: job.Reason, + }, nil + } else if agency.IsKeyNotFound(err) { + continue + } else { + return CleanoutJobStatus{}, maskAny(err) + } + } + // Job not found in any states + return CleanoutJobStatus{ + reason: "job not found", + }, nil +} diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index 5645d327d..7774a63ce 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -32,7 +32,9 @@ import ( "time" driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/agency" "github.com/arangodb/go-driver/http" + "github.com/arangodb/go-driver/jwt" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" @@ -108,6 +110,38 @@ func CreateArangodDatabaseClient(ctx context.Context, cli corev1.CoreV1Interface return c, nil } +// CreateArangodAgencyClient creates a go-driver client for accessing the agents of the given deployment. +func CreateArangodAgencyClient(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment) (agency.Agency, error) { + var dnsNames []string + for _, m := range apiObject.Status.Members.Agents { + dnsName := k8sutil.CreatePodDNSName(apiObject, api.ServerGroupAgents.AsRole(), m.ID) + dnsNames = append(dnsNames, dnsName) + } + connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, dnsNames) + if err != nil { + return nil, maskAny(err) + } + agencyConn, err := agency.NewAgencyConnection(connConfig) + if err != nil { + return nil, maskAny(err) + } + auth, err := createArangodClientAuthentication(ctx, cli, apiObject) + if err != nil { + return nil, maskAny(err) + } + if auth != nil { + agencyConn, err = agencyConn.SetAuthentication(auth) + if err != nil { + return nil, maskAny(err) + } + } + a, err := agency.NewAgency(agencyConn) + if err != nil { + return nil, maskAny(err) + } + return a, nil +} + // CreateArangodImageIDClient creates a go-driver client for an ArangoDB instance // running in an Image-ID pod. func CreateArangodImageIDClient(ctx context.Context, deployment k8sutil.APIObject, role, id string) (driver.Client, error) { @@ -122,6 +156,34 @@ func CreateArangodImageIDClient(ctx context.Context, deployment k8sutil.APIObjec // CreateArangodClientForDNSName creates a go-driver client for a given DNS name. func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsName string) (driver.Client, error) { + connConfig, err := createArangodHTTPConfigForDNSNames(ctx, cli, apiObject, []string{dnsName}) + if err != nil { + return nil, maskAny(err) + } + // TODO deal with TLS with proper CA checking + conn, err := http.NewConnection(connConfig) + if err != nil { + return nil, maskAny(err) + } + + // Create client + config := driver.ClientConfig{ + Connection: conn, + } + auth, err := createArangodClientAuthentication(ctx, cli, apiObject) + if err != nil { + return nil, maskAny(err) + } + config.Authentication = auth + c, err := driver.NewClient(config) + if err != nil { + return nil, maskAny(err) + } + return c, nil +} + +// createArangodHTTPConfigForDNSNames creates a go-driver HTTP connection config for a given DNS names. +func createArangodHTTPConfigForDNSNames(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment, dnsNames []string) (http.ConnectionConfig, error) { scheme := "http" transport := sharedHTTPTransport if apiObject != nil && apiObject.Spec.IsSecure() { @@ -129,33 +191,30 @@ func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interfa transport = sharedHTTPSTransport } connConfig := http.ConnectionConfig{ - Endpoints: []string{scheme + "://" + net.JoinHostPort(dnsName, strconv.Itoa(k8sutil.ArangoPort))}, Transport: transport, DontFollowRedirect: true, } - // TODO deal with TLS with proper CA checking - conn, err := http.NewConnection(connConfig) - if err != nil { - return nil, maskAny(err) + for _, dnsName := range dnsNames { + connConfig.Endpoints = append(connConfig.Endpoints, scheme+"://"+net.JoinHostPort(dnsName, strconv.Itoa(k8sutil.ArangoPort))) } + return connConfig, nil +} - // Create client - config := driver.ClientConfig{ - Connection: conn, - } +// createArangodClientAuthentication creates a go-driver authentication for the servers in the given deployment. +func createArangodClientAuthentication(ctx context.Context, cli corev1.CoreV1Interface, apiObject *api.ArangoDeployment) (driver.Authentication, error) { if apiObject != nil && apiObject.Spec.IsAuthenticated() { // Authentication is enabled. // Should we skip using it? if ctx.Value(skipAuthenticationKey{}) == nil { - s, err := k8sutil.GetJWTSecret(cli, apiObject.Spec.Authentication.GetJWTSecretName(), apiObject.GetNamespace()) + s, err := k8sutil.GetTokenSecret(cli, apiObject.Spec.Authentication.GetJWTSecretName(), apiObject.GetNamespace()) if err != nil { return nil, maskAny(err) } - jwt, err := CreateArangodJwtAuthorizationHeader(s) + jwt, err := jwt.CreateArangodJwtAuthorizationHeader(s, "kube-arangodb") if err != nil { return nil, maskAny(err) } - config.Authentication = driver.RawAuthentication(jwt) + return driver.RawAuthentication(jwt), nil } } else { // Authentication is not enabled. @@ -164,9 +223,5 @@ func createArangodClientForDNSName(ctx context.Context, cli corev1.CoreV1Interfa return nil, maskAny(fmt.Errorf("Authentication is required by context, but not provided in API object")) } } - c, err := driver.NewClient(config) - if err != nil { - return nil, maskAny(err) - } - return c, nil + return nil, nil } diff --git a/pkg/util/arangod/endpoint.go b/pkg/util/arangod/endpoint.go index d5f2d0641..61a37f6d6 100644 --- a/pkg/util/arangod/endpoint.go +++ b/pkg/util/arangod/endpoint.go @@ -26,7 +26,7 @@ import "net/url" // IsSameEndpoint returns true when the 2 given endpoints // refer to the same server. -func IsSameEndpoint(a, b string) bool { +func IsSameEndpoint_(a, b string) bool { if a == b { return true } diff --git a/pkg/util/constants/constants.go b/pkg/util/constants/constants.go index b11f14dad..ceed6f90f 100644 --- a/pkg/util/constants/constants.go +++ b/pkg/util/constants/constants.go @@ -28,14 +28,27 @@ const ( EnvOperatorPodNamespace = "MY_POD_NAMESPACE" EnvOperatorPodIP = "MY_POD_IP" - EnvArangodJWTSecret = "ARANGOD_JWT_SECRET" // Contains JWT secret for the ArangoDB cluster - EnvArangoSyncJWTSecret = "ARANGOSYNC_JWT_SECRET" // Contains JWT secret for the ArangoSync masters + EnvArangodJWTSecret = "ARANGOD_JWT_SECRET" // Contains JWT secret for the ArangoDB cluster + EnvArangoSyncMonitoringToken = "ARANGOSYNC_MONITORING_TOKEN" // Constains monitoring token for ArangoSync servers SecretEncryptionKey = "key" // Key in a Secret.Data used to store an 32-byte encryption key - SecretKeyJWT = "token" // Key inside a Secret used to hold a JW token + SecretKeyToken = "token" // Key inside a Secret used to hold a JWT or monitoring token SecretCACertificate = "ca.crt" // Key in Secret.data used to store a PEM encoded CA certificate (public key) SecretCAKey = "ca.key" // Key in Secret.data used to store a PEM encoded CA private key SecretTLSKeyfile = "tls.keyfile" // Key in Secret.data used to store a PEM encoded TLS certificate in the format used by ArangoDB (`--ssl.keyfile`) + + SecretUsername = "username" // Key in Secret.data used to store a username used for basic authentication + SecretPassword = "password" // Key in Secret.data used to store a password used for basic authentication + + SecretAccessPackageYaml = "accessPackage.yaml" // Key in Secret.data used to store a YAML encoded access package + + FinalizerDeplRemoveChildFinalizers = "database.arangodb.com/remove-child-finalizers" // Finalizer added to ArangoDeployment, indicating the need to remove finalizers from all children + FinalizerDeplReplStopSync = "replication.database.arangodb.com/stop-sync" // Finalizer added to ArangoDeploymentReplication, indicating the need to stop synchronization + FinalizerPodAgencyServing = "agent.database.arangodb.com/agency-serving" // Finalizer added to Agents, indicating the need for keeping enough agents alive + FinalizerPodDrainDBServer = "dbserver.database.arangodb.com/drain" // Finalizer added to DBServers, indicating the need for draining that dbserver + FinalizerPVCMemberExists = "pvc.database.arangodb.com/member-exists" // Finalizer added to PVCs, indicating the need to keep is as long as its member exists + + AnnotationEnforceAntiAffinity = "database.arangodb.com/enforce-anti-affinity" // Key of annotation added to PVC. Value is a boolean "true" or "false" ) diff --git a/pkg/util/k8sutil/constants.go b/pkg/util/k8sutil/constants.go index 86b8de7dc..d4524d201 100644 --- a/pkg/util/k8sutil/constants.go +++ b/pkg/util/k8sutil/constants.go @@ -24,7 +24,9 @@ package k8sutil const ( // Arango constants - ArangoPort = 8529 + ArangoPort = 8529 + ArangoSyncMasterPort = 8629 + ArangoSyncWorkerPort = 8729 // K8s constants ClusterIPNone = "None" diff --git a/pkg/util/k8sutil/container.go b/pkg/util/k8sutil/container.go new file mode 100644 index 000000000..b018eff97 --- /dev/null +++ b/pkg/util/k8sutil/container.go @@ -0,0 +1,38 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package k8sutil + +import ( + "k8s.io/api/core/v1" +) + +// GetContainerByName returns the container in the given pod with the given name. +// Returns false if not found. +func GetContainerByName(p *v1.Pod, name string) (v1.Container, bool) { + for _, c := range p.Spec.Containers { + if c.Name == name { + return c, true + } + } + return v1.Container{}, false +} diff --git a/pkg/util/k8sutil/dns.go b/pkg/util/k8sutil/dns.go index 2fb4e7171..3356d2202 100644 --- a/pkg/util/k8sutil/dns.go +++ b/pkg/util/k8sutil/dns.go @@ -39,3 +39,9 @@ func CreateDatabaseClientServiceDNSName(deployment metav1.Object) string { return CreateDatabaseClientServiceName(deployment.GetName()) + "." + deployment.GetNamespace() + ".svc" } + +// CreateSyncMasterClientServiceDNSName returns the DNS of the syncmaster client service. +func CreateSyncMasterClientServiceDNSName(deployment metav1.Object) string { + return CreateSyncMasterClientServiceName(deployment.GetName()) + "." + + deployment.GetNamespace() + ".svc" +} diff --git a/pkg/util/k8sutil/events.go b/pkg/util/k8sutil/events.go index c0210add8..201d4c3ba 100644 --- a/pkg/util/k8sutil/events.go +++ b/pkg/util/k8sutil/events.go @@ -24,25 +24,31 @@ package k8sutil import ( "fmt" - "os" "strings" - "time" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - "github.com/arangodb/kube-arangodb/pkg/util/constants" + "k8s.io/apimachinery/pkg/runtime" ) +// Event is used to create events using an EventRecorder. +type Event struct { + InvolvedObject runtime.Object + Type string + Reason string + Message string +} + // APIObject helps to abstract an object from our custom API. type APIObject interface { + runtime.Object metav1.Object // AsOwner creates an OwnerReference for the given deployment AsOwner() metav1.OwnerReference } // NewMemberAddEvent creates an event indicating that a member was added. -func NewMemberAddEvent(memberName, role string, apiObject APIObject) *v1.Event { +func NewMemberAddEvent(memberName, role string, apiObject APIObject) *Event { event := newDeploymentEvent(apiObject) event.Type = v1.EventTypeNormal event.Reason = fmt.Sprintf("New %s Added", strings.Title(role)) @@ -51,7 +57,7 @@ func NewMemberAddEvent(memberName, role string, apiObject APIObject) *v1.Event { } // NewMemberRemoveEvent creates an event indicating that an existing member was removed. -func NewMemberRemoveEvent(memberName, role string, apiObject APIObject) *v1.Event { +func NewMemberRemoveEvent(memberName, role string, apiObject APIObject) *Event { event := newDeploymentEvent(apiObject) event.Type = v1.EventTypeNormal event.Reason = fmt.Sprintf("%s Removed", strings.Title(role)) @@ -60,7 +66,7 @@ func NewMemberRemoveEvent(memberName, role string, apiObject APIObject) *v1.Even } // NewPodCreatedEvent creates an event indicating that a pod has been created -func NewPodCreatedEvent(podName, role string, apiObject APIObject) *v1.Event { +func NewPodCreatedEvent(podName, role string, apiObject APIObject) *Event { event := newDeploymentEvent(apiObject) event.Type = v1.EventTypeNormal event.Reason = fmt.Sprintf("Pod Of %s Created", strings.Title(role)) @@ -69,7 +75,7 @@ func NewPodCreatedEvent(podName, role string, apiObject APIObject) *v1.Event { } // NewPodGoneEvent creates an event indicating that a pod is missing -func NewPodGoneEvent(podName, role string, apiObject APIObject) *v1.Event { +func NewPodGoneEvent(podName, role string, apiObject APIObject) *Event { event := newDeploymentEvent(apiObject) event.Type = v1.EventTypeNormal event.Reason = fmt.Sprintf("Pod Of %s Gone", strings.Title(role)) @@ -79,7 +85,7 @@ func NewPodGoneEvent(podName, role string, apiObject APIObject) *v1.Event { // NewImmutableFieldEvent creates an event indicating that an attempt was made to change a field // that is immutable. -func NewImmutableFieldEvent(fieldName string, apiObject APIObject) *v1.Event { +func NewImmutableFieldEvent(fieldName string, apiObject APIObject) *Event { event := newDeploymentEvent(apiObject) event.Type = v1.EventTypeNormal event.Reason = "Immutable Field Change" @@ -88,7 +94,7 @@ func NewImmutableFieldEvent(fieldName string, apiObject APIObject) *v1.Event { } // NewPodsSchedulingFailureEvent creates an event indicating that one of more cannot be scheduled. -func NewPodsSchedulingFailureEvent(unscheduledPodNames []string, apiObject APIObject) *v1.Event { +func NewPodsSchedulingFailureEvent(unscheduledPodNames []string, apiObject APIObject) *Event { event := newDeploymentEvent(apiObject) event.Type = v1.EventTypeNormal event.Reason = "Pods Scheduling Failure" @@ -98,7 +104,7 @@ func NewPodsSchedulingFailureEvent(unscheduledPodNames []string, apiObject APIOb // NewPodsSchedulingResolvedEvent creates an event indicating that an earlier problem with // pod scheduling has been resolved. -func NewPodsSchedulingResolvedEvent(apiObject APIObject) *v1.Event { +func NewPodsSchedulingResolvedEvent(apiObject APIObject) *Event { event := newDeploymentEvent(apiObject) event.Type = v1.EventTypeNormal event.Reason = "Pods Scheduling Resolved" @@ -107,7 +113,7 @@ func NewPodsSchedulingResolvedEvent(apiObject APIObject) *v1.Event { } // NewSecretsChangedEvent creates an event indicating that one of more secrets have changed. -func NewSecretsChangedEvent(changedSecretNames []string, apiObject APIObject) *v1.Event { +func NewSecretsChangedEvent(changedSecretNames []string, apiObject APIObject) *Event { event := newDeploymentEvent(apiObject) event.Type = v1.EventTypeNormal event.Reason = "Secrets changed" @@ -117,7 +123,7 @@ func NewSecretsChangedEvent(changedSecretNames []string, apiObject APIObject) *v // NewSecretsRestoredEvent creates an event indicating that all secrets have been restored // to their original values. -func NewSecretsRestoredEvent(apiObject APIObject) *v1.Event { +func NewSecretsRestoredEvent(apiObject APIObject) *Event { event := newDeploymentEvent(apiObject) event.Type = v1.EventTypeNormal event.Reason = "Secrets restored" @@ -125,8 +131,68 @@ func NewSecretsRestoredEvent(apiObject APIObject) *v1.Event { return event } +// NewAccessPackageCreatedEvent creates an event indicating that a secret containing an access package +// has been created. +func NewAccessPackageCreatedEvent(apiObject APIObject, apSecretName string) *Event { + event := newDeploymentEvent(apiObject) + event.Type = v1.EventTypeNormal + event.Reason = "Access package created" + event.Message = fmt.Sprintf("An access package named %s has been created", apSecretName) + return event +} + +// NewAccessPackageDeletedEvent creates an event indicating that a secret containing an access package +// has been deleted. +func NewAccessPackageDeletedEvent(apiObject APIObject, apSecretName string) *Event { + event := newDeploymentEvent(apiObject) + event.Type = v1.EventTypeNormal + event.Reason = "Access package deleted" + event.Message = fmt.Sprintf("An access package named %s has been deleted", apSecretName) + return event +} + +// NewPlanTimeoutEvent creates an event indicating that an item on a reconciliation plan did not +// finish before its deadline. +func NewPlanTimeoutEvent(apiObject APIObject, itemType, memberID, role string) *Event { + event := newDeploymentEvent(apiObject) + event.Type = v1.EventTypeNormal + event.Reason = "Reconciliation Plan Timeout" + event.Message = fmt.Sprintf("An plan item of type %s or member %s with role %s did not finish in time", itemType, memberID, role) + return event +} + +// NewPlanAbortedEvent creates an event indicating that an item on a reconciliation plan wants to abort +// the entire plan. +func NewPlanAbortedEvent(apiObject APIObject, itemType, memberID, role string) *Event { + event := newDeploymentEvent(apiObject) + event.Type = v1.EventTypeNormal + event.Reason = "Reconciliation Plan Aborted" + event.Message = fmt.Sprintf("An plan item of type %s or member %s with role %s wants to abort the plan", itemType, memberID, role) + return event +} + +// NewCannotChangeStorageClassEvent creates an event indicating that an item would need to use a different StorageClass, +// but this is not possible for the given reason. +func NewCannotChangeStorageClassEvent(apiObject APIObject, memberID, role, subReason string) *Event { + event := newDeploymentEvent(apiObject) + event.Type = v1.EventTypeNormal + event.Reason = fmt.Sprintf("%s Member StorageClass Cannot Change", strings.Title(role)) + event.Message = fmt.Sprintf("Member %s with role %s should use a different StorageClass, but is cannot because: %s", memberID, role, subReason) + return event +} + +// NewDowntimeNotAllowedEvent creates an event indicating that an operation cannot be executed because downtime +// is currently not allowed. +func NewDowntimeNotAllowedEvent(apiObject APIObject, operation string) *Event { + event := newDeploymentEvent(apiObject) + event.Type = v1.EventTypeNormal + event.Reason = "Downtime Operation Postponed" + event.Message = fmt.Sprintf("The '%s' operation is postponed because downtime it not allowed. Set `spec.downtimeAllowed` to true to execute this operation", operation) + return event +} + // NewErrorEvent creates an even of type error. -func NewErrorEvent(reason string, err error, apiObject APIObject) *v1.Event { +func NewErrorEvent(reason string, err error, apiObject APIObject) *Event { event := newDeploymentEvent(apiObject) event.Type = v1.EventTypeWarning event.Reason = strings.Title(reason) @@ -135,28 +201,8 @@ func NewErrorEvent(reason string, err error, apiObject APIObject) *v1.Event { } // newDeploymentEvent creates a new event for the given api object & owner. -func newDeploymentEvent(apiObject APIObject) *v1.Event { - t := time.Now() - owner := apiObject.AsOwner() - return &v1.Event{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: apiObject.GetName() + "-", - Namespace: apiObject.GetNamespace(), - }, - InvolvedObject: v1.ObjectReference{ - APIVersion: owner.APIVersion, - Kind: owner.Kind, - Name: owner.Name, - Namespace: apiObject.GetNamespace(), - UID: owner.UID, - ResourceVersion: apiObject.GetResourceVersion(), - }, - Source: v1.EventSource{ - Component: os.Getenv(constants.EnvOperatorPodName), - }, - // Each deployment event is unique so it should not be collapsed with other events - FirstTimestamp: metav1.Time{Time: t}, - LastTimestamp: metav1.Time{Time: t}, - Count: int32(1), +func newDeploymentEvent(apiObject runtime.Object) *Event { + return &Event{ + InvolvedObject: apiObject, } } diff --git a/pkg/util/k8sutil/finalizers.go b/pkg/util/k8sutil/finalizers.go new file mode 100644 index 000000000..836329584 --- /dev/null +++ b/pkg/util/k8sutil/finalizers.go @@ -0,0 +1,144 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package k8sutil + +import ( + "github.com/rs/zerolog" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +const ( + maxRemoveFinalizersAttempts = 50 +) + +// RemovePodFinalizers removes the given finalizers from the given pod. +func RemovePodFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1.Pod, finalizers []string, ignoreNotFound bool) error { + pods := kubecli.CoreV1().Pods(p.GetNamespace()) + getFunc := func() (metav1.Object, error) { + result, err := pods.Get(p.GetName(), metav1.GetOptions{}) + if err != nil { + return nil, maskAny(err) + } + return result, nil + } + updateFunc := func(updated metav1.Object) error { + updatedPod := updated.(*v1.Pod) + result, err := pods.Update(updatedPod) + if err != nil { + return maskAny(err) + } + *p = *result + return nil + } + if err := RemoveFinalizers(log, finalizers, getFunc, updateFunc, ignoreNotFound); err != nil { + return maskAny(err) + } + return nil +} + +// RemovePVCFinalizers removes the given finalizers from the given PVC. +func RemovePVCFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1.PersistentVolumeClaim, finalizers []string, ignoreNotFound bool) error { + pvcs := kubecli.CoreV1().PersistentVolumeClaims(p.GetNamespace()) + getFunc := func() (metav1.Object, error) { + result, err := pvcs.Get(p.GetName(), metav1.GetOptions{}) + if err != nil { + return nil, maskAny(err) + } + return result, nil + } + updateFunc := func(updated metav1.Object) error { + updatedPVC := updated.(*v1.PersistentVolumeClaim) + result, err := pvcs.Update(updatedPVC) + if err != nil { + return maskAny(err) + } + *p = *result + return nil + } + if err := RemoveFinalizers(log, finalizers, getFunc, updateFunc, ignoreNotFound); err != nil { + return maskAny(err) + } + return nil +} + +// RemoveFinalizers is a helper used to remove finalizers from an object. +// The functions tries to get the object using the provided get function, +// then remove the given finalizers and update the update using the given update function. +// In case of an update conflict, the functions tries again. +func RemoveFinalizers(log zerolog.Logger, finalizers []string, getFunc func() (metav1.Object, error), updateFunc func(metav1.Object) error, ignoreNotFound bool) error { + attempts := 0 + for { + attempts++ + obj, err := getFunc() + if err != nil { + if IsNotFound(err) && ignoreNotFound { + // Object no longer found and we're allowed to ignore that. + return nil + } + log.Warn().Err(err).Msg("Failed to get resource") + return maskAny(err) + } + original := obj.GetFinalizers() + if len(original) == 0 { + // We're done + return nil + } + newList := make([]string, 0, len(original)) + shouldRemove := func(f string) bool { + for _, x := range finalizers { + if x == f { + return true + } + } + return false + } + for _, f := range original { + if !shouldRemove(f) { + newList = append(newList, f) + } + } + if len(newList) < len(original) { + obj.SetFinalizers(newList) + if err := updateFunc(obj); IsConflict(err) { + if attempts > maxRemoveFinalizersAttempts { + log.Warn().Err(err).Msg("Failed to update resource with fewer finalizers after many attempts") + return maskAny(err) + } else { + // Try again + continue + } + } else if IsNotFound(err) && ignoreNotFound { + // Object no longer found and we're allowed to ignore that. + return nil + } else if err != nil { + log.Warn().Err(err).Msg("Failed to update resource with fewer finalizers") + return maskAny(err) + } + } else { + log.Debug().Msg("No finalizers needed removal. Resource unchanged") + } + return nil + } +} diff --git a/pkg/util/k8sutil/images.go b/pkg/util/k8sutil/images.go new file mode 100644 index 000000000..f5f2327b6 --- /dev/null +++ b/pkg/util/k8sutil/images.go @@ -0,0 +1,38 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package k8sutil + +import "strings" + +const ( + dockerPullableImageIDPrefix = "docker-pullable://" +) + +// ConvertImageID2Image converts a ImageID from a ContainerStatus to an Image that can be used +// in a Container specification. +func ConvertImageID2Image(imageID string) string { + if strings.HasPrefix(imageID, dockerPullableImageIDPrefix) { + return imageID[len(dockerPullableImageIDPrefix):] + } + return imageID +} diff --git a/pkg/util/k8sutil/names.go b/pkg/util/k8sutil/names.go index d75836352..4a242fedc 100644 --- a/pkg/util/k8sutil/names.go +++ b/pkg/util/k8sutil/names.go @@ -30,7 +30,7 @@ import ( var ( resourceNameRE = regexp.MustCompile(`^([0-9\-\.a-z])+$`) - arangodPrefixes = []string{"CRDN-", "PRMR-", "AGNT-"} + arangodPrefixes = []string{"CRDN-", "PRMR-", "AGNT-", "SNGL-"} ) // ValidateOptionalResourceName validates a kubernetes resource name. diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 8700e0f5d..000021937 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -24,23 +24,37 @@ package k8sutil import ( "fmt" + "math" + "os" "path/filepath" "strings" "time" + "github.com/arangodb/kube-arangodb/pkg/util/constants" "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) const ( + InitDataContainerName = "init-data" + InitLifecycleContainerName = "init-lifecycle" + ServerContainerName = "server" alpineImage = "alpine" arangodVolumeName = "arangod-data" tlsKeyfileVolumeName = "tls-keyfile" + lifecycleVolumeName = "lifecycle" + clientAuthCAVolumeName = "client-auth-ca" + clusterJWTSecretVolumeName = "cluster-jwt" + masterJWTSecretVolumeName = "master-jwt" rocksdbEncryptionVolumeName = "rocksdb-encryption" ArangodVolumeMountDir = "/data" RocksDBEncryptionVolumeMountDir = "/secrets/rocksdb/encryption" TLSKeyfileVolumeMountDir = "/secrets/tls" + LifecycleVolumeMountDir = "/lifecycle/tools" + ClientAuthCAVolumeMountDir = "/secrets/client-auth/ca" + ClusterJWTSecretVolumeMountDir = "/secrets/cluster/jwt" + MasterJWTSecretVolumeMountDir = "/secrets/master/jwt" ) // EnvValue is a helper structure for environment variable sources. @@ -104,6 +118,17 @@ func IsPodNotScheduledFor(pod *v1.Pod, timeout time.Duration) bool { condition.LastTransitionTime.Time.Add(timeout).Before(time.Now()) } +// IsPodMarkedForDeletion returns true if the pod has been marked for deletion. +func IsPodMarkedForDeletion(pod *v1.Pod) bool { + return pod.DeletionTimestamp != nil +} + +// IsPodTerminating returns true if the pod has been marked for deletion +// but is still running. +func IsPodTerminating(pod *v1.Pod) bool { + return IsPodMarkedForDeletion(pod) && pod.Status.Phase == v1.PodRunning +} + // IsArangoDBImageIDAndVersionPod returns true if the given pod is used for fetching image ID and ArangoDB version of an image func IsArangoDBImageIDAndVersionPod(p v1.Pod) bool { role, found := p.GetLabels()[LabelKeyRole] @@ -142,6 +167,13 @@ func CreateTLSKeyfileSecretName(deploymentName, role, id string) string { return CreatePodName(deploymentName, role, id, "-tls-keyfile") } +// lifecycleVolumeMounts creates a volume mount structure for shared lifecycle emptyDir. +func lifecycleVolumeMounts() []v1.VolumeMount { + return []v1.VolumeMount{ + {Name: lifecycleVolumeName, MountPath: LifecycleVolumeMountDir}, + } +} + // arangodVolumeMounts creates a volume mount structure for arangod. func arangodVolumeMounts() []v1.VolumeMount { return []v1.VolumeMount{ @@ -159,6 +191,36 @@ func tlsKeyfileVolumeMounts() []v1.VolumeMount { } } +// clientAuthCACertificateVolumeMounts creates a volume mount structure for a client-auth CA certificate (ca.crt). +func clientAuthCACertificateVolumeMounts() []v1.VolumeMount { + return []v1.VolumeMount{ + { + Name: clientAuthCAVolumeName, + MountPath: ClientAuthCAVolumeMountDir, + }, + } +} + +// masterJWTVolumeMounts creates a volume mount structure for a master JWT secret (token). +func masterJWTVolumeMounts() []v1.VolumeMount { + return []v1.VolumeMount{ + { + Name: masterJWTSecretVolumeName, + MountPath: MasterJWTSecretVolumeMountDir, + }, + } +} + +// clusterJWTVolumeMounts creates a volume mount structure for a cluster JWT secret (token). +func clusterJWTVolumeMounts() []v1.VolumeMount { + return []v1.VolumeMount{ + { + Name: clusterJWTSecretVolumeName, + MountPath: ClusterJWTSecretVolumeMountDir, + }, + } +} + // rocksdbEncryptionVolumeMounts creates a volume mount structure for a RocksDB encryption key. func rocksdbEncryptionVolumeMounts() []v1.VolumeMount { return []v1.VolumeMount{ @@ -202,12 +264,14 @@ func arangodInitContainer(name, id, engine string, requireUUID bool) v1.Containe } // arangodContainer creates a container configured to run `arangod`. -func arangodContainer(name string, image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig) v1.Container { +func arangodContainer(image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig, + lifecycle *v1.Lifecycle, lifecycleEnvVars []v1.EnvVar) v1.Container { c := v1.Container{ Command: append([]string{"/usr/sbin/arangod"}, args...), - Name: name, + Name: ServerContainerName, Image: image, ImagePullPolicy: imagePullPolicy, + Lifecycle: lifecycle, Ports: []v1.ContainerPort{ { Name: "server", @@ -226,17 +290,23 @@ func arangodContainer(name string, image string, imagePullPolicy v1.PullPolicy, if readinessProbe != nil { c.ReadinessProbe = readinessProbe.Create() } + if lifecycle != nil { + c.Env = append(c.Env, lifecycleEnvVars...) + c.VolumeMounts = append(c.VolumeMounts, lifecycleVolumeMounts()...) + } return c } // arangosyncContainer creates a container configured to run `arangosync`. -func arangosyncContainer(name string, image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig) v1.Container { +func arangosyncContainer(image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, + lifecycle *v1.Lifecycle, lifecycleEnvVars []v1.EnvVar) v1.Container { c := v1.Container{ Command: append([]string{"/usr/sbin/arangosync"}, args...), - Name: name, + Name: ServerContainerName, Image: image, ImagePullPolicy: imagePullPolicy, + Lifecycle: lifecycle, Ports: []v1.ContainerPort{ { Name: "server", @@ -251,22 +321,89 @@ func arangosyncContainer(name string, image string, imagePullPolicy v1.PullPolic if livenessProbe != nil { c.LivenessProbe = livenessProbe.Create() } + if lifecycle != nil { + c.Env = append(c.Env, lifecycleEnvVars...) + c.VolumeMounts = append(c.VolumeMounts, lifecycleVolumeMounts()...) + } return c } +// newLifecycle creates a lifecycle structure with preStop handler. +func newLifecycle() (*v1.Lifecycle, []v1.EnvVar, []v1.Volume, error) { + binaryPath, err := os.Executable() + if err != nil { + return nil, nil, nil, maskAny(err) + } + exePath := filepath.Join(LifecycleVolumeMountDir, filepath.Base(binaryPath)) + lifecycle := &v1.Lifecycle{ + PreStop: &v1.Handler{ + Exec: &v1.ExecAction{ + Command: append([]string{exePath}, "lifecycle", "preStop"), + }, + }, + } + envVars := []v1.EnvVar{ + v1.EnvVar{ + Name: constants.EnvOperatorPodName, + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.name", + }, + }, + }, + v1.EnvVar{ + Name: constants.EnvOperatorPodNamespace, + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "metadata.namespace", + }, + }, + }, + } + vols := []v1.Volume{ + v1.Volume{ + Name: lifecycleVolumeName, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }, + } + return lifecycle, envVars, vols, nil +} + +// initLifecycleContainer creates an init-container to copy the lifecycle binary +// to a shared volume. +func initLifecycleContainer(image string) (v1.Container, error) { + binaryPath, err := os.Executable() + if err != nil { + return v1.Container{}, maskAny(err) + } + c := v1.Container{ + Command: append([]string{binaryPath}, "lifecycle", "copy", "--target", LifecycleVolumeMountDir), + Name: InitLifecycleContainerName, + Image: image, + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: lifecycleVolumeMounts(), + } + return c, nil +} + // newPod creates a basic Pod for given settings. -func newPod(deploymentName, ns, role, id, podName string) v1.Pod { +func newPod(deploymentName, ns, role, id, podName string, finalizers []string, tolerations []v1.Toleration, serviceAccountName string) v1.Pod { hostname := CreatePodHostName(deploymentName, role, id) p := v1.Pod{ ObjectMeta: metav1.ObjectMeta{ - Name: podName, - Labels: LabelsForDeployment(deploymentName, role), + Name: podName, + Labels: LabelsForDeployment(deploymentName, role), + Finalizers: finalizers, }, Spec: v1.PodSpec{ - Hostname: hostname, - Subdomain: CreateHeadlessServiceName(deploymentName), - RestartPolicy: v1.RestartPolicyNever, + Hostname: hostname, + Subdomain: CreateHeadlessServiceName(deploymentName), + RestartPolicy: v1.RestartPolicyNever, + Tolerations: tolerations, + ServiceAccountName: serviceAccountName, }, } return p @@ -276,16 +413,34 @@ func newPod(deploymentName, ns, role, id, podName string) v1.Pod { // If the pod already exists, nil is returned. // If another error occurs, that error is returned. func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, - role, id, podName, pvcName, image string, imagePullPolicy v1.PullPolicy, - engine string, requireUUID bool, - args []string, env map[string]EnvValue, - livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig, + role, id, podName, pvcName, image, lifecycleImage string, imagePullPolicy v1.PullPolicy, + engine string, requireUUID bool, terminationGracePeriod time.Duration, + args []string, env map[string]EnvValue, finalizers []string, + livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig, tolerations []v1.Toleration, serviceAccountName string, tlsKeyfileSecretName, rocksdbEncryptionSecretName string) error { // Prepare basic pod - p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id, podName) + p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id, podName, finalizers, tolerations, serviceAccountName) + terminationGracePeriodSeconds := int64(math.Ceil(terminationGracePeriod.Seconds())) + p.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds + + // Add lifecycle container + var lifecycle *v1.Lifecycle + var lifecycleEnvVars []v1.EnvVar + var lifecycleVolumes []v1.Volume + if lifecycleImage != "" { + c, err := initLifecycleContainer(lifecycleImage) + if err != nil { + return maskAny(err) + } + p.Spec.InitContainers = append(p.Spec.InitContainers, c) + lifecycle, lifecycleEnvVars, lifecycleVolumes, err = newLifecycle() + if err != nil { + return maskAny(err) + } + } // Add arangod container - c := arangodContainer("arangod", image, imagePullPolicy, args, env, livenessProbe, readinessProbe) + c := arangodContainer(image, imagePullPolicy, args, env, livenessProbe, readinessProbe, lifecycle, lifecycleEnvVars) if tlsKeyfileSecretName != "" { c.VolumeMounts = append(c.VolumeMounts, tlsKeyfileVolumeMounts()...) } @@ -346,6 +501,9 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy p.Spec.Volumes = append(p.Spec.Volumes, vol) } + // Lifecycle volumes (if any) + p.Spec.Volumes = append(p.Spec.Volumes, lifecycleVolumes...) + // Add (anti-)affinity p.Spec.Affinity = createAffinity(deployment.GetName(), role, !developmentMode, "") @@ -358,15 +516,101 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy // CreateArangoSyncPod creates a Pod that runs `arangosync`. // If the pod already exists, nil is returned. // If another error occurs, that error is returned. -func CreateArangoSyncPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, role, id, podName, image string, imagePullPolicy v1.PullPolicy, - args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, affinityWithRole string) error { +func CreateArangoSyncPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, role, id, podName, image, lifecycleImage string, imagePullPolicy v1.PullPolicy, + terminationGracePeriod time.Duration, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, tolerations []v1.Toleration, serviceAccountName string, + tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName, affinityWithRole string) error { // Prepare basic pod - p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id, podName) + p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id, podName, nil, tolerations, serviceAccountName) + terminationGracePeriodSeconds := int64(math.Ceil(terminationGracePeriod.Seconds())) + p.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds + + // Add lifecycle container + var lifecycle *v1.Lifecycle + var lifecycleEnvVars []v1.EnvVar + var lifecycleVolumes []v1.Volume + if lifecycleImage != "" { + c, err := initLifecycleContainer(lifecycleImage) + if err != nil { + return maskAny(err) + } + p.Spec.InitContainers = append(p.Spec.InitContainers, c) + lifecycle, lifecycleEnvVars, lifecycleVolumes, err = newLifecycle() + if err != nil { + return maskAny(err) + } + } + + // Lifecycle volumes (if any) + p.Spec.Volumes = append(p.Spec.Volumes, lifecycleVolumes...) // Add arangosync container - c := arangosyncContainer("arangosync", image, imagePullPolicy, args, env, livenessProbe) + c := arangosyncContainer(image, imagePullPolicy, args, env, livenessProbe, lifecycle, lifecycleEnvVars) + if tlsKeyfileSecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, tlsKeyfileVolumeMounts()...) + } + if clientAuthCASecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, clientAuthCACertificateVolumeMounts()...) + } + if masterJWTSecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, masterJWTVolumeMounts()...) + } + if clusterJWTSecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, clusterJWTVolumeMounts()...) + } p.Spec.Containers = append(p.Spec.Containers, c) + // TLS keyfile secret mount (if any) + if tlsKeyfileSecretName != "" { + vol := v1.Volume{ + Name: tlsKeyfileVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: tlsKeyfileSecretName, + }, + }, + } + p.Spec.Volumes = append(p.Spec.Volumes, vol) + } + + // Client Authentication certificate secret mount (if any) + if clientAuthCASecretName != "" { + vol := v1.Volume{ + Name: clientAuthCAVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: clientAuthCASecretName, + }, + }, + } + p.Spec.Volumes = append(p.Spec.Volumes, vol) + } + + // Master JWT secret mount (if any) + if masterJWTSecretName != "" { + vol := v1.Volume{ + Name: masterJWTSecretVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: masterJWTSecretName, + }, + }, + } + p.Spec.Volumes = append(p.Spec.Volumes, vol) + } + + // Cluster JWT secret mount (if any) + if clusterJWTSecretName != "" { + vol := v1.Volume{ + Name: clusterJWTSecretVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: clusterJWTSecretName, + }, + }, + } + p.Spec.Volumes = append(p.Spec.Volumes, vol) + } + // Add (anti-)affinity p.Spec.Affinity = createAffinity(deployment.GetName(), role, !developmentMode, affinityWithRole) diff --git a/pkg/util/k8sutil/probes.go b/pkg/util/k8sutil/probes.go index ca8b67d6c..2b51d0825 100644 --- a/pkg/util/k8sutil/probes.go +++ b/pkg/util/k8sutil/probes.go @@ -35,6 +35,18 @@ type HTTPProbeConfig struct { Secure bool // Value for an Authorization header (can be empty) Authorization string + // Port to inspect (defaults to ArangoPort) + Port int + // Number of seconds after the container has started before liveness probes are initiated (defaults to 30) + InitialDelaySeconds int32 + // Number of seconds after which the probe times out (defaults to 2). + TimeoutSeconds int32 + // How often (in seconds) to perform the probe (defaults to 10). + PeriodSeconds int32 + // Minimum consecutive successes for the probe to be considered successful after having failed (defaults to 1). + SuccessThreshold int32 + // Minimum consecutive failures for the probe to be considered failed after having succeeded (defaults to 3). + FailureThreshold int32 } // Create creates a probe from given config @@ -50,19 +62,25 @@ func (config HTTPProbeConfig) Create() *v1.Probe { Value: config.Authorization, }) } + def := func(value, defaultValue int32) int32 { + if value != 0 { + return value + } + return defaultValue + } return &v1.Probe{ Handler: v1.Handler{ HTTPGet: &v1.HTTPGetAction{ Path: config.LocalPath, - Port: intstr.FromInt(ArangoPort), + Port: intstr.FromInt(int(def(int32(config.Port), ArangoPort))), Scheme: scheme, HTTPHeaders: headers, }, }, - InitialDelaySeconds: 30, // Wait 30s before first probe - TimeoutSeconds: 2, // Timeout of each probe is 2s - PeriodSeconds: 10, // Interval between probes is 10s - SuccessThreshold: 1, // Single probe is enough to indicate success - FailureThreshold: 3, // Need 3 failed probes to consider a failed state + InitialDelaySeconds: def(config.InitialDelaySeconds, 30), // Wait 30s before first probe + TimeoutSeconds: def(config.TimeoutSeconds, 2), // Timeout of each probe is 2s + PeriodSeconds: def(config.PeriodSeconds, 10), // Interval between probes is 10s + SuccessThreshold: def(config.SuccessThreshold, 1), // Single probe is enough to indicate success + FailureThreshold: def(config.FailureThreshold, 3), // Need 3 failed probes to consider a failed state } } diff --git a/pkg/util/k8sutil/probes_test.go b/pkg/util/k8sutil/probes_test.go index d3302509d..a993354a6 100644 --- a/pkg/util/k8sutil/probes_test.go +++ b/pkg/util/k8sutil/probes_test.go @@ -34,7 +34,7 @@ func TestCreate(t *testing.T) { secret := "the secret" // http - config := HTTPProbeConfig{path, false, secret} + config := HTTPProbeConfig{path, false, secret, 0, 0, 0, 0, 0, 0} probe := config.Create() assert.Equal(t, probe.InitialDelaySeconds, int32(30)) @@ -50,8 +50,18 @@ func TestCreate(t *testing.T) { assert.Equal(t, probe.Handler.HTTPGet.Scheme, v1.URISchemeHTTP) // https - config = HTTPProbeConfig{path, true, secret} + config = HTTPProbeConfig{path, true, secret, 0, 0, 0, 0, 0, 0} probe = config.Create() assert.Equal(t, probe.Handler.HTTPGet.Scheme, v1.URISchemeHTTPS) + + // http, custom timing + config = HTTPProbeConfig{path, false, secret, 0, 1, 2, 3, 4, 5} + probe = config.Create() + + assert.Equal(t, probe.InitialDelaySeconds, int32(1)) + assert.Equal(t, probe.TimeoutSeconds, int32(2)) + assert.Equal(t, probe.PeriodSeconds, int32(3)) + assert.Equal(t, probe.SuccessThreshold, int32(4)) + assert.Equal(t, probe.FailureThreshold, int32(5)) } diff --git a/pkg/util/k8sutil/pvc.go b/pkg/util/k8sutil/pvc.go index cfed48981..2bfc36d5a 100644 --- a/pkg/util/k8sutil/pvc.go +++ b/pkg/util/k8sutil/pvc.go @@ -23,11 +23,20 @@ package k8sutil import ( + "strconv" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" ) +// IsPersistentVolumeClaimMarkedForDeletion returns true if the pod has been marked for deletion. +func IsPersistentVolumeClaimMarkedForDeletion(pvc *v1.PersistentVolumeClaim) bool { + return pvc.DeletionTimestamp != nil +} + // CreatePersistentVolumeClaimName returns the name of the persistent volume claim for a member with // a given id in a deployment with a given name. func CreatePersistentVolumeClaimName(deploymentName, role, id string) string { @@ -37,13 +46,17 @@ func CreatePersistentVolumeClaimName(deploymentName, role, id string) string { // CreatePersistentVolumeClaim creates a persistent volume claim with given name and configuration. // If the pvc already exists, nil is returned. // If another error occurs, that error is returned. -func CreatePersistentVolumeClaim(kubecli kubernetes.Interface, pvcName, deploymentName, ns, storageClassName, role string, resources v1.ResourceRequirements, owner metav1.OwnerReference) error { +func CreatePersistentVolumeClaim(kubecli kubernetes.Interface, pvcName, deploymentName, ns, storageClassName, role string, enforceAntiAffinity bool, resources v1.ResourceRequirements, finalizers []string, owner metav1.OwnerReference) error { labels := LabelsForDeployment(deploymentName, role) volumeMode := v1.PersistentVolumeFilesystem pvc := &v1.PersistentVolumeClaim{ ObjectMeta: metav1.ObjectMeta{ - Name: pvcName, - Labels: labels, + Name: pvcName, + Labels: labels, + Finalizers: finalizers, + Annotations: map[string]string{ + constants.AnnotationEnforceAntiAffinity: strconv.FormatBool(enforceAntiAffinity), + }, }, Spec: v1.PersistentVolumeClaimSpec{ AccessModes: []v1.PersistentVolumeAccessMode{ diff --git a/pkg/util/k8sutil/secrets.go b/pkg/util/k8sutil/secrets.go index 7dca11481..a2db083b7 100644 --- a/pkg/util/k8sutil/secrets.go +++ b/pkg/util/k8sutil/secrets.go @@ -71,26 +71,68 @@ func CreateEncryptionKeySecret(cli corev1.CoreV1Interface, secretName, namespace return nil } +// ValidateCACertificateSecret checks that a secret with given name in given namespace +// exists and it contains a 'ca.crt' data field. +func ValidateCACertificateSecret(cli corev1.CoreV1Interface, secretName, namespace string) error { + s, err := cli.Secrets(namespace).Get(secretName, metav1.GetOptions{}) + if err != nil { + return maskAny(err) + } + // Check `ca.crt` field + _, found := s.Data[constants.SecretCACertificate] + if !found { + return maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretCACertificate, secretName)) + } + return nil +} + +// GetCACertficateSecret loads a secret with given name in the given namespace +// and extracts the `ca.crt` field. +// If the secret does not exists the field is missing, +// an error is returned. +// Returns: certificate, error +func GetCACertficateSecret(cli corev1.CoreV1Interface, secretName, namespace string) (string, error) { + s, err := cli.Secrets(namespace).Get(secretName, metav1.GetOptions{}) + if err != nil { + return "", maskAny(err) + } + // Load `ca.crt` field + cert, found := s.Data[constants.SecretCACertificate] + if !found { + return "", maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretCACertificate, secretName)) + } + return string(cert), nil +} + // GetCASecret loads a secret with given name in the given namespace // and extracts the `ca.crt` & `ca.key` field. // If the secret does not exists or one of the fields is missing, // an error is returned. -// Returns: certificate, private-key, error -func GetCASecret(cli corev1.CoreV1Interface, secretName, namespace string) (string, string, error) { +// Returns: certificate, private-key, isOwnedByDeployment, error +func GetCASecret(cli corev1.CoreV1Interface, secretName, namespace string, ownerRef *metav1.OwnerReference) (string, string, bool, error) { s, err := cli.Secrets(namespace).Get(secretName, metav1.GetOptions{}) if err != nil { - return "", "", maskAny(err) + return "", "", false, maskAny(err) + } + isOwned := false + if ownerRef != nil { + for _, x := range s.GetOwnerReferences() { + if x.UID == ownerRef.UID { + isOwned = true + break + } + } } // Load `ca.crt` field cert, found := s.Data[constants.SecretCACertificate] if !found { - return "", "", maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretCACertificate, secretName)) + return "", "", isOwned, maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretCACertificate, secretName)) } priv, found := s.Data[constants.SecretCAKey] if !found { - return "", "", maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretCAKey, secretName)) + return "", "", isOwned, maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretCAKey, secretName)) } - return string(cert), string(priv), nil + return string(cert), string(priv), isOwned, nil } // CreateCASecret creates a secret used to store a PEM encoded CA certificate & private key. @@ -151,30 +193,45 @@ func CreateTLSKeyfileSecret(cli corev1.CoreV1Interface, secretName, namespace st return nil } -// GetJWTSecret loads the JWT secret from a Secret with given name. -func GetJWTSecret(cli corev1.CoreV1Interface, secretName, namespace string) (string, error) { +// ValidateTokenSecret checks that a secret with given name in given namespace +// exists and it contains a 'token' data field. +func ValidateTokenSecret(cli corev1.CoreV1Interface, secretName, namespace string) error { + s, err := cli.Secrets(namespace).Get(secretName, metav1.GetOptions{}) + if err != nil { + return maskAny(err) + } + // Check `token` field + _, found := s.Data[constants.SecretKeyToken] + if !found { + return maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretKeyToken, secretName)) + } + return nil +} + +// GetTokenSecret loads the token secret from a Secret with given name. +func GetTokenSecret(cli corev1.CoreV1Interface, secretName, namespace string) (string, error) { s, err := cli.Secrets(namespace).Get(secretName, metav1.GetOptions{}) if err != nil { return "", maskAny(err) } // Take the first data from the token key - data, found := s.Data[constants.SecretKeyJWT] + data, found := s.Data[constants.SecretKeyToken] if !found { - return "", maskAny(fmt.Errorf("No '%s' data found in secret '%s'", constants.SecretKeyJWT, secretName)) + return "", maskAny(fmt.Errorf("No '%s' data found in secret '%s'", constants.SecretKeyToken, secretName)) } return string(data), nil } -// CreateJWTSecret creates a secret with given name in given namespace +// CreateTokenSecret creates a secret with given name in given namespace // with a given token as value. -func CreateJWTSecret(cli corev1.CoreV1Interface, secretName, namespace, token string, ownerRef *metav1.OwnerReference) error { +func CreateTokenSecret(cli corev1.CoreV1Interface, secretName, namespace, token string, ownerRef *metav1.OwnerReference) error { // Create secret secret := &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ Name: secretName, }, Data: map[string][]byte{ - constants.SecretKeyJWT: []byte(token), + constants.SecretKeyToken: []byte(token), }, } // Attach secret to owner @@ -185,3 +242,25 @@ func CreateJWTSecret(cli corev1.CoreV1Interface, secretName, namespace, token st } return nil } + +// GetBasicAuthSecret loads a secret with given name in the given namespace +// and extracts the `username` & `password` field. +// If the secret does not exists or one of the fields is missing, +// an error is returned. +// Returns: username, password, error +func GetBasicAuthSecret(cli corev1.CoreV1Interface, secretName, namespace string) (string, string, error) { + s, err := cli.Secrets(namespace).Get(secretName, metav1.GetOptions{}) + if err != nil { + return "", "", maskAny(err) + } + // Load `ca.crt` field + username, found := s.Data[constants.SecretUsername] + if !found { + return "", "", maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretUsername, secretName)) + } + password, found := s.Data[constants.SecretPassword] + if !found { + return "", "", maskAny(fmt.Errorf("No '%s' found in secret '%s'", constants.SecretPassword, secretName)) + } + return string(username), string(password), nil +} diff --git a/pkg/util/k8sutil/secrets_test.go b/pkg/util/k8sutil/secrets_test.go index 18cf85216..476974ab3 100644 --- a/pkg/util/k8sutil/secrets_test.go +++ b/pkg/util/k8sutil/secrets_test.go @@ -86,8 +86,8 @@ func TestCreateEncryptionKeySecret(t *testing.T) { assert.Error(t, CreateEncryptionKeySecret(cli, "short-key", "ns", key)) } -// TestGetJWTSecret tests GetJWTSecret. -func TestGetJWTSecret(t *testing.T) { +// TestGetTokenSecret tests GetTokenSecret. +func TestGetTokenSecret(t *testing.T) { cli := mocks.NewCore() // Prepare mock @@ -104,17 +104,17 @@ func TestGetJWTSecret(t *testing.T) { }, nil) m.On("Get", "notfound", mock.Anything).Return(nil, apierrors.NewNotFound(schema.GroupResource{}, "notfound")) - token, err := GetJWTSecret(cli, "good", "ns") + token, err := GetTokenSecret(cli, "good", "ns") assert.NoError(t, err) assert.Equal(t, token, "foo") - _, err = GetJWTSecret(cli, "no-token", "ns") + _, err = GetTokenSecret(cli, "no-token", "ns") assert.Error(t, err) - _, err = GetJWTSecret(cli, "notfound", "ns") + _, err = GetTokenSecret(cli, "notfound", "ns") assert.True(t, IsNotFound(err)) } -// TestCreateJWTSecret tests CreateJWTSecret -func TestCreateJWTSecret(t *testing.T) { +// TestCreateTokenSecret tests CreateTokenSecret +func TestCreateTokenSecret(t *testing.T) { cli := mocks.NewCore() // Prepare mock @@ -130,6 +130,6 @@ func TestCreateJWTSecret(t *testing.T) { } }).Return(nil, nil) - assert.NoError(t, CreateJWTSecret(cli, "good", "ns", "token", nil)) - assert.NoError(t, CreateJWTSecret(cli, "with-owner", "ns", "token", &metav1.OwnerReference{})) + assert.NoError(t, CreateTokenSecret(cli, "good", "ns", "token", nil)) + assert.NoError(t, CreateTokenSecret(cli, "with-owner", "ns", "token", &metav1.OwnerReference{})) } diff --git a/pkg/util/k8sutil/services.go b/pkg/util/k8sutil/services.go index 16cdeccc3..56d2548a5 100644 --- a/pkg/util/k8sutil/services.go +++ b/pkg/util/k8sutil/services.go @@ -23,6 +23,8 @@ package k8sutil import ( + "strconv" + "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -40,6 +42,12 @@ func CreateDatabaseClientServiceName(deploymentName string) string { return deploymentName } +// CreateDatabaseExternalAccessServiceName returns the name of the service used to access the database from +// output the kubernetes cluster. +func CreateDatabaseExternalAccessServiceName(deploymentName string) string { + return deploymentName + "-ea" +} + // CreateSyncMasterClientServiceName returns the name of the service used by syncmaster clients for the given // deployment name. func CreateSyncMasterClientServiceName(deploymentName string) string { @@ -61,9 +69,9 @@ func CreateHeadlessService(kubecli kubernetes.Interface, deployment metav1.Objec Port: ArangoPort, }, } - publishNotReadyAddresses := false - sessionAffinity := v1.ServiceAffinityNone - newlyCreated, err := createService(kubecli, svcName, deploymentName, deployment.GetNamespace(), ClusterIPNone, "", ports, publishNotReadyAddresses, sessionAffinity, owner) + publishNotReadyAddresses := true + serviceType := v1.ServiceTypeClusterIP + newlyCreated, err := createService(kubecli, svcName, deploymentName, deployment.GetNamespace(), ClusterIPNone, "", serviceType, ports, "", publishNotReadyAddresses, owner) if err != nil { return "", false, maskAny(err) } @@ -90,32 +98,31 @@ func CreateDatabaseClientService(kubecli kubernetes.Interface, deployment metav1 } else { role = "coordinator" } - publishNotReadyAddresses := true - sessionAffinity := v1.ServiceAffinityClientIP - newlyCreated, err := createService(kubecli, svcName, deploymentName, deployment.GetNamespace(), "", role, ports, publishNotReadyAddresses, sessionAffinity, owner) + serviceType := v1.ServiceTypeClusterIP + publishNotReadyAddresses := false + newlyCreated, err := createService(kubecli, svcName, deploymentName, deployment.GetNamespace(), "", role, serviceType, ports, "", publishNotReadyAddresses, owner) if err != nil { return "", false, maskAny(err) } return svcName, newlyCreated, nil } -// CreateSyncMasterClientService prepares and creates a service in k8s, used by syncmaster clients within the k8s cluster. +// CreateExternalAccessService prepares and creates a service in k8s, used to access the database/sync from outside k8s cluster. // If the service already exists, nil is returned. // If another error occurs, that error is returned. // The returned bool is true if the service is created, or false when the service already existed. -func CreateSyncMasterClientService(kubecli kubernetes.Interface, deployment metav1.Object, owner metav1.OwnerReference) (string, bool, error) { +func CreateExternalAccessService(kubecli kubernetes.Interface, svcName, role string, deployment metav1.Object, serviceType v1.ServiceType, port, nodePort int, loadBalancerIP string, owner metav1.OwnerReference) (string, bool, error) { deploymentName := deployment.GetName() - svcName := CreateSyncMasterClientServiceName(deploymentName) ports := []v1.ServicePort{ v1.ServicePort{ Name: "server", Protocol: v1.ProtocolTCP, - Port: ArangoPort, + Port: int32(port), + NodePort: int32(nodePort), }, } - publishNotReadyAddresses := true - sessionAffinity := v1.ServiceAffinityNone - newlyCreated, err := createService(kubecli, svcName, deploymentName, deployment.GetNamespace(), "", "syncmaster", ports, publishNotReadyAddresses, sessionAffinity, owner) + publishNotReadyAddresses := false + newlyCreated, err := createService(kubecli, svcName, deploymentName, deployment.GetNamespace(), "", role, serviceType, ports, loadBalancerIP, publishNotReadyAddresses, owner) if err != nil { return "", false, maskAny(err) } @@ -126,8 +133,8 @@ func CreateSyncMasterClientService(kubecli kubernetes.Interface, deployment meta // If the service already exists, nil is returned. // If another error occurs, that error is returned. // The returned bool is true if the service is created, or false when the service already existed. -func createService(kubecli kubernetes.Interface, svcName, deploymentName, ns, clusterIP, role string, - ports []v1.ServicePort, publishNotReadyAddresses bool, sessionAffinity v1.ServiceAffinity, owner metav1.OwnerReference) (bool, error) { +func createService(kubecli kubernetes.Interface, svcName, deploymentName, ns, clusterIP, role string, serviceType v1.ServiceType, + ports []v1.ServicePort, loadBalancerIP string, publishNotReadyAddresses bool, owner metav1.OwnerReference) (bool, error) { labels := LabelsForDeployment(deploymentName, role) svc := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -137,15 +144,16 @@ func createService(kubecli kubernetes.Interface, svcName, deploymentName, ns, cl // This annotation is deprecated, PublishNotReadyAddresses is // used instead. We leave the annotation in for a while. // See https://github.com/kubernetes/kubernetes/pull/49061 - TolerateUnreadyEndpointsAnnotation: "true", + TolerateUnreadyEndpointsAnnotation: strconv.FormatBool(publishNotReadyAddresses), }, }, Spec: v1.ServiceSpec{ + Type: serviceType, Ports: ports, Selector: labels, ClusterIP: clusterIP, PublishNotReadyAddresses: publishNotReadyAddresses, - SessionAffinity: sessionAffinity, + LoadBalancerIP: loadBalancerIP, }, } addOwnerRefToObject(svc.GetObjectMeta(), &owner) diff --git a/pkg/util/k8sutil/tolerations.go b/pkg/util/k8sutil/tolerations.go new file mode 100644 index 000000000..9609c186a --- /dev/null +++ b/pkg/util/k8sutil/tolerations.go @@ -0,0 +1,67 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package k8sutil + +import ( + "time" + + "k8s.io/api/core/v1" +) + +const ( + TolerationKeyNodeNotReady = "node.kubernetes.io/not-ready" + TolerationKeyNodeAlphaUnreachable = "node.alpha.kubernetes.io/unreachable" + TolerationKeyNodeUnreachable = "node.kubernetes.io/unreachable" +) + +// TolerationDuration is a duration spec for tolerations. +type TolerationDuration struct { + Forever bool + TimeSpan time.Duration +} + +// NewNoExecuteToleration is a helper to create a Toleration with +// Key=key, Operator='Exists' Effect='NoExecute', TolerationSeconds=tolerationDuration.Seconds(). +func NewNoExecuteToleration(key string, duration TolerationDuration) v1.Toleration { + t := v1.Toleration{ + Key: key, + Operator: "Exists", + Effect: "NoExecute", + } + if !duration.Forever { + tolerationSeconds := int64(duration.TimeSpan.Seconds()) + t.TolerationSeconds = &tolerationSeconds + } + return t +} + +// AddTolerationIfNotFound adds the given tolerations, if no such toleration has been set in the given source. +func AddTolerationIfNotFound(source []v1.Toleration, toAdd v1.Toleration) []v1.Toleration { + for _, t := range source { + if (t.Key == toAdd.Key || len(t.Key) == 0) && (t.Effect == toAdd.Effect || len(t.Effect) == 0) { + // Toleration alread exists, do not add + return source + } + } + return append(source, toAdd) +} diff --git a/scripts/kube_create_storage.sh b/scripts/kube_create_storage.sh index 1649b1ab3..15befaabf 100755 --- a/scripts/kube_create_storage.sh +++ b/scripts/kube_create_storage.sh @@ -20,3 +20,5 @@ if [ "${REQUIRE_LOCAL_STORAGE}" = "1" ]; then else echo "No local storage needed for this cluster" fi +echo "Found $(kubectl get pv | wc -l) PersistentVolumes" +echo "Found $(kubectl get pv | grep Available | wc -l) available PersistentVolumes" diff --git a/tests/acceptance/activefailover.yaml b/tests/acceptance/activefailover.yaml new file mode 100644 index 000000000..84eb32ad8 --- /dev/null +++ b/tests/acceptance/activefailover.yaml @@ -0,0 +1,7 @@ +apiVersion: "database.arangodb.com/v1alpha" +kind: "ArangoDeployment" +metadata: + name: "acceptance-activefailover" +spec: + mode: ActiveFailover + image: arangodb/arangodb:3.3.10 diff --git a/tests/acceptance/cluster-sync.yaml b/tests/acceptance/cluster-sync.yaml new file mode 100644 index 000000000..25cd357bb --- /dev/null +++ b/tests/acceptance/cluster-sync.yaml @@ -0,0 +1,9 @@ +apiVersion: "database.arangodb.com/v1alpha" +kind: "ArangoDeployment" +metadata: + name: "acceptance-cluster" +spec: + mode: Cluster + image: + sync: + enabled: true diff --git a/tests/acceptance/cluster.yaml b/tests/acceptance/cluster.yaml new file mode 100644 index 000000000..ad0797765 --- /dev/null +++ b/tests/acceptance/cluster.yaml @@ -0,0 +1,7 @@ +apiVersion: "database.arangodb.com/v1alpha" +kind: "ArangoDeployment" +metadata: + name: "acceptance-cluster" +spec: + mode: Cluster + image: arangodb/arangodb:3.3.10 diff --git a/tests/acceptance/local-storage.yaml b/tests/acceptance/local-storage.yaml new file mode 100644 index 000000000..569221d93 --- /dev/null +++ b/tests/acceptance/local-storage.yaml @@ -0,0 +1,9 @@ +apiVersion: "storage.arangodb.com/v1alpha" +kind: "ArangoLocalStorage" +metadata: + name: "acceptance-local-storage" +spec: + storageClass: + name: acceptance + localPath: + - /var/lib/acceptance-test diff --git a/tests/acceptance/single.yaml b/tests/acceptance/single.yaml new file mode 100644 index 000000000..fcedd778a --- /dev/null +++ b/tests/acceptance/single.yaml @@ -0,0 +1,7 @@ +apiVersion: "database.arangodb.com/v1alpha" +kind: "ArangoDeployment" +metadata: + name: "acceptance-single" +spec: + mode: Single + image: arangodb/arangodb:3.3.10 diff --git a/tests/auth_test.go b/tests/auth_test.go index 66c2b9fb7..611190989 100644 --- a/tests/auth_test.go +++ b/tests/auth_test.go @@ -100,7 +100,7 @@ func TestAuthenticationSingleCustomSecret(t *testing.T) { depl.Spec.SetDefaults(depl.GetName()) // Create secret - if err := k8sutil.CreateJWTSecret(kubecli.CoreV1(), depl.Spec.Authentication.GetJWTSecretName(), ns, "foo", nil); err != nil { + if err := k8sutil.CreateTokenSecret(kubecli.CoreV1(), depl.Spec.Authentication.GetJWTSecretName(), ns, "foo", nil); err != nil { t.Fatalf("Create JWT secret failed: %v", err) } @@ -239,7 +239,7 @@ func TestAuthenticationClusterCustomSecret(t *testing.T) { depl.Spec.SetDefaults(depl.GetName()) // Create secret - if err := k8sutil.CreateJWTSecret(kubecli.CoreV1(), depl.Spec.Authentication.GetJWTSecretName(), ns, "foo", nil); err != nil { + if err := k8sutil.CreateTokenSecret(kubecli.CoreV1(), depl.Spec.Authentication.GetJWTSecretName(), ns, "foo", nil); err != nil { t.Fatalf("Create JWT secret failed: %v", err) } diff --git a/tests/cursor_test.go b/tests/cursor_test.go index 036f30a49..d5ca20dea 100644 --- a/tests/cursor_test.go +++ b/tests/cursor_test.go @@ -29,7 +29,7 @@ import ( "time" "github.com/dchest/uniuri" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" @@ -71,10 +71,7 @@ func TestCursorSingle(t *testing.T) { } // Check server role - assert.NoError(t, client.SynchronizeEndpoints(ctx)) - role, err := client.ServerRole(ctx) - assert.NoError(t, err) - assert.Equal(t, driver.ServerRoleSingle, role) + require.NoError(t, testServerRole(ctx, client, driver.ServerRoleSingle)) // Run cursor tests runCursorTests(t, client) @@ -83,9 +80,9 @@ func TestCursorSingle(t *testing.T) { removeDeployment(c, depl.GetName(), ns) } -// TestCursorResilientSingle tests the creating of a resilientsingle server deployment +// TestCursorActiveFailover tests the creating of a ActiveFailover server deployment // with default settings. -func TestCursorResilientSingle(t *testing.T) { +func TestCursorActiveFailover(t *testing.T) { longOrSkip(t) c := client.MustNewInCluster() kubecli := mustNewKubeClient(t) @@ -93,7 +90,7 @@ func TestCursorResilientSingle(t *testing.T) { // Prepare deployment config depl := newDeployment("test-cur-rs-" + uniuri.NewLen(4)) - depl.Spec.Mode = api.NewMode(api.DeploymentModeResilientSingle) + depl.Spec.Mode = api.NewMode(api.DeploymentModeActiveFailover) // Create deployment _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) @@ -114,14 +111,11 @@ func TestCursorResilientSingle(t *testing.T) { // Wait for single server available if err := waitUntilVersionUp(client, nil); err != nil { - t.Fatalf("ResilientSingle servers not running returning version in time: %v", err) + t.Fatalf("ActiveFailover servers not running returning version in time: %v", err) } // Check server role - assert.NoError(t, client.SynchronizeEndpoints(ctx)) - role, err := client.ServerRole(ctx) - assert.NoError(t, err) - assert.Equal(t, driver.ServerRoleSingleActive, role) + require.NoError(t, testServerRole(ctx, client, driver.ServerRoleSingleActive)) // Run cursor tests runCursorTests(t, client) @@ -165,10 +159,7 @@ func TestCursorCluster(t *testing.T) { } // Check server role - assert.NoError(t, client.SynchronizeEndpoints(ctx)) - role, err := client.ServerRole(ctx) - assert.NoError(t, err) - assert.Equal(t, driver.ServerRoleCoordinator, role) + require.NoError(t, testServerRole(ctx, client, driver.ServerRoleCoordinator)) // Run cursor tests runCursorTests(t, client) diff --git a/tests/deployments_test.go b/tests/deployments_test.go index cc945c5bc..b3c97e750 100644 --- a/tests/deployments_test.go +++ b/tests/deployments_test.go @@ -46,14 +46,14 @@ func TestDeploymentSingleRocksDB(t *testing.T) { deploymentSubTest(t, api.DeploymentModeSingle, api.StorageEngineRocksDB) } -// test deployment resilient single server mmfiles -func TestDeploymentResilientSingleMMFiles(t *testing.T) { - deploymentSubTest(t, api.DeploymentModeResilientSingle, api.StorageEngineMMFiles) +// test deployment active-failover server mmfiles +func TestDeploymentActiveFailoverMMFiles(t *testing.T) { + deploymentSubTest(t, api.DeploymentModeActiveFailover, api.StorageEngineMMFiles) } -// test deployment resilient single server rocksdb -func TestDeploymentResilientSingleRocksDB(t *testing.T) { - deploymentSubTest(t, api.DeploymentModeResilientSingle, api.StorageEngineRocksDB) +// test deployment active-failover server rocksdb +func TestDeploymentActiveFailoverRocksDB(t *testing.T) { + deploymentSubTest(t, api.DeploymentModeActiveFailover, api.StorageEngineRocksDB) } // test deployment cluster mmfiles diff --git a/tests/duration/Dockerfile b/tests/duration/Dockerfile new file mode 100644 index 000000000..b00043c4f --- /dev/null +++ b/tests/duration/Dockerfile @@ -0,0 +1,5 @@ +FROM scratch + +ADD bin/arangodb_operator_duration_test /usr/bin/ + +ENTRYPOINT [ "/usr/bin/arangodb_operator_duration_test" ] \ No newline at end of file diff --git a/tests/duration/README.md b/tests/duration/README.md new file mode 100644 index 000000000..79f9e1024 --- /dev/null +++ b/tests/duration/README.md @@ -0,0 +1,32 @@ +# Kube-ArangoDB duration test + +This test is a simple application that keeps accessing the database with various requests. + +## Building + +In root of kube-arangodb repository, run: + +```bash +make docker-duration-test +``` + +## Running + +Start an ArangoDB `Cluster` deployment. + +Run: + +```bash +kubectl run \ + --image=${DOCKERNAMESPACE}/kube-arangodb-durationtest:dev \ + --image-pull-policy=Always duration-test \ + -- \ + --cluster=https://..svc:8529 \ + --username=root +``` + +To remove the test, run: + +```bash +kubectl delete -n deployment/duration-test +``` diff --git a/tests/duration/main.go b/tests/duration/main.go new file mode 100644 index 000000000..2ddfc0b6f --- /dev/null +++ b/tests/duration/main.go @@ -0,0 +1,132 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package main + +import ( + "context" + "crypto/tls" + "flag" + "fmt" + "log" + "os" + "os/signal" + "strings" + "syscall" + "time" + + driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/http" + "github.com/pkg/errors" + + "github.com/arangodb/kube-arangodb/pkg/util/retry" +) + +const ( + defaultTestDuration = time.Hour * 24 * 7 // 7 days +) + +var ( + maskAny = errors.WithStack + userName string + password string + clusterEndpoints string + testDuration time.Duration +) + +func init() { + flag.StringVar(&userName, "username", "", "Authenticating username") + flag.StringVar(&password, "password", "", "Authenticating password") + flag.StringVar(&clusterEndpoints, "cluster", "", "Endpoints for database cluster") + flag.DurationVar(&testDuration, "duration", defaultTestDuration, "Duration of the test") +} + +func main() { + flag.Parse() + + // Create clients & wait for cluster available + client, err := createClusterClient(clusterEndpoints, userName, password) + if err != nil { + log.Fatalf("Failed to create cluster client: %#v\n", err) + } + if err := waitUntilClusterUp(client); err != nil { + log.Fatalf("Failed to reach cluster: %#v\n", err) + } + + // Start running tests + ctx, cancel := context.WithCancel(context.Background()) + sigChannel := make(chan os.Signal) + signal.Notify(sigChannel, os.Interrupt, syscall.SIGTERM) + go handleSignal(sigChannel, cancel) + runTestLoop(ctx, client, testDuration) +} + +// createClusterClient creates a configuration, connection and client for +// one of the two ArangoDB clusters in the test. It uses the go-driver. +// It needs a list of endpoints. +func createClusterClient(endpoints string, user string, password string) (driver.Client, error) { + // This will always use HTTP, and user and password authentication + config := http.ConnectionConfig{ + Endpoints: strings.Split(endpoints, ","), + TLSConfig: &tls.Config{InsecureSkipVerify: true}, + } + connection, err := http.NewConnection(config) + if err != nil { + return nil, maskAny(err) + } + clientCfg := driver.ClientConfig{ + Connection: connection, + Authentication: driver.BasicAuthentication(user, password), + } + client, err := driver.NewClient(clientCfg) + if err != nil { + return nil, maskAny(err) + } + return client, nil +} + +func waitUntilClusterUp(c driver.Client) error { + op := func() error { + ctx := context.Background() + if _, err := c.Version(ctx); err != nil { + return maskAny(err) + } + return nil + } + if err := retry.Retry(op, time.Minute); err != nil { + return maskAny(err) + } + return nil +} + +// handleSignal listens for termination signals and stops this process on termination. +func handleSignal(sigChannel chan os.Signal, cancel context.CancelFunc) { + signalCount := 0 + for s := range sigChannel { + signalCount++ + fmt.Println("Received signal:", s) + if signalCount > 1 { + os.Exit(1) + } + cancel() + } +} diff --git a/tests/duration/simple/check.go b/tests/duration/simple/check.go new file mode 100644 index 000000000..ea769eeef --- /dev/null +++ b/tests/duration/simple/check.go @@ -0,0 +1,50 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +import "context" + +// isDocumentEqualTo reads an existing document and checks that it is equal to the given document. +// Returns: (isEqual,currentRevision,error) +func (t *simpleTest) isDocumentEqualTo(c *collection, key string, expected UserDocument) (bool, string, error) { + ctx := context.Background() + var result UserDocument + t.log.Info().Msgf("Checking existing document '%s' from '%s'...", key, c.name) + col, err := t.db.Collection(ctx, c.name) + if err != nil { + return false, "", maskAny(err) + } + m, err := col.ReadDocument(ctx, key, &result) + if err != nil { + // This is a failure + t.log.Error().Msgf("Failed to read document '%s' from '%s': %v", key, c.name, err) + return false, "", maskAny(err) + } + // Compare document against expected document + if result.Equals(expected) { + // Found an exact match + return true, m.Rev, nil + } + t.log.Info().Msgf("Document '%s' in '%s' returned different values: got %q expected %q", key, c.name, result, expected) + return false, m.Rev, nil +} diff --git a/tests/duration/simple/error.go b/tests/duration/simple/error.go new file mode 100644 index 000000000..cf34f2807 --- /dev/null +++ b/tests/duration/simple/error.go @@ -0,0 +1,31 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +import ( + "github.com/pkg/errors" +) + +var ( + maskAny = errors.WithStack +) diff --git a/tests/duration/simple/simple.go b/tests/duration/simple/simple.go new file mode 100644 index 000000000..053ac6301 --- /dev/null +++ b/tests/duration/simple/simple.go @@ -0,0 +1,689 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +import ( + "context" + "fmt" + "io" + "math/rand" + "os" + "sort" + "sync" + "sync/atomic" + "time" + + driver "github.com/arangodb/go-driver" + "github.com/pkg/errors" + "github.com/rs/zerolog" + + "github.com/arangodb/kube-arangodb/tests/duration/test" +) + +type SimpleConfig struct { + MaxDocuments int + MaxCollections int +} + +const ( + initialDocumentCount = 999 +) + +type simpleTest struct { + SimpleConfig + activeMutex sync.Mutex + logPath string + reportDir string + log zerolog.Logger + listener test.TestListener + stop chan struct{} + active bool + pauseRequested bool + paused bool + client driver.Client + db driver.Database + failures int + actions int + collections map[string]*collection + collectionsMutex sync.Mutex + lastCollectionIndex int32 + readExistingCounter counter + readExistingWrongRevisionCounter counter + readNonExistingCounter counter + createCounter counter + createCollectionCounter counter + removeExistingCollectionCounter counter + updateExistingCounter counter + updateExistingWrongRevisionCounter counter + updateNonExistingCounter counter + replaceExistingCounter counter + replaceExistingWrongRevisionCounter counter + replaceNonExistingCounter counter + deleteExistingCounter counter + deleteExistingWrongRevisionCounter counter + deleteNonExistingCounter counter + importCounter counter + queryCreateCursorCounter counter + queryNextBatchCounter counter + queryNextBatchNewCoordinatorCounter counter + queryLongRunningCounter counter + rebalanceShardsCounter counter + queryUpdateCounter counter + queryUpdateLongRunningCounter counter +} + +type counter struct { + succeeded int + failed int +} + +type collection struct { + name string + existingDocs map[string]UserDocument +} + +// NewSimpleTest creates a simple test +func NewSimpleTest(log zerolog.Logger, reportDir string, config SimpleConfig) test.TestScript { + return &simpleTest{ + SimpleConfig: config, + reportDir: reportDir, + log: log, + collections: make(map[string]*collection), + } +} + +// Name returns the name of the script +func (t *simpleTest) Name() string { + return "simple" +} + +// Start triggers the test script to start. +// It should spwan actions in a go routine. +func (t *simpleTest) Start(client driver.Client, listener test.TestListener) error { + t.activeMutex.Lock() + defer t.activeMutex.Unlock() + + if t.active { + // No restart unless needed + return nil + } + + t.listener = listener + t.client = client + ctx := context.Background() + db, err := client.Database(ctx, "_system") + if err != nil { + return maskAny(err) + } + t.db = db + + // Cleanup of old data + for i := 1; i <= t.MaxCollections; i++ { + col, err := db.Collection(ctx, t.getCollectionName(i)) + if err == nil { + if err := col.Remove(ctx); err != nil { + return errors.Wrapf(err, "Failed to remove collection %s", col.Name()) + } + } else if !driver.IsNotFound(err) { + return maskAny(err) + } + } + + t.active = true + go t.testLoop() + return nil +} + +// Stop any running test. This should not return until tests are actually stopped. +func (t *simpleTest) Stop() error { + t.activeMutex.Lock() + defer t.activeMutex.Unlock() + + if !t.active { + // No active, nothing to stop + return nil + } + + stop := make(chan struct{}) + t.stop = stop + <-stop + return nil +} + +// Interrupt the tests, but be prepared to continue. +func (t *simpleTest) Pause() error { + t.pauseRequested = true + return nil +} + +// Resume running the tests, where Pause interrupted it. +func (t *simpleTest) Resume() error { + t.pauseRequested = false + return nil +} + +// Status returns the current status of the test +func (t *simpleTest) Status() test.TestStatus { + cc := func(name string, c counter) test.Counter { + return test.Counter{ + Name: name, + Succeeded: c.succeeded, + Failed: c.failed, + } + } + + status := test.TestStatus{ + Active: t.active && !t.paused, + Pausing: t.pauseRequested && t.paused, + Failures: t.failures, + Actions: t.actions, + Counters: []test.Counter{ + cc("#collections created", t.createCollectionCounter), + cc("#collections removed", t.removeExistingCollectionCounter), + cc("#documents created", t.createCounter), + cc("#existing documents read", t.readExistingCounter), + cc("#existing documents updated", t.updateExistingCounter), + cc("#existing documents replaced", t.replaceExistingCounter), + cc("#existing documents removed", t.deleteExistingCounter), + cc("#existing documents wrong revision read", t.readExistingWrongRevisionCounter), + cc("#existing documents wrong revision updated", t.updateExistingWrongRevisionCounter), + cc("#existing documents wrong revision replaced", t.replaceExistingWrongRevisionCounter), + cc("#existing documents wrong revision removed", t.deleteExistingWrongRevisionCounter), + cc("#non-existing documents read", t.readNonExistingCounter), + cc("#non-existing documents updated", t.updateNonExistingCounter), + cc("#non-existing documents replaced", t.replaceNonExistingCounter), + cc("#non-existing documents removed", t.deleteNonExistingCounter), + cc("#import operations", t.importCounter), + cc("#create AQL cursor operations", t.queryCreateCursorCounter), + cc("#fetch next AQL cursor batch operations", t.queryNextBatchCounter), + cc("#fetch next AQL cursor batch after coordinator change operations", t.queryNextBatchNewCoordinatorCounter), + cc("#long running AQL query operations", t.queryLongRunningCounter), + cc("#rebalance shards operations", t.rebalanceShardsCounter), + cc("#update AQL query operations", t.queryUpdateCounter), + cc("#long running update AQL query operations", t.queryUpdateLongRunningCounter), + }, + } + + t.collectionsMutex.Lock() + for _, c := range t.collections { + status.Messages = append(status.Messages, + fmt.Sprintf("Current #documents in %s: %d", c.name, len(c.existingDocs)), + ) + } + t.collectionsMutex.Unlock() + + return status +} + +// CollectLogs copies all logging info to the given writer. +func (t *simpleTest) CollectLogs(w io.Writer) error { + if logPath := t.logPath; logPath == "" { + // Nothing to log yet + return nil + } else { + rd, err := os.Open(logPath) + if err != nil { + return maskAny(err) + } + defer rd.Close() + if _, err := io.Copy(w, rd); err != nil { + return maskAny(err) + } + return nil + } +} + +func (t *simpleTest) shouldStop() bool { + // Should we stop? + if stop := t.stop; stop != nil { + stop <- struct{}{} + return true + } + return false +} + +type UserDocument struct { + Key string `json:"_key"` + rev string // Note that we do not export this field! + Value int `json:"value"` + Name string `json:"name"` + Odd bool `json:"odd"` +} + +// Equals returns true when the value fields of `d` and `other` are the equal. +func (d UserDocument) Equals(other UserDocument) bool { + return d.Value == other.Value && + d.Name == other.Name && + d.Odd == other.Odd +} + +func (t *simpleTest) reportFailure(f test.Failure) { + t.failures++ + t.listener.ReportFailure(f) +} + +func (t *simpleTest) testLoop() { + t.active = true + t.actions = 0 + defer func() { t.active = false }() + + if err := t.createAndInitCollection(); err != nil { + t.log.Error().Msgf("Failed to create&init first collection: %v. Giving up", err) + return + } + + var plan []int + planIndex := 0 + for { + // Should we stop + if t.shouldStop() { + return + } + if t.pauseRequested { + t.paused = true + time.Sleep(time.Second * 2) + continue + } + t.paused = false + t.actions++ + if plan == nil || planIndex >= len(plan) { + plan = createTestPlan(20) // Update when more tests are added + planIndex = 0 + } + + switch plan[planIndex] { + case 0: + // Create collection with initial data + if len(t.collections) < t.MaxCollections && rand.Intn(100)%2 == 0 { + if err := t.createAndInitCollection(); err != nil { + t.log.Error().Msgf("Failed to create&init collection: %v", err) + } + } + planIndex++ + + case 1: + // Remove an existing collection + if len(t.collections) > 1 && rand.Intn(100)%2 == 0 { + c := t.selectRandomCollection() + if err := t.removeExistingCollection(c); err != nil { + t.log.Error().Msgf("Failed to remove existing collection: %#v", err) + } + } + planIndex++ + + case 2: + // Create a random document + if len(t.collections) > 0 { + c := t.selectRandomCollection() + if len(c.existingDocs) < t.MaxDocuments { + userDoc := UserDocument{ + Key: c.createNewKey(true), + Value: rand.Int(), + Name: fmt.Sprintf("User %d", time.Now().Nanosecond()), + Odd: time.Now().Nanosecond()%2 == 1, + } + if rev, err := t.createDocument(c, userDoc, userDoc.Key); err != nil { + t.log.Error().Msgf("Failed to create document: %#v", err) + } else { + userDoc.rev = rev + c.existingDocs[userDoc.Key] = userDoc + + // Now try to read it, it must exist + //t.client.SetCoordinator("") + if _, err := t.readExistingDocument(c, userDoc.Key, rev, false, false); err != nil { + t.log.Error().Msgf("Failed to read just-created document '%s': %#v", userDoc.Key, err) + } + } + } + } + planIndex++ + + case 3: + // Read a random existing document + if len(t.collections) > 0 { + c := t.selectRandomCollection() + if len(c.existingDocs) > 0 { + randomKey, rev := c.selectRandomKey() + if _, err := t.readExistingDocument(c, randomKey, rev, false, false); err != nil { + t.log.Error().Msgf("Failed to read existing document '%s': %#v", randomKey, err) + } + } + } + planIndex++ + + case 4: + // Read a random existing document but with wrong revision + planIndex++ + + case 5: + // Read a random non-existing document + if len(t.collections) > 0 { + c := t.selectRandomCollection() + randomKey := c.createNewKey(false) + if err := t.readNonExistingDocument(c.name, randomKey); err != nil { + t.log.Error().Msgf("Failed to read non-existing document '%s': %#v", randomKey, err) + } + } + planIndex++ + + case 6: + // Remove a random existing document + if len(t.collections) > 0 { + c := t.selectRandomCollection() + if len(c.existingDocs) > 0 { + randomKey, rev := c.selectRandomKey() + if err := t.removeExistingDocument(c.name, randomKey, rev); err != nil { + t.log.Error().Msgf("Failed to remove existing document '%s': %#v", randomKey, err) + } else { + // Remove succeeded, key should no longer exist + c.removeExistingKey(randomKey) + + // Now try to read it, it should not exist + //t.client.SetCoordinator("") + if err := t.readNonExistingDocument(c.name, randomKey); err != nil { + t.log.Error().Msgf("Failed to read just-removed document '%s': %#v", randomKey, err) + } + } + } + } + planIndex++ + + case 7: + // Remove a random existing document but with wrong revision + planIndex++ + + case 8: + // Remove a random non-existing document + if len(t.collections) > 0 { + c := t.selectRandomCollection() + randomKey := c.createNewKey(false) + if err := t.removeNonExistingDocument(c.name, randomKey); err != nil { + t.log.Error().Msgf("Failed to remove non-existing document '%s': %#v", randomKey, err) + } + } + planIndex++ + + case 9: + // Update a random existing document + if len(t.collections) > 0 { + c := t.selectRandomCollection() + if len(c.existingDocs) > 0 { + randomKey, rev := c.selectRandomKey() + if newRev, err := t.updateExistingDocument(c, randomKey, rev); err != nil { + t.log.Error().Msgf("Failed to update existing document '%s': %#v", randomKey, err) + } else { + // Updated succeeded, now try to read it, it should exist and be updated + //t.client.SetCoordinator("") + if _, err := t.readExistingDocument(c, randomKey, newRev, false, false); err != nil { + t.log.Error().Msgf("Failed to read just-updated document '%s': %#v", randomKey, err) + } + } + } + } + planIndex++ + + case 10: + // Update a random existing document but with wrong revision + planIndex++ + + case 11: + // Update a random non-existing document + if len(t.collections) > 0 { + c := t.selectRandomCollection() + randomKey := c.createNewKey(false) + if err := t.updateNonExistingDocument(c.name, randomKey); err != nil { + t.log.Error().Msgf("Failed to update non-existing document '%s': %#v", randomKey, err) + } + } + planIndex++ + + case 12: + // Replace a random existing document + if len(t.collections) > 0 { + c := t.selectRandomCollection() + if len(c.existingDocs) > 0 { + randomKey, rev := c.selectRandomKey() + if newRev, err := t.replaceExistingDocument(c, randomKey, rev); err != nil { + t.log.Error().Msgf("Failed to replace existing document '%s': %#v", randomKey, err) + } else { + // Replace succeeded, now try to read it, it should exist and be replaced + //t.client.SetCoordinator("") + if _, err := t.readExistingDocument(c, randomKey, newRev, false, false); err != nil { + t.log.Error().Msgf("Failed to read just-replaced document '%s': %#v", randomKey, err) + } + } + } + } + planIndex++ + + case 13: + // Replace a random existing document but with wrong revision + planIndex++ + + case 14: + // Replace a random non-existing document + if len(t.collections) > 0 { + c := t.selectRandomCollection() + randomKey := c.createNewKey(false) + if err := t.replaceNonExistingDocument(c.name, randomKey); err != nil { + t.log.Error().Msgf("Failed to replace non-existing document '%s': %#v", randomKey, err) + } + } + planIndex++ + + case 15: + // Query documents + planIndex++ + + case 16: + // Query documents (long running) + if len(t.collections) > 0 { + c := t.selectRandomCollection() + if err := t.queryDocumentsLongRunning(c); err != nil { + t.log.Error().Msgf("Failed to query (long running) documents: %#v", err) + } + } + planIndex++ + + case 17: + // Rebalance shards + if err := t.rebalanceShards(); err != nil { + t.log.Error().Msgf("Failed to rebalance shards: %#v", err) + } + planIndex++ + + case 18: + // AQL update query + if len(t.collections) > 0 { + c := t.selectRandomCollection() + if len(c.existingDocs) > 0 { + randomKey, _ := c.selectRandomKey() + if newRev, err := t.queryUpdateDocuments(c, randomKey); err != nil { + t.log.Error().Msgf("Failed to update document using AQL query: %#v", err) + } else { + // Updated succeeded, now try to read it (anywhere), it should exist and be updated + //t.client.SetCoordinator("") + if _, err := t.readExistingDocument(c, randomKey, newRev, false, false); err != nil { + t.log.Error().Msgf("Failed to read just-updated document '%s': %#v", randomKey, err) + } + } + } + } + planIndex++ + + case 19: + // Long running AQL update query + if len(t.collections) > 0 { + c := t.selectRandomCollection() + if len(c.existingDocs) > 0 { + randomKey, _ := c.selectRandomKey() + if newRev, err := t.queryUpdateDocumentsLongRunning(c, randomKey); err != nil { + t.log.Error().Msgf("Failed to update document using long running AQL query: %#v", err) + } else { + // Updated succeeded, now try to read it (anywhere), it should exist and be updated + //t.client.SetCoordinator("") + if _, err := t.readExistingDocument(c, randomKey, newRev, false, false); err != nil { + t.log.Error().Msgf("Failed to read just-updated document '%s': %#v", randomKey, err) + } + } + } + } + planIndex++ + } + time.Sleep(time.Second * 2) + } +} + +// createTestPlan creates an int-array of 'steps' long with all values from 0..steps-1 in random order. +func createTestPlan(steps int) []int { + plan := make([]int, steps) + for i := 0; i < steps; i++ { + plan[i] = i + } + test.Shuffle(sort.IntSlice(plan)) + return plan +} + +// createNewCollectionName returns a new (unique) collection name +func (t *simpleTest) createNewCollectionName() string { + index := atomic.AddInt32(&t.lastCollectionIndex, 1) + return t.getCollectionName(int(index)) +} + +// getCollectionName returns a collection name with given index +func (t *simpleTest) getCollectionName(index int) string { + return fmt.Sprintf("simple_user_%d", index) +} + +func (t *simpleTest) selectRandomCollection() *collection { + index := rand.Intn(len(t.collections)) + for _, c := range t.collections { + if index == 0 { + return c + } + index-- + } + return nil // This should never be reached when len(t.collections) > 0 +} + +func (t *simpleTest) registerCollection(c *collection) { + t.collectionsMutex.Lock() + defer t.collectionsMutex.Unlock() + t.collections[c.name] = c +} + +func (t *simpleTest) unregisterCollection(c *collection) { + t.collectionsMutex.Lock() + defer t.collectionsMutex.Unlock() + delete(t.collections, c.name) +} + +func (t *simpleTest) createAndInitCollection() error { + c := &collection{ + name: t.createNewCollectionName(), + existingDocs: make(map[string]UserDocument), + } + if err := t.createCollection(c, 9, 2); err != nil { + t.reportFailure(test.NewFailure("Creating collection '%s' failed: %v", c.name, err)) + return maskAny(err) + } + t.registerCollection(c) + t.createCollectionCounter.succeeded++ + t.actions++ + + // Import documents + if err := t.importDocuments(c); err != nil { + t.reportFailure(test.NewFailure("Failed to import documents: %#v", err)) + } + t.actions++ + + // Check imported documents + for k := range c.existingDocs { + if t.shouldStop() || t.pauseRequested { + return nil + } + if _, err := t.readExistingDocument(c, k, "", true, false); err != nil { + t.reportFailure(test.NewFailure("Failed to read existing document '%s': %#v", k, err)) + } + t.actions++ + } + + // Create sample users + for i := 0; i < initialDocumentCount; i++ { + if t.shouldStop() || t.pauseRequested { + return nil + } + userDoc := UserDocument{ + Key: fmt.Sprintf("doc%05d", i), + Value: i, + Name: fmt.Sprintf("User %d", i), + Odd: i%2 == 1, + } + if rev, err := t.createDocument(c, userDoc, userDoc.Key); err != nil { + t.reportFailure(test.NewFailure("Failed to create document: %#v", err)) + } else { + userDoc.rev = rev + c.existingDocs[userDoc.Key] = userDoc + } + t.actions++ + } + return nil +} + +func (c *collection) createNewKey(record bool) string { + for { + key := fmt.Sprintf("newkey%07d", rand.Int31n(100*1000)) + _, found := c.existingDocs[key] + if !found { + if record { + c.existingDocs[key] = UserDocument{} + } + return key + } + } +} + +func (c *collection) removeExistingKey(key string) { + delete(c.existingDocs, key) +} + +func (c *collection) selectRandomKey() (string, string) { + index := rand.Intn(len(c.existingDocs)) + for k, v := range c.existingDocs { + if index == 0 { + return k, v.rev + } + index-- + } + return "", "" // This should never be reached when len(t.existingDocs) > 0 +} + +func (c *collection) selectWrongRevision(key string) (string, bool) { + correctRev := c.existingDocs[key].rev + for _, v := range c.existingDocs { + if v.rev != correctRev && v.rev != "" { + return v.rev, true + } + } + return "", false // This should never be reached when len(t.existingDocs) > 1 +} diff --git a/tests/duration/simple/simple_collection.go b/tests/duration/simple/simple_collection.go new file mode 100644 index 000000000..71ddaab0c --- /dev/null +++ b/tests/duration/simple/simple_collection.go @@ -0,0 +1,94 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +import ( + "context" + "fmt" + + driver "github.com/arangodb/go-driver" + + "github.com/arangodb/kube-arangodb/tests/duration/test" +) + +// createCollection creates a new collection. +// The operation is expected to succeed. +func (t *simpleTest) createCollection(c *collection, numberOfShards, replicationFactor int) error { + ctx := context.Background() + opts := &driver.CreateCollectionOptions{ + NumberOfShards: numberOfShards, + ReplicationFactor: replicationFactor, + } + t.log.Info().Msgf("Creating collection '%s' with numberOfShards=%d, replicationFactor=%d...", c.name, numberOfShards, replicationFactor) + if _, err := t.db.CreateCollection(ctx, c.name, opts); err != nil { + // This is a failure + t.reportFailure(test.NewFailure("Failed to create collection '%s': %v", c.name, err)) + return maskAny(err) + } else if driver.IsConflict(err) { + // Duplicate name, check if that is correct + if exists, checkErr := t.collectionExists(c); checkErr != nil { + t.log.Error().Msgf("Failed to check if collection exists: %v", checkErr) + t.reportFailure(test.NewFailure("Failed to create collection '%s': %v and cannot check existance: %v", c.name, err, checkErr)) + return maskAny(err) + } else if !exists { + // Collection has not been created, so 409 status is really wrong + t.reportFailure(test.NewFailure("Failed to create collection '%s': 409 reported but collection does not exist", c.name)) + return maskAny(fmt.Errorf("Create collection reported 409, but collection does not exist")) + } + } + t.log.Info().Msgf("Creating collection '%s' with numberOfShards=%d, replicationFactor=%d succeeded", c.name, numberOfShards, replicationFactor) + return nil +} + +// removeCollection remove an existing collection. +// The operation is expected to succeed. +func (t *simpleTest) removeExistingCollection(c *collection) error { + ctx := context.Background() + t.log.Info().Msgf("Removing collection '%s'...", c.name) + col, err := t.db.Collection(ctx, c.name) + if err != nil { + return maskAny(err) + } + if err := col.Remove(ctx); err != nil { + // This is a failure + t.removeExistingCollectionCounter.failed++ + t.reportFailure(test.NewFailure("Failed to remove collection '%s': %v", c.name, err)) + return maskAny(err) + } + t.removeExistingCollectionCounter.succeeded++ + t.log.Info().Msgf("Removing collection '%s' succeeded", c.name) + t.unregisterCollection(c) + return nil +} + +// collectionExists tries to fetch information about the collection to see if it exists. +func (t *simpleTest) collectionExists(c *collection) (bool, error) { + ctx := context.Background() + t.log.Info().Msgf("Checking collection '%s'...", c.name) + if found, err := t.db.CollectionExists(ctx, c.name); err != nil { + // This is a failure + return false, maskAny(err) + } else { + return found, nil + } +} diff --git a/tests/duration/simple/simple_create.go b/tests/duration/simple/simple_create.go new file mode 100644 index 000000000..af5ce6bb5 --- /dev/null +++ b/tests/duration/simple/simple_create.go @@ -0,0 +1,50 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +import ( + "context" + + "github.com/arangodb/kube-arangodb/tests/duration/test" +) + +// createDocument creates a new document. +// The operation is expected to succeed. +func (t *simpleTest) createDocument(c *collection, document interface{}, key string) (string, error) { + ctx := context.Background() + col, err := t.db.Collection(ctx, c.name) + if err != nil { + return "", maskAny(err) + } + t.log.Info().Msgf("Creating document '%s' in '%s'...", key, c.name) + m, err := col.CreateDocument(ctx, document) + if err != nil { + // This is a failure + t.createCounter.failed++ + t.reportFailure(test.NewFailure("Failed to create document with key '%s' in collection '%s': %v", key, c.name, err)) + return "", maskAny(err) + } + t.createCounter.succeeded++ + t.log.Info().Msgf("Creating document '%s' in '%s' succeeded", key, c.name) + return m.Rev, nil +} diff --git a/tests/duration/simple/simple_import.go b/tests/duration/simple/simple_import.go new file mode 100644 index 000000000..02dcaeb8d --- /dev/null +++ b/tests/duration/simple/simple_import.go @@ -0,0 +1,79 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +import ( + "bytes" + "context" + "fmt" + + "github.com/arangodb/kube-arangodb/tests/duration/test" +) + +// createImportDocument creates a #document based import file. +func (t *simpleTest) createImportDocument() ([]byte, []UserDocument) { + buf := &bytes.Buffer{} + docs := make([]UserDocument, 0, 10000) + fmt.Fprintf(buf, `[ "_key", "value", "name", "odd" ]`) + fmt.Fprintln(buf) + for i := 0; i < 10000; i++ { + key := fmt.Sprintf("docimp%05d", i) + userDoc := UserDocument{ + Key: key, + Value: i, + Name: fmt.Sprintf("Imported %d", i), + Odd: i%2 == 0, + } + docs = append(docs, userDoc) + fmt.Fprintf(buf, `[ "%s", %d, "%s", %v ]`, userDoc.Key, userDoc.Value, userDoc.Name, userDoc.Odd) + fmt.Fprintln(buf) + } + return buf.Bytes(), docs +} + +// importDocuments imports a bulk set of documents. +// The operation is expected to succeed. +func (t *simpleTest) importDocuments(c *collection) error { + ctx := context.Background() + col, err := t.db.Collection(ctx, c.name) + if err != nil { + return maskAny(err) + } + _, docs := t.createImportDocument() + t.log.Info().Msgf("Importing %d documents ('%s' - '%s') into '%s'...", len(docs), docs[0].Key, docs[len(docs)-1].Key, c.name) + _, errs, err := col.CreateDocuments(ctx, docs) + if err != nil { + // This is a failure + t.importCounter.failed++ + t.reportFailure(test.NewFailure("Failed to import documents in collection '%s': %v", c.name, err)) + return maskAny(err) + } + for i, d := range docs { + if errs[i] == nil { + c.existingDocs[d.Key] = d + } + } + t.importCounter.succeeded++ + t.log.Info().Msgf("Importing %d documents ('%s' - '%s') into '%s' succeeded", len(docs), docs[0].Key, docs[len(docs)-1].Key, c.name) + return nil +} diff --git a/tests/duration/simple/simple_query.go b/tests/duration/simple/simple_query.go new file mode 100644 index 000000000..9757bb93e --- /dev/null +++ b/tests/duration/simple/simple_query.go @@ -0,0 +1,66 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +import ( + "context" + "fmt" + + driver "github.com/arangodb/go-driver" + + "github.com/arangodb/kube-arangodb/tests/duration/test" +) + +// queryDocumentsLongRunning runs a long running AQL query. +// The operation is expected to succeed. +func (t *simpleTest) queryDocumentsLongRunning(c *collection) error { + if len(c.existingDocs) < 10 { + t.log.Info().Msgf("Skipping query test, we need 10 or more documents") + return nil + } + + ctx := context.Background() + ctx = driver.WithQueryCount(ctx) + + t.log.Info().Msgf("Creating long running AQL query for '%s'...", c.name) + query := fmt.Sprintf("FOR d IN %s LIMIT 10 RETURN {d:d, s:SLEEP(2)}", c.name) + cursor, err := t.db.Query(ctx, query, nil) + if err != nil { + // This is a failure + t.queryLongRunningCounter.failed++ + t.reportFailure(test.NewFailure("Failed to create long running AQL cursor in collection '%s': %v", c.name, err)) + return maskAny(err) + } + cursor.Close() + resultCount := cursor.Count() + t.queryLongRunningCounter.succeeded++ + t.log.Info().Msgf("Creating long running AQL query for collection '%s' succeeded", c.name) + + // We should've fetched all documents, check result count + if resultCount != 10 { + t.reportFailure(test.NewFailure("Number of documents was %d, expected 10", resultCount)) + return maskAny(fmt.Errorf("Number of documents was %d, expected 10", resultCount)) + } + + return nil +} diff --git a/tests/duration/simple/simple_query_update.go b/tests/duration/simple/simple_query_update.go new file mode 100644 index 000000000..059d83001 --- /dev/null +++ b/tests/duration/simple/simple_query_update.go @@ -0,0 +1,115 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +import ( + "context" + "fmt" + "time" + + driver "github.com/arangodb/go-driver" + + "github.com/arangodb/kube-arangodb/tests/duration/test" +) + +// queryUpdateDocuments runs an AQL update query. +// The operation is expected to succeed. +func (t *simpleTest) queryUpdateDocuments(c *collection, key string) (string, error) { + ctx := context.Background() + ctx = driver.WithQueryCount(ctx) + + t.log.Info().Msgf("Creating update AQL query for collection '%s'...", c.name) + newName := fmt.Sprintf("AQLUpdate name %s", time.Now()) + query := fmt.Sprintf("UPDATE \"%s\" WITH { name: \"%s\" } IN %s RETURN NEW", key, newName, c.name) + cursor, err := t.db.Query(ctx, query, nil) + if err != nil { + // This is a failure + t.queryUpdateCounter.failed++ + t.reportFailure(test.NewFailure("Failed to create update AQL cursor in collection '%s': %v", c.name, err)) + return "", maskAny(err) + } + var resultDocument UserDocument + m, err := cursor.ReadDocument(ctx, &resultDocument) + if err != nil { + // This is a failure + t.queryUpdateCounter.failed++ + t.reportFailure(test.NewFailure("Failed to read document from cursor in collection '%s': %v", c.name, err)) + return "", maskAny(err) + } + resultCount := cursor.Count() + cursor.Close() + if resultCount != 1 { + // This is a failure + t.queryUpdateCounter.failed++ + t.reportFailure(test.NewFailure("Failed to create update AQL cursor in collection '%s': expected 1 result, got %d", c.name, resultCount)) + return "", maskAny(fmt.Errorf("Number of documents was %d, expected 1", resultCount)) + } + + // Update document + c.existingDocs[key] = resultDocument + t.queryUpdateCounter.succeeded++ + t.log.Info().Msgf("Creating update AQL query for collection '%s' succeeded", c.name) + + return m.Rev, nil +} + +// queryUpdateDocumentsLongRunning runs a long running AQL update query. +// The operation is expected to succeed. +func (t *simpleTest) queryUpdateDocumentsLongRunning(c *collection, key string) (string, error) { + ctx := context.Background() + ctx = driver.WithQueryCount(ctx) + + t.log.Info().Msgf("Creating long running update AQL query for collection '%s'...", c.name) + newName := fmt.Sprintf("AQLLongRunningUpdate name %s", time.Now()) + query := fmt.Sprintf("UPDATE \"%s\" WITH { name: \"%s\", unknown: SLEEP(15) } IN %s RETURN NEW", key, newName, c.name) + cursor, err := t.db.Query(ctx, query, nil) + if err != nil { + // This is a failure + t.queryUpdateLongRunningCounter.failed++ + t.reportFailure(test.NewFailure("Failed to create long running update AQL cursor in collection '%s': %v", c.name, err)) + return "", maskAny(err) + } + var resultDocument UserDocument + m, err := cursor.ReadDocument(ctx, &resultDocument) + if err != nil { + // This is a failure + t.queryUpdateCounter.failed++ + t.reportFailure(test.NewFailure("Failed to read document from cursor in collection '%s': %v", c.name, err)) + return "", maskAny(err) + } + resultCount := cursor.Count() + cursor.Close() + if resultCount != 1 { + // This is a failure + t.queryUpdateLongRunningCounter.failed++ + t.reportFailure(test.NewFailure("Failed to create long running update AQL cursor in collection '%s': expected 1 result, got %d", c.name, resultCount)) + return "", maskAny(fmt.Errorf("Number of documents was %d, expected 1", resultCount)) + } + + // Update document + c.existingDocs[key] = resultDocument + t.queryUpdateLongRunningCounter.succeeded++ + t.log.Info().Msgf("Creating long running update AQL query for collection '%s' succeeded", c.name) + + return m.Rev, nil +} diff --git a/tests/duration/simple/simple_read.go b/tests/duration/simple/simple_read.go new file mode 100644 index 000000000..131d0b15f --- /dev/null +++ b/tests/duration/simple/simple_read.go @@ -0,0 +1,88 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +import ( + "context" + "fmt" + + driver "github.com/arangodb/go-driver" + + "github.com/arangodb/kube-arangodb/tests/duration/test" +) + +// readExistingDocument reads an existing document with an optional explicit revision. +// The operation is expected to succeed. +func (t *simpleTest) readExistingDocument(c *collection, key, rev string, updateRevision, skipExpectedValueCheck bool) (string, error) { + ctx := context.Background() + var result UserDocument + col, err := t.db.Collection(ctx, c.name) + if err != nil { + return "", maskAny(err) + } + m, err := col.ReadDocument(ctx, key, &result) + if err != nil { + // This is a failure + t.readExistingCounter.failed++ + t.reportFailure(test.NewFailure("Failed to read existing document '%s' in collection '%s': %v", key, c.name, err)) + return "", maskAny(err) + } + // Compare document against expected document + if !skipExpectedValueCheck { + expected := c.existingDocs[key] + if result.Value != expected.Value || result.Name != expected.Name || result.Odd != expected.Odd { + // This is a failure + t.readExistingCounter.failed++ + t.reportFailure(test.NewFailure("Read existing document '%s' returned different values '%s': got %q expected %q", key, c.name, result, expected)) + return "", maskAny(fmt.Errorf("Read returned invalid values")) + } + } + if updateRevision { + // Store read document so we have the last revision + c.existingDocs[key] = result + } + t.readExistingCounter.succeeded++ + t.log.Info().Msgf("Reading existing document '%s' from '%s' succeeded", key, c.name) + return m.Rev, nil +} + +// readNonExistingDocument reads a non-existing document. +// The operation is expected to fail. +func (t *simpleTest) readNonExistingDocument(collectionName string, key string) error { + ctx := context.Background() + var result UserDocument + t.log.Info().Msgf("Reading non-existing document '%s' from '%s'...", key, collectionName) + col, err := t.db.Collection(ctx, collectionName) + if err != nil { + return maskAny(err) + } + if _, err := col.ReadDocument(ctx, key, &result); !driver.IsNotFound(err) { + // This is a failure + t.readNonExistingCounter.failed++ + t.reportFailure(test.NewFailure("Failed to read non-existing document '%s' in collection '%s': %v", key, collectionName, err)) + return maskAny(err) + } + t.readNonExistingCounter.succeeded++ + t.log.Info().Msgf("Reading non-existing document '%s' from '%s' succeeded", key, collectionName) + return nil +} diff --git a/tests/duration/simple/simple_rebalance.go b/tests/duration/simple/simple_rebalance.go new file mode 100644 index 000000000..330a32afa --- /dev/null +++ b/tests/duration/simple/simple_rebalance.go @@ -0,0 +1,40 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +// rebalanceShards attempts to rebalance shards over the existing servers. +// The operation is expected to succeed. +func (t *simpleTest) rebalanceShards() error { + /*opts := struct{}{} + operationTimeout, retryTimeout := t.OperationTimeout, t.RetryTimeout + t.log.Info().Msgf("Rebalancing shards...") + if _, err := t.client.Post("/_admin/cluster/rebalanceShards", nil, nil, opts, "", nil, []int{202}, []int{400, 403, 503}, operationTimeout, retryTimeout); err != nil { + // This is a failure + t.rebalanceShardsCounter.failed++ + t.reportFailure(test.NewFailure("Failed to rebalance shards: %v", err)) + return maskAny(err) + } + t.rebalanceShardsCounter.succeeded++ + t.log.Info().Msgf("Rebalancing shards succeeded")*/ + return nil +} diff --git a/tests/duration/simple/simple_remove.go b/tests/duration/simple/simple_remove.go new file mode 100644 index 000000000..17cd5e518 --- /dev/null +++ b/tests/duration/simple/simple_remove.go @@ -0,0 +1,71 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +import ( + "context" + + driver "github.com/arangodb/go-driver" + + "github.com/arangodb/kube-arangodb/tests/duration/test" +) + +// removeExistingDocument removes an existing document with an optional explicit revision. +// The operation is expected to succeed. +func (t *simpleTest) removeExistingDocument(collectionName string, key, rev string) error { + ctx := context.Background() + col, err := t.db.Collection(ctx, collectionName) + if err != nil { + return maskAny(err) + } + t.log.Info().Msgf("Removing existing document '%s' from '%s'...", key, collectionName) + if _, err := col.RemoveDocument(ctx, key); err != nil { + // This is a failure + t.deleteExistingCounter.failed++ + t.reportFailure(test.NewFailure("Failed to delete existing document '%s' in collection '%s': %v", key, collectionName, err)) + return maskAny(err) + } + t.deleteExistingCounter.succeeded++ + t.log.Info().Msgf("Removing existing document '%s' from '%s' succeeded", key, collectionName) + return nil +} + +// removeNonExistingDocument removes a non-existing document. +// The operation is expected to fail. +func (t *simpleTest) removeNonExistingDocument(collectionName string, key string) error { + ctx := context.Background() + col, err := t.db.Collection(ctx, collectionName) + if err != nil { + return maskAny(err) + } + t.log.Info().Msgf("Removing non-existing document '%s' from '%s'...", key, collectionName) + if _, err := col.RemoveDocument(ctx, key); !driver.IsNotFound(err) { + // This is a failure + t.deleteNonExistingCounter.failed++ + t.reportFailure(test.NewFailure("Failed to delete non-existing document '%s' in collection '%s': %v", key, collectionName, err)) + return maskAny(err) + } + t.deleteNonExistingCounter.succeeded++ + t.log.Info().Msgf("Removing non-existing document '%s' from '%s' succeeded", key, collectionName) + return nil +} diff --git a/tests/duration/simple/simple_replace.go b/tests/duration/simple/simple_replace.go new file mode 100644 index 000000000..852968d6f --- /dev/null +++ b/tests/duration/simple/simple_replace.go @@ -0,0 +1,92 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +import ( + "context" + "fmt" + "math/rand" + "time" + + driver "github.com/arangodb/go-driver" + + "github.com/arangodb/kube-arangodb/tests/duration/test" +) + +// replaceExistingDocument replaces an existing document with an optional explicit revision. +// The operation is expected to succeed. +func (t *simpleTest) replaceExistingDocument(c *collection, key, rev string) (string, error) { + ctx := context.Background() + col, err := t.db.Collection(ctx, c.name) + if err != nil { + return "", maskAny(err) + } + newName := fmt.Sprintf("Updated name %s", time.Now()) + t.log.Info().Msgf("Replacing existing document '%s' in '%s' (name -> '%s')...", key, c.name, newName) + newDoc := UserDocument{ + Key: key, + Name: fmt.Sprintf("Replaced named %s", key), + Value: rand.Int(), + Odd: rand.Int()%2 == 0, + } + m, err := col.ReplaceDocument(ctx, key, newDoc) + if err != nil { + // This is a failure + t.replaceExistingCounter.failed++ + t.reportFailure(test.NewFailure("Failed to replace existing document '%s' in collection '%s': %v", key, c.name, err)) + return "", maskAny(err) + } + // Update internal doc + newDoc.rev = m.Rev + c.existingDocs[key] = newDoc + t.replaceExistingCounter.succeeded++ + t.log.Info().Msgf("Replacing existing document '%s' in '%s' (name -> '%s') succeeded", key, c.name, newName) + return m.Rev, nil +} + +// replaceNonExistingDocument replaces a non-existing document. +// The operation is expected to fail. +func (t *simpleTest) replaceNonExistingDocument(collectionName string, key string) error { + ctx := context.Background() + col, err := t.db.Collection(ctx, collectionName) + if err != nil { + return maskAny(err) + } + newName := fmt.Sprintf("Updated non-existing name %s", time.Now()) + t.log.Info().Msgf("Replacing non-existing document '%s' in '%s' (name -> '%s')...", key, collectionName, newName) + newDoc := UserDocument{ + Key: key, + Name: fmt.Sprintf("Replaced named %s", key), + Value: rand.Int(), + Odd: rand.Int()%2 == 0, + } + if _, err := col.ReplaceDocument(ctx, key, newDoc); !driver.IsNotFound(err) { + // This is a failure + t.replaceNonExistingCounter.failed++ + t.reportFailure(test.NewFailure("Failed to replace non-existing document '%s' in collection '%s': %v", key, collectionName, err)) + return maskAny(err) + } + t.replaceNonExistingCounter.succeeded++ + t.log.Info().Msgf("Replacing non-existing document '%s' in '%s' (name -> '%s') succeeded", key, collectionName, newName) + return nil +} diff --git a/tests/duration/simple/simple_update.go b/tests/duration/simple/simple_update.go new file mode 100644 index 000000000..b8f3f9d63 --- /dev/null +++ b/tests/duration/simple/simple_update.go @@ -0,0 +1,87 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package simple + +import ( + "context" + "fmt" + "time" + + driver "github.com/arangodb/go-driver" + + "github.com/arangodb/kube-arangodb/tests/duration/test" +) + +// updateExistingDocument updates an existing document with an optional explicit revision. +// The operation is expected to succeed. +func (t *simpleTest) updateExistingDocument(c *collection, key, rev string) (string, error) { + ctx := context.Background() + col, err := t.db.Collection(ctx, c.name) + if err != nil { + return "", maskAny(err) + } + newName := fmt.Sprintf("Updated name %s", time.Now()) + t.log.Info().Msgf("Updating existing document '%s' in '%s' (name -> '%s')...", key, c.name, newName) + delta := map[string]interface{}{ + "name": newName, + } + doc := c.existingDocs[key] + m, err := col.UpdateDocument(ctx, key, delta) + if err != nil { + // This is a failure + t.updateExistingCounter.failed++ + t.reportFailure(test.NewFailure("Failed to update existing document '%s' in collection '%s': %v", key, c.name, err)) + return "", maskAny(err) + } + // Update internal doc + doc.Name = newName + doc.rev = m.Rev + c.existingDocs[key] = doc + t.updateExistingCounter.succeeded++ + t.log.Info().Msgf("Updating existing document '%s' in '%s' (name -> '%s') succeeded", key, c.name, newName) + return m.Rev, nil +} + +// updateNonExistingDocument updates a non-existing document. +// The operation is expected to fail. +func (t *simpleTest) updateNonExistingDocument(collectionName string, key string) error { + ctx := context.Background() + col, err := t.db.Collection(ctx, collectionName) + if err != nil { + return maskAny(err) + } + newName := fmt.Sprintf("Updated non-existing name %s", time.Now()) + t.log.Info().Msgf("Updating non-existing document '%s' in '%s' (name -> '%s')...", key, collectionName, newName) + delta := map[string]interface{}{ + "name": newName, + } + if _, err := col.UpdateDocument(ctx, key, delta); !driver.IsNotFound(err) { + // This is a failure + t.updateNonExistingCounter.failed++ + t.reportFailure(test.NewFailure("Failed to update non-existing document '%s' in collection '%s': %v", key, collectionName, err)) + return maskAny(err) + } + t.updateNonExistingCounter.succeeded++ + t.log.Info().Msgf("Updating non-existing document '%s' in '%s' (name -> '%s') succeeded", key, collectionName, newName) + return nil +} diff --git a/tests/duration/test/shuffle.go b/tests/duration/test/shuffle.go new file mode 100644 index 000000000..98892f167 --- /dev/null +++ b/tests/duration/test/shuffle.go @@ -0,0 +1,43 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package test + +import "math/rand" + +// A type, typically a collection, that satisfies shuffle.Interface can be +// shuffled by the routines in this package. +type Interface interface { + // Len is the number of elements in the collection. + Len() int + // Swap swaps the elements with indexes i and j. + Swap(i, j int) +} + +// Shuffle shuffles Data. +func Shuffle(data Interface) { + n := data.Len() + for i := n - 1; i >= 0; i-- { + j := rand.Intn(i + 1) + data.Swap(i, j) + } +} diff --git a/tests/duration/test/test.go b/tests/duration/test/test.go new file mode 100644 index 000000000..b99c5639e --- /dev/null +++ b/tests/duration/test/test.go @@ -0,0 +1,66 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package test + +import ( + "fmt" + + driver "github.com/arangodb/go-driver" +) + +type TestScript interface { + Start(client driver.Client, listener TestListener) error + Stop() error + Pause() error + Resume() error + Status() TestStatus +} + +type TestListener interface { + ReportFailure(Failure) +} + +type Counter struct { + Name string + Succeeded int + Failed int +} + +type TestStatus struct { + Active bool + Pausing bool + Failures int + Actions int + Counters []Counter + Messages []string +} + +type Failure struct { + Message string +} + +func NewFailure(msg string, args ...interface{}) Failure { + return Failure{ + Message: fmt.Sprintf(msg, args...), + } +} diff --git a/tests/duration/test_listener.go b/tests/duration/test_listener.go new file mode 100644 index 000000000..8e141fd26 --- /dev/null +++ b/tests/duration/test_listener.go @@ -0,0 +1,88 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package main + +import ( + "sync" + "time" + + "github.com/rs/zerolog" + + "github.com/arangodb/kube-arangodb/tests/duration/test" +) + +const ( + recentFailureTimeout = time.Hour // Disregard failures old than this timeout + requiredRecentFailureSpread = time.Minute * 5 // How far apart the first and last recent failure must be + requiredRecentFailures = 30 // At least so many recent failures are needed to fail the test +) + +type testListener struct { + mutex sync.Mutex + Log zerolog.Logger + FailedCallback func() + recentFailures []time.Time + failed bool +} + +var _ test.TestListener = &testListener{} + +// ReportFailure logs the given failure and keeps track of recent failure timestamps. +func (l *testListener) ReportFailure(f test.Failure) { + l.Log.Error().Msg(f.Message) + + // Remove all old recent failures + l.mutex.Lock() + defer l.mutex.Unlock() + for { + if len(l.recentFailures) == 0 { + break + } + isOld := l.recentFailures[0].Add(recentFailureTimeout).Before(time.Now()) + if isOld { + // Remove first entry + l.recentFailures = l.recentFailures[1:] + } else { + // First failure is not old, keep the list as is + break + } + } + l.recentFailures = append(l.recentFailures, time.Now()) + + // Detect failed state + if len(l.recentFailures) > requiredRecentFailures { + spread := l.recentFailures[len(l.recentFailures)-1].Sub(l.recentFailures[0]) + if spread > requiredRecentFailureSpread { + l.failed = true + if l.FailedCallback != nil { + l.FailedCallback() + } + } + } +} + +// IsFailed returns true when the number of recent failures +// has gone above the set maximum, false otherwise. +func (l *testListener) IsFailed() bool { + return l.failed +} diff --git a/tests/duration/test_loop.go b/tests/duration/test_loop.go new file mode 100644 index 000000000..acdbaaa9f --- /dev/null +++ b/tests/duration/test_loop.go @@ -0,0 +1,124 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package main + +import ( + "context" + "os" + "time" + + driver "github.com/arangodb/go-driver" + "github.com/rs/zerolog" + + "github.com/arangodb/kube-arangodb/tests/duration/simple" + t "github.com/arangodb/kube-arangodb/tests/duration/test" +) + +var ( + delayBeforeCompare = time.Minute + testPeriod = time.Minute * 2 + systemCollectionsToIgnore = map[string]bool{ + "_appbundles": true, + "_apps": true, + "_jobs": true, + "_queues": true, + "_routing": true, + "_statistics": true, + "_statisticsRaw": true, + "_statistics15": true, + } +) + +// runTestLoop keeps running tests until the given context is canceled. +func runTestLoop(ctx context.Context, client driver.Client, duration time.Duration) { + log := zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger() + endTime := time.Now().Add(duration) + reportDir := "." + tests := []t.TestScript{} + tests = append(tests, simple.NewSimpleTest(log, reportDir, simple.SimpleConfig{ + MaxDocuments: 500, + MaxCollections: 50, + })) + + log.Info().Msg("Starting tests") + listener := &testListener{ + Log: log, + FailedCallback: func() { + log.Fatal().Msg("Too many recent failures. Aborting test") + }, + } + for _, tst := range tests { + if err := tst.Start(client, listener); err != nil { + log.Fatal().Err(err).Msg("Failed to start test") + } + } + for { + if err := ctx.Err(); err != nil { + return + } + + // Check end time + if time.Now().After(endTime) { + log.Info().Msgf("Test has run for %s. We're done", duration) + return + } + + // Run tests + log.Info().Msg("Running tests...") + select { + case <-time.After(testPeriod): + // Continue + case <-ctx.Done(): + return + } + + // Pause tests + log.Info().Msg("Pause tests") + for _, tst := range tests { + if err := tst.Pause(); err != nil { + log.Fatal().Err(err).Msg("Failed to pause test") + } + } + + // Wait for tests to really pause + log.Info().Msg("Waiting for tests to reach pausing state") + for _, tst := range tests { + for !tst.Status().Pausing { + select { + case <-time.After(time.Second): + // Continue + case <-ctx.Done(): + return + } + } + } + + // Resume tests + log.Info().Msg("Resuming tests") + for _, tst := range tests { + if err := tst.Resume(); err != nil { + log.Fatal().Err(err).Msg("Failed to resume test") + } + } + } +} diff --git a/tests/persistent_volumes_test.go b/tests/persistent_volumes_test.go new file mode 100644 index 000000000..af0cb99ad --- /dev/null +++ b/tests/persistent_volumes_test.go @@ -0,0 +1,82 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Jan Christoph Uhde +// +package tests + +import ( + "fmt" + "strings" + "testing" + + "github.com/dchest/uniuri" + "github.com/stretchr/testify/assert" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + kubeArangoClient "github.com/arangodb/kube-arangodb/pkg/client" + //"github.com/arangodb/kube-arangodb/pkg/util" +) + +// TODO - add description +func TestPersistence(t *testing.T) { + longOrSkip(t) + + k8sNameSpace := getNamespace(t) + k8sClient := mustNewKubeClient(t) + + volumesList, err := k8sClient.CoreV1().PersistentVolumes().List(metav1.ListOptions{}) + assert.NoError(t, err, "error while listing volumes") + claimsList, err := k8sClient.CoreV1().PersistentVolumeClaims(k8sNameSpace).List(metav1.ListOptions{}) + assert.NoError(t, err, "error while listing volume claims") + + fmt.Printf("----------------------------------------") + fmt.Printf("%v %v", volumesList, claimsList) + fmt.Printf("----------------------------------------") + fmt.Printf("%v %v", len(volumesList.Items), len(claimsList.Items)) + fmt.Printf("----------------------------------------") + + mode := api.DeploymentModeCluster + engine := api.StorageEngineRocksDB + + deploymentClient := kubeArangoClient.MustNewInCluster() + deploymentTemplate := newDeployment(strings.Replace(fmt.Sprintf("tpers-%s-%s-%s", mode[:2], engine[:2], uniuri.NewLen(4)), ".", "", -1)) + deploymentTemplate.Spec.Mode = api.NewMode(mode) + deploymentTemplate.Spec.StorageEngine = api.NewStorageEngine(engine) + deploymentTemplate.Spec.TLS = api.TLSSpec{} + //deploymentTemplate.Spec.Environment = api.NewEnvironment(api.EnvironmentDevelopment) + //deploymentTemplate.Spec.Image = util.NewString("arangodb/arangodb:3.3.4") + //deploymentTemplate.Spec.DBServers.Count = util.NewInt(numNodes + 1) + deploymentTemplate.Spec.SetDefaults(deploymentTemplate.GetName()) // this must be last + assert.NoError(t, deploymentTemplate.Spec.Validate()) + + // Create deployment + _, err = deploymentClient.DatabaseV1alpha().ArangoDeployments(k8sNameSpace).Create(deploymentTemplate) + assert.NoError(t, err, "failed to create deplyment: %s", err) + + _, err = waitUntilDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace, deploymentIsReady()) + assert.NoError(t, err, fmt.Sprintf("Deployment not running in time: %s", err)) // <-- fails here at the moment + + // TODO - add tests that check the number of volumes and claims + + // Cleanup + removeDeployment(deploymentClient, deploymentTemplate.GetName(), k8sNameSpace) +} diff --git a/tests/resilience_test.go b/tests/resilience_test.go index 5144d7250..7958e25a8 100644 --- a/tests/resilience_test.go +++ b/tests/resilience_test.go @@ -166,8 +166,9 @@ func TestResiliencePVC(t *testing.T) { // Delete one pvc after the other apiObject.ForeachServerGroup(func(group api.ServerGroup, spec api.ServerGroupSpec, status *api.MemberStatusList) error { - if group == api.ServerGroupCoordinators { + if group != api.ServerGroupAgents { // Coordinators have no PVC + // DBServers will be cleaned out and create a new member return nil } for _, m := range *status { @@ -179,6 +180,10 @@ func TestResiliencePVC(t *testing.T) { if err := kubecli.CoreV1().PersistentVolumeClaims(ns).Delete(m.PersistentVolumeClaimName, &metav1.DeleteOptions{}); err != nil { t.Fatalf("Failed to delete pvc %s: %v", m.PersistentVolumeClaimName, err) } + // Now delete the pod as well, otherwise the PVC will only have a deletion timestamp but its finalizers will stay on. + if err := kubecli.CoreV1().Pods(ns).Delete(m.PodName, &metav1.DeleteOptions{}); err != nil { + t.Fatalf("Failed to delete pod %s: %v", m.PodName, err) + } // Wait for pvc to return with different UID op := func() error { pvc, err := kubecli.CoreV1().PersistentVolumeClaims(ns).Get(m.PersistentVolumeClaimName, metav1.GetOptions{}) diff --git a/tests/scale_test.go b/tests/scale_test.go index 6e8bc3079..38b1a0998 100644 --- a/tests/scale_test.go +++ b/tests/scale_test.go @@ -177,3 +177,105 @@ func TestScaleCluster(t *testing.T) { // Cleanup removeDeployment(c, depl.GetName(), ns) } + +// TestScaleClusterWithSync tests scaling a cluster deployment with sync enabled. +func TestScaleClusterWithSync(t *testing.T) { + longOrSkip(t) + img := getEnterpriseImageOrSkip(t) + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) + + // Prepare deployment config + depl := newDeployment("test-scale-sync" + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + depl.Spec.Image = util.NewString(img) + depl.Spec.Sync.Enabled = util.NewBool(true) + depl.Spec.SetDefaults(depl.GetName()) // this must be last + + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer deferedCleanupDeployment(c, depl.GetName(), ns) + + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } + + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) + + // Create a syncmaster client + syncClient := mustNewArangoSyncClient(ctx, kubecli, apiObject, t) + + // Wait for cluster to be completely ready + if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error { + return clusterHealthEqualsSpec(h, apiObject.Spec) + }); err != nil { + t.Fatalf("Cluster not running in expected health in time: %v", err) + } + + // Wait for syncmasters to be available + if err := waitUntilSyncVersionUp(syncClient, nil); err != nil { + t.Fatalf("SyncMasters not running returning version in time: %v", err) + } + + // Add 1 DBServer, 2 SyncMasters, 1 syncworker + updated, err := updateDeployment(c, depl.GetName(), ns, func(spec *api.DeploymentSpec) { + spec.DBServers.Count = util.NewInt(4) + spec.SyncMasters.Count = util.NewInt(5) + spec.SyncWorkers.Count = util.NewInt(4) + }) + if err != nil { + t.Fatalf("Failed to update deployment: %v", err) + } + + // Wait for cluster to reach new size + if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error { + return clusterHealthEqualsSpec(h, updated.Spec) + }); err != nil { + t.Fatalf("Cluster not running, after scale-up, in expected health in time: %v", err) + } + // Check number of syncmasters + if err := waitUntilSyncMasterCountReached(syncClient, updated.Spec.SyncMasters.GetCount()); err != nil { + t.Fatalf("Unexpected #syncmasters, after scale-up: %v", err) + } + // Check number of syncworkers + if err := waitUntilSyncWorkerCountReached(syncClient, updated.Spec.SyncWorkers.GetCount()); err != nil { + t.Fatalf("Unexpected #syncworkers, after scale-up: %v", err) + } + + // Remove 1 DBServer, 2 SyncMasters & 1 SyncWorker + updated, err = updateDeployment(c, depl.GetName(), ns, func(spec *api.DeploymentSpec) { + spec.DBServers.Count = util.NewInt(3) + spec.SyncMasters.Count = util.NewInt(3) + spec.SyncWorkers.Count = util.NewInt(3) + }) + if err != nil { + t.Fatalf("Failed to update deployment: %v", err) + } + + // Wait for cluster to reach new size + if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error { + return clusterHealthEqualsSpec(h, updated.Spec) + }); err != nil { + t.Fatalf("Cluster not running, after scale-down, in expected health in time: %v", err) + } + // Check number of syncmasters + if err := waitUntilSyncMasterCountReached(syncClient, updated.Spec.SyncMasters.GetCount()); err != nil { + t.Fatalf("Unexpected #syncmasters, after scale-up: %v", err) + } + // Check number of syncworkers + if err := waitUntilSyncWorkerCountReached(syncClient, updated.Spec.SyncWorkers.GetCount()); err != nil { + t.Fatalf("Unexpected #syncworkers, after scale-up: %v", err) + } + + // Cleanup + removeDeployment(c, depl.GetName(), ns) +} diff --git a/tests/service_account_test.go b/tests/service_account_test.go new file mode 100644 index 000000000..cd0379127 --- /dev/null +++ b/tests/service_account_test.go @@ -0,0 +1,292 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package tests + +import ( + "context" + "strings" + "testing" + + "github.com/dchest/uniuri" + "github.com/stretchr/testify/assert" + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + driver "github.com/arangodb/go-driver" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/client" + "github.com/arangodb/kube-arangodb/pkg/util" +) + +// TestServiceAccountSingle tests the creating of a single server deployment +// with default settings using a custom service account. +func TestServiceAccountSingle(t *testing.T) { + longOrSkip(t) + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) + + // Prepare service account + namePrefix := "test-sa-sng-" + saName := mustCreateServiceAccount(kubecli, namePrefix, ns, t) + defer deleteServiceAccount(kubecli, saName, ns) + + // Prepare deployment config + depl := newDeployment(namePrefix + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeSingle) + depl.Spec.Single.ServiceAccountName = util.NewString(saName) + + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer removeDeployment(c, depl.GetName(), ns) + + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } + + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) + + // Wait for single server available + if err := waitUntilVersionUp(client, nil); err != nil { + t.Fatalf("Single server not running returning version in time: %v", err) + } + + // Check service account name + checkMembersUsingServiceAccount(kubecli, ns, apiObject.Status.Members.Single, saName, t) + + // Check server role + assert.NoError(t, testServerRole(ctx, client, driver.ServerRoleSingle)) +} + +// TestServiceAccountActiveFailover tests the creating of a ActiveFailover server deployment +// with default settings using a custom service account. +func TestServiceAccountActiveFailover(t *testing.T) { + longOrSkip(t) + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) + + // Prepare service account + namePrefix := "test-sa-rs-" + saName := mustCreateServiceAccount(kubecli, namePrefix, ns, t) + defer deleteServiceAccount(kubecli, saName, ns) + + // Prepare deployment config + depl := newDeployment(namePrefix + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeActiveFailover) + depl.Spec.Single.ServiceAccountName = util.NewString(saName) + depl.Spec.Agents.ServiceAccountName = util.NewString(saName) + + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer removeDeployment(c, depl.GetName(), ns) + + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } + + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) + + // Wait for single server available + if err := waitUntilVersionUp(client, nil); err != nil { + t.Fatalf("ActiveFailover servers not running returning version in time: %v", err) + } + + // Check service account name + checkMembersUsingServiceAccount(kubecli, ns, apiObject.Status.Members.Single, saName, t) + checkMembersUsingServiceAccount(kubecli, ns, apiObject.Status.Members.Agents, saName, t) + + // Check server role + assert.NoError(t, testServerRole(ctx, client, driver.ServerRoleSingleActive)) +} + +// TestServiceAccountCluster tests the creating of a cluster deployment +// with default settings using a custom service account. +func TestServiceAccountCluster(t *testing.T) { + longOrSkip(t) + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) + + // Prepare service account + namePrefix := "test-sa-cls-" + saName := mustCreateServiceAccount(kubecli, namePrefix, ns, t) + defer deleteServiceAccount(kubecli, saName, ns) + + // Prepare deployment config + depl := newDeployment(namePrefix + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + depl.Spec.Agents.ServiceAccountName = util.NewString(saName) + depl.Spec.DBServers.ServiceAccountName = util.NewString(saName) + depl.Spec.Coordinators.ServiceAccountName = util.NewString(saName) + + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer removeDeployment(c, depl.GetName(), ns) + + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } + + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) + + // Wait for cluster to be available + if err := waitUntilVersionUp(client, nil); err != nil { + t.Fatalf("Cluster not running returning version in time: %v", err) + } + + // Check service account name + checkMembersUsingServiceAccount(kubecli, ns, apiObject.Status.Members.Agents, saName, t) + checkMembersUsingServiceAccount(kubecli, ns, apiObject.Status.Members.Coordinators, saName, t) + checkMembersUsingServiceAccount(kubecli, ns, apiObject.Status.Members.DBServers, saName, t) + + // Check server role + assert.NoError(t, testServerRole(ctx, client, driver.ServerRoleCoordinator)) +} + +// TestServiceAccountClusterWithSync tests the creating of a cluster deployment +// with default settings and sync enabled using a custom service account. +func TestServiceAccountClusterWithSync(t *testing.T) { + longOrSkip(t) + img := getEnterpriseImageOrSkip(t) + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) + + // Prepare service account + namePrefix := "test-sa-cls-sync-" + saName := mustCreateServiceAccount(kubecli, namePrefix, ns, t) + defer deleteServiceAccount(kubecli, saName, ns) + + // Prepare deployment config + depl := newDeployment(namePrefix + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + depl.Spec.Image = util.NewString(img) + depl.Spec.Sync.Enabled = util.NewBool(true) + depl.Spec.Agents.ServiceAccountName = util.NewString(saName) + depl.Spec.DBServers.ServiceAccountName = util.NewString(saName) + depl.Spec.Coordinators.ServiceAccountName = util.NewString(saName) + depl.Spec.SyncMasters.ServiceAccountName = util.NewString(saName) + depl.Spec.SyncWorkers.ServiceAccountName = util.NewString(saName) + + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer removeDeployment(c, depl.GetName(), ns) + + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } + + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) + + // Wait for cluster to be available + if err := waitUntilVersionUp(client, nil); err != nil { + t.Fatalf("Cluster not running returning version in time: %v", err) + } + + // Create a syncmaster client + syncClient := mustNewArangoSyncClient(ctx, kubecli, apiObject, t) + + // Wait for syncmasters to be available + if err := waitUntilSyncVersionUp(syncClient, nil); err != nil { + t.Fatalf("SyncMasters not running returning version in time: %v", err) + } + + // Check service account name + checkMembersUsingServiceAccount(kubecli, ns, apiObject.Status.Members.Agents, saName, t) + checkMembersUsingServiceAccount(kubecli, ns, apiObject.Status.Members.Coordinators, saName, t) + checkMembersUsingServiceAccount(kubecli, ns, apiObject.Status.Members.DBServers, saName, t) + checkMembersUsingServiceAccount(kubecli, ns, apiObject.Status.Members.SyncMasters, saName, t) + checkMembersUsingServiceAccount(kubecli, ns, apiObject.Status.Members.SyncWorkers, saName, t) + + // Check server role + assert.NoError(t, testServerRole(ctx, client, driver.ServerRoleCoordinator)) +} + +// mustCreateServiceAccount creates an empty service account with random name and returns +// its name. On error, the test is failed. +func mustCreateServiceAccount(kubecli kubernetes.Interface, namePrefix, ns string, t *testing.T) string { + s := v1.ServiceAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: strings.ToLower(namePrefix + uniuri.NewLen(4)), + }, + } + if _, err := kubecli.CoreV1().ServiceAccounts(ns).Create(&s); err != nil { + t.Fatalf("Failed to create service account: %v", err) + } + return s.GetName() +} + +// deleteServiceAccount deletes a service account with given name in given namespace. +func deleteServiceAccount(kubecli kubernetes.Interface, name, ns string) error { + if err := kubecli.CoreV1().ServiceAccounts(ns).Delete(name, &metav1.DeleteOptions{}); err != nil { + return maskAny(err) + } + return nil +} + +// checkMembersUsingServiceAccount checks the serviceAccountName of the pods of all members +// to ensure that is equal to the given serviceAccountName. +func checkMembersUsingServiceAccount(kubecli kubernetes.Interface, ns string, members []api.MemberStatus, serviceAccountName string, t *testing.T) { + pods := kubecli.CoreV1().Pods(ns) + for _, m := range members { + if p, err := pods.Get(m.PodName, metav1.GetOptions{}); err != nil { + t.Errorf("Failed to get pod for member '%s': %v", m.ID, err) + } else if p.Spec.ServiceAccountName != serviceAccountName { + t.Errorf("Expected pod '%s' to have serviceAccountName '%s', got '%s'", p.GetName(), serviceAccountName, p.Spec.ServiceAccountName) + } + } +} diff --git a/tests/simple_test.go b/tests/simple_test.go index 060cca7d1..ff5e20a48 100644 --- a/tests/simple_test.go +++ b/tests/simple_test.go @@ -32,6 +32,7 @@ import ( driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/client" + "github.com/arangodb/kube-arangodb/pkg/util" ) // TestSimpleSingle tests the creating of a single server deployment @@ -69,22 +70,19 @@ func TestSimpleSingle(t *testing.T) { } // Check server role - assert.NoError(t, client.SynchronizeEndpoints(ctx)) - role, err := client.ServerRole(ctx) - assert.NoError(t, err) - assert.Equal(t, driver.ServerRoleSingle, role) + assert.NoError(t, testServerRole(ctx, client, driver.ServerRoleSingle)) } -// TestSimpleResilientSingle tests the creating of a resilientsingle server deployment +// TestSimpleActiveFailover tests the creating of a ActiveFailover server deployment // with default settings. -func TestSimpleResilientSingle(t *testing.T) { +func TestSimpleActiveFailover(t *testing.T) { c := client.MustNewInCluster() kubecli := mustNewKubeClient(t) ns := getNamespace(t) // Prepare deployment config depl := newDeployment("test-rs-" + uniuri.NewLen(4)) - depl.Spec.Mode = api.NewMode(api.DeploymentModeResilientSingle) + depl.Spec.Mode = api.NewMode(api.DeploymentModeActiveFailover) // Create deployment _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) @@ -106,14 +104,11 @@ func TestSimpleResilientSingle(t *testing.T) { // Wait for single server available if err := waitUntilVersionUp(client, nil); err != nil { - t.Fatalf("ResilientSingle servers not running returning version in time: %v", err) + t.Fatalf("ActiveFailover servers not running returning version in time: %v", err) } // Check server role - assert.NoError(t, client.SynchronizeEndpoints(ctx)) - role, err := client.ServerRole(ctx) - assert.NoError(t, err) - assert.Equal(t, driver.ServerRoleSingleActive, role) + assert.NoError(t, testServerRole(ctx, client, driver.ServerRoleSingleActive)) } // TestSimpleCluster tests the creating of a cluster deployment @@ -145,14 +140,60 @@ func TestSimpleCluster(t *testing.T) { ctx := context.Background() client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) - // Wait for single server available + // Wait for cluster to be available + if err := waitUntilVersionUp(client, nil); err != nil { + t.Fatalf("Cluster not running returning version in time: %v", err) + } + + // Check server role + assert.NoError(t, testServerRole(ctx, client, driver.ServerRoleCoordinator)) +} + +// TestSimpleClusterWithSync tests the creating of a cluster deployment +// with default settings and sync enabled. +func TestSimpleClusterWithSync(t *testing.T) { + img := getEnterpriseImageOrSkip(t) + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) + + // Prepare deployment config + depl := newDeployment("test-cls-sync-" + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + depl.Spec.Image = util.NewString(img) + depl.Spec.Sync.Enabled = util.NewBool(true) + + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer removeDeployment(c, depl.GetName(), ns) + + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } + + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) + + // Wait for cluster to be available if err := waitUntilVersionUp(client, nil); err != nil { t.Fatalf("Cluster not running returning version in time: %v", err) } + // Create a syncmaster client + syncClient := mustNewArangoSyncClient(ctx, kubecli, apiObject, t) + + // Wait for syncmasters to be available + if err := waitUntilSyncVersionUp(syncClient, nil); err != nil { + t.Fatalf("SyncMasters not running returning version in time: %v", err) + } + // Check server role - assert.NoError(t, client.SynchronizeEndpoints(ctx)) - role, err := client.ServerRole(ctx) - assert.NoError(t, err) - assert.Equal(t, driver.ServerRoleCoordinator, role) + assert.NoError(t, testServerRole(ctx, client, driver.ServerRoleCoordinator)) } diff --git a/tests/sync_test.go b/tests/sync_test.go new file mode 100644 index 000000000..ec3ea4af4 --- /dev/null +++ b/tests/sync_test.go @@ -0,0 +1,136 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package tests + +import ( + "context" + "fmt" + "testing" + + "github.com/dchest/uniuri" + + driver "github.com/arangodb/go-driver" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/client" + "github.com/arangodb/kube-arangodb/pkg/util" +) + +// TestSyncToggleEnabled tests a normal cluster and enables sync later. +// Once sync is active, it is disabled again. +func TestSyncToggleEnabled(t *testing.T) { + longOrSkip(t) + img := getEnterpriseImageOrSkip(t) + c := client.MustNewInCluster() + kubecli := mustNewKubeClient(t) + ns := getNamespace(t) + + // Prepare deployment config + depl := newDeployment("test-sync-toggle-" + uniuri.NewLen(4)) + depl.Spec.Mode = api.NewMode(api.DeploymentModeCluster) + depl.Spec.Image = util.NewString(img) + + // Create deployment + _, err := c.DatabaseV1alpha().ArangoDeployments(ns).Create(depl) + if err != nil { + t.Fatalf("Create deployment failed: %v", err) + } + // Prepare cleanup + defer deferedCleanupDeployment(c, depl.GetName(), ns) + + // Wait for deployment to be ready + apiObject, err := waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) + if err != nil { + t.Fatalf("Deployment not running in time: %v", err) + } + + // Create a database client + ctx := context.Background() + client := mustNewArangodDatabaseClient(ctx, kubecli, apiObject, t) + + // Wait for cluster to be completely ready + if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error { + return clusterHealthEqualsSpec(h, apiObject.Spec) + }); err != nil { + t.Fatalf("Cluster not running in expected health in time: %v", err) + } + + // Enable sync + updated, err := updateDeployment(c, depl.GetName(), ns, func(spec *api.DeploymentSpec) { + spec.Sync.Enabled = util.NewBool(true) + }) + if err != nil { + t.Fatalf("Failed to update deployment: %v", err) + } + + // Wait until sync jwt secret has been created + if _, err := waitUntilSecret(kubecli, updated.Spec.Sync.Authentication.GetJWTSecretName(), ns, nil, deploymentReadyTimeout); err != nil { + t.Fatalf("Sync JWT secret not created in time: %v", err) + } + + // Create a syncmaster client + syncClient := mustNewArangoSyncClient(ctx, kubecli, apiObject, t) + + // Wait for syncmasters to be available + if err := waitUntilSyncVersionUp(syncClient, nil); err != nil { + t.Fatalf("SyncMasters not running returning version in time: %v", err) + } + + // Wait for cluster to reach new size + if err := waitUntilClusterHealth(client, func(h driver.ClusterHealth) error { + return clusterHealthEqualsSpec(h, updated.Spec) + }); err != nil { + t.Fatalf("Cluster not running, after scale-up, in expected health in time: %v", err) + } + // Check number of syncmasters + if err := waitUntilSyncMasterCountReached(syncClient, 3); err != nil { + t.Fatalf("Unexpected #syncmasters, after enabling sync: %v", err) + } + // Check number of syncworkers + if err := waitUntilSyncWorkerCountReached(syncClient, 3); err != nil { + t.Fatalf("Unexpected #syncworkers, after enabling sync: %v", err) + } + + // Disable sync + updated, err = updateDeployment(c, depl.GetName(), ns, func(spec *api.DeploymentSpec) { + spec.Sync.Enabled = util.NewBool(false) + }) + if err != nil { + t.Fatalf("Failed to update deployment: %v", err) + } + + // Wait for deployment to have no more syncmasters & workers + if _, err := waitUntilDeployment(c, depl.GetName(), ns, func(apiObject *api.ArangoDeployment) error { + if cnt := len(apiObject.Status.Members.SyncMasters); cnt > 0 { + return maskAny(fmt.Errorf("Expected 0 syncmasters, got %d", cnt)) + } + if cnt := len(apiObject.Status.Members.SyncWorkers); cnt > 0 { + return maskAny(fmt.Errorf("Expected 0 syncworkers, got %d", cnt)) + } + return nil + }); err != nil { + t.Fatalf("Failed to reach deployment state without syncmasters & syncworkers: %v", err) + } + + // Cleanup + removeDeployment(c, depl.GetName(), ns) +} diff --git a/tests/test_util.go b/tests/test_util.go index f13c262c8..3922b9a7c 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -25,18 +25,24 @@ package tests import ( "context" "fmt" + "net" "os" + "strconv" "strings" + "sync" "testing" "time" "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" + "github.com/arangodb/arangosync/client" + "github.com/arangodb/arangosync/tasks" + driver "github.com/arangodb/go-driver" "github.com/pkg/errors" + "github.com/rs/zerolog" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" "github.com/arangodb/kube-arangodb/pkg/util/arangod" @@ -45,11 +51,13 @@ import ( ) const ( - deploymentReadyTimeout = time.Minute * 2 + deploymentReadyTimeout = time.Minute * 4 ) var ( - maskAny = errors.WithStack + maskAny = errors.WithStack + syncClientCache client.ClientCache + showEnterpriseImageOnce sync.Once ) // longOrSkip checks the short test flag. @@ -64,9 +72,13 @@ func longOrSkip(t *testing.T) { // getEnterpriseImageOrSkip returns the docker image used for enterprise // tests. If empty, enterprise tests are skipped. func getEnterpriseImageOrSkip(t *testing.T) string { - image := os.Getenv("ENTERPRISEIMAGE") + image := strings.TrimSpace(os.Getenv("ENTERPRISEIMAGE")) if image == "" { t.Skip("Skipping test because ENTERPRISEIMAGE is not set") + } else { + showEnterpriseImageOnce.Do(func() { + t.Logf("Using enterprise image: %s", image) + }) } return image } @@ -97,6 +109,32 @@ func mustNewArangodDatabaseClient(ctx context.Context, kubecli kubernetes.Interf return c } +// mustNewArangoSyncClient creates a new arangosync client, with all syncmasters +// as endpoint. It is failing the test on errors. +func mustNewArangoSyncClient(ctx context.Context, kubecli kubernetes.Interface, apiObject *api.ArangoDeployment, t *testing.T) client.API { + ns := apiObject.GetNamespace() + secretName := apiObject.Spec.Sync.Authentication.GetJWTSecretName() + jwtToken, err := k8sutil.GetTokenSecret(kubecli.CoreV1(), secretName, ns) + if err != nil { + t.Fatalf("Failed to get sync jwt secret '%s': %s", secretName, err) + } + + // Fetch service DNS name + dnsName := k8sutil.CreateSyncMasterClientServiceDNSName(apiObject) + ep := client.Endpoint{"https://" + net.JoinHostPort(dnsName, strconv.Itoa(k8sutil.ArangoSyncMasterPort))} + + // Build client + log := zerolog.Logger{} + tlsAuth := tasks.TLSAuthentication{} + auth := client.NewAuthentication(tlsAuth, jwtToken) + insecureSkipVerify := true + c, err := syncClientCache.GetClient(log, ep, auth, insecureSkipVerify) + if err != nil { + t.Fatalf("Failed to get sync client: %s", err) + } + return c +} + // getNamespace returns the kubernetes namespace in which to run tests. func getNamespace(t *testing.T) string { ns := os.Getenv("TEST_NAMESPACE") @@ -247,6 +285,70 @@ func waitUntilVersionUp(cli driver.Client, predicate func(driver.VersionInfo) er return nil } +// waitUntilSyncVersionUp waits until the syncmasters responds to +// an `/_api/version` request without an error. An additional Predicate +// can do a check on the VersionInfo object returned by the server. +func waitUntilSyncVersionUp(cli client.API, predicate func(client.VersionInfo) error) error { + ctx := context.Background() + + op := func() error { + if version, err := cli.Version(ctx); err != nil { + return maskAny(err) + } else if predicate != nil { + return predicate(version) + } + return nil + } + + if err := retry.Retry(op, deploymentReadyTimeout); err != nil { + return maskAny(err) + } + + return nil +} + +// waitUntilSyncMasterCountReached waits until the number of syncmasters +// is equal to the given number. +func waitUntilSyncMasterCountReached(cli client.API, expectedSyncMasters int) error { + ctx := context.Background() + + op := func() error { + if list, err := cli.Master().Masters(ctx); err != nil { + return maskAny(err) + } else if len(list) != expectedSyncMasters { + return maskAny(fmt.Errorf("Expected %d syncmasters, got %d", expectedSyncMasters, len(list))) + } + return nil + } + + if err := retry.Retry(op, deploymentReadyTimeout); err != nil { + return maskAny(err) + } + + return nil +} + +// waitUntilSyncWorkerCountReached waits until the number of syncworkers +// is equal to the given number. +func waitUntilSyncWorkerCountReached(cli client.API, expectedSyncWorkers int) error { + ctx := context.Background() + + op := func() error { + if list, err := cli.Master().RegisteredWorkers(ctx); err != nil { + return maskAny(err) + } else if len(list) != expectedSyncWorkers { + return maskAny(fmt.Errorf("Expected %d syncworkers, got %d", expectedSyncWorkers, len(list))) + } + return nil + } + + if err := retry.Retry(op, deploymentReadyTimeout); err != nil { + return maskAny(err) + } + + return nil +} + // creates predicate to be used in waitUntilVersionUp func createEqualVersionsPredicate(version driver.Version) func(driver.VersionInfo) error { return func(infoFromServer driver.VersionInfo) error { @@ -353,7 +455,7 @@ func waitUntilArangoDeploymentHealthy(deployment *api.ArangoDeployment, DBClient if err := waitUntilVersionUp(DBClient, checkVersionPredicate); err != nil { return maskAny(fmt.Errorf("Single Server not running in time: %s", err)) } - case api.DeploymentModeResilientSingle: + case api.DeploymentModeActiveFailover: if err := waitUntilVersionUp(DBClient, checkVersionPredicate); err != nil { return maskAny(fmt.Errorf("Single Server not running in time: %s", err)) } @@ -404,7 +506,30 @@ func waitUntilArangoDeploymentHealthy(deployment *api.ArangoDeployment, DBClient } } default: - return maskAny(fmt.Errorf("DeploymentMode %s is not supported!", mode)) + return maskAny(fmt.Errorf("DeploymentMode %s is not supported", mode)) + } + return nil +} + +// testServerRole performs a synchronize endpoints and then requests the server role. +// On success, the role is compared with the given expected role. +// When the requests fail or the role is not equal to the expected role, an error is returned. +func testServerRole(ctx context.Context, client driver.Client, expectedRole driver.ServerRole) error { + op := func(ctx context.Context) error { + if err := client.SynchronizeEndpoints(ctx); err != nil { + return maskAny(err) + } + role, err := client.ServerRole(ctx) + if err != nil { + return maskAny(err) + } + if role != expectedRole { + return retry.Permanent(fmt.Errorf("Unexpected server role: Expected '%s', got '%s'", expectedRole, role)) + } + return nil + } + if err := retry.RetryWithContext(ctx, op, time.Second*20); err != nil { + return maskAny(err) } return nil } diff --git a/tests/upgrade_test.go b/tests/upgrade_test.go index 660397ee8..f97152fe3 100644 --- a/tests/upgrade_test.go +++ b/tests/upgrade_test.go @@ -44,14 +44,14 @@ func TestUpgradeSingleMMFiles32to33(t *testing.T) { // upgradeSubTest(t, api.DeploymentModeSingle, api.StorageEngineRocksDB, "3.3.4", "3.4.0") // } -/*// test upgrade resilient single server rocksdb 3.3 -> 3.4 -func TestUpgradeResilientSingleRocksDB33to34(t *testing.T) { - upgradeSubTest(t, api.DeploymentModeResilientSingle, api.StorageEngineRocksDB, "3.3.5", "3.4.0") +/*// test upgrade active-failover server rocksdb 3.3 -> 3.4 +func TestUpgradeActiveFailoverRocksDB33to34(t *testing.T) { + upgradeSubTest(t, api.DeploymentModeActiveFailover, api.StorageEngineRocksDB, "3.3.5", "3.4.0") }*/ -// // test upgrade resilient single server mmfiles 3.3 -> 3.4 -// func TestUpgradeResilientSingleMMFiles33to34(t *testing.T) { -// upgradeSubTest(t, api.DeploymentModeResilientSingle, api.StorageEngineMMFiles, "3.3.0", "3.4.0") +// // test upgrade active-failover server mmfiles 3.3 -> 3.4 +// func TestUpgradeActiveFailoverMMFiles33to34(t *testing.T) { +// upgradeSubTest(t, api.DeploymentModeActiveFailover, api.StorageEngineMMFiles, "3.3.0", "3.4.0") // } // test upgrade cluster rocksdb 3.2 -> 3.3 @@ -69,9 +69,9 @@ func TestDowngradeSingleMMFiles333to332(t *testing.T) { upgradeSubTest(t, api.DeploymentModeSingle, api.StorageEngineMMFiles, "3.3.3", "3.3.2") } -// test downgrade resilient single server rocksdb 3.3.3 -> 3.3.2 -func TestDowngradeResilientSingleRocksDB333to332(t *testing.T) { - upgradeSubTest(t, api.DeploymentModeResilientSingle, api.StorageEngineRocksDB, "3.3.3", "3.3.2") +// test downgrade ActiveFailover server rocksdb 3.3.3 -> 3.3.2 +func TestDowngradeActiveFailoverRocksDB333to332(t *testing.T) { + upgradeSubTest(t, api.DeploymentModeActiveFailover, api.StorageEngineRocksDB, "3.3.3", "3.3.2") } // test downgrade cluster rocksdb 3.3.3 -> 3.3.2 diff --git a/tools/manifests/manifest_builder.go b/tools/manifests/manifest_builder.go index 06c40f010..1a1c5467c 100644 --- a/tools/manifests/manifest_builder.go +++ b/tools/manifests/manifest_builder.go @@ -41,19 +41,24 @@ var ( OutputSuffix string TemplatesDir string - Namespace string - Image string - ImagePullPolicy string - ImageSHA256 bool - DeploymentOperatorName string - StorageOperatorName string - RBAC bool - AllowChaos bool + Namespace string + Image string + ImagePullPolicy string + ImageSHA256 bool + DeploymentOperatorName string + DeploymentReplicationOperatorName string + StorageOperatorName string + RBAC bool + AllowChaos bool } deploymentTemplateNames = []string{ "rbac.yaml", "deployment.yaml", } + deploymentReplicationTemplateNames = []string{ + "rbac.yaml", + "deployment-replication.yaml", + } storageTemplateNames = []string{ "rbac.yaml", "deployment.yaml", @@ -71,6 +76,7 @@ func init() { pflag.StringVar(&options.ImagePullPolicy, "image-pull-policy", "IfNotPresent", "Pull policy of the ArangoDB operator image") pflag.BoolVar(&options.ImageSHA256, "image-sha256", true, "Use SHA256 syntax for image") pflag.StringVar(&options.DeploymentOperatorName, "deployment-operator-name", "arango-deployment-operator", "Name of the ArangoDeployment operator deployment") + pflag.StringVar(&options.DeploymentReplicationOperatorName, "deployment-replication-operator-name", "arango-deployment-replication-operator", "Name of the ArangoDeploymentReplication operator deployment") pflag.StringVar(&options.StorageOperatorName, "storage-operator-name", "arango-storage-operator", "Name of the ArangoLocalStorage operator deployment") pflag.BoolVar(&options.RBAC, "rbac", true, "Use role based access control") pflag.BoolVar(&options.AllowChaos, "allow-chaos", false, "If set, allows chaos in deployments") @@ -79,12 +85,13 @@ func init() { } type TemplateOptions struct { - Image string - ImagePullPolicy string - RBAC bool - Deployment ResourceOptions - Storage ResourceOptions - Test CommonOptions + Image string + ImagePullPolicy string + RBAC bool + Deployment ResourceOptions + DeploymentReplication ResourceOptions + Storage ResourceOptions + Test CommonOptions } type CommonOptions struct { @@ -128,9 +135,10 @@ func main() { // Prepare templates to include templateNameSet := map[string][]string{ - "deployment": deploymentTemplateNames, - "storage": storageTemplateNames, - "test": testTemplateNames, + "deployment": deploymentTemplateNames, + "deployment-replication": deploymentReplicationTemplateNames, + "storage": storageTemplateNames, + "test": testTemplateNames, } // Process templates @@ -154,6 +162,21 @@ func main() { OperatorDeploymentName: "arango-deployment-operator", AllowChaos: options.AllowChaos, }, + DeploymentReplication: ResourceOptions{ + User: CommonOptions{ + Namespace: options.Namespace, + RoleName: "arango-deployment-replications", + RoleBindingName: "arango-deployment-replications", + ServiceAccountName: "default", + }, + Operator: CommonOptions{ + Namespace: options.Namespace, + RoleName: "arango-deployment-replication-operator", + RoleBindingName: "arango-deployment-replication-operator", + ServiceAccountName: "default", + }, + OperatorDeploymentName: "arango-deployment-replication-operator", + }, Storage: ResourceOptions{ User: CommonOptions{ Namespace: options.Namespace,