feat(engine, ui): view and edit DLE config for logical jobs via UI (#359)

* add a mechanism to read and update configuration file using UI
* validate configuration before applying
* tests DB source connection by passed parameters
* reload DLE configs after updating the DLE file
* store backups of the config file before reloading the configuration (10, by default)
* run retrieval jobs to create a snapshot after the successful configuration in UI
* manage state of not configured retrieval (#408)

Author: yuridevx

.gitignore

@@ -13,3 +13,5 @@ engine/bin/
 /engine/configs/ci_checker.yml
 /packer/example.com.key
 cloudformation/output/*.yaml
+
+engine/meta

cloudformation/dle_cf_template.yaml

@@ -404,6 +404,10 @@ Resources:
           dle_config_path="/home/ubuntu/.dblab/engine/configs"
           dle_meta_path="/home/ubuntu/.dblab/engine/meta"
           postgres_conf_path="/home/ubuntu/.dblab/postgres_conf"
+          
+          # Create a special marker file to identify that the DLE is running as a "managed" instance
+          # (e.g., launched using Marketplace or Terraform), and has not yet been configured.
+          touch $dle_meta_path/pending.retrieval
 
           yq e -i '
           .global.debug=${DLEDebugMode} |

engine/.dockerignore

@@ -0,0 +1 @@
+meta/

engine/.golangci.yml

@@ -25,7 +25,7 @@ linters-settings:
     lang-version: "1.17"
     extra-rules: false
   gosimple:
-    go: "1.17"
+    go: "1.18"
     checks: [ "all" ]
   goimports:
     local-prefixes: gitlab.com/postgres-ai/database-lab

engine/Makefile

@@ -5,6 +5,8 @@ RUN_CI_BINARY = run-ci
 CLI_BINARY = dblab
 GOARCH = amd64
 
+PWD= $(shell pwd)
+
 COMMIT?=$(shell git rev-parse HEAD)
 BUILD_TIME?=$(shell date -u '+%Y%m%d-%H%M')
 VERSION=$(shell git describe --tags --match "v*" 2>/dev/null || echo "${COMMIT}")
@@ -32,7 +34,7 @@ help: ## Display the help message
 all: clean build ## Build all binary components of the project
 
 install-lint: ## Install the linter to $GOPATH/bin which is expected to be in $PATH
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.45.0
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.45.2
 
 run-lint: ## Run linters
 	golangci-lint run
@@ -77,4 +79,20 @@ fmt: ## Format code
 clean: ## Remove compiled binaries from the local bin/ directory
 	rm -f bin/*
 
-.PHONY: help all build test run-lint install-lint lint fmt clean build-image build-dle build-ci-checker build-client build-ci-checker
+run-dle: build-dle
+	docker run \
+	--rm \
+	--volume /tmp:/tmp \
+	--volume /var/run/docker.sock:/var/run/docker.sock \
+	--volume /var/lib/dblab:/var/lib/dblab/:rshared \
+	--volume /var/lib/dblab/dblab_pool/dump:/var/lib/dblab/dblab_pool/dump \
+	--volume /sys/kernel/debug:/sys/kernel/debug:rw \
+	--volume /lib/modules:/lib/modules:ro \
+    --volume $(PWD)/configs:/home/dblab/configs:ro \
+    --volume $(PWD)/meta:/home/dblab/meta \
+	--volume /proc:/host_proc:ro \
+	--env DOCKER_API_VERSION=1.39 \
+	-p "2345:2345" \
+	dblab_server:local
+
+.PHONY: help all build test run-lint install-lint lint fmt clean build-image build-dle build-ci-checker build-client build-ci-checker run-dle

engine/api/swagger-spec/dblab_server_swagger.yaml

@@ -486,6 +486,112 @@ paths:
             $ref: "#/definitions/Error"
 
 
+  /admin/config:
+    post:
+      tags:
+        - "config"
+      summary: "Set instance configuration"
+      description: ""
+      operationId: "setConfig"
+      consumes:
+        - "application/json"
+      produces:
+        - "application/json"
+      parameters:
+        - in: header
+          name: Verification-Token
+          type: string
+          required: true
+        - in: body
+          name: body
+          description: "Set configuration object"
+          required: true
+          schema:
+            $ref: '#/definitions/Config'
+      responses:
+        200:
+          description: "Successful operation"
+          schema:
+            $ref: "#/definitions/Config"
+        500:
+          description: "Internal server error"
+          schema:
+            $ref: "#/definitions/Error"
+    get:
+      tags:
+        - "config"
+      summary: "Get instance configuration"
+      description: ""
+      operationId: "getConfig"
+      produces:
+        - "application/json"
+      parameters:
+        - in: header
+          name: Verification-Token
+          type: string
+          required: true
+      responses:
+        200:
+          description: "Successful operation"
+          schema:
+            $ref: "#/definitions/Config"
+        500:
+          description: "Internal server error"
+          schema:
+            $ref: "#/definitions/Error"
+
+  /admin/config.yaml:
+    get:
+      tags:
+        - "config"
+      summary: "Get the config of the instance"
+      description: ""
+      operationId: "getConfigYaml"
+      produces:
+        - "application/yaml"
+      parameters:
+        - in: header
+          name: Verification-Token
+          type: string
+          required: true
+      responses:
+        200:
+          description: "Successful operation"
+          schema:
+            $ref: "#/definitions/Config"
+        500:
+          description: "Internal server error"
+          schema:
+            $ref: "#/definitions/Error"
+
+  /admin/test-db-source:
+    post:
+      tags:
+        - "config"
+      summary: "Test source database"
+      description: ""
+      operationId: "testDBConnection"
+      consumes:
+        - "application/json"
+      parameters:
+        - in: header
+          name: Verification-Token
+          type: string
+          required: true
+        - in: body
+          name: body
+          description: "Connection DB object"
+          required: true
+          schema:
+            $ref: '#/definitions/Connection'
+      responses:
+        200:
+          description: "Successful operation"
+        500:
+          description: "Internal server error"
+          schema:
+            $ref: "#/definitions/Error"
+
 definitions:
   Instance:
     type: "object"
@@ -878,6 +984,24 @@ definitions:
       hint:
         type: "string"
 
+  Config:
+    type: object
+
+  Connection:
+    type: "object"
+    properties:
+      host:
+        type: "string"
+      port:
+        type: "string"
+      dbname:
+        type: "string"
+      username:
+        type: "string"
+      password:
+        type: "string"
+
+
 externalDocs:
   description: "Database Lab Docs"
   url: "https://gitlab.com/postgres-ai/docs/tree/master/docs/database-lab"

engine/cmd/database-lab/main.go

@@ -54,6 +54,8 @@ func main() {
 		log.Fatal(errors.WithMessage(err, "failed to parse config"))
 	}
 
+	config.ApplyGlobals(cfg)
+
 	docker, err := client.NewClientWithOpts(client.FromEnv)
 	if err != nil {
 		log.Fatal("Failed to create a Docker client:", err)
@@ -163,12 +165,29 @@ func main() {
 	})
 
 	embeddedUI := embeddedui.New(cfg.EmbeddedUI, engProps, runner, docker)
-	server := srv.NewServer(&cfg.Server, &cfg.Global, engProps, docker, cloningSvc, provisioner, retrievalSvc, platformSvc,
-		obs, est, pm, tm, tokenHolder, embeddedUI)
-	shutdownCh := setShutdownListener()
 
 	logCleaner := diagnostic.NewLogCleaner()
 
+	reloadConfigFn := func(server *srv.Server) error {
+		return reloadConfig(
+			ctx,
+			provisioner,
+			tm,
+			retrievalSvc,
+			pm,
+			cloningSvc,
+			platformSvc,
+			est,
+			embeddedUI,
+			server,
+			logCleaner,
+		)
+	}
+
+	server := srv.NewServer(&cfg.Server, &cfg.Global, engProps, docker, cloningSvc, provisioner, retrievalSvc, platformSvc,
+		obs, est, pm, tm, tokenHolder, embeddedUI, reloadConfigFn)
+	shutdownCh := setShutdownListener()
+
 	go setReloadListener(ctx, provisioner, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server, logCleaner)
 
 	server.InitHandlers()
@@ -256,6 +275,8 @@ func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *
 		return err
 	}
 
+	config.ApplyGlobals(cfg)
+
 	if err := provision.IsValidConfig(cfg.Provision); err != nil {
 		return err
 	}

engine/configs/config.example.logical_generic.yml

@@ -219,7 +219,7 @@ retrieval:
         # Option for specifying the database list that must be copied.
         # By default, DLE dumps and restores all available databases.
         # Do not specify the databases section to take all databases.
-        # databases:
+        databases:
         #   database1:
         # Options for a partial dump.
         # Do not specify the tables section to dump all available tables.

engine/configs/config.example.logical_rds_iam.yml

@@ -221,7 +221,7 @@ retrieval:
         # Option for specifying the database list that must be copied.
         # By default, DLE dumps and restores all available databases.
         # Do not specify the databases section to take all databases.
-        # databases:
+        databases:
         #   database1:
         #     # Option for a partial dump. Do not specify the tables section to dump all available tables.
         #     tables:

engine/go.mod

@@ -1,6 +1,6 @@
 module gitlab.com/postgres-ai/database-lab/v3
 
-go 1.17
+go 1.18
 
 require (
 	github.com/AlekSi/pointer v1.1.0
@@ -33,6 +33,7 @@ require (
 	golang.org/x/mod v0.5.1
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
 	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
 
 require (
@@ -77,7 +78,6 @@ require (
 	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
 	google.golang.org/grpc v1.43.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
 
 // Include the single version of the dependency to clean up go.sum from old revisions.

engine/internal/retrieval/engine/postgres/logical/dump.go

@@ -397,7 +397,7 @@ func (d *DumpJob) Run(ctx context.Context) (err error) {
 			return errors.Wrap(err, "failed to recalculate statistics after restore")
 		}
 
-		if err := tools.RunCheckpoint(ctx, d.dockerClient, containerID, d.globalCfg.Database.User(), d.globalCfg.Database.DBName); err != nil {
+		if err := tools.RunCheckpoint(ctx, d.dockerClient, containerID, d.globalCfg.Database.User(), d.globalCfg.Database.Name()); err != nil {
 			return errors.Wrap(err, "failed to run checkpoint before stop")
 		}
 

engine/internal/retrieval/engine/postgres/logical/restore.go

@@ -286,7 +286,7 @@ func (r *RestoreJob) Run(ctx context.Context) (err error) {
 		return errors.Wrap(err, "failed to recalculate statistics after restore")
 	}
 
-	if err := tools.RunCheckpoint(ctx, r.dockerClient, containerID, r.globalCfg.Database.User(), r.globalCfg.Database.DBName); err != nil {
+	if err := tools.RunCheckpoint(ctx, r.dockerClient, containerID, r.globalCfg.Database.User(), r.globalCfg.Database.Name()); err != nil {
 		return errors.Wrap(err, "failed to run checkpoint before stop")
 	}
 

engine/internal/retrieval/engine/postgres/tools/cont/container.go

@@ -57,6 +57,8 @@ const (
 	DBLabRestoreLabel = "dblab_restore"
 	// DBLabEmbeddedUILabel defines a label value for embedded UI containers.
 	DBLabEmbeddedUILabel = "dblab_embedded_ui"
+	// DBLabFoundationLabel defines a label value to mark foundation containers.
+	DBLabFoundationLabel = "dblab_foundation"
 
 	// DBLabRunner defines a label to mark runner containers.
 	DBLabRunner = "dblab_runner"