Merge branch '240-run-ci' into 'master'

feat: provide service to automatically validate DB migrations in the CI pipeline (postgres-ai#240)

Closes postgres-ai#240

See merge request postgres-ai/database-lab!275

Author: agneum

.gitignore

@@ -8,3 +8,4 @@
 /deploy/
 
 /configs/config.yml
+/configs/run_ci.yaml

.gitlab-ci.yml

@@ -124,6 +124,17 @@ build-image-feature-server:
     DOCKER_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab-server"
     TAGS: "${DOCKER_NAME}:${CI_COMMIT_REF_SLUG}"
 
+build-image-feature-ci-checker:
+  <<: *build_image_definition
+  <<: *only_feature
+  variables:
+    REGISTRY_USER: "${CI_REGISTRY_USER}"
+    REGISTRY_PASSWORD: "${CI_REGISTRY_PASSWORD}"
+    REGISTRY: "${CI_REGISTRY}"
+    DOCKER_FILE: "Dockerfile.ci-checker"
+    DOCKER_NAME: "registry.gitlab.com/postgres-ai/database-lab/dblab-ci-checker"
+    TAGS: "${DOCKER_NAME}:${CI_COMMIT_REF_SLUG}"
+
 build-image-feature-client:
   <<: *build_image_definition
   <<: *only_feature

.golangci.yml

@@ -99,4 +99,4 @@ issues:
   max-issues-per-linter: 0
   max-same-issues: 0
 
-  new-from-rev: 66fcab89
+  new-from-rev: 9e951ebe

Dockerfile.ci-checker

@@ -0,0 +1,11 @@
+FROM docker:19.03.14
+
+# Install dependencies.
+RUN apk update && apk add --no-cache bash
+
+WORKDIR /home/dblab
+
+COPY ./bin/run-ci ./bin/run-ci
+COPY ./configs ./configs
+
+CMD ./bin/run-ci

Dockerfile.runci

@@ -0,0 +1,9 @@
+FROM sqitch/sqitch:1.0.0
+
+ENTRYPOINT ["/bin/bash"]
+
+RUN echo "#!/bin/bash" > /tmp/pg_start.sh && chmod a+x /tmp/pg_start.sh \
+    && echo "/bin/bash -c \"trap : TERM INT; sleep infinity & wait\"" \
+      >> /tmp/pg_start.sh
+
+CMD ["/tmp/pg_start.sh"]

Makefile

@@ -1,6 +1,7 @@
 .DEFAULT_GOAL = all
 
 SERVER_BINARY = dblab-server
+RUN_CI_BINARY = run-ci
 CLI_BINARY = dblab
 GOARCH = amd64
 
@@ -38,8 +39,12 @@ lint: install-lint run-lint
 
 build:
 	${GOBUILD} -o bin/${SERVER_BINARY} ./cmd/database-lab/main.go
+	${GOBUILD} -o bin/${RUN_CI_BINARY} ./cmd/runci/main.go
 	${GOBUILD} -o bin/${CLI_BINARY} ./cmd/cli/main.go
 
+build-ci-checker:
+	${GOBUILD} -o bin/${RUN_CI_BINARY} ./cmd/runci/main.go
+
 build-client:
 	$(foreach GOOS, $(CLIENT_PLATFORMS),\
 		$(foreach GOARCH, $(ARCHITECTURES), \

cmd/cli/commands/clone/actions.go

@@ -8,16 +8,21 @@ package clone
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/url"
+	"os"
+	"path"
 	"strings"
 	"sync"
 
+	"github.com/pkg/errors"
 	"github.com/urfave/cli/v2"
 
 	"gitlab.com/postgres-ai/database-lab/v2/cmd/cli/commands"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/client/dblabapi/types"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/log"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/models"
+	"gitlab.com/postgres-ai/database-lab/v2/pkg/observer"
 )
 
 // list runs a request to list clones of an instance.
@@ -291,14 +296,40 @@ func downloadArtifact(cliCtx *cli.Context) error {
 	cloneID := cliCtx.String("clone-id")
 	sessionID := cliCtx.String("session-id")
 	artifactType := cliCtx.String("artifact-type")
-	outputFile := cliCtx.String("output")
+	outputPath := cliCtx.String("output")
 
-	artifactPath, err := dblabClient.DownloadArtifact(cliCtx.Context, cloneID, sessionID, artifactType, outputFile)
+	body, err := dblabClient.DownloadArtifact(cliCtx.Context, cloneID, sessionID, artifactType)
 	if err != nil {
 		return err
 	}
 
-	_, err = fmt.Fprintf(cliCtx.App.Writer, "The file has been successfully downloaded: %s\n", artifactPath)
+	defer func() {
+		if err := body.Close(); err != nil {
+			log.Err(err)
+		}
+	}()
+
+	if outputPath == "" {
+		wd, err := os.Getwd()
+		if err != nil {
+			return err
+		}
+
+		outputPath = path.Join(wd, observer.BuildArtifactFilename(artifactType))
+	}
+
+	artifactFile, err := os.Create(outputPath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create file %s", outputPath)
+	}
+
+	defer func() { _ = artifactFile.Close() }()
+
+	if _, err := io.Copy(artifactFile, body); err != nil {
+		return err
+	}
+
+	_, err = fmt.Fprintf(cliCtx.App.Writer, "The file has been successfully downloaded: %s\n", outputPath)
 
 	return err
 }

cmd/cli/main.go

@@ -27,7 +27,7 @@ func main() {
 		},
 		Before: loadEnvironmentParams,
 		Commands: joinCommands(
-			// Global commands.
+			// Config commands.
 			global.List(),
 
 			// Database Lab API.

cmd/database-lab/main.go

@@ -16,11 +16,14 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/network"
 	"github.com/docker/docker/client"
 	"github.com/pkg/errors"
 	"github.com/rs/xid"
 
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/config"
+	"gitlab.com/postgres-ai/database-lab/v2/pkg/config/global"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/estimator"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/log"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/observer"
@@ -33,6 +36,7 @@ import (
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/services/provision/resources"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/services/provision/runners"
 	"gitlab.com/postgres-ai/database-lab/v2/pkg/srv"
+	"gitlab.com/postgres-ai/database-lab/v2/pkg/util/networks"
 	"gitlab.com/postgres-ai/database-lab/v2/version"
 )
 
@@ -67,6 +71,63 @@ func main() {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	hostname := os.Getenv("HOSTNAME")
+	if hostname == "" {
+		log.Err("hostname is empty")
+	}
+
+	networkID := "network_" + instanceID
+
+	internalNetwork, err := dockerCLI.NetworkCreate(ctx, networkID, types.NetworkCreate{
+		Labels: map[string]string{
+			"instance": instanceID,
+			"app":      networks.DLEApp,
+			"type":     networks.InternalType,
+		},
+		Attachable: true,
+		Internal:   true,
+	})
+	if err != nil {
+		log.Errf(err.Error())
+		return
+	}
+
+	defer func() {
+		networkInspect, err := dockerCLI.NetworkInspect(context.Background(), internalNetwork.ID, types.NetworkInspectOptions{})
+		if err != nil {
+			log.Errf(err.Error())
+			return
+		}
+
+		for _, resource := range networkInspect.Containers {
+			log.Dbg("Disconnecting container: ", resource.Name)
+
+			if err := dockerCLI.NetworkDisconnect(context.Background(), internalNetwork.ID, resource.Name, true); err != nil {
+				log.Errf(err.Error())
+				return
+			}
+		}
+
+		if err := dockerCLI.NetworkRemove(context.Background(), internalNetwork.ID); err != nil {
+			log.Errf(err.Error())
+			return
+		}
+	}()
+
+	log.Dbg("New network: ", internalNetwork.ID)
+
+	if err := dockerCLI.NetworkConnect(ctx, internalNetwork.ID, hostname, &network.EndpointSettings{}); err != nil {
+		log.Errf(err.Error())
+		return
+	}
+
+	defer func() {
+		if err := dockerCLI.NetworkDisconnect(context.Background(), internalNetwork.ID, hostname, true); err != nil {
+			log.Errf(err.Error())
+			return
+		}
+	}()
+
 	// Create a platform service to make requests to Platform.
 	platformSvc, err := platform.New(ctx, cfg.Platform)
 	if err != nil {
@@ -95,7 +156,7 @@ func main() {
 	}
 
 	// Create a cloning service to provision new clones.
-	provisionSvc, err := provision.New(ctx, &cfg.Provision, dbCfg, dockerCLI, pm)
+	provisionSvc, err := provision.New(ctx, &cfg.Provision, dbCfg, dockerCLI, pm, internalNetwork.ID)
 	if err != nil {
 		log.Errf(errors.WithMessage(err, `error in the "provision" section of the config`).Error())
 	}
@@ -113,7 +174,7 @@ func main() {
 
 	go removeObservingClones(obsCh, obs)
 
-	server := srv.NewServer(&cfg.Server, obs, cloningSvc, platformSvc, dockerCLI, est)
+	server := srv.NewServer(&cfg.Server, &cfg.Global, obs, cloningSvc, platformSvc, dockerCLI, est, pm)
 
 	reloadCh := setReloadListener()
 
@@ -220,7 +281,7 @@ func setShutdownListener() chan os.Signal {
 	return c
 }
 
-func shutdownDatabaseLabEngine(ctx context.Context, dockerCLI *client.Client, global config.Global, fsp *resources.Pool) {
+func shutdownDatabaseLabEngine(ctx context.Context, dockerCLI *client.Client, global global.Config, fsp *resources.Pool) {
 	log.Msg("Stopping control containers")
 
 	if err := cont.StopControlContainers(ctx, dockerCLI, global.InstanceID, fsp.DataDir()); err != nil {