chore: get rid of estimator (#492)

* remove estimator package
* remove estimator client
* remove estimator API endpoint
* remove estimator's dependencies from Dockerfiles
* clean up configs

Author: agneum

engine/Dockerfile.dblab-server

@@ -6,13 +6,6 @@ FROM docker:20.10.12
 RUN apk update \
   && apk add zfs=2.1.4-r0 --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main \
   && apk add --no-cache lvm2 bash util-linux
-RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.13/main' >> /etc/apk/repositories \
-  && echo 'http://dl-cdn.alpinelinux.org/alpine/v3.13/community' >> /etc/apk/repositories \
-  && apk add bcc-tools=0.18.0-r0 bcc-doc=0.18.0-r0 && ln -s $(which python3) /usr/bin/python \
-  # TODO: remove after release the PR: https://github.com/iovisor/bcc/pull/3286 (issue: https://github.com/iovisor/bcc/issues/3099)
-  && wget https://raw.githubusercontent.com/iovisor/bcc/master/tools/biosnoop.py -O /usr/share/bcc/tools/biosnoop
-
-ENV PATH="${PATH}:/usr/share/bcc/tools"
 
 WORKDIR /home/dblab
 

engine/Dockerfile.dblab-server-debug

@@ -18,13 +18,6 @@ FROM docker:20.10.12
 RUN apk update \
   && apk add zfs=2.1.4-r0 --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/edge/main \
   && apk add --no-cache lvm2 bash util-linux
-RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.13/main' >> /etc/apk/repositories \
-  && echo 'http://dl-cdn.alpinelinux.org/alpine/v3.13/community' >> /etc/apk/repositories \
-  && apk add bcc-tools=0.18.0-r0 bcc-doc=0.18.0-r0 && ln -s $(which python3) /usr/bin/python \
-  # TODO: remove after release the PR: https://github.com/iovisor/bcc/pull/3286 (issue: https://github.com/iovisor/bcc/issues/3099)
-  && wget https://raw.githubusercontent.com/iovisor/bcc/master/tools/biosnoop.py -O /usr/share/bcc/tools/biosnoop
-
-ENV PATH="${PATH}:/usr/share/bcc/tools"
 
 WORKDIR /home/dblab
 

engine/Dockerfile.dblab-server-zfs08

@@ -5,13 +5,6 @@ FROM docker:20.10.12
 # Install dependencies.
 RUN apk update && apk add zfs=0.8.4-r0 --no-cache --repository=https://dl-cdn.alpinelinux.org/alpine/v3.12/main \
   && apk add --no-cache lvm2 bash util-linux
-RUN echo 'http://dl-cdn.alpinelinux.org/alpine/v3.13/main' >> /etc/apk/repositories \
-  && echo 'http://dl-cdn.alpinelinux.org/alpine/v3.13/community' >> /etc/apk/repositories \
-  && apk add bcc-tools=0.18.0-r0 bcc-doc=0.18.0-r0 && ln -s $(which python3) /usr/bin/python \
-  # TODO: remove after release the PR: https://github.com/iovisor/bcc/pull/3286 (issue: https://github.com/iovisor/bcc/issues/3099)
-  && wget https://raw.githubusercontent.com/iovisor/bcc/master/tools/biosnoop.py -O /usr/share/bcc/tools/biosnoop
-
-ENV PATH="${PATH}:/usr/share/bcc/tools"
 
 WORKDIR /home/dblab
 

engine/api/swagger-spec/dblab_server_swagger.yaml

@@ -419,48 +419,6 @@ paths:
           schema:
             $ref: "#/definitions/Error"
 
-  /estimate:
-    get:
-      tags:
-        - "observation"
-      summary: "Run clone estimator"
-      description: ""
-      operationId: "runEstimator"
-      consumes:
-        - "application/json"
-      produces:
-        - "application/json"
-      parameters:
-        - in: header
-          name: Verification-Token
-          type: string
-          required: true
-        - in: path
-          required: true
-          name: "clone_id"
-          type: "string"
-          description: "Clone ID"
-        - in: path
-          required: true
-          name: "pid"
-          type: "string"
-          description: "Process ID"
-      responses:
-        200:
-          description: "The estimation session has been finished successfully."
-        400:
-          description: "Bad request"
-          schema:
-            $ref: "#/definitions/Error"
-        404:
-          description: "Not found"
-          schema:
-            $ref: "#/definitions/Error"
-        500:
-          description: "Internal server error"
-          schema:
-            $ref: "#/definitions/Error"
-
   /instance/retrieval:
     get:
       tags:

engine/cmd/database-lab/main.go

@@ -25,7 +25,6 @@ import (
 	"gitlab.com/postgres-ai/database-lab/v3/internal/cloning"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/diagnostic"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/embeddedui"
-	"gitlab.com/postgres-ai/database-lab/v3/internal/estimator"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/observer"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/platform"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/provision"
@@ -158,7 +157,6 @@ func main() {
 	}
 
 	obs := observer.NewObserver(docker, &cfg.Observer, pm)
-	est := estimator.NewEstimator(&cfg.Estimator)
 
 	go removeObservingClones(observingChan, obs)
 
@@ -187,7 +185,6 @@ func main() {
 			pm,
 			cloningSvc,
 			platformSvc,
-			est,
 			embeddedUI,
 			server,
 			logCleaner,
@@ -196,10 +193,10 @@ func main() {
 	}
 
 	server := srv.NewServer(&cfg.Server, &cfg.Global, engProps, docker, cloningSvc, provisioner, retrievalSvc, platformSvc,
-		obs, est, pm, tm, tokenHolder, logFilter, embeddedUI, reloadConfigFn)
+		obs, pm, tm, tokenHolder, logFilter, embeddedUI, reloadConfigFn)
 	shutdownCh := setShutdownListener()
 
-	go setReloadListener(ctx, provisioner, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server,
+	go setReloadListener(ctx, provisioner, tm, retrievalSvc, pm, cloningSvc, platformSvc, embeddedUI, server,
 		logCleaner, logFilter)
 
 	server.InitHandlers()
@@ -281,8 +278,7 @@ func getEngineProperties(ctx context.Context, dockerCLI *client.Client, cfg *con
 
 func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *telemetry.Agent,
 	retrievalSvc *retrieval.Retrieval, pm *pool.Manager, cloningSvc *cloning.Base, platformSvc *platform.Service,
-	est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner,
-	filtering *log.Filtering) error {
+	embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner, filtering *log.Filtering) error {
 	cfg, err := config.LoadConfiguration()
 	if err != nil {
 		return err
@@ -327,23 +323,21 @@ func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *
 	retrievalSvc.Reload(ctx, newRetrievalConfig)
 	cloningSvc.Reload(cfg.Cloning)
 	platformSvc.Reload(newPlatformSvc)
-	est.Reload(cfg.Estimator)
 	server.Reload(cfg.Server)
 
 	return nil
 }
 
 func setReloadListener(ctx context.Context, provisionSvc *provision.Provisioner, tm *telemetry.Agent,
 	retrievalSvc *retrieval.Retrieval, pm *pool.Manager, cloningSvc *cloning.Base, platformSvc *platform.Service,
-	est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner,
-	logFilter *log.Filtering) {
+	embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner, logFilter *log.Filtering) {
 	reloadCh := make(chan os.Signal, 1)
 	signal.Notify(reloadCh, syscall.SIGHUP)
 
 	for range reloadCh {
 		log.Msg("Reloading configuration")
 
-		if err := reloadConfig(ctx, provisionSvc, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server,
+		if err := reloadConfig(ctx, provisionSvc, tm, retrievalSvc, pm, cloningSvc, platformSvc, embeddedUI, server,
 			cleaner, logFilter); err != nil {
 			log.Err("Failed to reload configuration", err)
 		}

engine/configs/config.example.logical_generic.yml

@@ -393,16 +393,3 @@ diagnostic:
 #    "select \\d+": "***"
 #    "[a-z0-9._%+\\-]+(@[a-z0-9.\\-]+\\.[a-z]{2,4})": "***$1"
 #
-# Tool to calculate timing difference between Database Lab and production environments.
-#estimator:
-#  # The ratio evaluating the timing difference for operations involving IO Read between Database Lab and production environments.
-#  readRatio: 1
-#
-#  # The ratio evaluating the timing difference for operations involving IO Write between Database Lab and production environments.
-#  writeRatio: 1
-#
-#  # Time interval of samples taken by the profiler.
-#  profilingInterval: 10ms
-#
-#  # The minimum number of samples sufficient to display the estimation results.
-#  sampleThreshold: 20

engine/configs/config.example.logical_rds_iam.yml

@@ -394,16 +394,3 @@ diagnostic:
 #    "select \\d+": "***"
 #    "[a-z0-9._%+\\-]+(@[a-z0-9.\\-]+\\.[a-z]{2,4})": "***$1"
 #
-# Tool to calculate timing difference between Database Lab and production environments.
-#estimator:
-#  # The ratio evaluating the timing difference for operations involving IO Read between Database Lab and production environments.
-#  readRatio: 1
-#
-#  # The ratio evaluating the timing difference for operations involving IO Write between Database Lab and production environments.
-#  writeRatio: 1
-#
-#  # Time interval of samples taken by the profiler.
-#  profilingInterval: 10ms
-#
-#  # The minimum number of samples sufficient to display the estimation results.
-#  sampleThreshold: 20

engine/configs/config.example.physical_generic.yml

@@ -346,16 +346,3 @@ diagnostic:
 #    "select \\d+": "***"
 #    "[a-z0-9._%+\\-]+(@[a-z0-9.\\-]+\\.[a-z]{2,4})": "***$1"
 #
-# Tool to calculate timing difference between Database Lab and production environments.
-#estimator:
-#  # The ratio evaluating the timing difference for operations involving IO Read between Database Lab and production environments.
-#  readRatio: 1
-#
-#  # The ratio evaluating the timing difference for operations involving IO Write between Database Lab and production environments.
-#  writeRatio: 1
-#
-#  # Time interval of samples taken by the profiler.
-#  profilingInterval: 10ms
-#
-#  # The minimum number of samples sufficient to display the estimation results.
-#  sampleThreshold: 20

engine/configs/config.example.physical_pgbackrest.yml

@@ -364,17 +364,3 @@ diagnostic:
 #    "regexp": "replace"
 #    "select \\d+": "***"
 #    "[a-z0-9._%+\\-]+(@[a-z0-9.\\-]+\\.[a-z]{2,4})": "***$1"
-#
-# Tool to calculate timing difference between Database Lab and production environments.
-#estimator:
-#  # The ratio evaluating the timing difference for operations involving IO Read between Database Lab and production environments.
-#  readRatio: 1
-#
-#  # The ratio evaluating the timing difference for operations involving IO Write between Database Lab and production environments.
-#  writeRatio: 1
-#
-#  # Time interval of samples taken by the profiler.
-#  profilingInterval: 10ms
-#
-#  # The minimum number of samples sufficient to display the estimation results.
-#  sampleThreshold: 20

engine/configs/config.example.physical_walg.yml

@@ -337,17 +337,3 @@ diagnostic:
 #    "regexp": "replace"
 #    "select \\d+": "***"
 #    "[a-z0-9._%+\\-]+(@[a-z0-9.\\-]+\\.[a-z]{2,4})": "***$1"
-#
-# Tool to calculate timing difference between Database Lab and production environments.
-#estimator:
-#  # The ratio evaluating the timing difference for operations involving IO Read between Database Lab and production environments.
-#  readRatio: 1
-#
-#  # The ratio evaluating the timing difference for operations involving IO Write between Database Lab and production environments.
-#  writeRatio: 1
-#
-#  # Time interval of samples taken by the profiler.
-#  profilingInterval: 10ms
-#
-#  # The minimum number of samples sufficient to display the estimation results.
-#  sampleThreshold: 20