Description
Hello. I am trying to configure image caching based on this example, but having some issues. When creating workspace, I notice the following errors during provisioning:
Failed to find cached image in repository "registry:5000/devcontainer-cache". It will be rebuilt in the next apply. Error: open devcontainer.json: open /workspaces/project/.devcontainer/devcontainer.json: no such file or directory
The workspace starts up fine, just rebuilds the image every time. The path from which it attempts to read devcontainer.json
is correct, but I don't see any git clone logs before that at all. The devcontainer.json should be read from the provided git repo after all. So I suspect this might be the cause? The git repo is private, but I've setup external authentication as instructed by the docs, so the agent is able to successfully access the repository. I assume the envbuilder should have access to the repo using the same mechanism, but perhaphs some extra setup is needed?
In any case I'm attaching provisioning logs as well as template code below. The agent logs are omitted since it behaves as expected, and successfully pushes the image at the end. Please let me know if there is anything else I should provide. Thank you for your time.
main.tf
terraform {
required_providers {
coder = {
source = "coder/coder"
version = "~> 2.3.0"
}
docker = {
source = "kreuzwerker/docker"
}
envbuilder = {
source = "coder/envbuilder"
}
}
}
provider "coder" {}
provider "docker" {}
provider "envbuilder" {}
data "coder_provisioner" "me" {}
data "coder_workspace" "me" {}
data "coder_workspace_owner" "me" {}
data "coder_parameter" "repo" {
name = "repo"
display_name = "Git repository"
description = "Select a repository to automatically clone and start working with a devcontainer."
order = 1
}
data "coder_parameter" "fallback_image" {
default = "codercom/enterprise-base:ubuntu"
description = "This image runs if the devcontainer fails to build."
display_name = "Fallback Image"
mutable = true
name = "fallback_image"
order = 3
}
data "coder_parameter" "devcontainer_builder" {
description = <<-EOF
Image that will build the devcontainer.
We highly recommend using a specific release as the `:latest` tag will change.
Find the latest version of Envbuilder here: https://github.com/coder/envbuilder/pkgs/container/envbuilder
EOF
display_name = "Devcontainer Builder"
mutable = true
name = "devcontainer_builder"
default = "ghcr.io/coder/envbuilder:1.1.0"
order = 4
}
### https://coder.com/docs/admin/external-auth
data "coder_external_auth" "github" {
id = "primary-github"
}
locals {
project_dir = "/workspaces/project"
container_name = "coder-${data.coder_workspace_owner.me.name}-${lower(data.coder_workspace.me.name)}"
devcontainer_builder_image = data.coder_parameter.devcontainer_builder.value
repo_url = data.coder_parameter.repo.value
# Use a container registry as a cache to speed up builds.
cache_repo = "registry:5000/devcontainer-cache"
# Path to a docker config.json containing credentials to the provided cache repo, if required.
cache_repo_docker_config_path = ""
# Enable this option if your cache registry does not serve HTTPS.
insecure_cache_repo = true
# The envbuilder provider requires a key-value map of environment variables.
# https://github.com/coder/envbuilder/blob/main/docs/env-variables.md
envbuilder_env = {
# ENVBUILDER_GIT_URL and ENVBUILDER_CACHE_REPO will be overridden by the provider
# if the cache repo is enabled.
"ENVBUILDER_WORKSPACE_FOLDER" : local.project_dir
"ENVBUILDER_DEVCONTAINER_DIR" : "${local.project_dir}/.devcontainer"
"ENVBUILDER_GIT_URL" : local.repo_url,
"ENVBUILDER_GIT_USERNAME" : data.coder_external_auth.github.access_token,
"ENVBUILDER_CACHE_REPO" : local.cache_repo,
"CODER_AGENT_TOKEN" : coder_agent.main.token,
# Use the docker gateway if the access URL is 127.0.0.1
"CODER_AGENT_URL" : replace(data.coder_workspace.me.access_url, "/localhost|127\\.0\\.0\\.1/", "host.docker.internal"),
# Use the docker gateway if the access URL is 127.0.0.1
"ENVBUILDER_INIT_SCRIPT" : replace(coder_agent.main.init_script, "/localhost|127\\.0\\.0\\.1/", "host.docker.internal"),
"ENVBUILDER_FALLBACK_IMAGE" : data.coder_parameter.fallback_image.value,
"ENVBUILDER_DOCKER_CONFIG_BASE64" : try(data.local_sensitive_file.cache_repo_dockerconfigjson[0].content_base64, ""),
"ENVBUILDER_PUSH_IMAGE" : local.cache_repo == "" ? "" : "true",
"ENVBUILDER_INSECURE" : "${local.insecure_cache_repo}",
}
# Convert the above map to the format expected by the docker provider.
docker_env = [
for k, v in local.envbuilder_env : "${k}=${v}"
]
}
data "local_sensitive_file" "cache_repo_dockerconfigjson" {
count = local.cache_repo_docker_config_path == "" ? 0 : 1
filename = local.cache_repo_docker_config_path
}
resource "docker_image" "devcontainer_builder_image" {
name = local.devcontainer_builder_image
keep_locally = true
}
resource "docker_volume" "workspaces" {
name = "coder-${data.coder_workspace.me.id}"
# Protect the volume from being deleted due to changes in attributes.
lifecycle {
ignore_changes = all
}
# Add labels in Docker to keep track of orphan resources.
labels {
label = "coder.owner"
value = data.coder_workspace_owner.me.name
}
labels {
label = "coder.owner_id"
value = data.coder_workspace_owner.me.id
}
labels {
label = "coder.workspace_id"
value = data.coder_workspace.me.id
}
# This field becomes outdated if the workspace is renamed but can
# be useful for debugging or cleaning out dangling volumes.
labels {
label = "coder.workspace_name_at_creation"
value = data.coder_workspace.me.name
}
}
# Check for the presence of a prebuilt image in the cache repo
# that we can use instead.
resource "envbuilder_cached_image" "cached" {
count = local.cache_repo == "" ? 0 : data.coder_workspace.me.start_count
builder_image = local.devcontainer_builder_image
git_url = local.repo_url
cache_repo = local.cache_repo
extra_env = local.envbuilder_env
insecure = local.insecure_cache_repo
verbose = true
}
resource "docker_container" "workspace" {
count = data.coder_workspace.me.start_count
image = local.cache_repo == "" ? local.devcontainer_builder_image : envbuilder_cached_image.cached.0.image
# Uses lower() to avoid Docker restriction on container names.
name = "coder-${data.coder_workspace_owner.me.name}-${lower(data.coder_workspace.me.name)}"
# Hostname makes the shell more user friendly: coder@my-workspace:~$
hostname = data.coder_workspace.me.name
# Use the environment specified by the envbuilder provider, if available.
env = local.cache_repo == "" ? local.docker_env : envbuilder_cached_image.cached.0.env
# network_mode = "host" # Uncomment if testing with a registry running on `localhost`.
host {
host = "host.docker.internal"
ip = "host-gateway"
}
volumes {
container_path = "/workspaces"
volume_name = docker_volume.workspaces.name
read_only = false
}
networks_advanced {
# Add to the common network with registry so the workspaces can be cached
name = "registry_network"
}
# Add labels in Docker to keep track of orphan resources.
labels {
label = "coder.owner"
value = data.coder_workspace_owner.me.name
}
labels {
label = "coder.owner_id"
value = data.coder_workspace_owner.me.id
}
labels {
label = "coder.workspace_id"
value = data.coder_workspace.me.id
}
labels {
label = "coder.workspace_name"
value = data.coder_workspace.me.name
}
}
resource "coder_agent" "main" {
arch = data.coder_provisioner.me.arch
os = "linux"
# This startup script allows you to make Git commits right away
# after creating a workspace. It sets up git user name
# and a private email address.
startup_script = <<-EOT
set -e
# Add any commands that should be executed at workspace startup (e.g install requirements, start a program, etc) here
echo "Configuring global git user name & email"
# Fetch GitHub user ID using the user's access token
GH_USER=$(curl -s -H "Authorization: Bearer ${data.coder_external_auth.github.access_token}" https://api.github.com/user)
echo "GH_USER: $GH_USER"
GH_USER_ID=$(echo $GH_USER | jq .id --raw-output)
GH_USER_NAME=$(echo $GH_USER | jq .login --raw-output)
# Configure Git with the user's no-reply email
git config --global user.email "$${GH_USER_ID}+$${GH_USER_NAME}@users.noreply.github.com"
git config --global user.name "$${GH_USER_NAME}"
# Print the result
echo "Email: $(git config --global user.email)"
echo "Name: $(git config --global user.name)"
EOT
# The following metadata blocks are optional. They are used to display
# information about your workspace in the dashboard. You can remove them
# if you don't want to display any information.
# For basic resources, you can use the `coder stat` command.
# If you need more control, you can write your own script.
metadata {
display_name = "CPU Usage"
key = "0_cpu_usage"
script = "coder stat cpu"
interval = 10
timeout = 1
}
metadata {
display_name = "RAM Usage"
key = "1_ram_usage"
script = "coder stat mem"
interval = 10
timeout = 1
}
metadata {
display_name = "Home Disk"
key = "3_home_disk"
script = "coder stat disk --path $HOME"
interval = 60
timeout = 1
}
metadata {
display_name = "CPU Usage (Host)"
key = "4_cpu_usage_host"
script = "coder stat cpu --host"
interval = 10
timeout = 1
}
metadata {
display_name = "Memory Usage (Host)"
key = "5_mem_usage_host"
script = "coder stat mem --host"
interval = 10
timeout = 1
}
metadata {
display_name = "Load Average (Host)"
key = "6_load_host"
# get load avg scaled by number of cores
script = <<EOT
echo "`cat /proc/loadavg | awk '{ print $1 }'` `nproc`" | awk '{ printf "%0.2f", $1/$2 }'
EOT
interval = 60
timeout = 1
}
metadata {
display_name = "Swap Usage (Host)"
key = "7_swap_host"
script = <<EOT
free -b | awk '/^Swap/ { printf("%.1f/%.1f", $3/1024.0/1024.0/1024.0, $2/1024.0/1024.0/1024.0) }'
EOT
interval = 10
timeout = 1
}
}
# See https://registry.coder.com/modules/jetbrains-gateway
module "jetbrains_gateway" {
count = data.coder_workspace.me.start_count
source = "registry.coder.com/modules/jetbrains-gateway/coder"
# JetBrains IDEs to make available for the user to select
jetbrains_ides = ["IU", "PY", "WS", "PS", "RD", "CL", "GO", "RM"]
default = "IU"
latest = true
# Default folder to open when starting a JetBrains IDE
folder = local.project_dir
# This ensures that the latest version of the module gets downloaded, you can also pin the module version to prevent breaking changes in production.
version = ">= 1.0.0"
agent_id = coder_agent.main.id
agent_name = "main"
order = 1
}
# See https://registry.coder.com/modules/code-server
module "code-server" {
count = data.coder_workspace.me.start_count
source = "registry.coder.com/modules/code-server/coder"
folder = local.project_dir
# This ensures that the latest version of the module gets downloaded, you can also pin the module version to prevent breaking changes in production.
version = ">= 1.0.0"
agent_id = coder_agent.main.id
order = 2
display_name = "Code - OSS Online"
}
resource "coder_metadata" "container_info" {
count = data.coder_workspace.me.start_count
resource_id = coder_agent.main.id
item {
key = "workspace image"
value = local.cache_repo == "" ? local.devcontainer_builder_image : envbuilder_cached_image.cached.0.image
}
item {
key = "git url"
value = local.repo_url
}
item {
key = "cache repo"
value = local.cache_repo == "" ? "not enabled" : local.cache_repo
}
}
Provisioning (build) logs
Initializing the backend...
Initializing modules...
Downloading registry.coder.com/modules/code-server/coder 1.1.0 for code-server...
- code-server in .terraform/modules/code-server
Downloading registry.coder.com/modules/jetbrains-gateway/coder 1.0.30 for jetbrains_gateway...
- jetbrains_gateway in .terraform/modules/jetbrains_gateway
Initializing provider plugins...
- Reusing previous version of coder/envbuilder from the dependency lock file
- Reusing previous version of coder/coder from the dependency lock file
- Reusing previous version of hashicorp/local from the dependency lock file
- Reusing previous version of hashicorp/http from the dependency lock file
- Reusing previous version of kreuzwerker/docker from the dependency lock file
- Using coder/envbuilder v1.0.0 from the shared cache directory
- Using coder/coder v2.3.0 from the shared cache directory
- Using hashicorp/local v2.5.2 from the shared cache directory
- Using hashicorp/http v3.5.0 from the shared cache directory
- Using kreuzwerker/docker v3.3.0 from the shared cache directory
Terraform has been successfully initialized!
You may now begin working with Terraform. Try running "terraform plan" to see
any changes that are required for your infrastructure. All Terraform commands
should now work.
If you ever set or change modules or backend configuration for Terraform,
rerun this command to reinitialize your working directory. If you forget, other
commands will detect it and remind you to do so if necessary.
Terraform 1.11.2
data.coder_external_auth.github: Refreshing...
data.coder_parameter.fallback_image: Refreshing...
data.coder_workspace_owner.me: Refreshing...
data.coder_parameter.repo: Refreshing...
data.coder_provisioner.me: Refreshing...
data.coder_workspace.me: Refreshing...
data.coder_parameter.devcontainer_builder: Refreshing...
data.coder_workspace.me: Refresh complete after 0s [id=623cef48-d3e9-4775-b02a-4f3813d2665d]
data.coder_workspace_owner.me: Refresh complete after 0s [id=d00ddf4a-a482-4be5-95dc-0bdd1a8e6b58]
data.coder_external_auth.github: Refresh complete after 0s [id=primary-github]
data.coder_provisioner.me: Refresh complete after 0s [id=6e7f4d95-d701-4344-98f4-1307aa5c2095]
data.coder_parameter.fallback_image: Refresh complete after 0s [id=4adf7653-c0ad-4d45-a674-313f33d1d1fa]
module.jetbrains_gateway[0].data.coder_workspace.me: Refreshing...
data.coder_parameter.devcontainer_builder: Refresh complete after 0s [id=beac1278-eb01-4dc9-9765-4c432bcf10c1]
data.coder_parameter.repo: Refresh complete after 0s [id=51ec89b5-4f6a-48f5-9eda-f29b81ebbd02]
module.jetbrains_gateway[0].data.coder_workspace_owner.me: Refreshing...
module.jetbrains_gateway[0].data.coder_workspace.me: Refresh complete after 0s [id=623cef48-d3e9-4775-b02a-4f3813d2665d]
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["RD"]: Refreshing...
module.jetbrains_gateway[0].data.coder_workspace_owner.me: Refresh complete after 0s [id=d00ddf4a-a482-4be5-95dc-0bdd1a8e6b58]
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["PY"]: Refreshing...
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["GO"]: Refreshing...
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["RM"]: Refreshing...
module.jetbrains_gateway[0].data.coder_parameter.jetbrains_ide: Refreshing...
module.jetbrains_gateway[0].data.coder_parameter.jetbrains_ide: Refresh complete after 0s [id=5ebe22b0-2f91-484c-bfb3-9d555f5cdd87]
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["WS"]: Refreshing...
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["IU"]: Refreshing...
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["CL"]: Refreshing...
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["PS"]: Refreshing...
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["RD"]: Refresh complete after 0s [id=https://data.services.jetbrains.com/products/releases?code=RD&latest=true&type=release]
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["PS"]: Refresh complete after 0s [id=https://data.services.jetbrains.com/products/releases?code=PS&latest=true&type=release]
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["WS"]: Refresh complete after 0s [id=https://data.services.jetbrains.com/products/releases?code=WS&latest=true&type=release]
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["RM"]: Refresh complete after 0s [id=https://data.services.jetbrains.com/products/releases?code=RM&latest=true&type=release]
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["GO"]: Refresh complete after 0s [id=https://data.services.jetbrains.com/products/releases?code=GO&latest=true&type=release]
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["CL"]: Refresh complete after 0s [id=https://data.services.jetbrains.com/products/releases?code=CL&latest=true&type=release]
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["IU"]: Refresh complete after 0s [id=https://data.services.jetbrains.com/products/releases?code=IU&latest=true&type=release]
module.jetbrains_gateway[0].data.http.jetbrains_ide_versions["PY"]: Refresh complete after 0s [id=https://data.services.jetbrains.com/products/releases?code=PY&latest=true&type=release]
coder_agent.main: Plan to create
docker_image.devcontainer_builder_image: Plan to create
docker_volume.workspaces: Plan to create
module.code-server[0].coder_app.code-server: Plan to create
module.code-server[0].coder_script.code-server: Plan to create
envbuilder_cached_image.cached[0]: Plan to create
coder_metadata.container_info[0]: Plan to create
docker_container.workspace[0]: Plan to create
module.jetbrains_gateway[0].coder_app.gateway: Plan to create
Plan: 9 to add, 0 to change, 0 to destroy.
Terraform 1.11.2
coder_agent.main: Plan to create
docker_image.devcontainer_builder_image: Plan to create
docker_volume.workspaces: Plan to create
module.code-server[0].coder_app.code-server: Plan to create
module.code-server[0].coder_script.code-server: Plan to create
envbuilder_cached_image.cached[0]: Plan to create
coder_metadata.container_info[0]: Plan to create
docker_container.workspace[0]: Plan to create
module.jetbrains_gateway[0].coder_app.gateway: Plan to create
coder_agent.main: Creating...
docker_image.devcontainer_builder_image: Creating...
docker_volume.workspaces: Creating...
coder_agent.main: Creation complete after 0s [id=693b22f6-bf40-4eea-92a9-1fdf055d244a]
docker_volume.workspaces: Creation complete after 0s [id=coder-623cef48-d3e9-4775-b02a-4f3813d2665d]
module.code-server[0].coder_app.code-server: Creating...
module.code-server[0].coder_script.code-server: Creating...
module.code-server[0].coder_app.code-server: Creation complete after 0s [id=d31385c4-ffcb-4913-a9c0-16b96714aed7]
docker_image.devcontainer_builder_image: Creation complete after 0s [id=sha256:0043aade3996c429a302de5f79050c63bf2a56a7286c5aef2cd65387bc870db5ghcr.io/coder/envbuilder:1.1.0]
module.code-server[0].coder_script.code-server: Creation complete after 0s [id=f43ef551-dfc2-4879-93e0-02c24cbba14d]
module.jetbrains_gateway[0].coder_app.gateway: Creating...
envbuilder_cached_image.cached[0]: Creating...
module.jetbrains_gateway[0].coder_app.gateway: Creation complete after 0s [id=0383eec7-b4bf-4a6f-abdd-0b4f113bcfca]
envbuilder_cached_image.cached[0]: Creation complete after 4s [id=00000000-0000-0000-0000-000000000000]
coder_metadata.container_info[0]: Creating...
coder_metadata.container_info[0]: Creation complete after 0s [id=cc836c98-35e8-499d-b78d-7ada2f95b991]
docker_container.workspace[0]: Creating...
docker_container.workspace[0]: Creation complete after 1s [id=94277f68ab9b0ce5d107f490bfa9f15bc9ffd86858acd52748814cbff0714b7d]
Warning: Cannot override required environment variable
on main.tf line 143, in resource "envbuilder_cached_image" "cached":
143: extra_env = local.envbuilder_env
The key "ENVBUILDER_GIT_URL" in extra_env cannot be overridden.
Warning: Cannot override required environment variable
on main.tf line 143, in resource "envbuilder_cached_image" "cached":
143: extra_env = local.envbuilder_env
The key "ENVBUILDER_CACHE_REPO" in extra_env cannot be overridden.
Warning: Overriding provider environment variable
on main.tf line 143, in resource "envbuilder_cached_image" "cached":
143: extra_env = local.envbuilder_env
The key "ENVBUILDER_INSECURE" in extra_env overrides an option set on the provider.
Warning: Cached image not found.
on main.tf line 138, in resource "envbuilder_cached_image" "cached":
138: resource "envbuilder_cached_image" "cached" {
Failed to find cached image in repository "registry:5000/devcontainer-cache". It will be rebuilt in the next apply. Error: open devcontainer.json: open /workspaces/project/.devcontainer/devcontainer.json: no such file or directory
Apply complete! Resources: 9 added, 0 changed, 0 destroyed.
Outputs: 0