From 8359c56e2e648c35318b824cd3b4fc31431ae479 Mon Sep 17 00:00:00 2001 From: sidecus <4399408+sidecus@users.noreply.github.com> Date: Mon, 22 Jul 2024 13:53:12 +0800 Subject: [PATCH 01/12] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 43fdad9..3497de5 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ Automatically setup package manager mirrors, including ubuntu, pypi and apk etc. "image": "mcr.microsoft.com/devcontainers/base:ubuntu", "features": { "ghcr.io/sidecus/devcontainer-features/magicmirror:1": { - "ubuntu_mirror": "your-ubuntu-mirror", - // "apk_mirror": "your-alpine-apk-mirror" - "pypi_mirror": "your-python-pypi-mirror", + "ubuntu_mirror": "", # only applicable for Ubuntu base image + "apk_mirror": "", # only applicable to Alpine base image + "pypi_mirror": "https:///pypi/web/simple/", } } } From 215c0ac661ee8477b6f547a2755449a5c207dff6 Mon Sep 17 00:00:00 2001 From: sidecus <4399408+sidecus@users.noreply.github.com> Date: Mon, 22 Jul 2024 13:53:42 +0800 Subject: [PATCH 02/12] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3497de5..a91acae 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@ Automatically setup package manager mirrors, including ubuntu, pypi and apk etc. "image": "mcr.microsoft.com/devcontainers/base:ubuntu", "features": { "ghcr.io/sidecus/devcontainer-features/magicmirror:1": { - "ubuntu_mirror": "", # only applicable for Ubuntu base image - "apk_mirror": "", # only applicable to Alpine base image + "ubuntu_mirror": "", // only applicable to Ubuntu based images + "apk_mirror": "", // only applicable to Alpine based images "pypi_mirror": "https:///pypi/web/simple/", } } From d8a52189a6dd17dc97c857f44694ba91f55a31c0 Mon Sep 17 00:00:00 2001 From: sidecus <4399408+sidecus@users.noreply.github.com> Date: Mon, 22 Jul 2024 15:43:55 +0800 Subject: [PATCH 03/12] add huggingface mirrors --- README.md | 1 + src/magicmirror/devcontainer-feature.json | 8 +++++++ src/magicmirror/install.sh | 27 ++++++++++++++++------- test/magicmirror/hf.sh | 17 ++++++++++++++ test/magicmirror/scenarios.json | 14 +++++++++--- 5 files changed, 56 insertions(+), 11 deletions(-) create mode 100644 test/magicmirror/hf.sh diff --git a/README.md b/README.md index a91acae..3a8b75d 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ Automatically setup package manager mirrors, including ubuntu, pypi and apk etc. "ubuntu_mirror": "", // only applicable to Ubuntu based images "apk_mirror": "", // only applicable to Alpine based images "pypi_mirror": "https:///pypi/web/simple/", + "huggingface_mirror": "https://dummy-hf-mirror.com/" } } } diff --git a/src/magicmirror/devcontainer-feature.json b/src/magicmirror/devcontainer-feature.json index 50ef5b8..ac5f161 100644 --- a/src/magicmirror/devcontainer-feature.json +++ b/src/magicmirror/devcontainer-feature.json @@ -27,6 +27,14 @@ ], "default": "", "description": "Alpine apk mirror" + }, + "huggingface_mirror": { + "type": "string", + "proposals": [ + "https://hf-mirror.com/" + ], + "default": "", + "description": "Huggingface mirror" } }, "installsAfter": [ diff --git a/src/magicmirror/install.sh b/src/magicmirror/install.sh index 4c1ecbf..dbb1797 100644 --- a/src/magicmirror/install.sh +++ b/src/magicmirror/install.sh @@ -11,7 +11,6 @@ echo "Activating feature 'magicmirror'" # For more details, see https://containers.dev/implementors/features#user-env-var echo "The effective dev container remoteUser is '$_REMOTE_USER'" echo "The effective dev container remoteUser's home directory is '$_REMOTE_USER_HOME'" - echo "The effective dev container containerUser is '$_CONTAINER_USER'" echo "The effective dev container containerUser's home directory is '$_CONTAINER_USER_HOME'" @@ -23,18 +22,19 @@ UBUNTU_MIRROR=${UBUNTU_MIRROR:-""} echo "UBUNTU_MIRROR: $UBUNTU_MIRROR" if [ "${OS}" = '"Ubuntu"' ] && [ -n "${UBUNTU_MIRROR}" ]; then echo "Creating /etc/apt/sources.list with mirror = ${UBUNTU_MIRROR}" - cp /etc/apt/sources.list /etc/apt/sources.list.bak - sed -i s/archive.ubuntu.com/$UBUNTU_MIRROR/g /etc/apt/sources.list - sed -i s/security.ubuntu.com/$UBUNTU_MIRROR/g /etc/apt/sources.list - sed -i s/ports.ubuntu.com/$UBUNTU_MIRROR/g /etc/apt/sources.list + cp /etc/apt/sources.list /etc/apt/sources.list.mm.bak + sed -i "s/archive.ubuntu.com/${UBUNTU_MIRROR}/g" /etc/apt/sources.list + sed -i "s/security.ubuntu.com/${UBUNTU_MIRROR}/g" /etc/apt/sources.list + sed -i "s/ports.ubuntu.com/${UBUNTU_MIRROR}/g" /etc/apt/sources.list fi # Pypi mirror. TODO[sidecus]: Check Python installation PYPI_MIRROR=${PYPI_MIRROR:-""} echo "PYPI_MIRROR: $PYPI_MIRROR" if [ -n "${PYPI_MIRROR}" ]; then - echo "Creating /etc/pip.conf with index-url = ${PYPI_MIRROR}" - echo "[global]" > /etc/pip.conf + echo "Seting up pypi mirror via /etc/pip.conf: index-url = ${PYPI_MIRROR}" + touch /etc/pip.conf && cp /etc/pip.conf /etc/pip.conf.mm.bak + echo "[global]" >> /etc/pip.conf echo "index-url = ${PYPI_MIRROR}" >> /etc/pip.conf fi @@ -43,5 +43,16 @@ APK_MIRROR=${APK_MIRROR:-""} echo "APK_MIRROR: $APK_MIRROR" if [ "${OS}" = '"Alpine Linux"' ] && [ -n "${APK_MIRROR}" ]; then echo "Creating /etc/apk/repositories with mirror = ${APK_MIRROR}" + cp /etc/apk/repositories /etc/apk/repositories.mm.bak sed -i "s/dl-cdn.alpinelinux.org/${APK_MIRROR}/g" /etc/apk/repositories -fi \ No newline at end of file +fi + +# Set Huggingface mirror if it's provided +HUGGINGFACE_MIRROR=${HUGGINGFACE_MIRROR:-""} +echo "HUGGINGFACE_MIRROR: $HUGGINGFACE_MIRROR" +if [ -n "${HUGGINGFACE_MIRROR}" ]; then + echo "Enabling Huggingface mirror: ${HUGGINGFACE_MIRROR}" + touch /etc/environment && cp /etc/environment /etc/environment.mm.bak + echo "HF_ENDPOINT=${HF_ENDPOINT}" >> /etc/environment + set -a; . /etc/environment; set +a; +fi diff --git a/test/magicmirror/hf.sh b/test/magicmirror/hf.sh new file mode 100644 index 0000000..3a59b27 --- /dev/null +++ b/test/magicmirror/hf.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# This test file will be executed against one of the scenarios devcontainer.json test that +# includes the 'magicmirror' feature with "huggingface_mirror" option. + +set -e + +# Optional: Import test library bundled with the devcontainer CLI +source dev-container-features-test-lib + +# Feature-specific tests +# The 'check' command comes from the dev-container-features-test-lib. +check "test HF mirror" bash -c '[ "${HF_ENDPOINT}"="https://dummy-hf-mirror.com/" ];' + +# Report results +# If any of the checks above exited with a non-zero exit code, the test will fail. +reportResults diff --git a/test/magicmirror/scenarios.json b/test/magicmirror/scenarios.json index c20b930..8e067a8 100644 --- a/test/magicmirror/scenarios.json +++ b/test/magicmirror/scenarios.json @@ -3,7 +3,7 @@ "image": "mcr.microsoft.com/devcontainers/base:ubuntu", "features": { "magicmirror": { - "ubuntu_mirror": "dummy-ubuntu-mirror" + "ubuntu_mirror": "dummy-ubuntu-mirror.com" } } }, @@ -11,7 +11,7 @@ "image": "mcr.microsoft.com/devcontainers/base:ubuntu", "features": { "magicmirror": { - "pypi_mirror": "dummy-pypi-mirror" + "pypi_mirror": "https://dummy-pypi-mirror.com/pypi/web/simple/" } } }, @@ -19,7 +19,15 @@ "image": "mcr.microsoft.com/devcontainers/base:alpine", "features": { "magicmirror": { - "apk_mirror": "dummy-apk-repo" + "apk_mirror": "dummy-apk-repo.com" + } + } + }, + "hf": { + "image": "mcr.microsoft.com/devcontainers/base:ubuntu", + "features": { + "magicmirror": { + "huggingface_mirror": "https://dummy-hf-mirror.com/" } } } From d6d2ad11813d7599ccde824e78c47d42c5b376de Mon Sep 17 00:00:00 2001 From: sidecus <4399408+sidecus@users.noreply.github.com> Date: Mon, 22 Jul 2024 15:47:13 +0800 Subject: [PATCH 04/12] update test case --- test/magicmirror/pypi.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/magicmirror/pypi.sh b/test/magicmirror/pypi.sh index 7580817..6744699 100644 --- a/test/magicmirror/pypi.sh +++ b/test/magicmirror/pypi.sh @@ -10,7 +10,7 @@ source dev-container-features-test-lib # Feature-specific tests # The 'check' command comes from the dev-container-features-test-lib. -check "test pypi mirror" bash -c "cat /etc/pip.conf | grep 'index-url = dummy-pypi-mirror'" +check "test pypi mirror" bash -c "cat /etc/pip.conf | grep 'index-url = https://dummy-pypi-mirror.com/pypi/web/simple/'" # Report results # If any of the checks above exited with a non-zero exit code, the test will fail. From f2443f7873d3b08b9c04671fd5cb62685914a5d8 Mon Sep 17 00:00:00 2001 From: sidecus <4399408+sidecus@users.noreply.github.com> Date: Mon, 22 Jul 2024 15:51:25 +0800 Subject: [PATCH 05/12] bump version --- src/magicmirror/devcontainer-feature.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/magicmirror/devcontainer-feature.json b/src/magicmirror/devcontainer-feature.json index ac5f161..d052bbc 100644 --- a/src/magicmirror/devcontainer-feature.json +++ b/src/magicmirror/devcontainer-feature.json @@ -1,7 +1,7 @@ { "name": "MagicMirror", "id": "magicmirror", - "version": "1.0.1", + "version": "1.0.2", "description": "Automatic mirror setup for package managers like apt, pypi, apk etc.", "options": { "pypi_mirror": { From 7c16a5ff28b7718f0a7c3ecb3c7aa248632a5c51 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 22 Jul 2024 07:56:01 +0000 Subject: [PATCH 06/12] Automated documentation update [skip ci] --- src/magicmirror/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/magicmirror/README.md b/src/magicmirror/README.md index a00b69c..b173bf1 100644 --- a/src/magicmirror/README.md +++ b/src/magicmirror/README.md @@ -18,6 +18,7 @@ Automatic mirror setup for package managers like apt, pypi, apk etc. | pypi_mirror | Python Pypi mirror | string | - | | ubuntu_mirror | Ubuntu mirror | string | - | | apk_mirror | Alpine apk mirror | string | - | +| huggingface_mirror | Huggingface mirror | string | - | From 67ad752e0949249d56e4900d5fbfe97b1e720eb7 Mon Sep 17 00:00:00 2001 From: sidecus <4399408+sidecus@users.noreply.github.com> Date: Tue, 23 Jul 2024 12:46:32 +0800 Subject: [PATCH 07/12] fix hf mirror bug and related test cases --- src/magicmirror/install.sh | 5 +++-- test/magicmirror/hf.sh | 2 +- test/magicmirror/test.sh | 8 +++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/magicmirror/install.sh b/src/magicmirror/install.sh index dbb1797..16a745b 100644 --- a/src/magicmirror/install.sh +++ b/src/magicmirror/install.sh @@ -28,7 +28,7 @@ if [ "${OS}" = '"Ubuntu"' ] && [ -n "${UBUNTU_MIRROR}" ]; then sed -i "s/ports.ubuntu.com/${UBUNTU_MIRROR}/g" /etc/apt/sources.list fi -# Pypi mirror. TODO[sidecus]: Check Python installation +# Pypi mirror. PYPI_MIRROR=${PYPI_MIRROR:-""} echo "PYPI_MIRROR: $PYPI_MIRROR" if [ -n "${PYPI_MIRROR}" ]; then @@ -53,6 +53,7 @@ echo "HUGGINGFACE_MIRROR: $HUGGINGFACE_MIRROR" if [ -n "${HUGGINGFACE_MIRROR}" ]; then echo "Enabling Huggingface mirror: ${HUGGINGFACE_MIRROR}" touch /etc/environment && cp /etc/environment /etc/environment.mm.bak - echo "HF_ENDPOINT=${HF_ENDPOINT}" >> /etc/environment + echo "HF_ENDPOINT=${HUGGINGFACE_MIRROR}" >> /etc/environment + # export the new env set -a; . /etc/environment; set +a; fi diff --git a/test/magicmirror/hf.sh b/test/magicmirror/hf.sh index 3a59b27..66b8cc0 100644 --- a/test/magicmirror/hf.sh +++ b/test/magicmirror/hf.sh @@ -10,7 +10,7 @@ source dev-container-features-test-lib # Feature-specific tests # The 'check' command comes from the dev-container-features-test-lib. -check "test HF mirror" bash -c '[ "${HF_ENDPOINT}"="https://dummy-hf-mirror.com/" ];' +check "test HF mirror" bash -c "test \"${HF_ENDPOINT}\" = \"https://dummy-hf-mirror.com/"" # Report results # If any of the checks above exited with a non-zero exit code, the test will fail. diff --git a/test/magicmirror/test.sh b/test/magicmirror/test.sh index 9beff66..f62de9b 100644 --- a/test/magicmirror/test.sh +++ b/test/magicmirror/test.sh @@ -20,6 +20,7 @@ # UBUNTU_MIRROR="" # PYPI_MIRROR="" # APK_MIRROR="" +# HUGGINGFACE_MIRROR="" # # These scripts are run as 'root' by default. Although that can be changed @@ -44,9 +45,10 @@ source dev-container-features-test-lib # Feature-specific tests # The 'check' command comes from the dev-container-features-test-lib. Syntax is... # check