Skip to content

Commit 5d58aeb

Browse files
johnstcnjohnstcn
committed
add CODER_INNER_USR_LIB_DIR override
1 parent 55c15f9 commit 5d58aeb

File tree

3 files changed

+36
-3
lines changed

3 files changed

+36
-3
lines changed

‎README.md

+4-2
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ The environment variables can be used to configure various aspects of the inner
2121
| `CODER_DOCKER_BRIDGE_CIDR` | The bridge CIDR to start the Docker daemon with. | false |
2222
| `CODER_MOUNTS` | A list of mounts to mount into the inner container. Mounts default to `rw`. Ex: `CODER_MOUNTS=/home/coder:/home/coder,/var/run/mysecret:/var/run/mysecret:ro` | false |
2323
| `CODER_USR_LIB_DIR` | The mountpoint of the host `/usr/lib` directory. Only required when using GPUs. | false |
24+
| `CODER_INNER_USR_LIB_DIR` | The inner /usr/lib mountpoint. This is automatically detected based on `/etc/os-release` in the inner image, but may optionally be overridden. | false |
2425
| `CODER_ADD_TUN` | If `CODER_ADD_TUN=true` add a TUN device to the inner container. | false |
2526
| `CODER_ADD_FUSE` | If `CODER_ADD_FUSE=true` add a FUSE device to the inner container. | false |
2627
| `CODER_ADD_GPU` | If `CODER_ADD_GPU=true` add detected GPUs and related files to the inner container. Requires setting `CODER_USR_LIB_DIR` and mounting in the hosts `/usr/lib/` directory. | false |
@@ -43,7 +44,7 @@ It is not possible to develop `envbox` effectively using a containerized environ
4344

4445
If a login is required to pull images from a private repository, create a secret following the instructions from the [Kubernetes Documentation](https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/#create-a-secret-by-providing-credentials-on-the-command-line) as such:
4546

46-
```
47+
```shell
4748
kubectl -n <coder namespace> create secret docker-registry regcred \
4849
--docker-server=<your-registry-server> \
4950
--docker-username=<your-name> \
@@ -53,7 +54,7 @@ kubectl -n <coder namespace> create secret docker-registry regcred \
5354

5455
Then reference the secret in your template as such:
5556

56-
```
57+
```shell
5758
env {
5859
name = "CODER_IMAGE_PULL_SECRET"
5960
value_from {
@@ -98,6 +99,7 @@ Here's an example Docker command to run a GPU-enabled workload in Envbox. Note t
9899
1) The NVidia container runtime must be installed on the host (`--runtime=nvidia`).
99100
2) `CODER_ADD_GPU=true` must be set to enable GPU-specific functionality.
100101
3) When `CODER_ADD_GPU` is set, it is required to also set `CODER_USR_LIB_DIR` to a location where the relvant host directory has been mounted inside the outer container. In the below example, this is `/usr/lib/x86_64-linux-gnu` on the underlying host. It is mounted into the container under the path `/var/coder/usr/lib`. We then set `CODER_USR_LIB_DIR=/var/coder/usr/lib`. The actual location inside the container is not important **as long as it does not overwrite any pre-existing directories containing system libraries**.
102+
4) The location to mount the libraries in the inner container is determined by the distribution ID in the `/etc/os-release` of the inner container. If the automatically determined location is incorrect, you can set it manually via `CODER_INNER_USR_LIB_DIR`.
101103
102104
> Note: this step is required in case user workloads require libraries from the underlying host that are not added in by the container runtime.
103105

‎cli/docker.go

+11-1
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ var (
9898
EnvMemory = "CODER_MEMORY"
9999
EnvAddGPU = "CODER_ADD_GPU"
100100
EnvUsrLibDir = "CODER_USR_LIB_DIR"
101+
EnvInnerUsrLibDir = "CODER_INNER_USR_LIB_DIR"
101102
EnvDockerConfig = "CODER_DOCKER_CONFIG"
102103
EnvDebug = "CODER_DEBUG"
103104
EnvDisableIDMappedMount = "CODER_DISABLE_IDMAPPED_MOUNT"
@@ -135,6 +136,7 @@ type flags struct {
135136
boostrapScript string
136137
containerMounts string
137138
hostUsrLibDir string
139+
innerUsrLibDir string
138140
dockerConfig string
139141
cpus int
140142
memory int
@@ -370,6 +372,7 @@ func dockerCmd() *cobra.Command {
370372
cliflag.StringVarP(cmd.Flags(), &flags.boostrapScript, "boostrap-script", "", EnvBootstrap, "", "The script to use to bootstrap the container. This should typically install and start the agent.")
371373
cliflag.StringVarP(cmd.Flags(), &flags.containerMounts, "mounts", "", EnvMounts, "", "Comma separated list of mounts in the form of '<source>:<target>[:options]' (e.g. /var/lib/docker:/var/lib/docker:ro,/usr/src:/usr/src).")
372374
cliflag.StringVarP(cmd.Flags(), &flags.hostUsrLibDir, "usr-lib-dir", "", EnvUsrLibDir, "", "The host /usr/lib mountpoint. Used to detect GPU drivers to mount into inner container.")
375+
cliflag.StringVarP(cmd.Flags(), &flags.innerUsrLibDir, "inner-usr-lib-dir", "", EnvInnerUsrLibDir, "", "The inner /usr/lib mountpoint. This is automatically detected based on /etc/os-release in the inner image, but may optionally be overridden.")
373376
cliflag.StringVarP(cmd.Flags(), &flags.dockerConfig, "docker-config", "", EnvDockerConfig, "/root/.docker/config.json", "The path to the docker config to consult when pulling an image.")
374377
cliflag.BoolVarP(cmd.Flags(), &flags.addTUN, "add-tun", "", EnvAddTun, false, "Add a TUN device to the inner container.")
375378
cliflag.BoolVarP(cmd.Flags(), &flags.addFUSE, "add-fuse", "", EnvAddFuse, false, "Add a FUSE device to the inner container.")
@@ -616,6 +619,13 @@ func runDockerCVM(ctx context.Context, log slog.Logger, client dockerutil.Client
616619
})
617620
}
618621

622+
innerUsrLibDir := imgMeta.UsrLibDir()
623+
if flags.innerUsrLibDir != "" {
624+
log.Info(ctx, "overriding auto-detected inner usr lib dir ",
625+
slog.F("before", innerUsrLibDir),
626+
slog.F("after", flags.innerUsrLibDir))
627+
innerUsrLibDir = flags.innerUsrLibDir
628+
}
619629
for _, bind := range binds {
620630
// If the bind has a path that points to the host-mounted /usr/lib
621631
// directory we need to remap it to /usr/lib inside the container.
@@ -624,7 +634,7 @@ func runDockerCVM(ctx context.Context, log slog.Logger, client dockerutil.Client
624634
mountpoint = filepath.Join(
625635
// Note: we used to mount into /usr/lib, but this can change
626636
// based on the distro inside the container.
627-
imgMeta.UsrLibDir(),
637+
innerUsrLibDir,
628638
strings.TrimPrefix(mountpoint, strings.TrimSuffix(flags.hostUsrLibDir, "/")),
629639
)
630640
}

‎integration/gpu_test.go

+21
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,27 @@ func TestDocker_Nvidia(t *testing.T) {
6363
_, err := execContainerCmd(ctx, t, ctID, "docker", "exec", "workspace_cvm", "nvidia-smi")
6464
require.NoError(t, err, "failed to run nvidia-smi in the inner container")
6565
})
66+
67+
t.Run("InnerUsrLibDirOverride", func(t *testing.T) {
68+
t.Parallel()
69+
ctx, cancel := context.WithCancel(context.Background())
70+
t.Cleanup(cancel)
71+
72+
// Start the envbox container.
73+
ctID := startEnvboxCmd(ctx, t, integrationtest.UbuntuImage, "root",
74+
"-v", "/usr/lib/x86_64-linux-gnu:/var/coder/usr/lib",
75+
"--env", "CODER_ADD_GPU=true",
76+
"--env", "CODER_USR_LIB_DIR=/var/coder/usr/lib",
77+
"--env", "CODER_INNER_USR_LIB_DIR=/usr/lib/coder",
78+
"--runtime=nvidia",
79+
"--gpus=all",
80+
)
81+
82+
// Assert that we can run nvidia-smi in the inner container.
83+
out, err := execContainerCmd(ctx, t, ctID, "docker", "exec", "workspace_cvm", "ls", "-l", "/usr/lib/coder")
84+
require.NoError(t, err, "inner usr lib dir override failed")
85+
require.Regexp(t, `(?i)(libgl|nvidia|vulkan|cuda)`, out)
86+
})
6687
}
6788

6889
// dockerRuntimes returns the list of container runtimes available on the host.

0 commit comments

Comments
 (0)