diff --git a/src/ollama_gemma-4-26B-A4B/.devcontainer.json b/src/ollama_gemma-4-26B-A4B/.devcontainer.json new file mode 100644 index 000000000..d81311e07 --- /dev/null +++ b/src/ollama_gemma-4-26B-A4B/.devcontainer.json @@ -0,0 +1,44 @@ +{ + "name": "Ollama Gemma 4 26B-A4B", + "dockerComposeFile": "docker-compose.yaml", + "service": "app", + "shutdownAction": "none", + "workspaceFolder": "/workspace", + "postCreateCommand": + "./startupscript/post-startup.sh abc /config \"${templateOption:cloud}\" \"${templateOption:login}\"; ./sudo-passwordless.sh abc; ./start-ollama.sh", + "postStartCommand": + "./startupscript/remount-on-restart.sh abc /config \"${templateOption:cloud}\" \"${templateOption:login}\"; ./start-ollama.sh", + "features": { + "ghcr.io/devcontainers/features/java:1": { + "version": "17" + }, + "ghcr.io/devcontainers/features/aws-cli:1": {}, + "ghcr.io/dhoeric/features/google-cloud-cli:1": {}, + "ghcr.io/anthropics/devcontainer-features/claude-code@sha256:cfc2e7d3e9fd3b9b01f8d5cb158508a884c8c0ede2e23ed10f32dea5d4ffe69a": {}, + "./.devcontainer/features/workbench-tools": { + "cloud": "${templateOption:cloud}", + "username": "abc", + "userHomeDir": "/config" + } + }, + "remoteUser": "root", + "customizations": { + "workbench": { + "opens": { + "extensions": [ + ".py", + ".ipynb", + ".sh", + ".md", + ".html", + ".csv", + ".json", + ".jsonc", + ".yaml", + ".yml" + ], + "fileUrlSuffix": "?payload=[[\"openFile\",\"vscode-remote:///config/{path}\"]]" + } + } + } +} diff --git a/src/ollama_gemma-4-26B-A4B/Dockerfile b/src/ollama_gemma-4-26B-A4B/Dockerfile new file mode 100644 index 000000000..a8ddc67d1 --- /dev/null +++ b/src/ollama_gemma-4-26B-A4B/Dockerfile @@ -0,0 +1,24 @@ +FROM lscr.io/linuxserver/code-server@sha256:7bd334657f13505abc1e20afeeee5670ad8f818e68853c810889184e597f3051 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends jq python3 python3-pip python3-venv tesseract-ocr \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /config/extensions /config/.cache \ + && chown -R abc:abc /config + +RUN apt-get update && apt-get install -y --no-install-recommends zstd \ + && curl -fsSL https://ollama.com/download/ollama-linux-amd64.tar.zst | tar --zstd -x -C /usr \ + && rm -rf /var/lib/apt/lists/* \ + && ollama --version + +RUN pip install --break-system-packages openai uv + +USER abc +ENV HOME=/config + +RUN /app/code-server/bin/code-server --extensions-dir /config/extensions --install-extension ms-python.python \ + && mkdir -p /config/data/User \ + && echo '{"workbench.colorTheme":"Default Dark Modern"}' > /config/data/User/settings.json + +USER root +WORKDIR /config diff --git a/src/ollama_gemma-4-26B-A4B/README.md b/src/ollama_gemma-4-26B-A4B/README.md new file mode 100644 index 000000000..7126a49d2 --- /dev/null +++ b/src/ollama_gemma-4-26B-A4B/README.md @@ -0,0 +1,78 @@ +# Ollama + Gemma 4 26B-A4B + +Code-server IDE with an Ollama inference server running Gemma 4 26B MoE (4B active parameters). Opens a browser-based VS Code editor on port 8443 with Ollama serving an OpenAI-compatible API on port 11434 inside the container. + +## Recommended VM Configuration + +| Model | GPU | Machine type | CPUs | RAM | Notes | +|---|---|---|---|---|---| +| **Gemma 4 26B MoE** | **A100 (40 GB)** | **n1-standard-8** | **8** | **30 GB** | **Default config for this app** | +| Gemma 4 26B MoE | A100 (80 GB) | n1-standard-8 | 8 | 30 GB | More VRAM headroom for longer contexts | + +## Changing the Model + +Update the `ollama pull` command in `start-ollama.sh` and the `model=` parameter in your Python scripts. See available models at https://ollama.com/library. + +## Usage + +Once the app is ready, open the code-server IDE and run: + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:11434/v1", api_key="unused") + +response = client.chat.completions.create( + model="gemma4:26b", + messages=[{"role": "user", "content": "Hello!"}], + max_tokens=100, +) +print(response.choices[0].message.content) +``` + +### Using `uv` + +1. Install uv + + ```sh + pip install uv + ``` + +2. Initialize the project + + ```sh + uv init local-ollama-gemma4-26b + ``` + +3. Add openai package + + ```sh + cd local-ollama-gemma4-26b/ + uv add openai + ``` + +4. Run the application + + ```sh + uv run main.py + ``` + +## Debugging + +Check Ollama server logs: + +```bash +tail -f ~/ollama-server.log +``` + +Verify the server is running: + +```bash +curl http://localhost:11434/v1/models +``` + +List downloaded models: + +```bash +ollama list +``` diff --git a/src/ollama_gemma-4-26B-A4B/devcontainer-template.json b/src/ollama_gemma-4-26B-A4B/devcontainer-template.json new file mode 100644 index 000000000..47edeabd1 --- /dev/null +++ b/src/ollama_gemma-4-26B-A4B/devcontainer-template.json @@ -0,0 +1,30 @@ +{ + "id": "ollama_gemma-4-26B-A4B", + "version": "0.0.1", + "name": "Ollama Gemma 4 26B-A4B", + "description": "Code-server IDE with Ollama inference server (Gemma 4 26B MoE). Requires A100 GPU.", + "documentationURL": "https://github.com/verily-src/workbench-app-devcontainers/tree/master/src/ollama_gemma-4-26B-A4B", + "licenseURL": "https://github.com/verily-src/workbench-app-devcontainers/blob/master/LICENSE", + "options": { + "cloud": { + "type": "string", + "description": "VM cloud environment", + "proposals": [ + "gcp" + ], + "default": "gcp" + }, + "login": { + "type": "string", + "description": "Whether to log in to workbench CLI", + "proposals": [ + "true", + "false" + ], + "default": "false" + } + }, + "platforms": [ + "Any" + ] +} diff --git a/src/ollama_gemma-4-26B-A4B/docker-compose.yaml b/src/ollama_gemma-4-26B-A4B/docker-compose.yaml new file mode 100644 index 000000000..d34c906f2 --- /dev/null +++ b/src/ollama_gemma-4-26B-A4B/docker-compose.yaml @@ -0,0 +1,32 @@ +services: + app: + container_name: "application-server" + build: + context: . + dockerfile: Dockerfile + restart: always + volumes: + - .:/workspace:cached + - work:/config:cached + ports: + - "8443:8443" + environment: + USER: "abc" + DEFAULT_WORKSPACE: "/config" + SUDO_PASSWORD: "pwd" + OLLAMA_KEEP_ALIVE: "-1" + OLLAMA_FLASH_ATTENTION: "true" + networks: + - app-network + cap_add: + - SYS_ADMIN + devices: + - /dev/fuse + security_opt: + - apparmor:unconfined + +networks: + app-network: + external: true +volumes: + work: diff --git a/src/ollama_gemma-4-26B-A4B/start-ollama.sh b/src/ollama_gemma-4-26B-A4B/start-ollama.sh new file mode 100755 index 000000000..170260662 --- /dev/null +++ b/src/ollama_gemma-4-26B-A4B/start-ollama.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +OLLAMA_LOG="/config/ollama-server.log" + +nohup ollama serve > "${OLLAMA_LOG}" 2>&1 & +sleep 2 + +echo "Pulling gemma4:26b model (this may take a few minutes)..." +ollama pull gemma4:26b >> "${OLLAMA_LOG}" 2>&1 + +echo "Preloading model into GPU memory..." +curl -s http://localhost:11434/api/generate -d '{"model":"gemma4:26b","prompt":"warmup","options":{"num_predict":1}}' > /dev/null 2>&1 + +echo "Ollama ready — model loaded — logs at ${OLLAMA_LOG}" diff --git a/src/ollama_gemma-4-26B-A4B/sudo-passwordless.sh b/src/ollama_gemma-4-26B-A4B/sudo-passwordless.sh new file mode 100755 index 000000000..d14bd0a17 --- /dev/null +++ b/src/ollama_gemma-4-26B-A4B/sudo-passwordless.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# This script is used to set up passwordless sudo for the core user on the VM. +# It requires to be run with root priviledges and USER_NAME to be set in the environment. +# It is typically called from post-startup.sh. + +USER_NAME="${1}" + +if [[ -z "${USER_NAME}" ]]; then + echo "Usage: $0 " + exit 1 +fi + +sudoers_file="/etc/sudoers" +sudoers_d_file="/etc/sudoers.d/${USER_NAME}" + +# Make sure user exists +if ! id "${USER_NAME}" &>/dev/null; then + echo "User ${USER_NAME} does not exist." + exit 1 +fi + +# Check if there's an old rule in the main sudoers file that requires a password +if grep -q "^${USER_NAME} ALL=(ALL:ALL) ALL" "${sudoers_file}"; then + echo "Found password-requiring rule for ${USER_NAME} in /etc/sudoers. Commenting it out." + + # Comment out the old rule in /etc/sudoers + sed -i "s/^${USER_NAME} ALL=(ALL:ALL) ALL/# ${USER_NAME} ALL=(ALL:ALL) ALL/" "${sudoers_file}" +fi + +echo "${USER_NAME} ALL=(ALL) NOPASSWD:ALL" > "${sudoers_d_file}" +chmod 440 "${sudoers_d_file}" + +echo "User ${USER_NAME} has been given passwordless sudo access." diff --git a/src/pep-ppp-prod/.devcontainer.json b/src/pep-ppp-prod/.devcontainer.json new file mode 100644 index 000000000..15f3c2131 --- /dev/null +++ b/src/pep-ppp-prod/.devcontainer.json @@ -0,0 +1,69 @@ +{ + "name": "pepcli-prod app", + "dockerComposeFile": "docker-compose.yaml", + "service": "app", + "shutdownAction": "none", + "workspaceFolder": "/workspace", + "postCreateCommand": "./startupscript/post-startup.sh root /config \"${templateOption:cloud}\" \"${templateOption:login}\"; ./sudo-passwordless.sh root; ./setup-claude-code.sh", + "postStartCommand": [ + "./startupscript/remount-on-restart.sh", + "root", + "/config", + "${templateOption:cloud}", + "${templateOption:login}" + ], + "features": { + "ghcr.io/devcontainers/features/java:1.6.3": { + "version": "17", + "jdkDistro": "tem" + }, + "ghcr.io/devcontainers/features/aws-cli:1.1.1": {}, + "ghcr.io/dhoeric/features/google-cloud-cli:1.0.1": {}, + "ghcr.io/coder/devcontainer-features/code-server:1": { + "port": 8080, + "host": "0.0.0.0", + "auth": "none" + }, + "ghcr.io/devcontainers/features/go:1": { + "version": "1.22" + }, + "ghcr.io/devcontainers/features/python:1": { + "version": "3.12" + }, + "ghcr.io/anthropics/devcontainer-features/claude-code:1": {} + }, + "remoteUser": "root", + "customizations": { + "vscode": { + "settings": { + "workbench.colorTheme": "Default Dark Modern" + } + }, + "workbench": { + "opens": { + "extensions": [ + ".c", + ".cjs", + ".cpp", + ".go", + ".java", + ".js", + ".mjs", + ".php", + ".scala", + ".sh", + ".ts", + ".md", + ".html", + ".csv", + ".json", + ".jsonc", + ".tsv", + ".xml", + ".yml" + ], + "fileUrlSuffix": "?payload=[[\"openFile\",\"vscode-remote:///config/{path}\"]]" + } + } + } +} \ No newline at end of file diff --git a/src/pep-ppp-prod/README.md b/src/pep-ppp-prod/README.md new file mode 100644 index 000000000..9909fd726 --- /dev/null +++ b/src/pep-ppp-prod/README.md @@ -0,0 +1,17 @@ + +# PEP CLI Prod (pep-ppp-prod) + +A Template to run PEP CLI prod on workbench. + +## Options + +| Options Id | Description | Type | Default Value | +|-----|-----|-----|-----| +| cloud | VM cloud environment | string | gcp | +| login | Whether to log in to workbench CLI | string | false | + + + +--- + +_Note: This file was auto-generated from the [devcontainer-template.json](https://github.com/verily-src/workbench-app-devcontainers/blob/main/src/pep-ppp-prod/devcontainer-template.json). Add additional notes to a `NOTES.md`._ \ No newline at end of file diff --git a/src/pep-ppp-prod/devcontainer-template.json b/src/pep-ppp-prod/devcontainer-template.json new file mode 100644 index 000000000..57ad3f700 --- /dev/null +++ b/src/pep-ppp-prod/devcontainer-template.json @@ -0,0 +1,23 @@ +{ + "id": "pepcli-prod", + "version": "0.0.1", + "name": "PEP CLI Prod", + "description": "A Template to run PEP CLI prod on workbench", + "documentationURL": "https://github.com/verily-src/workbench-app-devcontainers/tree/master/src/pepcli-prod", + "licenseURL": "https://github.com/verily-src/workbench-app-devcontainers/blob/master/LICENSE", + "options": { + "cloud": { + "type": "string", + "description": "VM cloud environment", + "proposals": ["gcp", "aws"], + "default": "gcp" + }, + "login": { + "type": "string", + "description": "Whether to log in to workbench CLI", + "proposals": ["true", "false"], + "default": "true" + } + }, + "platforms": ["Any"] +} diff --git a/src/pep-ppp-prod/docker-compose.yaml b/src/pep-ppp-prod/docker-compose.yaml new file mode 100644 index 000000000..206dfe008 --- /dev/null +++ b/src/pep-ppp-prod/docker-compose.yaml @@ -0,0 +1,28 @@ +version: "2.4" +services: + app: + container_name: "application-server" + image: "gitlabregistry.pep.cs.ru.nl/pep-public/core/ppp-prod" + restart: always + volumes: + - .:/workspace:cached + - work:/home/ubuntu:cached + ports: + - "8080:8080" + environment: + USER: "root" + DEFAULT_WORKSPACE: "/config" + SUDO_PASSWORD: "pwd" + networks: + - app-network + cap_add: + - SYS_ADMIN + devices: + - /dev/fuse + security_opt: + - apparmor:unconfined +networks: + app-network: + external: true +volumes: + work: diff --git a/src/pep-ppp-prod/setup-claude-code.sh b/src/pep-ppp-prod/setup-claude-code.sh new file mode 100755 index 000000000..a8bf39a77 --- /dev/null +++ b/src/pep-ppp-prod/setup-claude-code.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +mkdir -p ~/.claude && cat < ~/.claude/settings.json +{ + "env": { + "CLAUDE_CODE_USE_VERTEX": "1", + "CLOUD_ML_REGION": "us-east5", + "VERTEX_REGION_CLAUDE_4_7_OPUS": "us", + "VERTEX_REGION_CLAUDE_4_8_OPUS": "us", + "ANTHROPIC_VERTEX_PROJECT_ID": "wb-agile-aubergine-8187", + "CLAUDE_CODE_ENABLE_TELEMETRY": "1", + "DISABLE_TELEMETRY": "1", + "OTEL_LOG_USER_PROMPTS": "0", + "OTEL_LOG_TOOL_DETAILS": "1", + "OTEL_METRICS_EXPORTER": "otlp", + "OTEL_LOGS_EXPORTER": "otlp", + "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf", + "OTEL_EXPORTER_OTLP_ENDPOINT": "https://claude-otel-collector-usage-events-64mw6qm9.uc.gateway.dev", + "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "https://claude-otel-collector-usage-events-64mw6qm9.uc.gateway.dev/v1/traces", + "OTEL_EXPORTER_OTLP_METRICS_ENDPOINT": "https://claude-otel-collector-usage-events-64mw6qm9.uc.gateway.dev/v1/metrics", + "OTEL_EXPORTER_OTLP_HEADERS": "x-api-key=AIzaSyAY8kCGXpV3XHEDc3J5CoMk7XeCkb6HVuQ", + "OTEL_RESOURCE_ATTRIBUTES": "environment=workbench,username=\${WORKBENCH_USER_EMAIL%@*}" + }, + "permissions": { + "disableBypassPermissionsMode": "disable", + "deny": [ + "Read(.env)", + "Read(.env.*)" + ] + }, + "allowedMcpServers": [ + {"serverUrl": "https://mcp.atlassian.com/v1/mcp"}, + {"serverUrl": "https://mcp.figma.com/mcp"}, + {"serverUrl": "https://mcp.cypress.io/mcp"}, + {"serverUrl": "https://mcp.heymarvin.com"}, + {"serverCommand": ["xcrun", "mcpbridge"]}, + { + "serverCommand": [ + "npx", + "-y", + "@playwright/mcp@latest", + "--allowed-origins", + "https://dev.*.verily.com;https://test.*.verily.com;https://dev.*.verilyme.com;https://test.*.verilyme.com;https://localhost:*", + "--save-session", + "--extension" + ] + } + ], + "sandbox": { + "enabled": true, + "autoAllowBashIfSandboxed": true, + "excludedCommands": ["git"], + "allowUnsandboxedCommands": false + } +} +EOF + +mkdir -p /config/data/User +echo '{"workbench.colorTheme":"Default Dark Modern"}' > /config/data/User/settings.json diff --git a/src/pep-ppp-prod/sudo-passwordless.sh b/src/pep-ppp-prod/sudo-passwordless.sh new file mode 100644 index 000000000..d14bd0a17 --- /dev/null +++ b/src/pep-ppp-prod/sudo-passwordless.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# This script is used to set up passwordless sudo for the core user on the VM. +# It requires to be run with root priviledges and USER_NAME to be set in the environment. +# It is typically called from post-startup.sh. + +USER_NAME="${1}" + +if [[ -z "${USER_NAME}" ]]; then + echo "Usage: $0 " + exit 1 +fi + +sudoers_file="/etc/sudoers" +sudoers_d_file="/etc/sudoers.d/${USER_NAME}" + +# Make sure user exists +if ! id "${USER_NAME}" &>/dev/null; then + echo "User ${USER_NAME} does not exist." + exit 1 +fi + +# Check if there's an old rule in the main sudoers file that requires a password +if grep -q "^${USER_NAME} ALL=(ALL:ALL) ALL" "${sudoers_file}"; then + echo "Found password-requiring rule for ${USER_NAME} in /etc/sudoers. Commenting it out." + + # Comment out the old rule in /etc/sudoers + sed -i "s/^${USER_NAME} ALL=(ALL:ALL) ALL/# ${USER_NAME} ALL=(ALL:ALL) ALL/" "${sudoers_file}" +fi + +echo "${USER_NAME} ALL=(ALL) NOPASSWD:ALL" > "${sudoers_d_file}" +chmod 440 "${sudoers_d_file}" + +echo "User ${USER_NAME} has been given passwordless sudo access." diff --git a/src/vllm/.devcontainer.json b/src/vllm/.devcontainer.json new file mode 100644 index 000000000..a3fac8486 --- /dev/null +++ b/src/vllm/.devcontainer.json @@ -0,0 +1,49 @@ +{ + "name": "Ollama Gemma 4", + "dockerComposeFile": "docker-compose.yaml", + "service": "app", + "shutdownAction": "none", + "workspaceFolder": "/workspace", + "postCreateCommand": + "./startupscript/post-startup.sh abc /config \"${templateOption:cloud}\" \"${templateOption:login}\"; ./sudo-passwordless.sh abc; ./start-vllm.sh", + "postStartCommand": [ + "./startupscript/remount-on-restart.sh", + "abc", + "/config", + "${templateOption:cloud}", + "${templateOption:login}" + ], + "features": { + "ghcr.io/devcontainers/features/java:1": { + "version": "17" + }, + "ghcr.io/devcontainers/features/aws-cli:1": {}, + "ghcr.io/dhoeric/features/google-cloud-cli:1": {}, + "ghcr.io/anthropics/devcontainer-features/claude-code@sha256:cfc2e7d3e9fd3b9b01f8d5cb158508a884c8c0ede2e23ed10f32dea5d4ffe69a": {}, + "./.devcontainer/features/workbench-tools": { + "cloud": "${templateOption:cloud}", + "username": "abc", + "userHomeDir": "/config" + } + }, + "remoteUser": "root", + "customizations": { + "workbench": { + "opens": { + "extensions": [ + ".py", + ".ipynb", + ".sh", + ".md", + ".html", + ".csv", + ".json", + ".jsonc", + ".yaml", + ".yml" + ], + "fileUrlSuffix": "?payload=[[\"openFile\",\"vscode-remote:///config/{path}\"]]" + } + } + } +} diff --git a/src/vllm/Dockerfile b/src/vllm/Dockerfile new file mode 100644 index 000000000..5ada347b5 --- /dev/null +++ b/src/vllm/Dockerfile @@ -0,0 +1,24 @@ +FROM lscr.io/linuxserver/code-server@sha256:7bd334657f13505abc1e20afeeee5670ad8f818e68853c810889184e597f3051 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends jq python3 python3-pip python3-venv \ + && rm -rf /var/lib/apt/lists/* \ + && mkdir -p /config/extensions /config/.cache \ + && chown -R abc:abc /config + +RUN apt-get update && apt-get install -y --no-install-recommends zstd \ + && curl -fsSL https://ollama.com/download/ollama-linux-amd64.tar.zst | tar --zstd -x -C /usr \ + && rm -rf /var/lib/apt/lists/* \ + && ollama --version + +RUN pip install --break-system-packages openai + +USER abc +ENV HOME=/config + +RUN /app/code-server/bin/code-server --extensions-dir /config/extensions --install-extension ms-python.python \ + && mkdir -p /config/data/User \ + && echo '{"workbench.colorTheme":"Default Dark Modern"}' > /config/data/User/settings.json + +USER root +WORKDIR /config diff --git a/src/vllm/README.md b/src/vllm/README.md new file mode 100644 index 000000000..477b79cae --- /dev/null +++ b/src/vllm/README.md @@ -0,0 +1,80 @@ +# Ollama + Gemma 4 + +Code-server IDE with an Ollama inference server running Gemma 4 E4B. Opens a browser-based VS Code editor on port 8443 with Ollama serving an OpenAI-compatible API on port 11434 inside the container. + +## Recommended VM Configuration + +| Model | GPU | Machine type | CPUs | RAM | Notes | +|---|---|---|---|---|---| +| Gemma 4 E2B | T4 (16 GB) | n1-standard-2 | 2 | 7.5 GB | Smallest viable setup | +| **Gemma 4 E4B** | **T4 (16 GB)** | **n1-standard-4** | **4** | **15 GB** | **Default config, best value** | +| Gemma 4 E4B | L4 (24 GB) | n1-standard-4 | 4 | 15 GB | ~2x faster inference | +| Gemma 4 26B MoE | A100 (40 GB) | n1-standard-8 | 8 | 30 GB | Larger model, needs more VRAM | + +## Changing the Model + +Update the `ollama pull` command in `start-vllm.sh` and the `model=` parameter in your Python scripts. See available models at https://ollama.com/library. + +## Usage + +Once the app is ready, open the code-server IDE and run: + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:11434/v1", api_key="unused") + +response = client.chat.completions.create( + model="gemma4:e4b", + messages=[{"role": "user", "content": "Hello!"}], + max_tokens=100, +) +print(response.choices[0].message.content) +``` + +### Using `uv` + +1. Install uv + + ```sh + pip install uv + ``` + +2. Initialize the project + + ```sh + uv init local-vllm-gema4-test + ``` + +3. Add openai package + + ```sh + cd local-vllm-gema4-test/ + uv add openai + ``` + +4. Run the application + + ```sh + uv run main.py + ``` + +## Debugging + +Check Ollama server logs: + +```bash +tail -f ~/ollama-server.log +``` + +Verify the server is running: + +```bash +curl http://localhost:11434/v1/models +``` + +List downloaded models: + +```bash +ollama list +``` diff --git a/src/vllm/devcontainer-template.json b/src/vllm/devcontainer-template.json new file mode 100644 index 000000000..a8bf68b3a --- /dev/null +++ b/src/vllm/devcontainer-template.json @@ -0,0 +1,30 @@ +{ + "id": "vllm", + "version": "0.0.1", + "name": "vLLM", + "description": "Code-server IDE with Ollama inference server (Gemma 4 E4B). Supports CPU and GPU.", + "documentationURL": "https://github.com/verily-src/workbench-app-devcontainers/tree/master/src/vllm", + "licenseURL": "https://github.com/verily-src/workbench-app-devcontainers/blob/master/LICENSE", + "options": { + "cloud": { + "type": "string", + "description": "VM cloud environment", + "proposals": [ + "gcp" + ], + "default": "gcp" + }, + "login": { + "type": "string", + "description": "Whether to log in to workbench CLI", + "proposals": [ + "true", + "false" + ], + "default": "false" + } + }, + "platforms": [ + "Any" + ] +} diff --git a/src/vllm/docker-compose.yaml b/src/vllm/docker-compose.yaml new file mode 100644 index 000000000..bc09abbab --- /dev/null +++ b/src/vllm/docker-compose.yaml @@ -0,0 +1,31 @@ +services: + app: + container_name: "application-server" + build: + context: . + dockerfile: Dockerfile + restart: always + volumes: + - .:/workspace:cached + - work:/config:cached + ports: + - "8443:8443" + environment: + USER: "abc" + DEFAULT_WORKSPACE: "/config" + SUDO_PASSWORD: "pwd" + OLLAMA_KEEP_ALIVE: "-1" + networks: + - app-network + cap_add: + - SYS_ADMIN + devices: + - /dev/fuse + security_opt: + - apparmor:unconfined + +networks: + app-network: + external: true +volumes: + work: diff --git a/src/vllm/start-vllm.sh b/src/vllm/start-vllm.sh new file mode 100755 index 000000000..d5c6f8616 --- /dev/null +++ b/src/vllm/start-vllm.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -o errexit +set -o nounset +set -o pipefail + +OLLAMA_LOG="/config/ollama-server.log" + +nohup ollama serve > "${OLLAMA_LOG}" 2>&1 & +sleep 2 + +echo "Pulling gemma4:e4b model (this may take a few minutes)..." +ollama pull gemma4:e4b >> "${OLLAMA_LOG}" 2>&1 + +echo "Preloading model into GPU memory..." +curl -s http://localhost:11434/api/generate -d '{"model":"gemma4:e4b","prompt":"warmup","options":{"num_predict":1}}' > /dev/null 2>&1 + +echo "Ollama ready — model loaded — logs at ${OLLAMA_LOG}" diff --git a/src/vllm/sudo-passwordless.sh b/src/vllm/sudo-passwordless.sh new file mode 100755 index 000000000..d14bd0a17 --- /dev/null +++ b/src/vllm/sudo-passwordless.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# This script is used to set up passwordless sudo for the core user on the VM. +# It requires to be run with root priviledges and USER_NAME to be set in the environment. +# It is typically called from post-startup.sh. + +USER_NAME="${1}" + +if [[ -z "${USER_NAME}" ]]; then + echo "Usage: $0 " + exit 1 +fi + +sudoers_file="/etc/sudoers" +sudoers_d_file="/etc/sudoers.d/${USER_NAME}" + +# Make sure user exists +if ! id "${USER_NAME}" &>/dev/null; then + echo "User ${USER_NAME} does not exist." + exit 1 +fi + +# Check if there's an old rule in the main sudoers file that requires a password +if grep -q "^${USER_NAME} ALL=(ALL:ALL) ALL" "${sudoers_file}"; then + echo "Found password-requiring rule for ${USER_NAME} in /etc/sudoers. Commenting it out." + + # Comment out the old rule in /etc/sudoers + sed -i "s/^${USER_NAME} ALL=(ALL:ALL) ALL/# ${USER_NAME} ALL=(ALL:ALL) ALL/" "${sudoers_file}" +fi + +echo "${USER_NAME} ALL=(ALL) NOPASSWD:ALL" > "${sudoers_d_file}" +chmod 440 "${sudoers_d_file}" + +echo "User ${USER_NAME} has been given passwordless sudo access."