diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0ef5266..8e388b6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,6 +1,10 @@ # https://docs.github.com/en/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly - package-ecosystem: pip directory: / schedule: diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 03fe228..220ea48 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -8,11 +8,11 @@ jobs: # https://github.com/pre-commit/action pre-commit: name: Lint - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.0 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + - uses: pre-commit/action@v3.0.1 test: name: Test @@ -21,31 +21,22 @@ jobs: fail-fast: false matrix: python-version: - - '3.7' - - '3.10' + - '3.8' + - '3.13' env: PACKER_VERSION: 1.6.1 - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - name: Cache pip - id: cache - uses: actions/cache@v2 - with: - # This path is specific to Ubuntu - path: ~/.cache/pip - # Look to see if there is a cache hit for the corresponding requirements file - key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} - restore-keys: | - ${{ runner.os }}-pip- - ${{ runner.os }}- + cache: pip + cache-dependency-path: dev-requirements.txt - name: Cache packer id: cachepacker - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/bin key: ${{ runner.os }}-packer-${{ env.PACKER_VERSION }} @@ -64,4 +55,6 @@ jobs: - name: Run pytest run: pytest -v tests - name: Run repo2shellscript scripts - run: ./run-docker-test.sh + run: | + export "PATH=$HOME/bin:$PATH" + ./run-docker-test.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b91baf3..71d2b16 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,12 +1,16 @@ repos: - repo: https://github.com/psf/black - rev: 22.12.0 + rev: 25.1.0 hooks: - id: black args: [--target-version=py37] - repo: https://github.com/pycqa/flake8 - rev: 6.0.0 + rev: 7.3.0 hooks: - id: flake8 - # default black line length is 88 - args: [--max-line-length=88] + args: + # default black line length is 88 + - --max-line-length=88 + # black may include space before : + # black breaks before binary operator + - --ignore=E203,W503 diff --git a/repo2shellscript/resources/buildpack-deps:jammy.sh b/repo2shellscript/resources/buildpack-deps:jammy.sh new file mode 100644 index 0000000..b550f0b --- /dev/null +++ b/repo2shellscript/resources/buildpack-deps:jammy.sh @@ -0,0 +1,75 @@ +# https://github.com/docker-library/buildpack-deps/blob/91dd87eecfa0cf2ae7e793aedbaca682dfcf693d/ubuntu/jammy/Dockerfile +# With the addition of +# - sudo since it makes it easier to switch USER +# - DEBIAN_FRONTEND=noninteractive https://github.com/docker-library/buildpack-deps/blob/91dd87eecfa0cf2ae7e793aedbaca682dfcf693d/ubuntu/jammy/curl/Dockerfile#L17C3-L18 + +ARG DEBIAN_FRONTEND=noninteractive + +apt-get -qq update + +# buildpack-deps:jammy-curl +# buildpack-deps:jammy-scm +# buildpack-deps:jammy +# + sudo + +apt-get -qq install --yes --no-install-recommends \ + ca-certificates \ + curl \ + gnupg \ + netbase \ + wget \ + \ + tzdata \ + \ + git \ + mercurial \ + openssh-client \ + subversion \ + \ + procps \ + \ + autoconf \ + automake \ + bzip2 \ + default-libmysqlclient-dev \ + dpkg-dev \ + file \ + g++ \ + gcc \ + imagemagick \ + libbz2-dev \ + libc6-dev \ + libcurl4-openssl-dev \ + libdb-dev \ + libevent-dev \ + libffi-dev \ + libgdbm-dev \ + libglib2.0-dev \ + libgmp-dev \ + libjpeg-dev \ + libkrb5-dev \ + liblzma-dev \ + libmagickcore-dev \ + libmagickwand-dev \ + libmaxminddb-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libpng-dev \ + libpq-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + libtool \ + libwebp-dev \ + libxml2-dev \ + libxslt-dev \ + libyaml-dev \ + make \ + patch \ + unzip \ + xz-utils \ + zlib1g-dev \ + \ + sudo + +rm -rf /var/lib/apt/lists/* diff --git a/repo2shellscript/resources/repo2docker.pkr.hcl b/repo2shellscript/resources/repo2docker.pkr.hcl index 9a550c2..85ddcb0 100644 --- a/repo2shellscript/resources/repo2docker.pkr.hcl +++ b/repo2shellscript/resources/repo2docker.pkr.hcl @@ -1,5 +1,5 @@ source "docker" "repo" { - image = "ubuntu:18.04" + image = "${os_name}:${os_version}" commit = true changes = [ "ENTRYPOINT /usr/local/bin/repo2shellscript-start.bash", diff --git a/repo2shellscript/resources/repo2vagrant.pkr.hcl b/repo2shellscript/resources/repo2vagrant.pkr.hcl index bc6a0d9..126489c 100644 --- a/repo2shellscript/resources/repo2vagrant.pkr.hcl +++ b/repo2shellscript/resources/repo2vagrant.pkr.hcl @@ -1,6 +1,6 @@ source "vagrant" "virtualbox" { communicator = "ssh" - source_path = "ubuntu/bionic64" + source_path = "${os_name}/${os_version}64" # box_version = "v20200821.1.0" provider = "virtualbox" # add_force = true diff --git a/repo2shellscript/shellscript.py b/repo2shellscript/shellscript.py index 6999783..e2fbd36 100644 --- a/repo2shellscript/shellscript.py +++ b/repo2shellscript/shellscript.py @@ -77,6 +77,20 @@ def _docker_copy(copy, chown): return statement +def _get_os(base_image): + docker_library_prefix = "docker.io/library/" + value = base_image + if value.startswith(docker_library_prefix): + value = value[len(docker_library_prefix) :] + + os_image, _, os_version = value.partition(":") + os_name = os_image + if os_image == "buildpack-deps": + os_name = "ubuntu" + + return os_name, os_image, os_version + + def dockerfile_to_bash(dockerfile, buildargs, parentenv): """ Convert a Dockerfile to a bash script @@ -110,7 +124,7 @@ def dockerfile_to_bash(dockerfile, buildargs, parentenv): user = "root" assert len(parser.structure) == len(parser.context_structure) - for (d, ctx) in zip(parser.structure, parser.context_structure): + for d, ctx in zip(parser.structure, parser.context_structure): statement = "" instruction = d["instruction"] for line in d["content"].splitlines(): @@ -121,8 +135,11 @@ def dockerfile_to_bash(dockerfile, buildargs, parentenv): if instruction in ("EXPOSE", "COMMENT", "LABEL"): pass elif instruction == "FROM": + os_name, os_image, os_version = _get_os(d["value"]) try: - base_setup = pkg_resources.read_text(resources, f"{d['value']}.sh") + base_setup = pkg_resources.read_text( + resources, f"{os_image}:{os_version}.sh" + ) except FileNotFoundError: raise NotImplementedError(f"Base image {d['value']} not supported") statement += base_setup @@ -172,6 +189,8 @@ def dockerfile_to_bash(dockerfile, buildargs, parentenv): "bash": bash, "dir": currentdir, "env": runtimeenv, + "os_name": os_name, + "os_version": os_version, "start": f"{entrypoint} {cmd}", "user": user, } @@ -218,6 +237,8 @@ def __init__(self, *, parent): def build( self, *, + push=False, + load=False, buildargs=None, cache_from=None, container_limits=None, @@ -245,6 +266,9 @@ def build( else: jupyter_token = str(uuid4()) + # Ignore push and load args + # https://github.com/jupyterhub/repo2docker/pull/1421 + if kwargs: raise NotImplementedError("Additional kwargs not supported") # TODO: custom_context? @@ -317,6 +341,8 @@ def build( work_dir = r["dir"] or "~" template_args = { + "os_name": r["os_name"], + "os_version": r["os_version"], "user": r["user"], "start": r["start"], "systemd_environment": systemd_environment, @@ -344,8 +370,11 @@ def images(self): return [Image(tags=[tag]) for tag in images] def inspect_image(self, image): - assert os.path.exists(os.path.join(self.output_directory, image)) - return Image(tags=[image]) + if os.path.exists(os.path.join(self.output_directory, image)): + return Image(tags=[image]) + # https://github.com/jupyterhub/repo2docker/pull/1421 + # Return None if image doesn't exist + return None def push(self, image_spec): raise NotImplementedError("push() is not supported") diff --git a/run-docker-test.sh b/run-docker-test.sh index 934bffa..3934f86 100755 --- a/run-docker-test.sh +++ b/run-docker-test.sh @@ -6,6 +6,7 @@ IMAGE_NAME=test-repo2shellscript repo2docker --engine shellscript --user-id 1000 --user-name test --no-run --image-name $IMAGE_NAME https://github.com/binder-examples/conda # Test by building a Docker image with packer +packer version cd ./repo2shellscript-output/$IMAGE_NAME packer build repo2docker.pkr.hcl