GuilloteauQ/study-docker-repro-longevity
1
0
Fork 0
Code Issues 6 Pull Requests Actions Projects Releases Wiki Activity

Removed image name from the config file, now generated from the artifact URL (close #24). Fixed the "--no-cache" option when using Docker cache.

Browse Source
This commit is contained in:
antux18 2024-07-24 18:58:50 +02:00
parent d2768d5373
commit 48d3da9c7d
13 changed files with 40 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
artifacts/nickel/albab_k9db.ncl
View File

@ -3,7 +3,6 @@
type = "zip",
doi = "...",
comment = "",
image_name = "k9db/latest",
dockerfile_location = "K9db-osdi2023",
package_managers = [ "dpkg", "pip" ],
git_packages = [],

1
artifacts/nickel/breitweiser_high.ncl
View File

@ -3,7 +3,6 @@
type = "tar",
doi = "10.1145/3572848.3577480",
comment = "",
image_name = "bdm-publication-image-v7",
dockerfile_location = "bdm-paper-examples/docker",
package_managers = [ "dpkg" ],
git_packages = [

1
artifacts/nickel/isakov_taxonomy.ncl
View File

@ -3,7 +3,6 @@
type = "zip",
doi = "10.5555/3571885.3571906",
comment = "",
image_name = "taxo:artefact",
dockerfile_location = "SC22_artifacts_submission",
package_managers = [ "dpkg", "pip" ],
git_packages = [],

1
artifacts/nickel/moses_high.ncl
View File

@ -3,7 +3,6 @@
type = "zip",
doi = "10.1145/3572848.3577475",
comment = "",
image_name = "polygeist:artefact",
dockerfile_location = "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA",
package_managers = [ "dpkg", "pip" ],
git_packages = [

1
artifacts/nickel/parasyris_approximate.ncl
View File

@ -3,7 +3,6 @@
type = "zip",
doi = "10.5555/3571885.3571974",
comment = "The package 'cmake-3.14.0-Linux-x86_64' is not specified below, because it is installed using a Bash script included in the artifact archive itself. The file puppet_env.sh is missing and may hold information about a possible Python venv.",
image_name = "puppeteer:artefact",
dockerfile_location = "koparasy-HPAC-2723bb8/approx/puppeteer/container",
package_managers = [ "dpkg", "pip" ],
git_packages = [],

1
artifacts/nickel/shi_welder.ncl
View File

@ -3,7 +3,6 @@
type = "zip",
doi = "",
comment = "",
image_name = "welder_cuda",
dockerfile_location = "nnfusion-osdi2023welder",
package_managers = [ "dpkg", "pip", "conda" ],
git_packages = [

1
artifacts/nickel/template.ncl
View File

@ -3,7 +3,6 @@
type = "zip",
doi = "...",
comment = "Template for artifact configuration.",
image_name = "image:version",
dockerfile_location = "path/to/docker/folder",
package_managers = [ "dpkg", "rpm", "pacman", "pip", "conda" ],
git_packages = [

1
artifacts/nickel/test.ncl
View File

@ -3,7 +3,6 @@
type = "zip",
doi = "...",
comment = "",
image_name = "prog:latest",
dockerfile_location = "./",
package_managers = [ "dpkg", "pip" ],
git_packages = [

1
artifacts/nickel/wang_tgopt.ncl
View File

@ -3,7 +3,6 @@
type = "tar",
doi = "10.1145/3572848.3577490",
comment = "",
image_name = "tgopt:artefact",
dockerfile_location = "tgopt-artifact",
package_managers = [ "dpkg", "conda" ],
git_packages = [],

1
artifacts/nickel/ziogas_deinsum-cpu.ncl
View File

@ -3,7 +3,6 @@
type = "zip",
doi = "10.5555/3571885.3571918",
comment = "",
image_name = "deinsum:artefact-cpu",
dockerfile_location = "deinsum-sc22-artifact-7559901/docker_cpu",
package_managers = [ "dpkg", "pip" ],
git_packages = [

1
artifacts/nickel/ziogas_deinsum-gpu.ncl
View File

@ -3,7 +3,6 @@
type = "zip",
comment = "",
doi = "10.5555/3571885.3571918",
image_name = "deinsum:artefact_gpu",
dockerfile_location = "deinsum-sc22-artifact-7559901/docker_gpu",
package_managers = [ "dpkg", "pip" ],
git_packages = [

64
ecg.py
View File

@ -20,10 +20,12 @@ import pathlib
import logging
import datetime
import sys
import string
def trim(url) :
def trim(url):
"""
Trims given url for cache storage.
Trims given URL to make it contain only lowercase letters and numbers,
as well as with a maximum length of 128.
Parameters
----------
@ -36,10 +38,13 @@ def trim(url) :
Trimmed URL.
"""
trimmed = ""
for c in url.lower():
if c not in "/:;\\'\" *?":
url_lc = url.lower()
i = 0
while i < len(url_lc) and i < 128:
c = url_lc[i]
if c in string.ascii_lowercase or c in [str(x) for x in range(0, 10)]:
trimmed += c
i += 1
return trimmed
def download_file(url, dest):
@ -66,7 +71,7 @@ def download_file(url, dest):
hash_process = subprocess.run(f"sha256sum {file.name} | cut -d ' ' -f 1 | tr -d '\n'", capture_output=True, shell=True)
return hash_process.stdout.decode("utf-8")
def download_sources(config, arthashlog_path, dl_dir, use_cache):
def download_sources(config, arthashlog_path, dl_dir, artifact_name, use_cache):
"""
Downloads the source of the artifact in 'config'.
@ -81,6 +86,9 @@ def download_sources(config, arthashlog_path, dl_dir, use_cache):
dl_dir: str
Path to the directory where to download the artifact.
artifact_name: str
Name of the artifact.
use_cache: bool
Indicates whether the cache should be used or not.
@ -90,7 +98,6 @@ def download_sources(config, arthashlog_path, dl_dir, use_cache):
Path to the directory where the artifact is downloaded to.
"""
url = config["artifact_url"]
artifact_name = trim(url)
artifact_dir = os.path.join(dl_dir, artifact_name)
# Checking if artifact in cache. Not downloading if it is:
if not os.path.exists(artifact_dir) or not use_cache:
@ -165,7 +172,7 @@ def buildstatus_saver(output, buildstatus_path, config_path):
buildstatus_file.write(f"{artifact_name},{timestamp},unknown_error\n")
buildstatus_file.close()
def build_image(config, src_dir, docker_cache = False):
def build_image(config, src_dir, image_name, docker_cache = False):
"""
Builds the given Docker image in 'config'.
@ -177,6 +184,9 @@ def build_image(config, src_dir, docker_cache = False):
src_dir: str
Path to the directory where the artifact is stored.
image_name: str
Name of the Docker image.
docker_cache: bool
Enables or disables Docker 'build' cache.
@ -185,13 +195,13 @@ def build_image(config, src_dir, docker_cache = False):
return_code: bool, build_output: str
Return code and output of Docker 'build'.
"""
cache_arg = "--no-cache"
cache_arg = " --no-cache"
if docker_cache:
cache_arg = ""
name = config["image_name"]
logging.info(f"Starting building image {name}")
logging.info(f"Starting building image {image_name}")
path = os.path.join(src_dir, config["dockerfile_location"])
build_command = f"docker build {cache_arg} -t {config['image_name']} ."
# Using trimmed artifact URL as name:
build_command = f"docker build{cache_arg} -t {trim(config["artifact_url"])} ."
build_process = subprocess.run(build_command.split(" "), cwd=path, capture_output=True)
build_output = f"stdout:\n{build_process.stdout.decode('utf-8')}\nstderr:\n{build_process.stderr.decode('utf-8')}"
# build_output = build_process.stderr.decode("utf-8")
@ -201,7 +211,7 @@ def build_image(config, src_dir, docker_cache = False):
logging.info(f"Command '{build_command}' exited with code {return_code}")
return return_code, build_output
def check_env(config, src_dir, pkglist_path):
def check_env(config, src_dir, image_name, pkglist_path):
"""
Builds a list of all software packages installed in the
Docker image given in 'config', depending on the package managers
@ -215,6 +225,9 @@ def check_env(config, src_dir, pkglist_path):
src_dir: str
Path to the directory where the artifact is stored.
image_name: str
Name of the Docker image.
pkglist_path: str
Path to the package list file.
@ -258,7 +271,7 @@ def check_env(config, src_dir, pkglist_path):
listformat_cmd = pkgmgr_cmd[pkgmgr][2]
logging.info(f"Checking '{pkgmgr}'")
# pkglist_process = subprocess.run(["docker", "run", "--rm", config["image_name"]] + pkglist_cmd.split(" "), cwd=path, capture_output=True)
pkglist_process = subprocess.run(["docker", "run", "--rm", "--entrypoint", pkglist_cmd, config["image_name"]] + pkglist_cmdargs, cwd=path, capture_output=True)
pkglist_process = subprocess.run(["docker", "run", "--rm", "--entrypoint", pkglist_cmd, image_name] + pkglist_cmdargs, cwd=path, capture_output=True)
format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode('utf-8')}EOF", cwd=path, capture_output=True, shell=True)
pkglist = format_process.stdout.decode("utf-8")
pkglist_file.write(pkglist)
@ -266,7 +279,7 @@ def check_env(config, src_dir, pkglist_path):
# Git packages:
logging.info("Checking Git packages")
for repo in config["git_packages"]:
pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], "--entrypoint", gitcmd[0], config["image_name"]] + gitcmd[1].split(" "), cwd=path, capture_output=True)
pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], "--entrypoint", gitcmd[0], image_name] + gitcmd[1].split(" "), cwd=path, capture_output=True)
repo_row = f"{repo['name']},{pkglist_process.stdout.decode('utf-8')},git"
pkglist_file.write(f"{repo_row}\n")
@ -284,14 +297,14 @@ def check_env(config, src_dir, pkglist_path):
for venv in config["python_venvs"]:
pipcmd = pkgmgr_cmd["pip"][0]
pipcmd_args = pkgmgr_cmd["pip"][1]
pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", venv["path"], "--entrypoint", "source", config["image_name"], ".bin/activate", "&&", pipcmd] + pipcmd_args.split(" "), cwd=path, capture_output=True)
pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", venv["path"], "--entrypoint", "source", image_name, ".bin/activate", "&&", pipcmd] + pipcmd_args.split(" "), cwd=path, capture_output=True)
format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode('utf-8')}EOF", cwd=path, capture_output=True, shell=True)
pkglist = format_process.stdout.decode("utf-8")
pkglist_file.write(pkglist)
pkglist_file.close()
def remove_image(config):
def remove_image(config, image_name):
"""
Removes the Docker image given in 'config'.
@ -300,13 +313,15 @@ def remove_image(config):
config: dict
Parsed config file.
image_name: str
Name of the Docker image.
Returns
-------
None
"""
name = config["image_name"]
logging.info(f"Removing image '{name}'")
subprocess.run(["docker", "rmi", name], capture_output = True)
logging.info(f"Removing image '{image_name}'")
subprocess.run(["docker", "rmi", image_name], capture_output = True)
def main():
# Paths:
@ -395,11 +410,12 @@ def main():
else:
use_cache = True
dl_dir = cache_dir
artifact_dir = download_sources(config, arthashlog_path, dl_dir, use_cache)
return_code, build_output = build_image(config, artifact_dir, args.docker_cache)
artifact_name = trim(config["artifact_url"])
artifact_dir = download_sources(config, arthashlog_path, dl_dir, artifact_name, use_cache)
return_code, build_output = build_image(config, artifact_dir, artifact_name, args.docker_cache)
if return_code == 0:
check_env(config, artifact_dir, pkglist_path)
remove_image(config)
check_env(config, artifact_dir, artifact_name, pkglist_path)
remove_image(config, artifact_name)
# Creates file if not already:
pathlib.Path(buildstatus_path).touch()
else:

6
workflow/nickel/artifact_contract.ncl
View File

@ -20,9 +20,6 @@ in
URL = std.contract.from_predicate (
fun value => std.string.is_match conf.URL_REGEX value
),
ImageName = std.contract.from_predicate (
fun value => (std.string.is_match conf.IMAGENAME_REGEX value) && (std.string.length value < 128) # Length could be more than 128, but it's easier that way...
),
GitPackage = {
name
| doc "Name of the package for future identification"
@ -57,9 +54,6 @@ in
comment
| doc "Additional comments on the artifact that couldn't be written elsewhere"
| String,
image_name
| doc "Name to give the image when building"
| ImageName,
dockerfile_location
| doc "Path to the dockerfile in the artifact"
| FilePath,