diff --git a/artifacts/nickel/albab_k9db.ncl b/artifacts/nickel/albab_k9db.ncl index 35f9911..b07eab1 100644 --- a/artifacts/nickel/albab_k9db.ncl +++ b/artifacts/nickel/albab_k9db.ncl @@ -3,7 +3,6 @@ type = "zip", doi = "...", comment = "", - image_name = "k9db/latest", dockerfile_location = "K9db-osdi2023", package_managers = [ "dpkg", "pip" ], git_packages = [], diff --git a/artifacts/nickel/breitweiser_high.ncl b/artifacts/nickel/breitweiser_high.ncl index e6a1f4f..d1ed495 100644 --- a/artifacts/nickel/breitweiser_high.ncl +++ b/artifacts/nickel/breitweiser_high.ncl @@ -3,7 +3,6 @@ type = "tar", doi = "10.1145/3572848.3577480", comment = "", - image_name = "bdm-publication-image-v7", dockerfile_location = "bdm-paper-examples/docker", package_managers = [ "dpkg" ], git_packages = [ diff --git a/artifacts/nickel/isakov_taxonomy.ncl b/artifacts/nickel/isakov_taxonomy.ncl index 77e161b..1be6b09 100644 --- a/artifacts/nickel/isakov_taxonomy.ncl +++ b/artifacts/nickel/isakov_taxonomy.ncl @@ -3,7 +3,6 @@ type = "zip", doi = "10.5555/3571885.3571906", comment = "", - image_name = "taxo:artefact", dockerfile_location = "SC22_artifacts_submission", package_managers = [ "dpkg", "pip" ], git_packages = [], diff --git a/artifacts/nickel/moses_high.ncl b/artifacts/nickel/moses_high.ncl index d894a26..bb00b12 100644 --- a/artifacts/nickel/moses_high.ncl +++ b/artifacts/nickel/moses_high.ncl @@ -3,7 +3,6 @@ type = "zip", doi = "10.1145/3572848.3577475", comment = "", - image_name = "polygeist:artefact", dockerfile_location = "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA", package_managers = [ "dpkg", "pip" ], git_packages = [ diff --git a/artifacts/nickel/parasyris_approximate.ncl b/artifacts/nickel/parasyris_approximate.ncl index bbffbd6..24c6f98 100644 --- a/artifacts/nickel/parasyris_approximate.ncl +++ b/artifacts/nickel/parasyris_approximate.ncl @@ -3,7 +3,6 @@ type = "zip", doi = "10.5555/3571885.3571974", comment = "The package 'cmake-3.14.0-Linux-x86_64' is not specified below, because it is installed using a Bash script included in the artifact archive itself. The file puppet_env.sh is missing and may hold information about a possible Python venv.", - image_name = "puppeteer:artefact", dockerfile_location = "koparasy-HPAC-2723bb8/approx/puppeteer/container", package_managers = [ "dpkg", "pip" ], git_packages = [], diff --git a/artifacts/nickel/shi_welder.ncl b/artifacts/nickel/shi_welder.ncl index d09083e..4634a5f 100644 --- a/artifacts/nickel/shi_welder.ncl +++ b/artifacts/nickel/shi_welder.ncl @@ -3,7 +3,6 @@ type = "zip", doi = "", comment = "", - image_name = "welder_cuda", dockerfile_location = "nnfusion-osdi2023welder", package_managers = [ "dpkg", "pip", "conda" ], git_packages = [ diff --git a/artifacts/nickel/template.ncl b/artifacts/nickel/template.ncl index f6de31b..617bd0e 100644 --- a/artifacts/nickel/template.ncl +++ b/artifacts/nickel/template.ncl @@ -3,7 +3,6 @@ type = "zip", doi = "...", comment = "Template for artifact configuration.", - image_name = "image:version", dockerfile_location = "path/to/docker/folder", package_managers = [ "dpkg", "rpm", "pacman", "pip", "conda" ], git_packages = [ diff --git a/artifacts/nickel/test.ncl b/artifacts/nickel/test.ncl index ef73223..3c407c1 100644 --- a/artifacts/nickel/test.ncl +++ b/artifacts/nickel/test.ncl @@ -3,7 +3,6 @@ type = "zip", doi = "...", comment = "", - image_name = "prog:latest", dockerfile_location = "./", package_managers = [ "dpkg", "pip" ], git_packages = [ diff --git a/artifacts/nickel/wang_tgopt.ncl b/artifacts/nickel/wang_tgopt.ncl index b29138b..24c92fe 100644 --- a/artifacts/nickel/wang_tgopt.ncl +++ b/artifacts/nickel/wang_tgopt.ncl @@ -3,7 +3,6 @@ type = "tar", doi = "10.1145/3572848.3577490", comment = "", - image_name = "tgopt:artefact", dockerfile_location = "tgopt-artifact", package_managers = [ "dpkg", "conda" ], git_packages = [], diff --git a/artifacts/nickel/ziogas_deinsum-cpu.ncl b/artifacts/nickel/ziogas_deinsum-cpu.ncl index cc3c640..62d5c60 100644 --- a/artifacts/nickel/ziogas_deinsum-cpu.ncl +++ b/artifacts/nickel/ziogas_deinsum-cpu.ncl @@ -3,7 +3,6 @@ type = "zip", doi = "10.5555/3571885.3571918", comment = "", - image_name = "deinsum:artefact-cpu", dockerfile_location = "deinsum-sc22-artifact-7559901/docker_cpu", package_managers = [ "dpkg", "pip" ], git_packages = [ diff --git a/artifacts/nickel/ziogas_deinsum-gpu.ncl b/artifacts/nickel/ziogas_deinsum-gpu.ncl index 5cf0f58..51b6180 100644 --- a/artifacts/nickel/ziogas_deinsum-gpu.ncl +++ b/artifacts/nickel/ziogas_deinsum-gpu.ncl @@ -3,7 +3,6 @@ type = "zip", comment = "", doi = "10.5555/3571885.3571918", - image_name = "deinsum:artefact_gpu", dockerfile_location = "deinsum-sc22-artifact-7559901/docker_gpu", package_managers = [ "dpkg", "pip" ], git_packages = [ diff --git a/ecg.py b/ecg.py index e41e9ae..71d46d5 100755 --- a/ecg.py +++ b/ecg.py @@ -20,10 +20,12 @@ import pathlib import logging import datetime import sys +import string -def trim(url) : +def trim(url): """ - Trims given url for cache storage. + Trims given URL to make it contain only lowercase letters and numbers, + as well as with a maximum length of 128. Parameters ---------- @@ -36,10 +38,13 @@ def trim(url) : Trimmed URL. """ trimmed = "" - for c in url.lower(): - if c not in "/:;\\'\" *?": + url_lc = url.lower() + i = 0 + while i < len(url_lc) and i < 128: + c = url_lc[i] + if c in string.ascii_lowercase or c in [str(x) for x in range(0, 10)]: trimmed += c - + i += 1 return trimmed def download_file(url, dest): @@ -66,7 +71,7 @@ def download_file(url, dest): hash_process = subprocess.run(f"sha256sum {file.name} | cut -d ' ' -f 1 | tr -d '\n'", capture_output=True, shell=True) return hash_process.stdout.decode("utf-8") -def download_sources(config, arthashlog_path, dl_dir, use_cache): +def download_sources(config, arthashlog_path, dl_dir, artifact_name, use_cache): """ Downloads the source of the artifact in 'config'. @@ -81,6 +86,9 @@ def download_sources(config, arthashlog_path, dl_dir, use_cache): dl_dir: str Path to the directory where to download the artifact. + artifact_name: str + Name of the artifact. + use_cache: bool Indicates whether the cache should be used or not. @@ -90,7 +98,6 @@ def download_sources(config, arthashlog_path, dl_dir, use_cache): Path to the directory where the artifact is downloaded to. """ url = config["artifact_url"] - artifact_name = trim(url) artifact_dir = os.path.join(dl_dir, artifact_name) # Checking if artifact in cache. Not downloading if it is: if not os.path.exists(artifact_dir) or not use_cache: @@ -165,7 +172,7 @@ def buildstatus_saver(output, buildstatus_path, config_path): buildstatus_file.write(f"{artifact_name},{timestamp},unknown_error\n") buildstatus_file.close() -def build_image(config, src_dir, docker_cache = False): +def build_image(config, src_dir, image_name, docker_cache = False): """ Builds the given Docker image in 'config'. @@ -177,6 +184,9 @@ def build_image(config, src_dir, docker_cache = False): src_dir: str Path to the directory where the artifact is stored. + image_name: str + Name of the Docker image. + docker_cache: bool Enables or disables Docker 'build' cache. @@ -185,13 +195,13 @@ def build_image(config, src_dir, docker_cache = False): return_code: bool, build_output: str Return code and output of Docker 'build'. """ - cache_arg = "--no-cache" + cache_arg = " --no-cache" if docker_cache: cache_arg = "" - name = config["image_name"] - logging.info(f"Starting building image {name}") + logging.info(f"Starting building image {image_name}") path = os.path.join(src_dir, config["dockerfile_location"]) - build_command = f"docker build {cache_arg} -t {config['image_name']} ." + # Using trimmed artifact URL as name: + build_command = f"docker build{cache_arg} -t {trim(config["artifact_url"])} ." build_process = subprocess.run(build_command.split(" "), cwd=path, capture_output=True) build_output = f"stdout:\n{build_process.stdout.decode('utf-8')}\nstderr:\n{build_process.stderr.decode('utf-8')}" # build_output = build_process.stderr.decode("utf-8") @@ -201,7 +211,7 @@ def build_image(config, src_dir, docker_cache = False): logging.info(f"Command '{build_command}' exited with code {return_code}") return return_code, build_output -def check_env(config, src_dir, pkglist_path): +def check_env(config, src_dir, image_name, pkglist_path): """ Builds a list of all software packages installed in the Docker image given in 'config', depending on the package managers @@ -215,6 +225,9 @@ def check_env(config, src_dir, pkglist_path): src_dir: str Path to the directory where the artifact is stored. + image_name: str + Name of the Docker image. + pkglist_path: str Path to the package list file. @@ -258,7 +271,7 @@ def check_env(config, src_dir, pkglist_path): listformat_cmd = pkgmgr_cmd[pkgmgr][2] logging.info(f"Checking '{pkgmgr}'") # pkglist_process = subprocess.run(["docker", "run", "--rm", config["image_name"]] + pkglist_cmd.split(" "), cwd=path, capture_output=True) - pkglist_process = subprocess.run(["docker", "run", "--rm", "--entrypoint", pkglist_cmd, config["image_name"]] + pkglist_cmdargs, cwd=path, capture_output=True) + pkglist_process = subprocess.run(["docker", "run", "--rm", "--entrypoint", pkglist_cmd, image_name] + pkglist_cmdargs, cwd=path, capture_output=True) format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode('utf-8')}EOF", cwd=path, capture_output=True, shell=True) pkglist = format_process.stdout.decode("utf-8") pkglist_file.write(pkglist) @@ -266,7 +279,7 @@ def check_env(config, src_dir, pkglist_path): # Git packages: logging.info("Checking Git packages") for repo in config["git_packages"]: - pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], "--entrypoint", gitcmd[0], config["image_name"]] + gitcmd[1].split(" "), cwd=path, capture_output=True) + pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], "--entrypoint", gitcmd[0], image_name] + gitcmd[1].split(" "), cwd=path, capture_output=True) repo_row = f"{repo['name']},{pkglist_process.stdout.decode('utf-8')},git" pkglist_file.write(f"{repo_row}\n") @@ -284,14 +297,14 @@ def check_env(config, src_dir, pkglist_path): for venv in config["python_venvs"]: pipcmd = pkgmgr_cmd["pip"][0] pipcmd_args = pkgmgr_cmd["pip"][1] - pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", venv["path"], "--entrypoint", "source", config["image_name"], ".bin/activate", "&&", pipcmd] + pipcmd_args.split(" "), cwd=path, capture_output=True) + pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", venv["path"], "--entrypoint", "source", image_name, ".bin/activate", "&&", pipcmd] + pipcmd_args.split(" "), cwd=path, capture_output=True) format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode('utf-8')}EOF", cwd=path, capture_output=True, shell=True) pkglist = format_process.stdout.decode("utf-8") pkglist_file.write(pkglist) pkglist_file.close() -def remove_image(config): +def remove_image(config, image_name): """ Removes the Docker image given in 'config'. @@ -300,13 +313,15 @@ def remove_image(config): config: dict Parsed config file. + image_name: str + Name of the Docker image. + Returns ------- None """ - name = config["image_name"] - logging.info(f"Removing image '{name}'") - subprocess.run(["docker", "rmi", name], capture_output = True) + logging.info(f"Removing image '{image_name}'") + subprocess.run(["docker", "rmi", image_name], capture_output = True) def main(): # Paths: @@ -395,11 +410,12 @@ def main(): else: use_cache = True dl_dir = cache_dir - artifact_dir = download_sources(config, arthashlog_path, dl_dir, use_cache) - return_code, build_output = build_image(config, artifact_dir, args.docker_cache) + artifact_name = trim(config["artifact_url"]) + artifact_dir = download_sources(config, arthashlog_path, dl_dir, artifact_name, use_cache) + return_code, build_output = build_image(config, artifact_dir, artifact_name, args.docker_cache) if return_code == 0: - check_env(config, artifact_dir, pkglist_path) - remove_image(config) + check_env(config, artifact_dir, artifact_name, pkglist_path) + remove_image(config, artifact_name) # Creates file if not already: pathlib.Path(buildstatus_path).touch() else: diff --git a/workflow/nickel/artifact_contract.ncl b/workflow/nickel/artifact_contract.ncl index 6c18547..d6a5b97 100644 --- a/workflow/nickel/artifact_contract.ncl +++ b/workflow/nickel/artifact_contract.ncl @@ -20,9 +20,6 @@ in URL = std.contract.from_predicate ( fun value => std.string.is_match conf.URL_REGEX value ), - ImageName = std.contract.from_predicate ( - fun value => (std.string.is_match conf.IMAGENAME_REGEX value) && (std.string.length value < 128) # Length could be more than 128, but it's easier that way... - ), GitPackage = { name | doc "Name of the package for future identification" @@ -57,9 +54,6 @@ in comment | doc "Additional comments on the artifact that couldn't be written elsewhere" | String, - image_name - | doc "Name to give the image when building" - | ImageName, dockerfile_location | doc "Path to the dockerfile in the artifact" | FilePath,