Added support for other virtualization techs in the Nickel contract (close #28). Added Nix dependency for Snakemake diagram generation.
This commit is contained in:
parent
ff2fc2d365
commit
114d0e5816
@ -34,7 +34,7 @@ python3 ecg.py <config_file> -p <pkglist_path> -l <log_file> -b <build_status_fi
|
||||
```
|
||||
|
||||
Where:
|
||||
- `<config_file>` is the configuration file of the artifact in JSON format. An example is given in `artifacts_json/test.json`. WARNING: The name of the file (without the extension) must comply with the Docker image naming convention: only characters allowed are lowercase letters and numbers, separated with either one "." maximum, or two "_" maximum, or an unlimited number of "-", and should be of 128 characters maximum.
|
||||
- `<config_file>` is the configuration file of the artifact in JSON format. A template of the Nickel file to use to produce the JSON config file is given in `artifacts/nickel/template.ncl`. WARNING: The name of the file (without the extension) must comply with the Docker image naming convention: only characters allowed are lowercase letters and numbers, separated with either one "." maximum, or two "_" maximum, or an unlimited number of "-", and should be of 128 characters maximum.
|
||||
- `<pkglist_path>` is the path to the file where the package list generated by the program should be written.
|
||||
- `<log_file>` is the path to the file where to log the output of the program.
|
||||
- `<build_status_file>` is the path to the file where to write the build status of the Docker image given in the configuration file.
|
||||
|
@ -4,7 +4,8 @@
|
||||
type = "zip",
|
||||
doi = "...",
|
||||
comment = "",
|
||||
dockerfile_location = "K9db-osdi2023",
|
||||
virtualization = "docker",
|
||||
buildfile_dir = "K9db-osdi2023",
|
||||
package_managers = [ "dpkg", "pip" ],
|
||||
git_packages = [],
|
||||
misc_packages = [
|
||||
|
@ -4,7 +4,8 @@
|
||||
type = "tar",
|
||||
doi = "10.1145/3572848.3577480",
|
||||
comment = "",
|
||||
dockerfile_location = "bdm-paper-examples/docker",
|
||||
virtualization = "docker",
|
||||
buildfile_dir = "bdm-paper-examples/docker",
|
||||
package_managers = [ "dpkg" ],
|
||||
git_packages = [
|
||||
{ name = "pyenv", location = "~/.pyenv" }
|
||||
|
@ -4,7 +4,8 @@
|
||||
type = "zip",
|
||||
doi = "10.5555/3571885.3571906",
|
||||
comment = "",
|
||||
dockerfile_location = "SC22_artifacts_submission",
|
||||
virtualization = "docker",
|
||||
buildfile_dir = "SC22_artifacts_submission",
|
||||
package_managers = [ "dpkg", "pip" ],
|
||||
git_packages = [],
|
||||
misc_packages = [],
|
||||
|
@ -4,7 +4,8 @@
|
||||
type = "zip",
|
||||
doi = "10.1145/3572848.3577475",
|
||||
comment = "",
|
||||
dockerfile_location = "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA",
|
||||
virtualization = "docker",
|
||||
buildfile_dir = "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA",
|
||||
package_managers = [ "dpkg", "pip" ],
|
||||
git_packages = [
|
||||
{ name = "MocCUDA", location = "/root/MocCUDA" }
|
||||
|
@ -4,7 +4,8 @@
|
||||
type = "zip",
|
||||
doi = "10.5555/3571885.3571974",
|
||||
comment = "The package 'cmake-3.14.0-Linux-x86_64' is not specified below, because it is installed using a Bash script included in the artifact archive itself. The file puppet_env.sh is missing and may hold information about a possible Python venv.",
|
||||
dockerfile_location = "koparasy-HPAC-2723bb8/approx/puppeteer/container",
|
||||
virtualization = "docker",
|
||||
buildfile_dir = "koparasy-HPAC-2723bb8/approx/puppeteer/container",
|
||||
package_managers = [ "dpkg", "pip" ],
|
||||
git_packages = [],
|
||||
misc_packages = [
|
||||
|
@ -4,7 +4,8 @@
|
||||
type = "zip",
|
||||
doi = "",
|
||||
comment = "",
|
||||
dockerfile_location = "nnfusion-osdi2023welder",
|
||||
buildfile_dir = "nnfusion-osdi2023welder",
|
||||
virtualization = "docker",
|
||||
package_managers = [ "dpkg", "pip", "conda" ],
|
||||
git_packages = [
|
||||
{ name = "tvm", location = "/root/tvm" },
|
||||
|
@ -4,7 +4,8 @@
|
||||
type = "zip",
|
||||
doi = "...",
|
||||
comment = "Template for artifact configuration.",
|
||||
dockerfile_location = "path/to/docker/folder",
|
||||
virtualization = "docker",
|
||||
buildfile_dir = "path/to/docker/folder",
|
||||
package_managers = [ "dpkg", "rpm", "pacman", "pip", "conda" ],
|
||||
git_packages = [
|
||||
{ name = "pkg1", location = "path/to/git/repo" }
|
||||
|
@ -1,16 +1,17 @@
|
||||
{
|
||||
version = "1.0",
|
||||
artifact_url = "http://localhost/artifact.zip",
|
||||
artifact_url = "http://localhost.com/artifact.zip",
|
||||
type = "zip",
|
||||
doi = "...",
|
||||
comment = "",
|
||||
dockerfile_location = "./",
|
||||
virtualization = "docker",
|
||||
buildfile_dir = "./",
|
||||
package_managers = [ "dpkg", "pip" ],
|
||||
git_packages = [
|
||||
{ name = "pkg1", location = "/pkg1" }
|
||||
],
|
||||
misc_packages = [
|
||||
{ name = "mpkg1", url = "http://localhost/package1.zip" }
|
||||
{ name = "mpkg1", url = "http://localhost.com/package1.zip" }
|
||||
],
|
||||
python_venvs = []
|
||||
}
|
||||
|
@ -4,7 +4,8 @@
|
||||
type = "tar",
|
||||
doi = "10.1145/3572848.3577490",
|
||||
comment = "",
|
||||
dockerfile_location = "tgopt-artifact",
|
||||
virtualization = "docker",
|
||||
buildfile_dir = "tgopt-artifact",
|
||||
package_managers = [ "dpkg", "conda" ],
|
||||
git_packages = [],
|
||||
misc_packages = [
|
||||
|
@ -4,7 +4,8 @@
|
||||
type = "zip",
|
||||
doi = "10.5555/3571885.3571918",
|
||||
comment = "",
|
||||
dockerfile_location = "deinsum-sc22-artifact-7559901/docker_cpu",
|
||||
virtualization = "docker",
|
||||
buildfile_dir = "deinsum-sc22-artifact-7559901/docker_cpu",
|
||||
package_managers = [ "dpkg", "pip" ],
|
||||
git_packages = [
|
||||
{ name = "dace", location = "/dace" },
|
||||
|
@ -4,7 +4,8 @@
|
||||
type = "zip",
|
||||
comment = "",
|
||||
doi = "10.5555/3571885.3571918",
|
||||
dockerfile_location = "deinsum-sc22-artifact-7559901/docker_gpu",
|
||||
virtualization = "docker",
|
||||
buildfile_dir = "deinsum-sc22-artifact-7559901/docker_gpu",
|
||||
package_managers = [ "dpkg", "pip" ],
|
||||
git_packages = [
|
||||
{ name = "dace", location = "/dace" },
|
||||
|
5
ecg.py
5
ecg.py
@ -235,7 +235,7 @@ def build_image(config, src_dir, image_name, docker_cache = False):
|
||||
if docker_cache:
|
||||
cache_arg = ""
|
||||
logging.info(f"Starting building image {image_name}")
|
||||
path = os.path.join(src_dir, config["dockerfile_location"])
|
||||
path = os.path.join(src_dir, config["buildfile_dir"])
|
||||
# Using trimmed artifact URL as name:
|
||||
build_command = f"docker build{cache_arg} -t {image_name} ."
|
||||
build_process = subprocess.run(build_command.split(" "), cwd=path, capture_output=True)
|
||||
@ -296,7 +296,7 @@ def check_env(config, src_dir, image_name, pkglist_path):
|
||||
logging.info("Checking software environment")
|
||||
pkglist_file = open(pkglist_path, "w")
|
||||
# pkglist_file.write("package,version,package_manager\n")
|
||||
path = os.path.join(src_dir, config["dockerfile_location"])
|
||||
path = os.path.join(src_dir, config["buildfile_dir"])
|
||||
# Package managers:
|
||||
for pkgmgr in config["package_managers"]:
|
||||
# "--entrypoint" requires command and arguments to be separated.
|
||||
@ -316,6 +316,7 @@ def check_env(config, src_dir, image_name, pkglist_path):
|
||||
logging.info("Checking Git packages")
|
||||
for repo in config["git_packages"]:
|
||||
pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], "--entrypoint", gitcmd[0], image_name] + gitcmd[1].split(" "), cwd=path, capture_output=True)
|
||||
print(pkglist_process.stderr.decode('utf-8'))
|
||||
repo_row = f"{repo['name']},{pkglist_process.stdout.decode('utf-8')},git"
|
||||
pkglist_file.write(f"{repo_row}\n")
|
||||
|
||||
|
@ -21,6 +21,7 @@
|
||||
snakemake
|
||||
gawk
|
||||
nickel
|
||||
graphviz
|
||||
# TODO separate into several shells
|
||||
(python3.withPackages (ps: with ps; [
|
||||
requests
|
||||
|
@ -1,5 +1,9 @@
|
||||
#!/bin/bash
|
||||
|
||||
ARTIFACT=$1
|
||||
ARTIFACT_NAME=$1
|
||||
ARTIFACT_OUT="artifacts/json/$ARTIFACT_NAME.json"
|
||||
ARTIFACT_IN="artifacts/nickel/$ARTIFACT_NAME.ncl"
|
||||
CONTRACT="workflow/nickel/artifact_contract.ncl"
|
||||
|
||||
nickel export --format json --output artifacts/json/$ARTIFACT.json workflow/nickel/artifact_contract.ncl artifacts/nickel/$ARTIFACT.ncl
|
||||
# nickel export --format json --output artifacts/json/$ARTIFACT.json workflow/nickel/artifact_contract.ncl artifacts/nickel/$ARTIFACT.ncl
|
||||
nickel export --format json --output $ARTIFACT_OUT <<< 'let {Artifact, ..} = import "'$CONTRACT'" in ((import "'$ARTIFACT_IN'") | Artifact)'
|
@ -2,6 +2,7 @@ let
|
||||
conf = {
|
||||
ARCHIVE_FORMATS = ["zip", "tar"],
|
||||
PACKAGE_MANAGERS = ["dpkg", "rpm", "pacman", "pip", "conda"],
|
||||
VIRT_TECHS = ["docker"],
|
||||
FILEPATH_REGEX = "^[^\\x00]+$", # For UNIX, anything of length > 0 but without NULL characters, found at: https://stackoverflow.com/questions/537772/what-is-the-most-correct-regular-expression-for-a-unix-file-path
|
||||
URL_REGEX = "^https?:\\/\\/(?:www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b(?:[-a-zA-Z0-9()@:%_\\+.~#?&\\/=]*)$", # Found at: https://uibakery.io/regex-library/url
|
||||
IMAGENAME_REGEX = "^[a-z0-9]+(([.]{0,1}|[_]{0,2}|[-]*)[a-z0-9]+)*(:[a-zA-Z0-9_]+[a-zA-Z0-9._-]*){0,1}$", # Based on, with modifications: https://regexr.com/3bsog
|
||||
@ -24,6 +25,9 @@ in
|
||||
VersionNumber = std.contract.from_predicate (
|
||||
fun value => std.string.is_match conf.VERNUMBER_REGEX value
|
||||
),
|
||||
VirtTech = std.contract.from_predicate (
|
||||
fun value => std.array.any (fun x => x == value) conf.VIRT_TECHS
|
||||
),
|
||||
GitPackage = {
|
||||
name
|
||||
| doc "Name of the package for future identification"
|
||||
@ -61,8 +65,11 @@ in
|
||||
comment
|
||||
| doc "Additional comments on the artifact that couldn't be written elsewhere"
|
||||
| String,
|
||||
dockerfile_location
|
||||
| doc "Path to the dockerfile in the artifact"
|
||||
virtualization
|
||||
| doc "Virtualization technology used in the artifact. Possible values: docker"
|
||||
| VirtTech,
|
||||
buildfile_dir
|
||||
| doc "Path to the directory containing the file used to build the container in the artifact"
|
||||
| FilePath,
|
||||
package_managers
|
||||
| doc "Package Managers used in the container. Possible values: dpkg, rpm, pacman, pip, conda"
|
||||
|
Loading…
Reference in New Issue
Block a user