Added support for other virtualization techs in the Nickel contract (close #28). Added Nix dependency for Snakemake diagram generation.

README.md

@@ -34,7 +34,7 @@ python3 ecg.py <config_file> -p <pkglist_path> -l <log_file> -b <build_status_fi
 ```
 
 Where:
-- `<config_file>` is the configuration file of the artifact in JSON format. An example is given in `artifacts_json/test.json`. WARNING: The name of the file (without the extension) must comply with the Docker image naming convention: only characters allowed are lowercase letters and numbers, separated with either one "." maximum, or two "_" maximum, or an unlimited number of "-", and should be of 128 characters maximum.
+- `<config_file>` is the configuration file of the artifact in JSON format. A template of the Nickel file to use to produce the JSON config file is given in `artifacts/nickel/template.ncl`. WARNING: The name of the file (without the extension) must comply with the Docker image naming convention: only characters allowed are lowercase letters and numbers, separated with either one "." maximum, or two "_" maximum, or an unlimited number of "-", and should be of 128 characters maximum.
 - `<pkglist_path>` is the path to the file where the package list generated by the program should be written.
 - `<log_file>` is the path to the file where to log the output of the program.
 - `<build_status_file>` is the path to the file where to write the build status of the Docker image given in the configuration file.

artifacts/nickel/albab_k9db.ncl

@@ -4,7 +4,8 @@
   type = "zip",
   doi = "...",
   comment = "",
-  dockerfile_location = "K9db-osdi2023",
+  virtualization = "docker",
+  buildfile_dir = "K9db-osdi2023",
   package_managers = [ "dpkg", "pip" ],
   git_packages = [],
   misc_packages = [

artifacts/nickel/breitweiser_high.ncl

@@ -4,7 +4,8 @@
   type = "tar",
   doi = "10.1145/3572848.3577480",
   comment = "",
-  dockerfile_location = "bdm-paper-examples/docker",
+  virtualization = "docker",
+  buildfile_dir = "bdm-paper-examples/docker",
   package_managers = [ "dpkg" ],
   git_packages = [
     { name = "pyenv", location = "~/.pyenv" }

artifacts/nickel/isakov_taxonomy.ncl

@@ -4,7 +4,8 @@
   type = "zip",
   doi = "10.5555/3571885.3571906",
   comment = "",
-  dockerfile_location = "SC22_artifacts_submission",
+  virtualization = "docker",
+  buildfile_dir = "SC22_artifacts_submission",
   package_managers = [ "dpkg", "pip" ],
   git_packages = [],
   misc_packages = [],

artifacts/nickel/moses_high.ncl

@@ -4,7 +4,8 @@
   type = "zip",
   doi = "10.1145/3572848.3577475",
   comment = "",
-  dockerfile_location = "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA",
+  virtualization = "docker",
+  buildfile_dir = "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA",
   package_managers = [ "dpkg", "pip" ],
   git_packages = [
     { name = "MocCUDA", location = "/root/MocCUDA" }

artifacts/nickel/parasyris_approximate.ncl

@@ -4,7 +4,8 @@
   type = "zip",
   doi = "10.5555/3571885.3571974",
   comment = "The package 'cmake-3.14.0-Linux-x86_64' is not specified below, because it is installed using a Bash script included in the artifact archive itself. The file puppet_env.sh is missing and may hold information about a possible Python venv.",
-  dockerfile_location = "koparasy-HPAC-2723bb8/approx/puppeteer/container",
+  virtualization = "docker",
+  buildfile_dir = "koparasy-HPAC-2723bb8/approx/puppeteer/container",
   package_managers = [ "dpkg", "pip" ],
   git_packages = [],
   misc_packages = [

artifacts/nickel/shi_welder.ncl

@@ -4,7 +4,8 @@
   type = "zip",
   doi = "",
   comment = "",
-  dockerfile_location = "nnfusion-osdi2023welder",
+  buildfile_dir = "nnfusion-osdi2023welder",
+  virtualization = "docker",
   package_managers = [ "dpkg", "pip", "conda" ],
   git_packages = [
     { name = "tvm", location = "/root/tvm" },

artifacts/nickel/template.ncl

@@ -4,7 +4,8 @@
   type = "zip",
   doi = "...",
   comment = "Template for artifact configuration.",
-  dockerfile_location = "path/to/docker/folder",
+  virtualization = "docker",
+  buildfile_dir = "path/to/docker/folder",
   package_managers = [ "dpkg", "rpm", "pacman", "pip", "conda" ],
   git_packages = [
     { name = "pkg1", location = "path/to/git/repo" }

artifacts/nickel/test.ncl

@@ -1,16 +1,17 @@
 {
   version = "1.0",
-  artifact_url = "http://localhost/artifact.zip",
+  artifact_url = "http://localhost.com/artifact.zip",
   type = "zip",
   doi = "...",
   comment = "",
-  dockerfile_location = "./",
+  virtualization = "docker",
+  buildfile_dir = "./",
   package_managers = [ "dpkg", "pip" ],
   git_packages = [
     { name = "pkg1", location = "/pkg1" }
   ],
   misc_packages = [
-    { name = "mpkg1", url = "http://localhost/package1.zip" }
+    { name = "mpkg1", url = "http://localhost.com/package1.zip" }
   ],
   python_venvs = []
 }

artifacts/nickel/wang_tgopt.ncl

@@ -4,7 +4,8 @@
   type = "tar",
   doi = "10.1145/3572848.3577490",
   comment = "",
-  dockerfile_location = "tgopt-artifact",
+  virtualization = "docker",
+  buildfile_dir = "tgopt-artifact",
   package_managers = [ "dpkg", "conda" ],
   git_packages = [],
   misc_packages = [

artifacts/nickel/ziogas_deinsum-cpu.ncl

@@ -4,7 +4,8 @@
   type = "zip",
   doi = "10.5555/3571885.3571918",
   comment = "",
-  dockerfile_location = "deinsum-sc22-artifact-7559901/docker_cpu",
+  virtualization = "docker",
+  buildfile_dir = "deinsum-sc22-artifact-7559901/docker_cpu",
   package_managers = [ "dpkg", "pip" ],
   git_packages = [
     { name = "dace", location = "/dace" },

artifacts/nickel/ziogas_deinsum-gpu.ncl

@@ -4,7 +4,8 @@
   type = "zip",
   comment = "",
   doi = "10.5555/3571885.3571918",
-  dockerfile_location = "deinsum-sc22-artifact-7559901/docker_gpu",
+  virtualization = "docker",
+  buildfile_dir = "deinsum-sc22-artifact-7559901/docker_gpu",
   package_managers = [ "dpkg", "pip" ],
   git_packages = [
     { name = "dace", location = "/dace" },

ecg.py

@@ -235,7 +235,7 @@ def build_image(config, src_dir, image_name, docker_cache = False):
     if docker_cache:
         cache_arg = ""
     logging.info(f"Starting building image {image_name}")
-    path = os.path.join(src_dir, config["dockerfile_location"])
+    path = os.path.join(src_dir, config["buildfile_dir"])
     # Using trimmed artifact URL as name:
     build_command = f"docker build{cache_arg} -t {image_name} ."
     build_process = subprocess.run(build_command.split(" "), cwd=path, capture_output=True)
@@ -296,7 +296,7 @@ def check_env(config, src_dir, image_name, pkglist_path):
     logging.info("Checking software environment")
     pkglist_file = open(pkglist_path, "w")
     # pkglist_file.write("package,version,package_manager\n")
-    path = os.path.join(src_dir, config["dockerfile_location"])
+    path = os.path.join(src_dir, config["buildfile_dir"])
     # Package managers:
     for pkgmgr in config["package_managers"]:
         # "--entrypoint" requires command and arguments to be separated.
@@ -316,6 +316,7 @@ def check_env(config, src_dir, image_name, pkglist_path):
     logging.info("Checking Git packages")
     for repo in config["git_packages"]:
         pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], "--entrypoint", gitcmd[0], image_name] + gitcmd[1].split(" "), cwd=path, capture_output=True)
+        print(pkglist_process.stderr.decode('utf-8'))
         repo_row = f"{repo['name']},{pkglist_process.stdout.decode('utf-8')},git"
         pkglist_file.write(f"{repo_row}\n")
 

flake.nix

@@ -21,6 +21,7 @@
             snakemake
             gawk
             nickel
+            graphviz
 	    # TODO separate into several shells
             (python3.withPackages (ps: with ps; [
               requests

nickel.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
-ARTIFACT=$1
+ARTIFACT_NAME=$1
+ARTIFACT_OUT="artifacts/json/$ARTIFACT_NAME.json"
+ARTIFACT_IN="artifacts/nickel/$ARTIFACT_NAME.ncl"
+CONTRACT="workflow/nickel/artifact_contract.ncl"
 
-nickel export --format json --output artifacts/json/$ARTIFACT.json workflow/nickel/artifact_contract.ncl artifacts/nickel/$ARTIFACT.ncl
+# nickel export --format json --output artifacts/json/$ARTIFACT.json workflow/nickel/artifact_contract.ncl artifacts/nickel/$ARTIFACT.ncl
+nickel export --format json --output $ARTIFACT_OUT <<< 'let {Artifact, ..} = import "'$CONTRACT'" in ((import "'$ARTIFACT_IN'") | Artifact)'

workflow/nickel/artifact_contract.ncl

@@ -2,6 +2,7 @@ let
   conf = {
     ARCHIVE_FORMATS = ["zip", "tar"],
     PACKAGE_MANAGERS = ["dpkg", "rpm", "pacman", "pip", "conda"],
+    VIRT_TECHS = ["docker"],
     FILEPATH_REGEX = "^[^\\x00]+$", # For UNIX, anything of length > 0 but without NULL characters, found at: https://stackoverflow.com/questions/537772/what-is-the-most-correct-regular-expression-for-a-unix-file-path
     URL_REGEX = "^https?:\\/\\/(?:www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{1,256}\\.[a-zA-Z0-9()]{1,6}\\b(?:[-a-zA-Z0-9()@:%_\\+.~#?&\\/=]*)$", # Found at: https://uibakery.io/regex-library/url
     IMAGENAME_REGEX = "^[a-z0-9]+(([.]{0,1}|[_]{0,2}|[-]*)[a-z0-9]+)*(:[a-zA-Z0-9_]+[a-zA-Z0-9._-]*){0,1}$", # Based on, with modifications: https://regexr.com/3bsog
@@ -24,6 +25,9 @@ in
   VersionNumber = std.contract.from_predicate (
       fun value => std.string.is_match conf.VERNUMBER_REGEX value
   ),
+  VirtTech = std.contract.from_predicate (
+      fun value => std.array.any (fun x => x == value) conf.VIRT_TECHS
+  ),
   GitPackage = {
     name
       | doc "Name of the package for future identification"
@@ -61,8 +65,11 @@ in
     comment
       | doc "Additional comments on the artifact that couldn't be written elsewhere"
       | String,
-    dockerfile_location
-      | doc "Path to the dockerfile in the artifact"
+    virtualization
+      | doc "Virtualization technology used in the artifact. Possible values: docker"
+      | VirtTech,
+    buildfile_dir
+      | doc "Path to the directory containing the file used to build the container in the artifact"
       | FilePath,
     package_managers
       | doc "Package Managers used in the container. Possible values: dpkg, rpm, pacman, pip, conda"