Written Nickel + JSON artifact config files from their YAML version. Handling unknown errors during image building.

artifacts_json/breitweiser_high.json

@@ -0,0 +1,38 @@
+{
+  "artifact_url": "https://zenodo.org/record/7544675/files/SF2-code.tar.gz",
+  "dockerfile_location": "bdm-paper-examples/docker",
+  "doi": "10.1145/3572848.3577480",
+  "git_packages": [
+    {
+      "location": "~/.pyenv",
+      "name": "pyenv"
+    }
+  ],
+  "image_name": "bdm-publication-image-v7",
+  "misc_packages": [
+    {
+      "name": "NetLogo-6.2.0-64",
+      "type": "tar",
+      "url": "http://ccl.northwestern.edu/netlogo/6.2.0/NetLogo-6.2.0-64.tgz"
+    },
+    {
+      "name": "root_v6.22.06_python3.9_ubuntu-20.04",
+      "type": "tar",
+      "url": "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/root_v6.22.06_python3.9_ubuntu-20.04.tar.gz"
+    },
+    {
+      "name": "qt_v5.12.10_ubuntu-20.04",
+      "type": "tar",
+      "url": "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/qt_v5.12.10_ubuntu-20.04.tar.gz"
+    },
+    {
+      "name": "paraview_v5.9.0_ubuntu-20.04_default",
+      "type": "tar",
+      "url": "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/paraview_v5.9.0_ubuntu-20.04_default.tar.gz"
+    }
+  ],
+  "package_managers": [
+    "dpkg"
+  ],
+  "type": "tar"
+}

artifacts_json/isakov_taxonomy.json

@@ -0,0 +1,13 @@
+{
+  "artifact_url": "https://zenodo.org/record/6632461/files/SC22_artifacts_submission.zip",
+  "dockerfile_location": "SC22_artifacts_submission",
+  "doi": "10.5555/3571885.3571906",
+  "git_packages": [],
+  "image_name": "taxo:artefact",
+  "misc_packages": [],
+  "package_managers": [
+    "dpkg",
+    "pip"
+  ],
+  "type": "zip"
+}

artifacts_json/moses_high.json

@@ -0,0 +1,24 @@
+{
+  "artifact_url": "https://zenodo.org/record/7508499/files/wsmoses/PolygeistGPU-Docker-v0.2.1.zip",
+  "dockerfile_location": "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA",
+  "doi": "10.1145/3572848.3577475",
+  "git_packages": [
+    {
+      "location": "/root/MocCUDA",
+      "name": "MocCUDA"
+    }
+  ],
+  "image_name": "polygeist:artefact",
+  "misc_packages": [
+    {
+      "name": "cmake-3.23.1",
+      "type": "tar",
+      "url": "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz"
+    }
+  ],
+  "package_managers": [
+    "dpkg",
+    "pip"
+  ],
+  "type": "zip"
+}

artifacts_json/parasyris_approximate.json

@@ -0,0 +1,19 @@
+{
+  "artifact_url": "https://zenodo.org/record/6926481/files/koparasy/HPAC-v0.0.0-Puppeteer.zip",
+  "dockerfile_location": "koparasy-HPAC-2723bb8/approx/puppeteer/container",
+  "doi": "10.5555/3571885.3571974",
+  "git_packages": [],
+  "image_name": "puppeteer:artefact",
+  "misc_packages": [
+    {
+      "name": "HPAC",
+      "type": "zip",
+      "url": "https://github.com/koparasy/HPAC/archive/refs/heads/develop.zip"
+    }
+  ],
+  "package_managers": [
+    "dpkg",
+    "pip"
+  ],
+  "type": "zip"
+}

artifacts_json/wang_tgopt.json

@@ -0,0 +1,19 @@
+{
+  "artifact_url": "https://zenodo.org/record/7328505/files/tgopt-artifact.tgz",
+  "dockerfile_location": "tgopt-artifact",
+  "doi": "10.1145/3572848.3577490",
+  "git_packages": [],
+  "image_name": "tgopt:artefact",
+  "misc_packages": [
+    {
+      "name": "Miniconda3-py37_4.12.0-Linux-x86_64",
+      "type": "zip",
+      "url": "https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh"
+    }
+  ],
+  "package_managers": [
+    "dpkg",
+    "conda"
+  ],
+  "type": "tar"
+}

artifacts_json/ziogas_deinsum-cpu.json

@@ -0,0 +1,37 @@
+{
+  "artifact_url": "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
+  "dockerfile_location": "deinsum-sc22-artifact-7559901/docker_cpu",
+  "doi": "10.5555/3571885.3571918",
+  "git_packages": [
+    {
+      "location": "/dace",
+      "name": "dace"
+    },
+    {
+      "location": "/multilinear-algebra-sc22-artifact",
+      "name": "multilinear-algebra-sc22-artifact"
+    },
+    {
+      "location": "/ctf",
+      "name": "ctf"
+    }
+  ],
+  "image_name": "deinsum:artefact-cpu",
+  "misc_packages": [
+    {
+      "name": "mpich-3.1.4",
+      "type": "tar",
+      "url": "http://www.mpich.org/static/downloads/3.1.4/mpich-3.1.4.tar.gz"
+    },
+    {
+      "name": "Python-3.10.2",
+      "type": "tar",
+      "url": "https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz"
+    }
+  ],
+  "package_managers": [
+    "dpkg",
+    "pip"
+  ],
+  "type": "zip"
+}

artifacts_json/ziogas_deinsum-gpu.json

@@ -0,0 +1,32 @@
+{
+  "artifact_url": "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
+  "dockerfile_location": "deinsum-sc22-artifact-7559901/docker_gpu",
+  "doi": "10.5555/3571885.3571918",
+  "git_packages": [
+    {
+      "location": "/dace",
+      "name": "dace"
+    },
+    {
+      "location": "/multilinear-algebra-sc22-artifact",
+      "name": "multilinear-algebra-sc22-artifact"
+    },
+    {
+      "location": "/ctf",
+      "name": "ctf"
+    }
+  ],
+  "image_name": "deinsum:artefact_gpu",
+  "misc_packages": [
+    {
+      "name": "pip",
+      "type": "zip",
+      "url": "https://bootstrap.pypa.io/get-pip.py"
+    }
+  ],
+  "package_managers": [
+    "dpkg",
+    "pip"
+  ],
+  "type": "zip"
+}

artifacts_nickel/breitweiser_high.ncl

@@ -0,0 +1,34 @@
+let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
+{
+  artifact_url = "https://zenodo.org/record/7544675/files/SF2-code.tar.gz",
+  type = "tar",
+  doi = "10.1145/3572848.3577480",
+  image_name = "bdm-publication-image-v7",
+  dockerfile_location = "bdm-paper-examples/docker",
+  package_managers = [ "dpkg" ],
+  git_packages = [
+    { name = "pyenv", location = "~/.pyenv" }
+  ],
+  misc_packages = [
+    {
+      name = "NetLogo-6.2.0-64",
+      url = "http://ccl.northwestern.edu/netlogo/6.2.0/NetLogo-6.2.0-64.tgz",
+      type = "tar"
+    },
+    {
+      name = "root_v6.22.06_python3.9_ubuntu-20.04",
+      url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/root_v6.22.06_python3.9_ubuntu-20.04.tar.gz",
+      type = "tar"
+    },
+    {
+      name = "qt_v5.12.10_ubuntu-20.04",
+      url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/qt_v5.12.10_ubuntu-20.04.tar.gz",
+      type = "tar"
+    },
+    {
+      name = "paraview_v5.9.0_ubuntu-20.04_default",
+      url = "http://biodynamo-lfs.web.cern.ch/biodynamo-lfs/third-party/paraview_v5.9.0_ubuntu-20.04_default.tar.gz",
+      type = "tar"
+    },
+  ],
+} | Artifact

artifacts_nickel/isakov_taxonomy.ncl

@@ -0,0 +1,11 @@
+let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
+{
+  artifact_url = "https://zenodo.org/record/6632461/files/SC22_artifacts_submission.zip",
+  type = "zip",
+  doi = "10.5555/3571885.3571906",
+  image_name = "taxo:artefact",
+  dockerfile_location = "SC22_artifacts_submission",
+  package_managers = [ "dpkg", "pip" ],
+  git_packages = [],
+  misc_packages = [],
+} | Artifact

artifacts_nickel/moses_high.ncl

@@ -0,0 +1,15 @@
+let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
+{
+  artifact_url = "https://zenodo.org/record/7508499/files/wsmoses/PolygeistGPU-Docker-v0.2.1.zip",
+  type = "zip",
+  doi = "10.1145/3572848.3577475",
+  image_name = "polygeist:artefact",
+  dockerfile_location = "wsmoses-PolygeistGPU-Docker-ba18197/MocCUDA",
+  package_managers = [ "dpkg", "pip" ],
+  git_packages = [
+    { name = "MocCUDA", location = "/root/MocCUDA"}
+  ],
+  misc_packages = [
+    { name = "cmake-3.23.1", url = "https://github.com/Kitware/CMake/releases/download/v3.23.1/cmake-3.23.1.tar.gz", type = "tar" }
+  ],
+} | Artifact

artifacts_nickel/parasyris_approximate.ncl

@@ -0,0 +1,13 @@
+let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
+{
+  artifact_url = "https://zenodo.org/record/6926481/files/koparasy/HPAC-v0.0.0-Puppeteer.zip",
+  type = "zip",
+  doi = "10.5555/3571885.3571974",
+  image_name = "puppeteer:artefact",
+  dockerfile_location = "koparasy-HPAC-2723bb8/approx/puppeteer/container",
+  package_managers = [ "dpkg", "pip" ],
+  git_packages = [],
+  misc_packages = [
+    { name = "HPAC", url = "https://github.com/koparasy/HPAC/archive/refs/heads/develop.zip", type = "zip" } # CMake also but not downloaded, so cannot add to list...
+  ],
+} | Artifact

artifacts_nickel/test.ncl

@@ -1,3 +1,4 @@
+let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
 {
   artifact_url = "http://localhost/artifact.zip",
   type = "zip",
@@ -11,4 +12,4 @@
   misc_packages = [
     { name = "mpkg1", url = "http://localhost/package1.zip", type = "zip" }
   ],
-}
+} | Artifact

artifacts_nickel/wang_tgopt.ncl

@@ -0,0 +1,13 @@
+let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
+{
+  artifact_url = "https://zenodo.org/record/7328505/files/tgopt-artifact.tgz",
+  type = "tar",
+  doi = "10.1145/3572848.3577490",
+  image_name = "tgopt:artefact",
+  dockerfile_location = "tgopt-artifact",
+  package_managers = [ "dpkg", "conda" ],
+  git_packages = [],
+  misc_packages = [
+    {name = "Miniconda3-py37_4.12.0-Linux-x86_64", url = "https://repo.anaconda.com/miniconda/Miniconda3-py37_4.12.0-Linux-x86_64.sh", type = "zip"}
+  ],
+} | Artifact

artifacts_nickel/ziogas_deinsum-cpu.ncl

@@ -0,0 +1,26 @@
+let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
+{
+  artifact_url = "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
+  type = "zip",
+  doi = "10.5555/3571885.3571918",
+  image_name = "deinsum:artefact-cpu",
+  dockerfile_location = "deinsum-sc22-artifact-7559901/docker_cpu",
+  package_managers = [ "dpkg", "pip" ],
+  git_packages = [
+    { name = "dace", location = "/dace"},
+    { name = "multilinear-algebra-sc22-artifact", location = "/multilinear-algebra-sc22-artifact"},
+    { name = "ctf", location = "/ctf"}
+  ],
+  misc_packages = [
+    {
+      name = "mpich-3.1.4",
+      url = "http://www.mpich.org/static/downloads/3.1.4/mpich-3.1.4.tar.gz",
+      type = "tar"
+    },
+    {
+      name = "Python-3.10.2",
+      url = "https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz",
+      type = "tar"
+    }
+  ],
+} | Artifact

artifacts_nickel/ziogas_deinsum-gpu.ncl

@@ -0,0 +1,17 @@
+let { Artifact, .. } = import "../workflow/nickel/artifact_contract.ncl" in
+{
+  artifact_url = "https://zenodo.org/record/7004393/files/deinsum/sc22-artifact-0.4.zip",
+  type = "zip",
+  doi = "10.5555/3571885.3571918",
+  image_name = "deinsum:artefact_gpu",
+  dockerfile_location = "deinsum-sc22-artifact-7559901/docker_gpu",
+  package_managers = [ "dpkg", "pip" ],
+  git_packages = [
+    { name = "dace", location = "/dace"},
+    { name = "multilinear-algebra-sc22-artifact", location = "/multilinear-algebra-sc22-artifact"},
+    { name = "ctf", location = "/ctf"}
+  ],
+  misc_packages = [
+    { name = "pip", url = "https://bootstrap.pypa.io/get-pip.py", type = "zip"}
+  ],
+} | Artifact

blacklists/blacklist.csv

@@ -1 +1 @@
-example, IMAGE_NOT_FOUND, 0
+template, IMAGE_NOT_FOUND, 0

ecg.py

@@ -167,11 +167,18 @@ def buildstatus_saver(output):
     # # Writing header in case file didn't exist:
     # if not file_exists:
     #     buildstatus_file.write("yaml_path,timestamp,error")
+    unknown_error = True
     for error_cat, error in build_errors.items():
         if error in output:
+            unknown_error = False
             now = datetime.datetime.now()
             timestamp = str(datetime.datetime.timestamp(now))
             buildstatus_file.write(f"{config_path},{timestamp},{error_cat}\n")
+    print(unknown_error)
+    if unknown_error:
+        now = datetime.datetime.now()
+        timestamp = str(datetime.datetime.timestamp(now))
+        buildstatus_file.write(f"{config_path},{timestamp},unknown_error\n")
     buildstatus_file.close()
 
 def build_image(config, src_dir):
@@ -188,8 +195,8 @@ def build_image(config, src_dir):
 
     Returns
     -------
-    bool
+    return_code: bool, build_output: str
-        'True' if build successful, 'False' otherwise.
+        Return code and output of Docker 'build'.
     """
     name = config["image_name"]
     logging.info(f"Starting building image {name}")
@@ -202,8 +209,7 @@ def build_image(config, src_dir):
     logging.info(build_output)
     return_code = build_process.returncode
     logging.info(f"Command '{build_command}' exited with code {return_code}")
-    buildstatus_saver(build_output)
+    return return_code, build_output
-    return return_code == 0
 
 def check_env(config, src_dir):
     """
@@ -342,10 +348,12 @@ def main():
     config_file.close()
 
     src_dir = download_sources(config)
-    successful_build = build_image(config, src_dir)
+    return_code, build_output = build_image(config, src_dir)
-    if successful_build:
+    if return_code == 0:
         check_env(config, src_dir)
         remove_image(config)
+    else:
+        buildstatus_saver(build_output)
 
     if not use_cache:
         os.system(f"rm -rf {os.path.join(cachedir_path, trim(config['artifact_url']))}")