Fixed Nickel contract and check files. Fixed input/output files in Snakefile.

.gitignore

@@ -2,6 +2,7 @@ pkglists/*
 output/*
 cache/*
 examples/*
+.snakemake/*
 pkglist.csv
 log.txt
 build_status.csv

artifacts_json/example.json

@@ -0,0 +1,23 @@
+{
+  "artifact_url": "https://example.com/artifact.zip",
+  "dockerfile_location": "path/to/docker/folder",
+  "doi": "...",
+  "git_packages": [
+    {
+      "location": "path/to/git/repo",
+      "name": "pkg1"
+    }
+  ],
+  "image_name": "image:version",
+  "misc_packages": [
+    {
+      "name": "mpkg1",
+      "type": "zip",
+      "url": "https://"
+    }
+  ],
+  "package_managers": [
+    "dpkg"
+  ],
+  "type": "zip"
+}

artifacts_json/test.json

@@ -1,20 +1,24 @@
 {
   "artifact_url": "http://localhost/artifact.zip",
-  "type": "zip",
-  "doi": "XX.XXXX/XXXXXXX.XXXXXXX",
-  "image_name": "prog:latest",
   "dockerfile_location": "./",
+  "doi": "...",
+  "git_packages": [
+    {
+      "location": "/pkg1",
+      "name": "pkg1"
+    }
+  ],
+  "image_name": "prog:latest",
+  "misc_packages": [
+    {
+      "name": "mpkg1",
+      "type": "zip",
+      "url": "http://localhost/package1.zip"
+    }
+  ],
   "package_managers": [
     "dpkg",
     "pip"
   ],
-  "git_packages": [{
-    "name": "pkg1",
-    "location": "/pkg1"
-  }],
-  "misc_packages": [{
-    "name": "mpkg1",
-    "url": "http://localhost/package1.zip",
   "type": "zip"
-  }]
 }

artifacts_nickel/test.ncl

@@ -0,0 +1,14 @@
+{
+  artifact_url = "http://localhost/artifact.zip",
+  type = "zip",
+  doi = "...",
+  image_name = "prog:latest",
+  dockerfile_location = "./",
+  package_managers = [ "dpkg", "pip" ],
+  git_packages = [
+    { name = "pkg1", location = "/pkg1"}
+  ],
+  misc_packages = [
+    { name = "mpkg1", url = "http://localhost/package1.zip", type = "zip" }
+  ],
+}

artifacts_yaml/template.yaml

@@ -11,4 +11,4 @@ git_packages:
 misc_packages:
   - name: "mpkg1"
     url: "https://example.com/package1.zip"
-    type: "zip" # Possible values: zip, tgz
+    type: "zip" # Possible values: zip, tar

blacklists/blacklist.csv

@@ -1 +1 @@
-sc24_test, IMAGE_NOT_FOUND, 0
+example, IMAGE_NOT_FOUND, 0

check.ncl

@@ -1,4 +1,4 @@
-let { Artifact, .. } = import "artifact_contract.ncl" in
+let { Artifact, .. } = import "workflow/nickel/artifact_contract.ncl" in
 (
   (import "artifacts_nickel/example.ncl") | Artifact
 )

ecg.py

@@ -28,7 +28,7 @@ config_path = ""
 pkglist_path = "" # Package list being generated
 buildstatus_path = "" # Summary of the build process of the image
 arthashlog_path = "" # Log of the hash of the downloaded artifact
-cachedir_path = "" # Artifact cache directory
+cachedir_path = "cache" # Artifact cache directory
 
 # Commands to list installed packages along with their versions and the name
 # of the package manager, depending on the package managers.
@@ -189,9 +189,9 @@ def build_image(config, src_dir):
     name = config["image_name"]
     logging.info(f"Starting building image {name}")
     path = os.path.join(src_dir, config["dockerfile_location"])
-    build_command = f"docker build -t {config["image_name"]} ."
+    build_command = f"docker build -t {config['image_name']} ."
     build_process = subprocess.run(build_command.split(" "), cwd=path, capture_output=True)
-    build_output = f"stdout:\n{build_process.stdout.decode("utf-8")}\nstderr:\n{build_process.stderr.decode("utf-8")}"
+    build_output = f"stdout:\n{build_process.stdout.decode('utf-8')}\nstderr:\n{build_process.stderr.decode('utf-8')}"
     # build_output = build_process.stderr.decode("utf-8")
     logging.info(f"Output of '{build_command}':")
     logging.info(build_output)
@@ -227,23 +227,23 @@ def check_env(config, src_dir):
         listformat_cmd = pkgmgr_cmd[pkgmgr][1]
         logging.info(f"Checking '{pkgmgr}'")
         pkglist_process = subprocess.run(["docker", "run", "--rm", config["image_name"]] + pkglist_cmd.split(" "), cwd=path, capture_output=True)
-        format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode("utf-8")}EOF", cwd=path, capture_output=True, shell=True)
+        format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode('utf-8')}EOF", cwd=path, capture_output=True, shell=True)
         pkglist = format_process.stdout.decode("utf-8")
         pkglist_file.write(pkglist)
     if "git_packages" in config.keys():
         logging.info("Checking Git packages")
         for repo in config["git_packages"]:
             pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], config["image_name"]] + gitcmd.split(" "), cwd=path, capture_output=True)
-            repo_row = f"{repo["name"]},{pkglist_process.stdout.decode("utf-8")},git"
+            repo_row = f"{repo['name']},{pkglist_process.stdout.decode('utf-8')},git"
             pkglist_file.write(f"{repo_row}\n")
     if "misc_packages" in config.keys():
         logging.info("Checking packages obtained outside of a package manager or VCS")
         for pkg in config["misc_packages"]:
-            logging.info(f"Downloading package {pkg["name"]} from {pkg["url"]}")
+            logging.info(f"Downloading package {pkg['name']} from {pkg['url']}")
             pkg_file = tempfile.NamedTemporaryFile()
             pkg_path = pkg_file.name
             pkg_hash = download_file(pkg["url"], pkg_path)
-            pkg_row = f"{pkg["name"]},{pkg_hash},misc"
+            pkg_row = f"{pkg['name']},{pkg_hash},misc"
             pkglist_file.write(f"{pkg_row}\n")
     pkglist_file.close()
 
@@ -304,7 +304,7 @@ def main():
     parser.add_argument(
         "-c", "--cache-dir",
         help = "Path to the cache directory, where artifact that are downloaded will be stored for future usage.",
-        required = True
+        required = False
     )
     args = parser.parse_args()
 
@@ -314,6 +314,7 @@ def main():
     log_path = args.log_path
     buildstatus_path = args.build_summary
     arthashlog_path = args.artifact_hash
+    if args.cache_dir != None:
         cachedir_path = args.cache_dir
 
     # Setting up the log: will be displayed both on stdout and to the specified

workflow/Snakefile

@@ -14,7 +14,12 @@ ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST)
 rule all:
   input:
     expand("{folder}/{artifact}/{date}.csv",\
-           folder=["logs", "pkgs", "build_status", "artifact_hash"],\
+           folder=["pkgs", "build_status", "artifact_hash"],\
+           artifact=ARTIFACTS,\
+           date=DATE
+          ),
+    expand("{folder}/{artifact}/{date}.txt",\
+        folder=["logs"],\
         artifact=ARTIFACTS,\
         date=DATE
       ),

workflow/nickel/artifact_contract.ncl

@@ -31,7 +31,7 @@ in
       | ArchiveType,
   },
   Artifact = {
-    artefact_url
+    artifact_url
       | doc "URL where to download the artifact"
       | String,
     type
@@ -43,7 +43,7 @@ in
     image_name
       | doc "Name to give the image when building"
       | String,
-    location
+    dockerfile_location
       | doc "Path to the dockerfile in the artifact"
       | String,
     package_managers