Added support for cache. Added run and clean scripts.

.gitignore

@@ -1,5 +1,6 @@
 pkglists/*
 output/*
+cache/*
 pkglist.csv
 log.txt
 build_status.csv

clean.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+OUTPUT_PATH=output
+CACHE_DIR=cache
+
+rm -rf $OUTPUT_PATH $CACHE_DIR

ecg.py

@@ -25,8 +25,8 @@ import sys
 
 # Paths:
 pkglist_path = "pkglist.csv" # Package list being generated
-log_path = "log.txt" # Output of the program
 buildstatus_path = "build_status.csv" # Summary of the build process of the image
+cachedir_path = "cache" # Artifact cache directory
 
 # Commands to list installed packages along with their versions and the name
 # of the package manager, depending on the package managers.
@@ -48,6 +48,27 @@ pkgmgr_cmd = {
 # Command to obtain the latest commit hash in a git repository:
 gitcmd = "git log -n 1 --pretty=format:%H"
 
+def trim(url) :
+    """
+    Trims given url for cache storage.
+
+    Parameters
+    ----------
+    url: str
+        URL to trim.
+
+    Returns
+    -------
+    str
+        Trimmed URL.
+    """
+    trimmed = ""
+    for c in url.lower():
+        if c not in "/:;\\'\" *?":
+            trimmed += c
+
+    return trimmed
+
 def download_sources(config):
     """
     Downloads the source of the artifact in 'config'.
@@ -63,16 +84,22 @@ def download_sources(config):
         The directory where the artifact is downloaded to.
     """
     url = config["artifact_url"]
+    artifact_name = trim(url)
+    artifact_dir = cachedir_path + "/" + artifact_name
+    # Checking if artifact in cache. Not downloading if it is:
+    if not os.path.exists(artifact_dir):
         logging.info(f"Downloading artifact from {url}")
-    temp_dir = tempfile.TemporaryDirectory()
+        os.mkdir(artifact_dir)
         req = requests.get(url)
         if config["type"] == "zip":
             artifact = zipfile.ZipFile(io.BytesIO(req.content))
         elif config["type"] == "tgz":
             artifact = tarfile.open(fileobj=io.BytesIO(req.content))
-    logging.info(f"Extracting artifact at {temp_dir.name}")
+        logging.info(f"Extracting artifact at {artifact_dir}")
-    artifact.extractall(temp_dir.name)
+        artifact.extractall(artifact_dir)
-    return temp_dir
+    else:
+        logging.info(f"Cache found for {url}, skipping download")
+    return artifact_dir
 
 def build_image(config, src_dir):
     """
@@ -120,6 +147,7 @@ def check_env(config, src_dir):
     -------
     None
     """
+    logging.info("Checking software environment")
     pkglist_file = open(pkglist_path, "w")
     pkglist_file.write("Package,Version,Package manager\n")
     path = os.path.join(src_dir, config["dockerfile_location"])
@@ -165,7 +193,7 @@ def remove_image(config):
     subprocess.run(["docker", "rmi", name], capture_output = True)
 
 def main():
-    global pkglist_path, log_path, buildstatus_path
+    global pkglist_path, buildstatus_path, cachedir_path
 
     # Command line arguments parsing:
     parser = argparse.ArgumentParser(
@@ -199,12 +227,15 @@ def main():
     args = parser.parse_args()
 
     # Setting up the paths of the outputs:
+    log_path = "log.txt" # Output of the program
     if args.pkg_list != None:
         pkglist_path = args.pkg_list
     if args.log_path != None:
         log_path = args.log_path
     if args.build_summary != None:
         buildstatus_path = args.build_summary
+    if args.cache_dir != None:
+        cachedir_path = args.cache_dir
 
     # Setting up the log: will be displayed both on stdout and to the specified
     # file:
@@ -223,9 +254,9 @@ def main():
     #    logging.info(f"Output will be stored in {output}")
 
     src_dir = download_sources(config)
-    successful_build = build_image(config, src_dir.name)
+    successful_build = build_image(config, src_dir)
     if successful_build:
-        check_env(config, src_dir.name)
+        check_env(config, src_dir)
         remove_image(config)
 
 if __name__ == "__main__":

test_run.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+OUTPUT_PATH=output
+CACHE_DIR=cache
+TESTFILE=test.yaml
+
+if [ ! -d $OUTPUT_PATH ]
+then
+    mkdir $OUTPUT_PATH
+fi
+if [ ! -d $CACHE_DIR ]
+then
+    mkdir $CACHE_DIR
+fi
+
+./ecg.py $TESTFILE -p $OUTPUT_PATH/pkglist.csv -l $OUTPUT_PATH/log.txt -b $OUTPUT_PATH/build_status.csv -c $CACHE_DIR