- Config files now contain 1 Dockerfile only.

- Removed support for running ECG with multiple config files at the same time: will be called with a single one only
- Removed storage management of outputs, paths are now given using command line args.
- Started working on cache support.

.gitignore

@@ -1,2 +1,5 @@
 pkglists/*
 output/*
+pkglist.csv
+log.txt
+build_status.csv

ecg.py

@@ -21,10 +21,28 @@ import tarfile
 import pathlib
 import logging
 import datetime
+import sys
+
+class Logger(object):
+    """
+        Class to log Python output to both stdout and a file.
+    """
+    def __init__(self, path):
+        self.terminal = sys.stdout
+        self.log = open(path, "w")
+
+    def write(self, message):
+        self.terminal.write(message)
+        self.log.write(message)
+
+    def flush(self):
+        # For Python 3 compatibility:
+        pass
 
 # Paths:
-ROOTPATH = pathlib.Path(__file__).parent.absolute()
+pkglist_path = "pkglist.csv" # Package list being generated
-output_path = "./output/"
+log_path = "log.txt" # Output of the program
+buildstatus_path = "build_status.csv" # Summary of the build process of the image
 
 # Commands to list installed packages along with their versions and the name
 # of the package manager, depending on the package managers.
@@ -49,22 +67,6 @@ gitcmd = "git log -n 1 --pretty=format:%H"
 # Enables logging:
 logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
 
-# TODO: This will be used to log both to stdout and to a file:
-class Logger(object):
-    def __init__(self):
-        self.terminal = sys.stdout
-        self.log = open("log.txt", "w")
-
-    def write(self, message):
-        self.terminal.write(message)
-        self.log.write(message)
-
-    def flush(self):
-        # this flush method is needed for python 3 compatibility.
-        # this handles the flush command by doing nothing.
-        # you might want to specify some extra behavior here.
-        pass
-
 def download_sources(config):
     """
         Downloads the source of the artifact in 'config'.
@@ -98,8 +100,7 @@ def build_image(config, src_dir):
         Parameters
         ----------
         config: dict
-            Part of the parsed YAML config file concerning the Docker image
+            Parsed YAML config file.
-            to build.
 
         src_dir: tempfile.TemporaryDirectory
             The directory where the artifact is stored.
@@ -109,11 +110,10 @@ def build_image(config, src_dir):
         return_code: int
             Return code of the Docker 'build' command.
     """
-    name = config["name"]
+    name = config["image_name"]
     logging.info(f"Starting building image {name}")
-    path = os.path.join(src_dir, config["location"])
+    path = os.path.join(src_dir, config["dockerfile_location"])
-    build_command = "docker build -t " + config["name"] + " ."
+    build_command = "docker build -t " + config["image_name"] + " ."
-    # subprocess.check_call(config["build_command"].split(" "), cwd=path)
     build_process = subprocess.run(build_command.split(" "), cwd=path, capture_output=False)
     return_code = build_process.returncode
     logging.info(f"Command '{build_command}' exited with code {return_code}")
@@ -128,8 +128,7 @@ def check_env(config, src_dir):
         Parameters
         ----------
         config: dict
-            Part of the parsed YAML config file concerning the Docker image
+            Parsed YAML config file.
-            where to check the environment.
 
         src_dir: tempfile.TemporaryDirectory
             The directory where the artifact is stored.
@@ -138,19 +137,19 @@ def check_env(config, src_dir):
         -------
         None
     """
-    pkglist_file = open("pkglist.csv", "w")
+    pkglist_file = open(pkglist_path, "w")
     pkglist_file.write("Package,Version,Package manager\n")
-    path = os.path.join(src_dir, config["location"])
+    path = os.path.join(src_dir, config["dockerfile_location"])
     for pkgmgr in config["package_managers"]:
         logging.info(f"Checking '{pkgmgr}'")
-        pkglist_process = subprocess.run(["docker", "run", "--rm", config["name"]] + pkgmgr_cmd[pkgmgr][0].split(" "), cwd=path, capture_output=True)
+        pkglist_process = subprocess.run(["docker", "run", "--rm", config["image_name"]] + pkgmgr_cmd[pkgmgr][0].split(" "), cwd=path, capture_output=True)
         format_process = subprocess.run("cat << EOF | " + pkgmgr_cmd[pkgmgr][1] + "\n" + pkglist_process.stdout.decode("utf-8") + "EOF", cwd=path, capture_output=True, shell=True)
         pkglist = format_process.stdout.decode("utf-8")
         pkglist_file.write(pkglist)
     if "git_packages" in config.keys():
         logging.info("Checking Git packages")
         for repo in config["git_packages"]:
-            pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], config["name"]] + gitcmd.split(" "), cwd=path, capture_output=True)
+            pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], config["image_name"]] + gitcmd.split(" "), cwd=path, capture_output=True)
             repo_row = repo["name"] + "," + pkglist_process.stdout.decode("utf-8") + ",git"
             pkglist_file.write(repo_row + "\n")
     if "misc_packages" in config.keys():
@@ -172,46 +171,18 @@ def remove_image(config):
         Parameters
         ----------
         config: dict
-            Part of the parsed YAML config file concerning the Docker image
+            Parsed YAML config file.
-            to remove.
 
         Returns
         -------
         None
     """
-    name = config["name"]
+    name = config["image_name"]
     logging.info(f"Removing image '{name}'")
     subprocess.run(["docker", "rmi", name])
 
-def build_images(config, src_dir):
-    """
-        Builds all Docker images specified in 'config', checks software
-        environment if build is successful, then removes the images.
-
-        Parameters
-        ----------
-        config: dict
-            Parsed YAML config file.
-
-        src_dir: tempfile.TemporaryDirectory
-            The directory where the artifact is stored.
-
-        Returns
-        -------
-        None
-    """
-    for image in config["dockerfiles"]:
-        # Creating an output directory for this specific container:
-        pathlib.Path(image["name"]).mkdir(parents=True, exist_ok=True)
-        os.chdir(image["name"])
-
-        successful_build = build_image(image, src_dir)
-        if successful_build:
-            check_env(image, src_dir)
-            remove_image(image)
-
 def main():
-    global output_path
+    global pkglist_path, log_path, buildstatus_path
 
     # Command line arguments parsing:
     parser = argparse.ArgumentParser(
@@ -221,62 +192,53 @@ def main():
     )
     parser.add_argument(
         "config",
-        help = "The path to either a single configuration file, or a directory containing multiple configuration files if using '-d'. "
+        help = "The path to the configuration file of the artifact's Docker image."
-            "Note that all YAML files in this directory must be artifact configuration files."
     )
     parser.add_argument(
-        "-d", "--directory",
+        "-p", "--pkg-list",
-        action = "store_true",
+        help = "Path to the file where the package list generated by the program should be written."
-        help = "Set this option to specify the path of a directory containing multiple configuration files instead of a single file."
     )
     parser.add_argument(
-        "-o", "--output",
+        "-l", "--log-path",
-        help = "Path to the output directory."
+        help = "Path to the file where to log the output of the program."
     )
-    parser.add_argument('-v', '--verbose', action='store_true')
+    parser.add_argument(
+        "-b", "--build-summary",
+        help = "Path to the file where to write the build summary of the Docker image given in the configuration file."
+    )
+    parser.add_argument(
+        "-c", "--cache-dir",
+        help = "Path to the cache directory, where artifact that are downloaded will be stored for future usage."
+    )
+    parser.add_argument('-v', '--verbose',
+        action = 'store_true',
+        help = "Shows more details on what is being done.")
     args = parser.parse_args()
 
-    # Parsing the input YAML file(s) including the configuration of the
+    # Setting up the paths of the outputs:
-    # artifact(s):
+    if args.pkg_list != None:
-    configs = {}
+        pkglist_path = args.pkg_list
-    # If args.config is a directory:
+    if args.log_path != None:
-    if args.directory:
+        log_path = args.log_path
-        for f in os.listdir(args.config):
+    if args.build_summary != None:
-            if os.path.isfile(f) and f.endswith(".yaml"):
+        buildstatus_path = args.build_summary
-                config_file = open(f, "r")
-                configs[f] = yaml.safe_load(config_file)
-                config_file.close()
-    # If args.config is a simple file:
-    else:
-        config_file = open(args.config, "r")
-        configs[args.config] = yaml.safe_load(config_file)
-        config_file.close()
 
-    # Configuring output directory:
+    # Parsing the input YAML file including the configuration of
-    if args.output != None:
+    # the artifact's image:
-        output_path = args.output
+    config_file = open(args.config, "r")
-    pathlib.Path(output_path).mkdir(parents=True, exist_ok=True)
+    config = yaml.safe_load(config_file)
-    os.chdir(output_path)
+    config_file.close()
 
-    for c in configs:
+    verbose = args.verbose
-        logging.info(f"Working on {c}")
-        verbose = args.verbose
-        config = configs[c]
 
-        # Creating an output folder for this artifact:
+    # if verbose:
-        pathlib.Path(os.path.splitext(c)[0]).mkdir(parents=True, exist_ok=True)
+    #    logging.info(f"Output will be stored in {output}")
-        os.chdir(os.path.splitext(c)[0])
-        # Creating an output folder for this specific runtime:
-        now = datetime.datetime.now()
-        timestamp = str(datetime.datetime.timestamp(now))
-        pathlib.Path(timestamp).mkdir(parents=True, exist_ok=True)
-        os.chdir(timestamp)
 
-        # if verbose:
+    src_dir = download_sources(config)
-        #    logging.info(f"Output will be stored in {output}")
+    successful_build = build_image(config, src_dir.name)
-
+    if successful_build:
-        src_dir = download_sources(config)
+        check_env(config, src_dir.name)
-        build_images(config, src_dir.name)
+        remove_image(config)
 
 if __name__ == "__main__":
     main()

example.yaml

@@ -1,15 +1,14 @@
 artefact_url: "https://example.com/artifact.zip"
 type: "zip" # Possible values: zip, tgz
 doi: "XX.XXXX/XXXXXXX.XXXXXXX"
-dockerfiles:
+image_name: "image1:version"
-  - name: "image1:version"
+dockerfile_location: "path/to/docker/folder"
-    location: "path/to/docker/folder"
+package_managers:
-    package_managers:
+  - "dpkg" # Possible values: dpkg, rpm, pacman, pip, conda
-      - "dpkg" # Possible values: dpkg, rpm, pacman, pip, conda
+git_packages:
-    git_packages:
+  - name: "pkg1"
-      - name: "pkg1"
+    location: "path/to/git/repo"
-        location: "path/to/git/repo"
+misc_packages:
-    misc_packages:
+  - name: "mpkg1"
-      - name: "mpkg1"
+    url: "https://example.com/package1.zip"
-        url: "https://example.com/package1.zip"
+    type: "zip" # Possible values: zip, tgz
-        type: "zip" # Possible values: zip, tgz

test.yaml

@@ -1,16 +1,15 @@
 artifact_url: "http://localhost/artifact.zip"
 type: "zip"
 doi: "XX.XXXX/XXXXXXX.XXXXXXX"
-dockerfiles:
+image_name: "prog:latest"
-  - name: "prog:latest"
+dockerfile_location: "./"
-    location: "./"
+package_managers:
-    package_managers:
+  - "dpkg"
-      - "dpkg"
+  - "pip"
-      - "pip"
+git_packages:
-    git_packages:
+  - name: "pkg1"
-      - name: "pkg1"
+    location: "/pkg1"
-        location: "/pkg1"
+misc_packages:
-    misc_packages:
+  - name: "mpkg1"
-      - name: "mpkg1"
+    url: "http://localhost/package1.zip"
-        url: "http://localhost/package1.zip"
+    type: "zip"
-        type: "zip"