diff --git a/ecg.py b/ecg.py index de0cf1f..5e668f8 100755 --- a/ecg.py +++ b/ecg.py @@ -1,5 +1,13 @@ #!/bin/python3 +""" + ECG is program that automates software environment checking + in scientific artifacts. + + It is meant to be executed periodically to analyze variations in the + software environment of the artifact through time. +""" + import subprocess import json import yaml @@ -10,20 +18,42 @@ import requests import zipfile import io import tarfile - import pathlib -HEREPATH = pathlib.Path(__file__).parent.absolute() +import logging +# Paths: +HEREPATH = pathlib.Path(__file__).parent.absolute() # Where to store list of installed packages: PKGLISTS = "./pkglists/" -# Commands to list installed packages along with their versions and the name of the package manager, depending on the packages manager: -pkgmgr_cmd = {"dpkg":"dpkg -l | awk 'NR>5 {print $2 \",\" $3 \",\" \"dpkg\"}'", "rpm":"rpm -qa --queryformat '%{NAME},%{VERSION},rpm\\n'", "pacman":"pacman -Q | awk '{print $0 \",\" $1 \",pacman\"}'", "pip":"pip freeze | sed 's/==/,/g' | awk '{print $0 \",pip\"}'", "conda":"/root/.conda/bin/conda list -e | sed 's/=/ /g' | awk 'NR>3 {print $1 \",\" $2 \",conda\"}'"} +# Commands to list installed packages along with their versions and the name +# of the package manager, depending on the package managers: +pkgmgr_cmd = { + "dpkg":"dpkg -l | awk 'NR>5 {print $2 \",\" $3 \",\" \"dpkg\"}'", \ + "rpm":"rpm -qa --queryformat '%{NAME},%{VERSION},rpm\\n'", \ + "pacman":"pacman -Q | awk '{print $0 \",\" $1 \",pacman\"}'", \ + "pip":"pip freeze | sed 's/==/,/g' | awk '{print $0 \",pip\"}'", \ + "conda":"/root/.conda/bin/conda list -e | sed 's/=/ /g' | awk 'NR>3 {print $1 \",\" $2 \",conda\"}'", \ + "git":"" +} -import logging +# Enables logging: logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) def download_sources(config): + """ + Downloads the source of the artifact in 'config'. + + Parameters + ---------- + config: dict + Parsed YAML config file. + + Returns + ------- + tempdir: tempfile.TemporaryDirectory + The directory where the artifact is downloaded to. + """ url = config["artifact_url"] logging.info(f"Downloading sources from {url}") temp_dir = tempfile.TemporaryDirectory() @@ -37,6 +67,23 @@ def download_sources(config): return temp_dir def build_image(config, src_dir): + """ + Builds the given Docker image in 'config'. + + Parameters + ---------- + config: dict + Part of the parsed YAML config file concerning the Docker image + to build. + + src_dir: tempfile.TemporaryDirectory + The directory where the artifact is stored. + + Returns + ------- + return_code: int + Return code of the Docker 'build' command. + """ name = config["name"] logging.info(f"Starting building image {name}") path = os.path.join(src_dir, config["location"]) @@ -49,6 +96,24 @@ def build_image(config, src_dir): return return_code == 0 def check_env(config, src_dir): + """ + Builds a list of all software packages installed in the + Docker image given in 'config', depending on the package managers + specified in the configuration, then stores it in a CSV file. + + Parameters + ---------- + config: dict + Part of the parsed YAML config file concerning the Docker image + where to check the environment. + + src_dir: tempfile.TemporaryDirectory + The directory where the artifact is stored. + + Returns + ------- + None + """ pathlib.Path(PKGLISTS).mkdir(parents=True, exist_ok=True) path = os.path.join(src_dir, config["location"]) pkglist_file = open(PKGLISTS + "pkglist.csv", "w") @@ -59,14 +124,45 @@ def check_env(config, src_dir): pkglist = check_process.stdout.decode("utf-8") print(pkglist) pkglist_file.write(pkglist) + if "git_packages" in config.keys(): + print(config["git_packages"]) pkglist_file.close() def remove_image(config): + """ + Removes the Docker image given in 'config'. + + Parameters + ---------- + config: dict + Part of the parsed YAML config file concerning the Docker image + to remove. + + Returns + ------- + None + """ name = config["name"] logging.info(f"Removing image '{name}'") subprocess.run(["docker", "rmi", name]) def build_images(config, src_dir): + """ + Builds all Docker images specified in 'config', checks software + environment if build is successful, then removes the images. + + Parameters + ---------- + config: dict + Parsed YAML config file. + + src_dir: tempfile.TemporaryDirectory + The directory where the artifact is stored. + + Returns + ------- + None + """ for image in config["dockerfiles"]: successful_build = build_image(image, src_dir) if successful_build: @@ -74,13 +170,15 @@ def build_images(config, src_dir): remove_image(image) def main(): + # Command line arguments parser: parser = argparse.ArgumentParser( prog='ecg', description='Check if a dockerfile is still buildable') parser.add_argument('config') parser.add_argument('-v', '--verbose', action='store_true') - args = parser.parse_args() + + # Parsing the input YAML file including the configuration of the artifact: config = None with open(args.config, "r") as config_file: config = yaml.safe_load(config_file) diff --git a/example.yaml b/example.yaml index 6a2d8a1..0913ed3 100644 --- a/example.yaml +++ b/example.yaml @@ -5,7 +5,10 @@ dockerfiles: - name: "image1:version" location: "path/to/docker/folder" package_managers: - - "dpkg" # Possible values: dpkg, pip, conda + - "dpkg" # Possible values: dpkg, rpm, pacman, pip, conda + git_packages: + - name: "pkg1" + location: "path/to/git/package" - name: "image2:version" location: "path/to/docker/folder" package_managers: diff --git a/test.yaml b/test.yaml index 41031d0..104d467 100644 --- a/test.yaml +++ b/test.yaml @@ -7,3 +7,6 @@ dockerfiles: package_managers: - "dpkg" - "pip" + git_packages: + - name: "pkg1" + location: "./pkg1" \ No newline at end of file