GuilloteauQ/study-docker-repro-longevity
1
0
Fork 0
Code Issues 6 Pull Requests Actions Projects Releases Wiki Activity

Added support for cache. Added run and clean scripts.

Browse Source
This commit is contained in:
antux18 2024-07-12 12:10:03 +02:00
parent 27d0331c06
commit 23c4f7ede4
4 changed files with 108 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,5 +1,6 @@
pkglists/* pkglists/*
output/* output/*
cache/*
pkglist.csv pkglist.csv
log.txt log.txt
build_status.csv build_status.csv

6
clean.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/bash
OUTPUT_PATH=output
CACHE_DIR=cache
rm -rf $OUTPUT_PATH $CACHE_DIR

139
ecg.py
View File

@ -25,8 +25,8 @@ import sys
# Paths: # Paths:
pkglist_path = "pkglist.csv" # Package list being generated pkglist_path = "pkglist.csv" # Package list being generated
log_path = "log.txt" # Output of the program
buildstatus_path = "build_status.csv" # Summary of the build process of the image buildstatus_path = "build_status.csv" # Summary of the build process of the image
cachedir_path = "cache" # Artifact cache directory
# Commands to list installed packages along with their versions and the name # Commands to list installed packages along with their versions and the name
# of the package manager, depending on the package managers. # of the package manager, depending on the package managers.
@ -48,48 +48,75 @@ pkgmgr_cmd = {
# Command to obtain the latest commit hash in a git repository: # Command to obtain the latest commit hash in a git repository:
gitcmd = "git log -n 1 --pretty=format:%H" gitcmd = "git log -n 1 --pretty=format:%H"
def trim(url) :
"""
Trims given url for cache storage.
Parameters
----------
url: str
URL to trim.
Returns
-------
str
Trimmed URL.
"""
trimmed = ""
for c in url.lower():
if c not in "/:;\\'\" *?":
trimmed += c
return trimmed
def download_sources(config): def download_sources(config):
""" """
Downloads the source of the artifact in 'config'. Downloads the source of the artifact in 'config'.
Parameters Parameters
---------- ----------
config: dict config: dict
Parsed YAML config file. Parsed YAML config file.
Returns Returns
------- -------
temp_dir: tempfile.TemporaryDirectory temp_dir: tempfile.TemporaryDirectory
The directory where the artifact is downloaded to. The directory where the artifact is downloaded to.
""" """
url = config["artifact_url"] url = config["artifact_url"]
logging.info(f"Downloading artifact from {url}") artifact_name = trim(url)
temp_dir = tempfile.TemporaryDirectory() artifact_dir = cachedir_path + "/" + artifact_name
req = requests.get(url) # Checking if artifact in cache. Not downloading if it is:
if config["type"] == "zip": if not os.path.exists(artifact_dir):
artifact = zipfile.ZipFile(io.BytesIO(req.content)) logging.info(f"Downloading artifact from {url}")
elif config["type"] == "tgz": os.mkdir(artifact_dir)
artifact = tarfile.open(fileobj=io.BytesIO(req.content)) req = requests.get(url)
logging.info(f"Extracting artifact at {temp_dir.name}") if config["type"] == "zip":
artifact.extractall(temp_dir.name) artifact = zipfile.ZipFile(io.BytesIO(req.content))
return temp_dir elif config["type"] == "tgz":
artifact = tarfile.open(fileobj=io.BytesIO(req.content))
logging.info(f"Extracting artifact at {artifact_dir}")
artifact.extractall(artifact_dir)
else:
logging.info(f"Cache found for {url}, skipping download")
return artifact_dir
def build_image(config, src_dir): def build_image(config, src_dir):
""" """
Builds the given Docker image in 'config'. Builds the given Docker image in 'config'.
Parameters Parameters
---------- ----------
config: dict config: dict
Parsed YAML config file. Parsed YAML config file.
src_dir: tempfile.TemporaryDirectory src_dir: tempfile.TemporaryDirectory
The directory where the artifact is stored. The directory where the artifact is stored.
Returns Returns
------- -------
return_code: int return_code: int
Return code of the Docker 'build' command. Return code of the Docker 'build' command.
""" """
name = config["image_name"] name = config["image_name"]
logging.info(f"Starting building image {name}") logging.info(f"Starting building image {name}")
@ -104,22 +131,23 @@ def build_image(config, src_dir):
def check_env(config, src_dir): def check_env(config, src_dir):
""" """
Builds a list of all software packages installed in the Builds a list of all software packages installed in the
Docker image given in 'config', depending on the package managers Docker image given in 'config', depending on the package managers
specified in the configuration, then stores it in a CSV file. specified in the configuration, then stores it in a CSV file.
Parameters Parameters
---------- ----------
config: dict config: dict
Parsed YAML config file. Parsed YAML config file.
src_dir: tempfile.TemporaryDirectory src_dir: tempfile.TemporaryDirectory
The directory where the artifact is stored. The directory where the artifact is stored.
Returns Returns
------- -------
None None
""" """
logging.info("Checking software environment")
pkglist_file = open(pkglist_path, "w") pkglist_file = open(pkglist_path, "w")
pkglist_file.write("Package,Version,Package manager\n") pkglist_file.write("Package,Version,Package manager\n")
path = os.path.join(src_dir, config["dockerfile_location"]) path = os.path.join(src_dir, config["dockerfile_location"])
@ -149,23 +177,23 @@ def check_env(config, src_dir):
def remove_image(config): def remove_image(config):
""" """
Removes the Docker image given in 'config'. Removes the Docker image given in 'config'.
Parameters Parameters
---------- ----------
config: dict config: dict
Parsed YAML config file. Parsed YAML config file.
Returns Returns
------- -------
None None
""" """
name = config["image_name"] name = config["image_name"]
logging.info(f"Removing image '{name}'") logging.info(f"Removing image '{name}'")
subprocess.run(["docker", "rmi", name], capture_output = True) subprocess.run(["docker", "rmi", name], capture_output = True)
def main(): def main():
global pkglist_path, log_path, buildstatus_path global pkglist_path, buildstatus_path, cachedir_path
# Command line arguments parsing: # Command line arguments parsing:
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
@ -199,12 +227,15 @@ def main():
args = parser.parse_args() args = parser.parse_args()
# Setting up the paths of the outputs: # Setting up the paths of the outputs:
log_path = "log.txt" # Output of the program
if args.pkg_list != None: if args.pkg_list != None:
pkglist_path = args.pkg_list pkglist_path = args.pkg_list
if args.log_path != None: if args.log_path != None:
log_path = args.log_path log_path = args.log_path
if args.build_summary != None: if args.build_summary != None:
buildstatus_path = args.build_summary buildstatus_path = args.build_summary
if args.cache_dir != None:
cachedir_path = args.cache_dir
# Setting up the log: will be displayed both on stdout and to the specified # Setting up the log: will be displayed both on stdout and to the specified
# file: # file:
@ -223,9 +254,9 @@ def main():
# logging.info(f"Output will be stored in {output}") # logging.info(f"Output will be stored in {output}")
src_dir = download_sources(config) src_dir = download_sources(config)
successful_build = build_image(config, src_dir.name) successful_build = build_image(config, src_dir)
if successful_build: if successful_build:
check_env(config, src_dir.name) check_env(config, src_dir)
remove_image(config) remove_image(config)
if __name__ == "__main__": if __name__ == "__main__":

16
test_run.sh Executable file
View File

@ -0,0 +1,16 @@
#!/bin/bash
OUTPUT_PATH=output
CACHE_DIR=cache
TESTFILE=test.yaml
if [ ! -d $OUTPUT_PATH ]
then
mkdir $OUTPUT_PATH
fi
if [ ! -d $CACHE_DIR ]
then
mkdir $CACHE_DIR
fi
./ecg.py $TESTFILE -p $OUTPUT_PATH/pkglist.csv -l $OUTPUT_PATH/log.txt -b $OUTPUT_PATH/build_status.csv -c $CACHE_DIR