diff --git a/.gitignore b/.gitignore index e5489d0..8f89ce5 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ pkglists/* output/* cache/* examples/* +.snakemake/* pkglist.csv log.txt build_status.csv \ No newline at end of file diff --git a/artifacts_json/example.json b/artifacts_json/example.json new file mode 100644 index 0000000..a8f33eb --- /dev/null +++ b/artifacts_json/example.json @@ -0,0 +1,23 @@ +{ + "artifact_url": "https://example.com/artifact.zip", + "dockerfile_location": "path/to/docker/folder", + "doi": "...", + "git_packages": [ + { + "location": "path/to/git/repo", + "name": "pkg1" + } + ], + "image_name": "image:version", + "misc_packages": [ + { + "name": "mpkg1", + "type": "zip", + "url": "https://" + } + ], + "package_managers": [ + "dpkg" + ], + "type": "zip" +} diff --git a/artifacts_json/test.json b/artifacts_json/test.json index 6f5ff6b..1e6e029 100644 --- a/artifacts_json/test.json +++ b/artifacts_json/test.json @@ -1,20 +1,24 @@ { "artifact_url": "http://localhost/artifact.zip", - "type": "zip", - "doi": "XX.XXXX/XXXXXXX.XXXXXXX", - "image_name": "prog:latest", "dockerfile_location": "./", + "doi": "...", + "git_packages": [ + { + "location": "/pkg1", + "name": "pkg1" + } + ], + "image_name": "prog:latest", + "misc_packages": [ + { + "name": "mpkg1", + "type": "zip", + "url": "http://localhost/package1.zip" + } + ], "package_managers": [ "dpkg", "pip" ], - "git_packages": [{ - "name": "pkg1", - "location": "/pkg1" - }], - "misc_packages": [{ - "name": "mpkg1", - "url": "http://localhost/package1.zip", - "type": "zip" - }] -} \ No newline at end of file + "type": "zip" +} diff --git a/artifacts_nickel/test.ncl b/artifacts_nickel/test.ncl new file mode 100644 index 0000000..456ada9 --- /dev/null +++ b/artifacts_nickel/test.ncl @@ -0,0 +1,14 @@ +{ + artifact_url = "http://localhost/artifact.zip", + type = "zip", + doi = "...", + image_name = "prog:latest", + dockerfile_location = "./", + package_managers = [ "dpkg", "pip" ], + git_packages = [ + { name = "pkg1", location = "/pkg1"} + ], + misc_packages = [ + { name = "mpkg1", url = "http://localhost/package1.zip", type = "zip" } + ], +} \ No newline at end of file diff --git a/artifacts_yaml/template.yaml b/artifacts_yaml/template.yaml index b8d485c..d0fe17f 100644 --- a/artifacts_yaml/template.yaml +++ b/artifacts_yaml/template.yaml @@ -11,4 +11,4 @@ git_packages: misc_packages: - name: "mpkg1" url: "https://example.com/package1.zip" - type: "zip" # Possible values: zip, tgz + type: "zip" # Possible values: zip, tar diff --git a/blacklists/blacklist.csv b/blacklists/blacklist.csv index 5b44974..3e0404a 100644 --- a/blacklists/blacklist.csv +++ b/blacklists/blacklist.csv @@ -1 +1 @@ -sc24_test, IMAGE_NOT_FOUND, 0 +example, IMAGE_NOT_FOUND, 0 diff --git a/check.ncl b/check.ncl index aa87899..703854d 100644 --- a/check.ncl +++ b/check.ncl @@ -1,4 +1,4 @@ -let { Artifact, .. } = import "artifact_contract.ncl" in +let { Artifact, .. } = import "workflow/nickel/artifact_contract.ncl" in ( (import "artifacts_nickel/example.ncl") | Artifact ) diff --git a/ecg.py b/ecg.py index 47c274b..30c2bcd 100755 --- a/ecg.py +++ b/ecg.py @@ -28,7 +28,7 @@ config_path = "" pkglist_path = "" # Package list being generated buildstatus_path = "" # Summary of the build process of the image arthashlog_path = "" # Log of the hash of the downloaded artifact -cachedir_path = "" # Artifact cache directory +cachedir_path = "cache" # Artifact cache directory # Commands to list installed packages along with their versions and the name # of the package manager, depending on the package managers. @@ -189,9 +189,9 @@ def build_image(config, src_dir): name = config["image_name"] logging.info(f"Starting building image {name}") path = os.path.join(src_dir, config["dockerfile_location"]) - build_command = f"docker build -t {config["image_name"]} ." + build_command = f"docker build -t {config['image_name']} ." build_process = subprocess.run(build_command.split(" "), cwd=path, capture_output=True) - build_output = f"stdout:\n{build_process.stdout.decode("utf-8")}\nstderr:\n{build_process.stderr.decode("utf-8")}" + build_output = f"stdout:\n{build_process.stdout.decode('utf-8')}\nstderr:\n{build_process.stderr.decode('utf-8')}" # build_output = build_process.stderr.decode("utf-8") logging.info(f"Output of '{build_command}':") logging.info(build_output) @@ -227,23 +227,23 @@ def check_env(config, src_dir): listformat_cmd = pkgmgr_cmd[pkgmgr][1] logging.info(f"Checking '{pkgmgr}'") pkglist_process = subprocess.run(["docker", "run", "--rm", config["image_name"]] + pkglist_cmd.split(" "), cwd=path, capture_output=True) - format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode("utf-8")}EOF", cwd=path, capture_output=True, shell=True) + format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode('utf-8')}EOF", cwd=path, capture_output=True, shell=True) pkglist = format_process.stdout.decode("utf-8") pkglist_file.write(pkglist) if "git_packages" in config.keys(): logging.info("Checking Git packages") for repo in config["git_packages"]: pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], config["image_name"]] + gitcmd.split(" "), cwd=path, capture_output=True) - repo_row = f"{repo["name"]},{pkglist_process.stdout.decode("utf-8")},git" + repo_row = f"{repo['name']},{pkglist_process.stdout.decode('utf-8')},git" pkglist_file.write(f"{repo_row}\n") if "misc_packages" in config.keys(): logging.info("Checking packages obtained outside of a package manager or VCS") for pkg in config["misc_packages"]: - logging.info(f"Downloading package {pkg["name"]} from {pkg["url"]}") + logging.info(f"Downloading package {pkg['name']} from {pkg['url']}") pkg_file = tempfile.NamedTemporaryFile() pkg_path = pkg_file.name pkg_hash = download_file(pkg["url"], pkg_path) - pkg_row = f"{pkg["name"]},{pkg_hash},misc" + pkg_row = f"{pkg['name']},{pkg_hash},misc" pkglist_file.write(f"{pkg_row}\n") pkglist_file.close() @@ -304,7 +304,7 @@ def main(): parser.add_argument( "-c", "--cache-dir", help = "Path to the cache directory, where artifact that are downloaded will be stored for future usage.", - required = True + required = False ) args = parser.parse_args() @@ -314,7 +314,8 @@ def main(): log_path = args.log_path buildstatus_path = args.build_summary arthashlog_path = args.artifact_hash - cachedir_path = args.cache_dir + if args.cache_dir != None: + cachedir_path = args.cache_dir # Setting up the log: will be displayed both on stdout and to the specified # file: diff --git a/workflow/Snakefile b/workflow/Snakefile index ae119b5..f601f4a 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -14,10 +14,15 @@ ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST) rule all: input: expand("{folder}/{artifact}/{date}.csv",\ - folder=["logs", "pkgs", "build_status", "artifact_hash"],\ + folder=["pkgs", "build_status", "artifact_hash"],\ artifact=ARTIFACTS,\ date=DATE ), + expand("{folder}/{artifact}/{date}.txt",\ + folder=["logs"],\ + artifact=ARTIFACTS,\ + date=DATE + ), f"{BLACKLIST_FOLDER}/{DATE}.csv" rule check_artifact: diff --git a/workflow/nickel/artifact_contract.ncl b/workflow/nickel/artifact_contract.ncl index 346fb48..579bf27 100644 --- a/workflow/nickel/artifact_contract.ncl +++ b/workflow/nickel/artifact_contract.ncl @@ -31,7 +31,7 @@ in | ArchiveType, }, Artifact = { - artefact_url + artifact_url | doc "URL where to download the artifact" | String, type @@ -43,7 +43,7 @@ in image_name | doc "Name to give the image when building" | String, - location + dockerfile_location | doc "Path to the dockerfile in the artifact" | String, package_managers