Fixed Nickel contract and check files. Fixed input/output files in Snakefile.

This commit is contained in:
antux18 2024-07-16 17:35:09 +02:00
parent 69e447ab0a
commit 1aaa77cba3
10 changed files with 76 additions and 28 deletions

1
.gitignore vendored
View File

@ -2,6 +2,7 @@ pkglists/*
output/* output/*
cache/* cache/*
examples/* examples/*
.snakemake/*
pkglist.csv pkglist.csv
log.txt log.txt
build_status.csv build_status.csv

View File

@ -0,0 +1,23 @@
{
"artifact_url": "https://example.com/artifact.zip",
"dockerfile_location": "path/to/docker/folder",
"doi": "...",
"git_packages": [
{
"location": "path/to/git/repo",
"name": "pkg1"
}
],
"image_name": "image:version",
"misc_packages": [
{
"name": "mpkg1",
"type": "zip",
"url": "https://"
}
],
"package_managers": [
"dpkg"
],
"type": "zip"
}

View File

@ -1,20 +1,24 @@
{ {
"artifact_url": "http://localhost/artifact.zip", "artifact_url": "http://localhost/artifact.zip",
"type": "zip",
"doi": "XX.XXXX/XXXXXXX.XXXXXXX",
"image_name": "prog:latest",
"dockerfile_location": "./", "dockerfile_location": "./",
"doi": "...",
"git_packages": [
{
"location": "/pkg1",
"name": "pkg1"
}
],
"image_name": "prog:latest",
"misc_packages": [
{
"name": "mpkg1",
"type": "zip",
"url": "http://localhost/package1.zip"
}
],
"package_managers": [ "package_managers": [
"dpkg", "dpkg",
"pip" "pip"
], ],
"git_packages": [{ "type": "zip"
"name": "pkg1", }
"location": "/pkg1"
}],
"misc_packages": [{
"name": "mpkg1",
"url": "http://localhost/package1.zip",
"type": "zip"
}]
}

14
artifacts_nickel/test.ncl Normal file
View File

@ -0,0 +1,14 @@
{
artifact_url = "http://localhost/artifact.zip",
type = "zip",
doi = "...",
image_name = "prog:latest",
dockerfile_location = "./",
package_managers = [ "dpkg", "pip" ],
git_packages = [
{ name = "pkg1", location = "/pkg1"}
],
misc_packages = [
{ name = "mpkg1", url = "http://localhost/package1.zip", type = "zip" }
],
}

View File

@ -11,4 +11,4 @@ git_packages:
misc_packages: misc_packages:
- name: "mpkg1" - name: "mpkg1"
url: "https://example.com/package1.zip" url: "https://example.com/package1.zip"
type: "zip" # Possible values: zip, tgz type: "zip" # Possible values: zip, tar

View File

@ -1 +1 @@
sc24_test, IMAGE_NOT_FOUND, 0 example, IMAGE_NOT_FOUND, 0

1 sc24_test example IMAGE_NOT_FOUND 0

View File

@ -1,4 +1,4 @@
let { Artifact, .. } = import "artifact_contract.ncl" in let { Artifact, .. } = import "workflow/nickel/artifact_contract.ncl" in
( (
(import "artifacts_nickel/example.ncl") | Artifact (import "artifacts_nickel/example.ncl") | Artifact
) )

19
ecg.py
View File

@ -28,7 +28,7 @@ config_path = ""
pkglist_path = "" # Package list being generated pkglist_path = "" # Package list being generated
buildstatus_path = "" # Summary of the build process of the image buildstatus_path = "" # Summary of the build process of the image
arthashlog_path = "" # Log of the hash of the downloaded artifact arthashlog_path = "" # Log of the hash of the downloaded artifact
cachedir_path = "" # Artifact cache directory cachedir_path = "cache" # Artifact cache directory
# Commands to list installed packages along with their versions and the name # Commands to list installed packages along with their versions and the name
# of the package manager, depending on the package managers. # of the package manager, depending on the package managers.
@ -189,9 +189,9 @@ def build_image(config, src_dir):
name = config["image_name"] name = config["image_name"]
logging.info(f"Starting building image {name}") logging.info(f"Starting building image {name}")
path = os.path.join(src_dir, config["dockerfile_location"]) path = os.path.join(src_dir, config["dockerfile_location"])
build_command = f"docker build -t {config["image_name"]} ." build_command = f"docker build -t {config['image_name']} ."
build_process = subprocess.run(build_command.split(" "), cwd=path, capture_output=True) build_process = subprocess.run(build_command.split(" "), cwd=path, capture_output=True)
build_output = f"stdout:\n{build_process.stdout.decode("utf-8")}\nstderr:\n{build_process.stderr.decode("utf-8")}" build_output = f"stdout:\n{build_process.stdout.decode('utf-8')}\nstderr:\n{build_process.stderr.decode('utf-8')}"
# build_output = build_process.stderr.decode("utf-8") # build_output = build_process.stderr.decode("utf-8")
logging.info(f"Output of '{build_command}':") logging.info(f"Output of '{build_command}':")
logging.info(build_output) logging.info(build_output)
@ -227,23 +227,23 @@ def check_env(config, src_dir):
listformat_cmd = pkgmgr_cmd[pkgmgr][1] listformat_cmd = pkgmgr_cmd[pkgmgr][1]
logging.info(f"Checking '{pkgmgr}'") logging.info(f"Checking '{pkgmgr}'")
pkglist_process = subprocess.run(["docker", "run", "--rm", config["image_name"]] + pkglist_cmd.split(" "), cwd=path, capture_output=True) pkglist_process = subprocess.run(["docker", "run", "--rm", config["image_name"]] + pkglist_cmd.split(" "), cwd=path, capture_output=True)
format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode("utf-8")}EOF", cwd=path, capture_output=True, shell=True) format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode('utf-8')}EOF", cwd=path, capture_output=True, shell=True)
pkglist = format_process.stdout.decode("utf-8") pkglist = format_process.stdout.decode("utf-8")
pkglist_file.write(pkglist) pkglist_file.write(pkglist)
if "git_packages" in config.keys(): if "git_packages" in config.keys():
logging.info("Checking Git packages") logging.info("Checking Git packages")
for repo in config["git_packages"]: for repo in config["git_packages"]:
pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], config["image_name"]] + gitcmd.split(" "), cwd=path, capture_output=True) pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], config["image_name"]] + gitcmd.split(" "), cwd=path, capture_output=True)
repo_row = f"{repo["name"]},{pkglist_process.stdout.decode("utf-8")},git" repo_row = f"{repo['name']},{pkglist_process.stdout.decode('utf-8')},git"
pkglist_file.write(f"{repo_row}\n") pkglist_file.write(f"{repo_row}\n")
if "misc_packages" in config.keys(): if "misc_packages" in config.keys():
logging.info("Checking packages obtained outside of a package manager or VCS") logging.info("Checking packages obtained outside of a package manager or VCS")
for pkg in config["misc_packages"]: for pkg in config["misc_packages"]:
logging.info(f"Downloading package {pkg["name"]} from {pkg["url"]}") logging.info(f"Downloading package {pkg['name']} from {pkg['url']}")
pkg_file = tempfile.NamedTemporaryFile() pkg_file = tempfile.NamedTemporaryFile()
pkg_path = pkg_file.name pkg_path = pkg_file.name
pkg_hash = download_file(pkg["url"], pkg_path) pkg_hash = download_file(pkg["url"], pkg_path)
pkg_row = f"{pkg["name"]},{pkg_hash},misc" pkg_row = f"{pkg['name']},{pkg_hash},misc"
pkglist_file.write(f"{pkg_row}\n") pkglist_file.write(f"{pkg_row}\n")
pkglist_file.close() pkglist_file.close()
@ -304,7 +304,7 @@ def main():
parser.add_argument( parser.add_argument(
"-c", "--cache-dir", "-c", "--cache-dir",
help = "Path to the cache directory, where artifact that are downloaded will be stored for future usage.", help = "Path to the cache directory, where artifact that are downloaded will be stored for future usage.",
required = True required = False
) )
args = parser.parse_args() args = parser.parse_args()
@ -314,7 +314,8 @@ def main():
log_path = args.log_path log_path = args.log_path
buildstatus_path = args.build_summary buildstatus_path = args.build_summary
arthashlog_path = args.artifact_hash arthashlog_path = args.artifact_hash
cachedir_path = args.cache_dir if args.cache_dir != None:
cachedir_path = args.cache_dir
# Setting up the log: will be displayed both on stdout and to the specified # Setting up the log: will be displayed both on stdout and to the specified
# file: # file:

View File

@ -14,10 +14,15 @@ ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST)
rule all: rule all:
input: input:
expand("{folder}/{artifact}/{date}.csv",\ expand("{folder}/{artifact}/{date}.csv",\
folder=["logs", "pkgs", "build_status", "artifact_hash"],\ folder=["pkgs", "build_status", "artifact_hash"],\
artifact=ARTIFACTS,\ artifact=ARTIFACTS,\
date=DATE date=DATE
), ),
expand("{folder}/{artifact}/{date}.txt",\
folder=["logs"],\
artifact=ARTIFACTS,\
date=DATE
),
f"{BLACKLIST_FOLDER}/{DATE}.csv" f"{BLACKLIST_FOLDER}/{DATE}.csv"
rule check_artifact: rule check_artifact:

View File

@ -31,7 +31,7 @@ in
| ArchiveType, | ArchiveType,
}, },
Artifact = { Artifact = {
artefact_url artifact_url
| doc "URL where to download the artifact" | doc "URL where to download the artifact"
| String, | String,
type type
@ -43,7 +43,7 @@ in
image_name image_name
| doc "Name to give the image when building" | doc "Name to give the image when building"
| String, | String,
location dockerfile_location
| doc "Path to the dockerfile in the artifact" | doc "Path to the dockerfile in the artifact"
| String, | String,
package_managers package_managers