Fixed Nickel contract and check files. Fixed input/output files in Snakefile.
This commit is contained in:
parent
69e447ab0a
commit
1aaa77cba3
1
.gitignore
vendored
1
.gitignore
vendored
@ -2,6 +2,7 @@ pkglists/*
|
|||||||
output/*
|
output/*
|
||||||
cache/*
|
cache/*
|
||||||
examples/*
|
examples/*
|
||||||
|
.snakemake/*
|
||||||
pkglist.csv
|
pkglist.csv
|
||||||
log.txt
|
log.txt
|
||||||
build_status.csv
|
build_status.csv
|
23
artifacts_json/example.json
Normal file
23
artifacts_json/example.json
Normal file
@ -0,0 +1,23 @@
|
|||||||
|
{
|
||||||
|
"artifact_url": "https://example.com/artifact.zip",
|
||||||
|
"dockerfile_location": "path/to/docker/folder",
|
||||||
|
"doi": "...",
|
||||||
|
"git_packages": [
|
||||||
|
{
|
||||||
|
"location": "path/to/git/repo",
|
||||||
|
"name": "pkg1"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"image_name": "image:version",
|
||||||
|
"misc_packages": [
|
||||||
|
{
|
||||||
|
"name": "mpkg1",
|
||||||
|
"type": "zip",
|
||||||
|
"url": "https://"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"package_managers": [
|
||||||
|
"dpkg"
|
||||||
|
],
|
||||||
|
"type": "zip"
|
||||||
|
}
|
@ -1,20 +1,24 @@
|
|||||||
{
|
{
|
||||||
"artifact_url": "http://localhost/artifact.zip",
|
"artifact_url": "http://localhost/artifact.zip",
|
||||||
"type": "zip",
|
|
||||||
"doi": "XX.XXXX/XXXXXXX.XXXXXXX",
|
|
||||||
"image_name": "prog:latest",
|
|
||||||
"dockerfile_location": "./",
|
"dockerfile_location": "./",
|
||||||
|
"doi": "...",
|
||||||
|
"git_packages": [
|
||||||
|
{
|
||||||
|
"location": "/pkg1",
|
||||||
|
"name": "pkg1"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"image_name": "prog:latest",
|
||||||
|
"misc_packages": [
|
||||||
|
{
|
||||||
|
"name": "mpkg1",
|
||||||
|
"type": "zip",
|
||||||
|
"url": "http://localhost/package1.zip"
|
||||||
|
}
|
||||||
|
],
|
||||||
"package_managers": [
|
"package_managers": [
|
||||||
"dpkg",
|
"dpkg",
|
||||||
"pip"
|
"pip"
|
||||||
],
|
],
|
||||||
"git_packages": [{
|
"type": "zip"
|
||||||
"name": "pkg1",
|
}
|
||||||
"location": "/pkg1"
|
|
||||||
}],
|
|
||||||
"misc_packages": [{
|
|
||||||
"name": "mpkg1",
|
|
||||||
"url": "http://localhost/package1.zip",
|
|
||||||
"type": "zip"
|
|
||||||
}]
|
|
||||||
}
|
|
||||||
|
14
artifacts_nickel/test.ncl
Normal file
14
artifacts_nickel/test.ncl
Normal file
@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
artifact_url = "http://localhost/artifact.zip",
|
||||||
|
type = "zip",
|
||||||
|
doi = "...",
|
||||||
|
image_name = "prog:latest",
|
||||||
|
dockerfile_location = "./",
|
||||||
|
package_managers = [ "dpkg", "pip" ],
|
||||||
|
git_packages = [
|
||||||
|
{ name = "pkg1", location = "/pkg1"}
|
||||||
|
],
|
||||||
|
misc_packages = [
|
||||||
|
{ name = "mpkg1", url = "http://localhost/package1.zip", type = "zip" }
|
||||||
|
],
|
||||||
|
}
|
@ -11,4 +11,4 @@ git_packages:
|
|||||||
misc_packages:
|
misc_packages:
|
||||||
- name: "mpkg1"
|
- name: "mpkg1"
|
||||||
url: "https://example.com/package1.zip"
|
url: "https://example.com/package1.zip"
|
||||||
type: "zip" # Possible values: zip, tgz
|
type: "zip" # Possible values: zip, tar
|
||||||
|
@ -1 +1 @@
|
|||||||
sc24_test, IMAGE_NOT_FOUND, 0
|
example, IMAGE_NOT_FOUND, 0
|
||||||
|
|
@ -1,4 +1,4 @@
|
|||||||
let { Artifact, .. } = import "artifact_contract.ncl" in
|
let { Artifact, .. } = import "workflow/nickel/artifact_contract.ncl" in
|
||||||
(
|
(
|
||||||
(import "artifacts_nickel/example.ncl") | Artifact
|
(import "artifacts_nickel/example.ncl") | Artifact
|
||||||
)
|
)
|
||||||
|
19
ecg.py
19
ecg.py
@ -28,7 +28,7 @@ config_path = ""
|
|||||||
pkglist_path = "" # Package list being generated
|
pkglist_path = "" # Package list being generated
|
||||||
buildstatus_path = "" # Summary of the build process of the image
|
buildstatus_path = "" # Summary of the build process of the image
|
||||||
arthashlog_path = "" # Log of the hash of the downloaded artifact
|
arthashlog_path = "" # Log of the hash of the downloaded artifact
|
||||||
cachedir_path = "" # Artifact cache directory
|
cachedir_path = "cache" # Artifact cache directory
|
||||||
|
|
||||||
# Commands to list installed packages along with their versions and the name
|
# Commands to list installed packages along with their versions and the name
|
||||||
# of the package manager, depending on the package managers.
|
# of the package manager, depending on the package managers.
|
||||||
@ -189,9 +189,9 @@ def build_image(config, src_dir):
|
|||||||
name = config["image_name"]
|
name = config["image_name"]
|
||||||
logging.info(f"Starting building image {name}")
|
logging.info(f"Starting building image {name}")
|
||||||
path = os.path.join(src_dir, config["dockerfile_location"])
|
path = os.path.join(src_dir, config["dockerfile_location"])
|
||||||
build_command = f"docker build -t {config["image_name"]} ."
|
build_command = f"docker build -t {config['image_name']} ."
|
||||||
build_process = subprocess.run(build_command.split(" "), cwd=path, capture_output=True)
|
build_process = subprocess.run(build_command.split(" "), cwd=path, capture_output=True)
|
||||||
build_output = f"stdout:\n{build_process.stdout.decode("utf-8")}\nstderr:\n{build_process.stderr.decode("utf-8")}"
|
build_output = f"stdout:\n{build_process.stdout.decode('utf-8')}\nstderr:\n{build_process.stderr.decode('utf-8')}"
|
||||||
# build_output = build_process.stderr.decode("utf-8")
|
# build_output = build_process.stderr.decode("utf-8")
|
||||||
logging.info(f"Output of '{build_command}':")
|
logging.info(f"Output of '{build_command}':")
|
||||||
logging.info(build_output)
|
logging.info(build_output)
|
||||||
@ -227,23 +227,23 @@ def check_env(config, src_dir):
|
|||||||
listformat_cmd = pkgmgr_cmd[pkgmgr][1]
|
listformat_cmd = pkgmgr_cmd[pkgmgr][1]
|
||||||
logging.info(f"Checking '{pkgmgr}'")
|
logging.info(f"Checking '{pkgmgr}'")
|
||||||
pkglist_process = subprocess.run(["docker", "run", "--rm", config["image_name"]] + pkglist_cmd.split(" "), cwd=path, capture_output=True)
|
pkglist_process = subprocess.run(["docker", "run", "--rm", config["image_name"]] + pkglist_cmd.split(" "), cwd=path, capture_output=True)
|
||||||
format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode("utf-8")}EOF", cwd=path, capture_output=True, shell=True)
|
format_process = subprocess.run(f"cat << EOF | {listformat_cmd}\n{pkglist_process.stdout.decode('utf-8')}EOF", cwd=path, capture_output=True, shell=True)
|
||||||
pkglist = format_process.stdout.decode("utf-8")
|
pkglist = format_process.stdout.decode("utf-8")
|
||||||
pkglist_file.write(pkglist)
|
pkglist_file.write(pkglist)
|
||||||
if "git_packages" in config.keys():
|
if "git_packages" in config.keys():
|
||||||
logging.info("Checking Git packages")
|
logging.info("Checking Git packages")
|
||||||
for repo in config["git_packages"]:
|
for repo in config["git_packages"]:
|
||||||
pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], config["image_name"]] + gitcmd.split(" "), cwd=path, capture_output=True)
|
pkglist_process = subprocess.run(["docker", "run", "--rm", "-w", repo["location"], config["image_name"]] + gitcmd.split(" "), cwd=path, capture_output=True)
|
||||||
repo_row = f"{repo["name"]},{pkglist_process.stdout.decode("utf-8")},git"
|
repo_row = f"{repo['name']},{pkglist_process.stdout.decode('utf-8')},git"
|
||||||
pkglist_file.write(f"{repo_row}\n")
|
pkglist_file.write(f"{repo_row}\n")
|
||||||
if "misc_packages" in config.keys():
|
if "misc_packages" in config.keys():
|
||||||
logging.info("Checking packages obtained outside of a package manager or VCS")
|
logging.info("Checking packages obtained outside of a package manager or VCS")
|
||||||
for pkg in config["misc_packages"]:
|
for pkg in config["misc_packages"]:
|
||||||
logging.info(f"Downloading package {pkg["name"]} from {pkg["url"]}")
|
logging.info(f"Downloading package {pkg['name']} from {pkg['url']}")
|
||||||
pkg_file = tempfile.NamedTemporaryFile()
|
pkg_file = tempfile.NamedTemporaryFile()
|
||||||
pkg_path = pkg_file.name
|
pkg_path = pkg_file.name
|
||||||
pkg_hash = download_file(pkg["url"], pkg_path)
|
pkg_hash = download_file(pkg["url"], pkg_path)
|
||||||
pkg_row = f"{pkg["name"]},{pkg_hash},misc"
|
pkg_row = f"{pkg['name']},{pkg_hash},misc"
|
||||||
pkglist_file.write(f"{pkg_row}\n")
|
pkglist_file.write(f"{pkg_row}\n")
|
||||||
pkglist_file.close()
|
pkglist_file.close()
|
||||||
|
|
||||||
@ -304,7 +304,7 @@ def main():
|
|||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"-c", "--cache-dir",
|
"-c", "--cache-dir",
|
||||||
help = "Path to the cache directory, where artifact that are downloaded will be stored for future usage.",
|
help = "Path to the cache directory, where artifact that are downloaded will be stored for future usage.",
|
||||||
required = True
|
required = False
|
||||||
)
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
@ -314,7 +314,8 @@ def main():
|
|||||||
log_path = args.log_path
|
log_path = args.log_path
|
||||||
buildstatus_path = args.build_summary
|
buildstatus_path = args.build_summary
|
||||||
arthashlog_path = args.artifact_hash
|
arthashlog_path = args.artifact_hash
|
||||||
cachedir_path = args.cache_dir
|
if args.cache_dir != None:
|
||||||
|
cachedir_path = args.cache_dir
|
||||||
|
|
||||||
# Setting up the log: will be displayed both on stdout and to the specified
|
# Setting up the log: will be displayed both on stdout and to the specified
|
||||||
# file:
|
# file:
|
||||||
|
@ -14,10 +14,15 @@ ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST)
|
|||||||
rule all:
|
rule all:
|
||||||
input:
|
input:
|
||||||
expand("{folder}/{artifact}/{date}.csv",\
|
expand("{folder}/{artifact}/{date}.csv",\
|
||||||
folder=["logs", "pkgs", "build_status", "artifact_hash"],\
|
folder=["pkgs", "build_status", "artifact_hash"],\
|
||||||
artifact=ARTIFACTS,\
|
artifact=ARTIFACTS,\
|
||||||
date=DATE
|
date=DATE
|
||||||
),
|
),
|
||||||
|
expand("{folder}/{artifact}/{date}.txt",\
|
||||||
|
folder=["logs"],\
|
||||||
|
artifact=ARTIFACTS,\
|
||||||
|
date=DATE
|
||||||
|
),
|
||||||
f"{BLACKLIST_FOLDER}/{DATE}.csv"
|
f"{BLACKLIST_FOLDER}/{DATE}.csv"
|
||||||
|
|
||||||
rule check_artifact:
|
rule check_artifact:
|
||||||
|
@ -31,7 +31,7 @@ in
|
|||||||
| ArchiveType,
|
| ArchiveType,
|
||||||
},
|
},
|
||||||
Artifact = {
|
Artifact = {
|
||||||
artefact_url
|
artifact_url
|
||||||
| doc "URL where to download the artifact"
|
| doc "URL where to download the artifact"
|
||||||
| String,
|
| String,
|
||||||
type
|
type
|
||||||
@ -43,7 +43,7 @@ in
|
|||||||
image_name
|
image_name
|
||||||
| doc "Name to give the image when building"
|
| doc "Name to give the image when building"
|
||||||
| String,
|
| String,
|
||||||
location
|
dockerfile_location
|
||||||
| doc "Path to the dockerfile in the artifact"
|
| doc "Path to the dockerfile in the artifact"
|
||||||
| String,
|
| String,
|
||||||
package_managers
|
package_managers
|
||||||
|
Loading…
Reference in New Issue
Block a user