Fixed Nickel config files. Added artifact hash log on Snakefile + modified some variables names. Switched from 'tgz' to 'tar' for archive type.
This commit is contained in:
parent
842f9d32a7
commit
69e447ab0a
@ -1,9 +1,9 @@
|
||||
{
|
||||
artefact_url = "https://example.com/artifact.zip",
|
||||
artifact_url = "https://example.com/artifact.zip",
|
||||
type = "zip",
|
||||
doi = "...",
|
||||
image_name = "image:version",
|
||||
location = "path/to/docker/folder",
|
||||
dockerfile_location = "path/to/docker/folder",
|
||||
package_managers = [ "dpkg" ],
|
||||
git_packages = [
|
||||
{ name = "pkg1", location = "path/to/git/repo"}
|
||||
|
@ -1,5 +1,5 @@
|
||||
artifact_url: "https://zenodo.org/record/7544675/files/SF2-code.tar.gz"
|
||||
type: "tgz"
|
||||
type: "tar"
|
||||
doi: "10.1145/3572848.3577480"
|
||||
image_name: "bdm-publication-image-v7"
|
||||
dockerfile_location: "bdm-paper-examples/docker"
|
||||
|
@ -1,5 +1,5 @@
|
||||
artifact_url: "https://zenodo.org/record/7328505/files/tgopt-artifact.tgz"
|
||||
type: "tgz"
|
||||
type: "tar"
|
||||
doi: "10.1145/3572848.3577490"
|
||||
image_name: "tgopt:artefact"
|
||||
dockerfile_location: "tgopt-artifact"
|
||||
|
@ -1,5 +1,5 @@
|
||||
artifact_url: "https://example.com/artifact.zip"
|
||||
type: "zip" # Possible values: zip, tgz
|
||||
type: "zip" # Possible values: zip, tar
|
||||
doi: "XX.XXXX/XXXXXXX.XXXXXXX"
|
||||
image_name: "image1:version"
|
||||
dockerfile_location: "path/to/docker/folder"
|
||||
|
4
ecg.py
4
ecg.py
@ -119,7 +119,7 @@ def download_sources(config):
|
||||
"""
|
||||
url = config["artifact_url"]
|
||||
artifact_name = trim(url)
|
||||
artifact_dir = f"{cachedir_path}/{artifact_name}"
|
||||
artifact_dir = os.path.join(cachedir_path, artifact_name)
|
||||
# Checking if artifact in cache. Not downloading if it is:
|
||||
if not os.path.exists(artifact_dir):
|
||||
logging.info(f"Downloading artifact from {url}")
|
||||
@ -129,7 +129,7 @@ def download_sources(config):
|
||||
artifact_hash = download_file(url, artifact_path)
|
||||
if config["type"] == "zip":
|
||||
artifact = zipfile.ZipFile(artifact_path)
|
||||
elif config["type"] == "tgz":
|
||||
elif config["type"] == "tar":
|
||||
artifact = tarfile.open(artifact_path)
|
||||
logging.info(f"Extracting artifact at {artifact_dir}")
|
||||
artifact.extractall(artifact_dir)
|
||||
|
@ -14,9 +14,10 @@ ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST)
|
||||
rule all:
|
||||
input:
|
||||
expand("{folder}/{artifact}/{date}.csv",\
|
||||
folder=["logs", "pkgs", "status"],\
|
||||
folder=["logs", "pkgs", "build_status", "artifact_hash"],\
|
||||
artifact=ARTIFACTS,\
|
||||
date=DATE),
|
||||
date=DATE
|
||||
),
|
||||
f"{BLACKLIST_FOLDER}/{DATE}.csv"
|
||||
|
||||
rule check_artifact:
|
||||
@ -39,16 +40,17 @@ rule run_ecg:
|
||||
ecg="ecg.py",
|
||||
artifact=f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.{EXTENSION}"
|
||||
output:
|
||||
log = "logs/{artifact}/{date}.csv",
|
||||
log = "logs/{artifact}/{date}.txt",
|
||||
pkg = "pkgs/{artifact}/{date}.csv",
|
||||
status = "status/{artifact}/{date}.csv",
|
||||
build_status = "build_status/{artifact}/{date}.csv",
|
||||
artifact_hash = "artifact_hash/{artifact}/{date}.csv"
|
||||
shell:
|
||||
f"python3 {{input.ecg}} --log {{output.log}} --pkg {{output.pkg}} --status {{output.pkg}} {ARTIFACTS_FOLDER_JSON}/{{wildcards.artifact}}.{EXTENSION}"
|
||||
f"python3 {{input.ecg}} -l {{output.log}} -p {{output.pkg}} -b {{output.build_status}} -a {{output.artifact_hash}} {ARTIFACTS_FOLDER_JSON}/{{wildcards.artifact}}.{EXTENSION}"
|
||||
|
||||
rule update_blacklist:
|
||||
input:
|
||||
BLACKLIST,
|
||||
status=expand("status/{artifact}/{{date}}.csv",\
|
||||
build_status=expand("build_status/{artifact}/{{date}}.csv",\
|
||||
artifact=ARTIFACTS)
|
||||
output:
|
||||
f"{BLACKLIST_FOLDER}/{{date}}.csv"
|
||||
|
@ -1,7 +1,7 @@
|
||||
let
|
||||
conf = {
|
||||
ARCHIVE_FORMATS = [ "zip", "tar" ],
|
||||
PACKAGE_MANAGERS = [ "dpkg", "conda", "pip"]
|
||||
ARCHIVE_FORMATS = ["zip", "tar"],
|
||||
PACKAGE_MANAGERS = ["dpkg", "rpm", "pacman", "pip", "conda"]
|
||||
}
|
||||
in
|
||||
{
|
||||
@ -10,7 +10,7 @@ in
|
||||
),
|
||||
ArchiveType = std.contract.from_predicate (
|
||||
fun value => std.array.any (fun x => x == value) conf.ARCHIVE_FORMATS
|
||||
),
|
||||
),
|
||||
GitPackage = {
|
||||
name
|
||||
| doc "Name of the package for future identification"
|
||||
|
Loading…
x
Reference in New Issue
Block a user