From 69e447ab0aa34973b2acb13a28a9af7d456fa1fd Mon Sep 17 00:00:00 2001 From: antux18 Date: Tue, 16 Jul 2024 16:47:17 +0200 Subject: [PATCH] Fixed Nickel config files. Added artifact hash log on Snakefile + modified some variables names. Switched from 'tgz' to 'tar' for archive type. --- artifacts_nickel/example.ncl | 4 ++-- ...able_agent_based_simulation_with_BioDynaMo.yaml | 2 +- ...tions_for_temporal_graph_attention_network.yaml | 2 +- artifacts_yaml/template.yaml | 2 +- ecg.py | 4 ++-- workflow/Snakefile | 14 ++++++++------ workflow/nickel/artifact_contract.ncl | 6 +++--- 7 files changed, 18 insertions(+), 16 deletions(-) diff --git a/artifacts_nickel/example.ncl b/artifacts_nickel/example.ncl index 732f293..cf08657 100644 --- a/artifacts_nickel/example.ncl +++ b/artifacts_nickel/example.ncl @@ -1,9 +1,9 @@ { - artefact_url = "https://example.com/artifact.zip", + artifact_url = "https://example.com/artifact.zip", type = "zip", doi = "...", image_name = "image:version", - location = "path/to/docker/folder", + dockerfile_location = "path/to/docker/folder", package_managers = [ "dpkg" ], git_packages = [ { name = "pkg1", location = "path/to/git/repo"} diff --git a/artifacts_yaml/ppopp23/high_performance_and_scalable_agent_based_simulation_with_BioDynaMo.yaml b/artifacts_yaml/ppopp23/high_performance_and_scalable_agent_based_simulation_with_BioDynaMo.yaml index 3a1c109..306ffb2 100644 --- a/artifacts_yaml/ppopp23/high_performance_and_scalable_agent_based_simulation_with_BioDynaMo.yaml +++ b/artifacts_yaml/ppopp23/high_performance_and_scalable_agent_based_simulation_with_BioDynaMo.yaml @@ -1,5 +1,5 @@ artifact_url: "https://zenodo.org/record/7544675/files/SF2-code.tar.gz" -type: "tgz" +type: "tar" doi: "10.1145/3572848.3577480" image_name: "bdm-publication-image-v7" dockerfile_location: "bdm-paper-examples/docker" diff --git a/artifacts_yaml/ppopp23/tgopt_redundancy_aware-optimizations_for_temporal_graph_attention_network.yaml b/artifacts_yaml/ppopp23/tgopt_redundancy_aware-optimizations_for_temporal_graph_attention_network.yaml index bbfc792..b3177b2 100644 --- a/artifacts_yaml/ppopp23/tgopt_redundancy_aware-optimizations_for_temporal_graph_attention_network.yaml +++ b/artifacts_yaml/ppopp23/tgopt_redundancy_aware-optimizations_for_temporal_graph_attention_network.yaml @@ -1,5 +1,5 @@ artifact_url: "https://zenodo.org/record/7328505/files/tgopt-artifact.tgz" -type: "tgz" +type: "tar" doi: "10.1145/3572848.3577490" image_name: "tgopt:artefact" dockerfile_location: "tgopt-artifact" diff --git a/artifacts_yaml/template.yaml b/artifacts_yaml/template.yaml index a597ca6..b8d485c 100644 --- a/artifacts_yaml/template.yaml +++ b/artifacts_yaml/template.yaml @@ -1,5 +1,5 @@ artifact_url: "https://example.com/artifact.zip" -type: "zip" # Possible values: zip, tgz +type: "zip" # Possible values: zip, tar doi: "XX.XXXX/XXXXXXX.XXXXXXX" image_name: "image1:version" dockerfile_location: "path/to/docker/folder" diff --git a/ecg.py b/ecg.py index 42d9c7c..47c274b 100755 --- a/ecg.py +++ b/ecg.py @@ -119,7 +119,7 @@ def download_sources(config): """ url = config["artifact_url"] artifact_name = trim(url) - artifact_dir = f"{cachedir_path}/{artifact_name}" + artifact_dir = os.path.join(cachedir_path, artifact_name) # Checking if artifact in cache. Not downloading if it is: if not os.path.exists(artifact_dir): logging.info(f"Downloading artifact from {url}") @@ -129,7 +129,7 @@ def download_sources(config): artifact_hash = download_file(url, artifact_path) if config["type"] == "zip": artifact = zipfile.ZipFile(artifact_path) - elif config["type"] == "tgz": + elif config["type"] == "tar": artifact = tarfile.open(artifact_path) logging.info(f"Extracting artifact at {artifact_dir}") artifact.extractall(artifact_dir) diff --git a/workflow/Snakefile b/workflow/Snakefile index 961aa90..ae119b5 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -14,9 +14,10 @@ ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST) rule all: input: expand("{folder}/{artifact}/{date}.csv",\ - folder=["logs", "pkgs", "status"],\ + folder=["logs", "pkgs", "build_status", "artifact_hash"],\ artifact=ARTIFACTS,\ - date=DATE), + date=DATE + ), f"{BLACKLIST_FOLDER}/{DATE}.csv" rule check_artifact: @@ -39,16 +40,17 @@ rule run_ecg: ecg="ecg.py", artifact=f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.{EXTENSION}" output: - log = "logs/{artifact}/{date}.csv", + log = "logs/{artifact}/{date}.txt", pkg = "pkgs/{artifact}/{date}.csv", - status = "status/{artifact}/{date}.csv", + build_status = "build_status/{artifact}/{date}.csv", + artifact_hash = "artifact_hash/{artifact}/{date}.csv" shell: - f"python3 {{input.ecg}} --log {{output.log}} --pkg {{output.pkg}} --status {{output.pkg}} {ARTIFACTS_FOLDER_JSON}/{{wildcards.artifact}}.{EXTENSION}" + f"python3 {{input.ecg}} -l {{output.log}} -p {{output.pkg}} -b {{output.build_status}} -a {{output.artifact_hash}} {ARTIFACTS_FOLDER_JSON}/{{wildcards.artifact}}.{EXTENSION}" rule update_blacklist: input: BLACKLIST, - status=expand("status/{artifact}/{{date}}.csv",\ + build_status=expand("build_status/{artifact}/{{date}}.csv",\ artifact=ARTIFACTS) output: f"{BLACKLIST_FOLDER}/{{date}}.csv" diff --git a/workflow/nickel/artifact_contract.ncl b/workflow/nickel/artifact_contract.ncl index cbb50cd..346fb48 100644 --- a/workflow/nickel/artifact_contract.ncl +++ b/workflow/nickel/artifact_contract.ncl @@ -1,7 +1,7 @@ let conf = { - ARCHIVE_FORMATS = [ "zip", "tar" ], - PACKAGE_MANAGERS = [ "dpkg", "conda", "pip"] + ARCHIVE_FORMATS = ["zip", "tar"], + PACKAGE_MANAGERS = ["dpkg", "rpm", "pacman", "pip", "conda"] } in { @@ -10,7 +10,7 @@ in ), ArchiveType = std.contract.from_predicate ( fun value => std.array.any (fun x => x == value) conf.ARCHIVE_FORMATS - ), + ), GitPackage = { name | doc "Name of the package for future identification"