From a62d338bd03089482b1dbca1405300a7521d61fb Mon Sep 17 00:00:00 2001 From: Quentin Guilloteau Date: Thu, 29 Aug 2024 12:58:50 +0200 Subject: [PATCH] moving some config around --- artifacts/nickel/europar24/canon_solving.ncl | 13 ++ artifacts/nickel/europar24/geijer_how.ncl | 10 + artifacts/nickel/europar24/hiraga_peanuts.ncl | 17 ++ .../nickel/europar24/lee_accelerated.ncl | 10 + artifacts/nickel/europar24/wolff_fast.ncl | 11 + config/config.yaml | 2 + workflow/Snakefile | 202 ++++++++---------- workflow/utils.smk | 19 +- 8 files changed, 164 insertions(+), 120 deletions(-) create mode 100644 artifacts/nickel/europar24/canon_solving.ncl create mode 100644 artifacts/nickel/europar24/geijer_how.ncl create mode 100644 artifacts/nickel/europar24/hiraga_peanuts.ncl create mode 100644 artifacts/nickel/europar24/lee_accelerated.ncl create mode 100644 artifacts/nickel/europar24/wolff_fast.ncl diff --git a/artifacts/nickel/europar24/canon_solving.ncl b/artifacts/nickel/europar24/canon_solving.ncl new file mode 100644 index 0000000..3d95def --- /dev/null +++ b/artifacts/nickel/europar24/canon_solving.ncl @@ -0,0 +1,13 @@ +{ + version = "1.0", + artifact_url = "https://zenodo.org/records/11636529/files/artifact-pap130.zip", + type = "zip", + doi = "10.5281/zenodo.11636529", + conf_date = 2024, + virtualization = "docker", + buildfile_dir = ".docker", + package_managers = [ "dpkg" ], + misc_packages = [ + { name = "gurobi10.0.1_linux64", url = "https://packages.gurobi.com/10.0/gurobi10.0.1_linux64.tar.gz" } + ] +} diff --git a/artifacts/nickel/europar24/geijer_how.ncl b/artifacts/nickel/europar24/geijer_how.ncl new file mode 100644 index 0000000..f6c0985 --- /dev/null +++ b/artifacts/nickel/europar24/geijer_how.ncl @@ -0,0 +1,10 @@ +{ + version = "1.0", + artifact_url = "https://zenodo.org/records/11547063/files/artifact.zip", + type = "zip", + doi = "10.5281/zenodo.11547063", + conf_date = 2024, + virtualization = "docker", + buildfile_dir = "artifact", + package_managers = [ "dpkg", "pip" ] +} diff --git a/artifacts/nickel/europar24/hiraga_peanuts.ncl b/artifacts/nickel/europar24/hiraga_peanuts.ncl new file mode 100644 index 0000000..669cfe0 --- /dev/null +++ b/artifacts/nickel/europar24/hiraga_peanuts.ncl @@ -0,0 +1,17 @@ +{ + version = "1.0", + artifact_url = "https://zenodo.org/records/11558678/files/peanuts-playground.zip", + type = "zip", + doi = "10.5281/zenodo.11558678", + conf_date = 2024, + comment = "Files in /var/lib/apt/lists/ are removed.", + virtualization = "docker", + buildfile_dir = "./", + package_managers = [ "dpkg" ], + git_packages = [ + { name = "spack", location = "/home/vscode/.cache/spack" } + ], + misc_packages = [ + { name = "cmake-3.22.2-linux", url = "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.sh" } + ] +} diff --git a/artifacts/nickel/europar24/lee_accelerated.ncl b/artifacts/nickel/europar24/lee_accelerated.ncl new file mode 100644 index 0000000..633af41 --- /dev/null +++ b/artifacts/nickel/europar24/lee_accelerated.ncl @@ -0,0 +1,10 @@ +{ + version = "1.0", + artifact_url = "https://zenodo.org/records/11579181/files/bsa_spmm.zip", + type = "zip", + doi = "10.5281/zenodo.11579181", + conf_date = 2024, + comment = "Are there really Dockerfiles for this artifact?", + virtualization = "docker", + package_managers = [ "dpkg" ], +} diff --git a/artifacts/nickel/europar24/wolff_fast.ncl b/artifacts/nickel/europar24/wolff_fast.ncl new file mode 100644 index 0000000..7c5b42e --- /dev/null +++ b/artifacts/nickel/europar24/wolff_fast.ncl @@ -0,0 +1,11 @@ +{ + version = "1.0", + artifact_url = "https://zenodo.org/records/11775182/files/Euro-PAR_2024_paper_432.zip", + type = "zip", + doi = "10.5281/zenodo.11775182", + conf_date = 2024, + comment = "Files in /var/lib/apt/lists/ are removed.", + virtualization = "docker", + buildfile_dir = "./", + package_managers = [ "dpkg", "pip" ] +} diff --git a/config/config.yaml b/config/config.yaml index 0b3fcd4..2de43f8 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -12,3 +12,5 @@ max_duration: 60 checkpoint: 1 besteffort: True sleep_time: 30 + +conference: "europar24" diff --git a/workflow/Snakefile b/workflow/Snakefile index 1a99611..e4a450a 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -8,24 +8,18 @@ DATE = datetime.datetime.now().strftime("%Y%m%d") ARTIFACTS_FOLDER_NICKEL = config["folder_artifacts_nickel"] ARTIFACTS_FOLDER_JSON = config["folder_artifacts_json"] -BLACKLIST_FOLDER = config["folder_blacklists"] -EXTENSION = "json" SYSTEM = config["system"] +CONFERENCE = config["conference"] -ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST_FOLDER) +ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL + "/" + CONFERENCE) PREFIX = config["prefix"] ECG_OUTPUTS = ["pkgs", "build_status", "artifact_hash"] -SHELLS_ECG = { - "local": f"./{{input.ecg_wrapper}} {{input.ecg}} {ARTIFACTS_FOLDER_JSON}/{{wildcards.artifact}}.{EXTENSION} {{output.pkg}} {{output.build_status}} {{output.artifact_hash}} {{output.log}}", - "g5k": f"python3 {{input.execo_wrapper}} --path {os.getcwd()} --script {{input.oar_wrapper}} --site {config['site']} --cluster {config['cluster']} --max-duration {config['max_duration']} --checkpoint {config['checkpoint']} {'--besteffort' if config['besteffort'] else ''} --sleep_time {config['sleep_time']} --build_status_file {{output.build_status}} --artifact {{wildcards.artifact}} -- '" -} ANALYSIS_DIR = config["analysis_dir"] ANALYSIS_CATS = ["sources_stats", "pkgs_changes", "build_status", "artifact"] ANALYSIS_SCRIPTS_DIR = "analysis" ANALYSIS_WRAPPER = "workflow/scripts/analysis_wrapper.sh" -AGGREGATE_WRAPPER = "workflow/scripts/aggregate_wrapper.sh" ARTIFACT_ANALYSIS_DIRS = " ".join(expand(f"{PREFIX}/artifact_hash/{{artifact}}", artifact = ARTIFACTS )) @@ -44,33 +38,28 @@ PLOT_HEADERS = { rule all: input: - expand(f"{ANALYSIS_DIR}/{{analysis_cat}}/plot/line/{{date}}.pdf", - analysis_cat = ANALYSIS_CATS, - date = DATE - ), - expand(f"{ANALYSIS_DIR}/{{analysis_cat}}/plot/bar/{{date}}.pdf", - analysis_cat = ["sources_stats", "build_status", "artifact"], - date = DATE - ), - f"{BLACKLIST_FOLDER}/{DATE}.csv" + expand(f"{PREFIX}/{{conference}}/build_status/{{artifact}}/{{date}}.csv",\ + conference=config['conference'],\ + artifact=ARTIFACTS,\ + date = DATE) # Artifacts configuration files: rule check_all: input: - expand(f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json", artifact=ARTIFACTS) + expand(f"{ARTIFACTS_FOLDER_JSON}/{{conference}}/{{artifact}}.json", artifact=ARTIFACTS, conference=config['conference']) rule check_artifact: input: "flake.nix", "flake.lock", contract="workflow/nickel/artifact_contract.ncl", - artifact=f"{ARTIFACTS_FOLDER_NICKEL}/{{artifact}}.ncl" + artifact=f"{ARTIFACTS_FOLDER_NICKEL}/{{conference}}/{{artifact}}.ncl" output: - f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json" + f"{ARTIFACTS_FOLDER_JSON}/{{conference}}/{{artifact}}.json" shell: """ - nickel export --format json --output {output} <<< 'let {{Artifact, ..}} = import "{input.contract}" in ((import "{input.artifact}") | Artifact)' + nix develop .#nickel --command nickel export --format json --output {output} <<< 'let {{Artifact, ..}} = import "{input.contract}" in ((import "{input.artifact}") | Artifact)' """ # ECG: @@ -82,12 +71,12 @@ rule run_ecg: ecg="ecg/app/ecg.py", execo_wrapper="workflow/scripts/submission_g5k.py", oar_wrapper="workflow/scripts/ecg_oar_wrapper.oar.bash", - artifact=f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.{EXTENSION}" + artifact=f"{ARTIFACTS_FOLDER_JSON}/{{conference}}/{{artifact}}.json" output: - log = f"{PREFIX}/logs/{{artifact}}/{{date}}.txt", - pkg = f"{PREFIX}/pkgs/{{artifact}}/{{date}}.csv", - build_status = f"{PREFIX}/build_status/{{artifact}}/{{date}}.csv", - artifact_hash = f"{PREFIX}/artifact_hash/{{artifact}}/{{date}}.csv", + log = f"{PREFIX}/{{conference}}/logs/{{artifact}}/{{date}}.txt", + pkg = f"{PREFIX}/{{conference}}/pkgs/{{artifact}}/{{date}}.csv", + build_status = f"{PREFIX}/{{conference}}/build_status/{{artifact}}/{{date}}.csv", + artifact_hash = f"{PREFIX}/{{conference}}/artifact_hash/{{artifact}}/{{date}}.csv", shell: (f"python3 {{input.execo_wrapper}} --path {os.getcwd()} \ --script {{input.oar_wrapper}} \ @@ -100,94 +89,83 @@ rule run_ecg: --build_status_file {{output.build_status}} \ --artifact {{wildcards.artifact}} -- '" if SYSTEM == "g5k" else "") + \ """ - nix develop .#ecg-shell --command ecg -p {output.pkg} -b {output.build_status} -a {output.artifact_hash} {output.artifact} &> {output.log} || echo "{input.artifact}, `date +%s.%N`, script_crash" > {output.build_status} + nix shell .#ecg --command ecg -p {output.pkg} -b {output.build_status} -a {output.artifact_hash} {input.artifact} &> {output.log} || echo "{input.artifact}, `date +%s.%N`, script_crash" > {output.build_status} """ + \ ("'" if SYSTEM == "g5k" else "") -rule update_blacklist: - input: - build_status=expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv", - artifact=ARTIFACTS - ) - output: - f"{BLACKLIST_FOLDER}/{{date}}.csv" - shell: - # We need to ignore lines where build is successful: - f"cat {{input}} | grep -v ',success' > {{output}} || true" - # Analysis: -rule softenv_analysis: - wildcard_constraints: - date="\d+" - input: - expand(f"{PREFIX}/pkgs/{{artifact}}/{{{{date}}}}.csv", - artifact = ARTIFACTS - ) - output: - sources_stats = f"{ANALYSIS_DIR}/sources_stats/{{date}}.csv", - pkgs_changes = f"{ANALYSIS_DIR}/pkgs_changes/{{date}}.csv" - shell: - f""" - {ANALYSIS_WRAPPER} files {ANALYSIS_SCRIPTS_DIR}/softenv_analysis.py -t sources-stats {{output.sources_stats}} {{input}} - {ANALYSIS_WRAPPER} dirs {ANALYSIS_SCRIPTS_DIR}/softenv_analysis.py -t pkgs-changes {{output.pkgs_changes}} {SOFTENV_ANALYSIS_DIRS} - """ - -rule buildstatus_analysis: - wildcard_constraints: - date="\d+" - input: - expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv", - artifact = ARTIFACTS - ), - output: - f"{ANALYSIS_DIR}/build_status/{{date}}.csv" - shell: - f""" - {ANALYSIS_WRAPPER} files {ANALYSIS_SCRIPTS_DIR}/buildstatus_analysis.py {{output}} {{input}} - """ - -rule artifact_analysis: - wildcard_constraints: - date="\d+" - input: - expand(f"{PREFIX}/artifact_hash/{{artifact}}/{{{{date}}}}.csv", - artifact = ARTIFACTS - ) - output: - f"{ANALYSIS_DIR}/artifact/{{date}}.csv" - shell: - f""" - {ANALYSIS_WRAPPER} dirs {ANALYSIS_SCRIPTS_DIR}/artifact_analysis.py {{output}} {ARTIFACT_ANALYSIS_DIRS} - """ - -# Analysis aggregate: - -rule analysis_aggregate: - input: - expand(f"{ANALYSIS_DIR}/{{cat}}/aggregated/{{date}}.csv", - cat = ANALYSIS_CATS, - date = DATE - ) - -rule aggregate_by_type: - input: - data=f"{ANALYSIS_DIR}/{{type}}/{{date}}.csv", - script="workflow/scripts/aggregate_wrapper.sh" - output: - f"{ANALYSIS_DIR}/{{type}}/aggregated/{{date}}.csv" - shell: - f"{input.script} {ANALYSIS_DIR}/{{type}} {{output}}" - -# Plot: - -rule plot: - input: - script = "plot/plot.r", - data = f"{ANALYSIS_DIR}/{{type}}/aggregated/{{date}}.csv", - output: - f"{ANALYSIS_DIR}/{{type}}/{{plot}}/{{date}}.pdf" - params: - header = lambda w: PLOT_HEADERS[w.type] - shell: - "Rscript {input.script} {wildcards.plot} {input.data} {output} {params.header} timestamp" +#rule softenv_analysis: +# wildcard_constraints: +# date="\d+" +# input: +# expand(f"{PREFIX}{{conference}}/pkgs/{{artifact}}/{{{{date}}}}.csv", +# artifact = ARTIFACTS +# ) +# output: +# sources_stats = f"{ANALYSIS_DIR}/sources_stats/{{date}}.csv", +# pkgs_changes = f"{ANALYSIS_DIR}/pkgs_changes/{{date}}.csv" +# shell: +# f""" +# {ANALYSIS_WRAPPER} files {ANALYSIS_SCRIPTS_DIR}/softenv_analysis.py -t sources-stats {{output.sources_stats}} {{input}} +# {ANALYSIS_WRAPPER} dirs {ANALYSIS_SCRIPTS_DIR}/softenv_analysis.py -t pkgs-changes {{output.pkgs_changes}} {SOFTENV_ANALYSIS_DIRS} +# """ +# +#rule buildstatus_analysis: +# wildcard_constraints: +# date="\d+" +# input: +# expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv", +# artifact = ARTIFACTS +# ), +# output: +# f"{ANALYSIS_DIR}/build_status/{{date}}.csv" +# shell: +# f""" +# {ANALYSIS_WRAPPER} files {ANALYSIS_SCRIPTS_DIR}/buildstatus_analysis.py {{output}} {{input}} +# """ +# +#rule artifact_analysis: +# wildcard_constraints: +# date="\d+" +# input: +# expand(f"{PREFIX}/artifact_hash/{{artifact}}/{{{{date}}}}.csv", +# artifact = ARTIFACTS +# ) +# output: +# f"{ANALYSIS_DIR}/artifact/{{date}}.csv" +# shell: +# f""" +# {ANALYSIS_WRAPPER} dirs {ANALYSIS_SCRIPTS_DIR}/artifact_analysis.py {{output}} {ARTIFACT_ANALYSIS_DIRS} +# """ +# +## Analysis aggregate: +# +#rule analysis_aggregate: +# input: +# expand(f"{ANALYSIS_DIR}/{{cat}}/aggregated/{{date}}.csv", +# cat = ANALYSIS_CATS, +# date = DATE +# ) +# +#rule aggregate_by_type: +# input: +# data=f"{ANALYSIS_DIR}/{{type}}/{{date}}.csv", +# script="workflow/scripts/aggregate_wrapper.sh" +# output: +# f"{ANALYSIS_DIR}/{{type}}/aggregated/{{date}}.csv" +# shell: +# f"{{input.script}} {ANALYSIS_DIR}/{{type}} {{output}}" +# +## Plot: +# +#rule plot: +# input: +# script = "plot/plot.r", +# data = f"{ANALYSIS_DIR}/{{type}}/aggregated/{{date}}.csv", +# output: +# f"{ANALYSIS_DIR}/{{type}}/{{plot}}/{{date}}.pdf" +# params: +# header = lambda w: PLOT_HEADERS[w.type] +# shell: +# "Rscript {input.script} {wildcards.plot} {input.data} {output} {params.header} timestamp" diff --git a/workflow/utils.smk b/workflow/utils.smk index f695f08..9039e9c 100644 --- a/workflow/utils.smk +++ b/workflow/utils.smk @@ -13,11 +13,14 @@ def get_blacklisted(blacklist_dir_path): blacklisted.add(row[0]) return blacklisted -def get_artifacts_to_build(artifacts_folder, blacklist_dir_path): - blacklisted = get_blacklisted(blacklist_dir_path) - all_artifacts = set([os.path.splitext(a)[0] for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))]) - artifacts_to_build = list(all_artifacts.difference(blacklisted)) - if artifacts_to_build != []: - return list(all_artifacts.difference(blacklisted)) - else: - raise(Exception(f"There is no artifact to build! Either no artifact configuration files have been found, or they have all been blacklisted.")) +#def get_artifacts_to_build(artifacts_folder, blacklist_dir_path): +# blacklisted = get_blacklisted(blacklist_dir_path) +# all_artifacts = set([os.path.splitext(a)[0] for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))]) +# artifacts_to_build = list(all_artifacts.difference(blacklisted)) +# if artifacts_to_build != []: +# return list(all_artifacts.difference(blacklisted)) +# else: +# raise(Exception(f"There is no artifact to build! Either no artifact configuration files have been found, or they have all been blacklisted.")) + +def get_artifacts_to_build(artifacts_folder): + return [os.path.splitext(a)[0] for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))]