GuilloteauQ/study-docker-repro-longevity
1
0
Fork 0
Code Issues 6 Pull Requests Actions Projects Releases Wiki Activity

moving some config around

Browse Source
This commit is contained in:
Quentin Guilloteau 2024-08-29 12:58:50 +02:00
parent 31e2ff0ca5
commit a62d338bd0
8 changed files with 164 additions and 120 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
artifacts/nickel/europar24/canon_solving.ncl Normal file
View File

@ -0,0 +1,13 @@
{
version = "1.0",
artifact_url = "https://zenodo.org/records/11636529/files/artifact-pap130.zip",
type = "zip",
doi = "10.5281/zenodo.11636529",
conf_date = 2024,
virtualization = "docker",
buildfile_dir = ".docker",
package_managers = [ "dpkg" ],
misc_packages = [
{ name = "gurobi10.0.1_linux64", url = "https://packages.gurobi.com/10.0/gurobi10.0.1_linux64.tar.gz" }
]
}

10
artifacts/nickel/europar24/geijer_how.ncl Normal file
View File

@ -0,0 +1,10 @@
{
version = "1.0",
artifact_url = "https://zenodo.org/records/11547063/files/artifact.zip",
type = "zip",
doi = "10.5281/zenodo.11547063",
conf_date = 2024,
virtualization = "docker",
buildfile_dir = "artifact",
package_managers = [ "dpkg", "pip" ]
}

17
artifacts/nickel/europar24/hiraga_peanuts.ncl Normal file
View File

@ -0,0 +1,17 @@
{
version = "1.0",
artifact_url = "https://zenodo.org/records/11558678/files/peanuts-playground.zip",
type = "zip",
doi = "10.5281/zenodo.11558678",
conf_date = 2024,
comment = "Files in /var/lib/apt/lists/ are removed.",
virtualization = "docker",
buildfile_dir = "./",
package_managers = [ "dpkg" ],
git_packages = [
{ name = "spack", location = "/home/vscode/.cache/spack" }
],
misc_packages = [
{ name = "cmake-3.22.2-linux", url = "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.sh" }
]
}

10
artifacts/nickel/europar24/lee_accelerated.ncl Normal file
View File

@ -0,0 +1,10 @@
{
version = "1.0",
artifact_url = "https://zenodo.org/records/11579181/files/bsa_spmm.zip",
type = "zip",
doi = "10.5281/zenodo.11579181",
conf_date = 2024,
comment = "Are there really Dockerfiles for this artifact?",
virtualization = "docker",
package_managers = [ "dpkg" ],
}

11
artifacts/nickel/europar24/wolff_fast.ncl Normal file
View File

@ -0,0 +1,11 @@
{
version = "1.0",
artifact_url = "https://zenodo.org/records/11775182/files/Euro-PAR_2024_paper_432.zip",
type = "zip",
doi = "10.5281/zenodo.11775182",
conf_date = 2024,
comment = "Files in /var/lib/apt/lists/ are removed.",
virtualization = "docker",
buildfile_dir = "./",
package_managers = [ "dpkg", "pip" ]
}

2
config/config.yaml
View File

@ -12,3 +12,5 @@ max_duration: 60
checkpoint: 1 checkpoint: 1
besteffort: True besteffort: True
sleep_time: 30 sleep_time: 30
conference: "europar24"

202
workflow/Snakefile
View File

@ -8,24 +8,18 @@ DATE = datetime.datetime.now().strftime("%Y%m%d")
ARTIFACTS_FOLDER_NICKEL = config["folder_artifacts_nickel"] ARTIFACTS_FOLDER_NICKEL = config["folder_artifacts_nickel"]
ARTIFACTS_FOLDER_JSON = config["folder_artifacts_json"] ARTIFACTS_FOLDER_JSON = config["folder_artifacts_json"]
BLACKLIST_FOLDER = config["folder_blacklists"]
EXTENSION = "json"
SYSTEM = config["system"] SYSTEM = config["system"]
CONFERENCE = config["conference"]
ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST_FOLDER) ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL + "/" + CONFERENCE)
PREFIX = config["prefix"] PREFIX = config["prefix"]
ECG_OUTPUTS = ["pkgs", "build_status", "artifact_hash"] ECG_OUTPUTS = ["pkgs", "build_status", "artifact_hash"]
SHELLS_ECG = {
"local": f"./{{input.ecg_wrapper}} {{input.ecg}} {ARTIFACTS_FOLDER_JSON}/{{wildcards.artifact}}.{EXTENSION} {{output.pkg}} {{output.build_status}} {{output.artifact_hash}} {{output.log}}",
"g5k": f"python3 {{input.execo_wrapper}} --path {os.getcwd()} --script {{input.oar_wrapper}} --site {config['site']} --cluster {config['cluster']} --max-duration {config['max_duration']} --checkpoint {config['checkpoint']} {'--besteffort' if config['besteffort'] else ''} --sleep_time {config['sleep_time']} --build_status_file {{output.build_status}} --artifact {{wildcards.artifact}} -- '"
}
ANALYSIS_DIR = config["analysis_dir"] ANALYSIS_DIR = config["analysis_dir"]
ANALYSIS_CATS = ["sources_stats", "pkgs_changes", "build_status", "artifact"] ANALYSIS_CATS = ["sources_stats", "pkgs_changes", "build_status", "artifact"]
ANALYSIS_SCRIPTS_DIR = "analysis" ANALYSIS_SCRIPTS_DIR = "analysis"
ANALYSIS_WRAPPER = "workflow/scripts/analysis_wrapper.sh" ANALYSIS_WRAPPER = "workflow/scripts/analysis_wrapper.sh"
AGGREGATE_WRAPPER = "workflow/scripts/aggregate_wrapper.sh"
ARTIFACT_ANALYSIS_DIRS = " ".join(expand(f"{PREFIX}/artifact_hash/{{artifact}}", ARTIFACT_ANALYSIS_DIRS = " ".join(expand(f"{PREFIX}/artifact_hash/{{artifact}}",
artifact = ARTIFACTS artifact = ARTIFACTS
)) ))
@ -44,33 +38,28 @@ PLOT_HEADERS = {
rule all: rule all:
input: input:
expand(f"{ANALYSIS_DIR}/{{analysis_cat}}/plot/line/{{date}}.pdf", expand(f"{PREFIX}/{{conference}}/build_status/{{artifact}}/{{date}}.csv",\
analysis_cat = ANALYSIS_CATS, conference=config['conference'],\
date = DATE artifact=ARTIFACTS,\
), date = DATE)
expand(f"{ANALYSIS_DIR}/{{analysis_cat}}/plot/bar/{{date}}.pdf",
analysis_cat = ["sources_stats", "build_status", "artifact"],
date = DATE
),
f"{BLACKLIST_FOLDER}/{DATE}.csv"
# Artifacts configuration files: # Artifacts configuration files:
rule check_all: rule check_all:
input: input:
expand(f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json", artifact=ARTIFACTS) expand(f"{ARTIFACTS_FOLDER_JSON}/{{conference}}/{{artifact}}.json", artifact=ARTIFACTS, conference=config['conference'])
rule check_artifact: rule check_artifact:
input: input:
"flake.nix", "flake.nix",
"flake.lock", "flake.lock",
contract="workflow/nickel/artifact_contract.ncl", contract="workflow/nickel/artifact_contract.ncl",
artifact=f"{ARTIFACTS_FOLDER_NICKEL}/{{artifact}}.ncl" artifact=f"{ARTIFACTS_FOLDER_NICKEL}/{{conference}}/{{artifact}}.ncl"
output: output:
f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json" f"{ARTIFACTS_FOLDER_JSON}/{{conference}}/{{artifact}}.json"
shell: shell:
""" """
nickel export --format json --output {output} <<< 'let {{Artifact, ..}} = import "{input.contract}" in ((import "{input.artifact}") | Artifact)' nix develop .#nickel --command nickel export --format json --output {output} <<< 'let {{Artifact, ..}} = import "{input.contract}" in ((import "{input.artifact}") | Artifact)'
""" """
# ECG: # ECG:
@ -82,12 +71,12 @@ rule run_ecg:
ecg="ecg/app/ecg.py", ecg="ecg/app/ecg.py",
execo_wrapper="workflow/scripts/submission_g5k.py", execo_wrapper="workflow/scripts/submission_g5k.py",
oar_wrapper="workflow/scripts/ecg_oar_wrapper.oar.bash", oar_wrapper="workflow/scripts/ecg_oar_wrapper.oar.bash",
artifact=f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.{EXTENSION}" artifact=f"{ARTIFACTS_FOLDER_JSON}/{{conference}}/{{artifact}}.json"
output: output:
log = f"{PREFIX}/logs/{{artifact}}/{{date}}.txt", log = f"{PREFIX}/{{conference}}/logs/{{artifact}}/{{date}}.txt",
pkg = f"{PREFIX}/pkgs/{{artifact}}/{{date}}.csv", pkg = f"{PREFIX}/{{conference}}/pkgs/{{artifact}}/{{date}}.csv",
build_status = f"{PREFIX}/build_status/{{artifact}}/{{date}}.csv", build_status = f"{PREFIX}/{{conference}}/build_status/{{artifact}}/{{date}}.csv",
artifact_hash = f"{PREFIX}/artifact_hash/{{artifact}}/{{date}}.csv", artifact_hash = f"{PREFIX}/{{conference}}/artifact_hash/{{artifact}}/{{date}}.csv",
shell: shell:
(f"python3 {{input.execo_wrapper}} --path {os.getcwd()} \ (f"python3 {{input.execo_wrapper}} --path {os.getcwd()} \
--script {{input.oar_wrapper}} \ --script {{input.oar_wrapper}} \
@ -100,94 +89,83 @@ rule run_ecg:
--build_status_file {{output.build_status}} \ --build_status_file {{output.build_status}} \
--artifact {{wildcards.artifact}} -- '" if SYSTEM == "g5k" else "") + \ --artifact {{wildcards.artifact}} -- '" if SYSTEM == "g5k" else "") + \
""" """
nix develop .#ecg-shell --command ecg -p {output.pkg} -b {output.build_status} -a {output.artifact_hash} {output.artifact} &> {output.log} || echo "{input.artifact}, `date +%s.%N`, script_crash" > {output.build_status} nix shell .#ecg --command ecg -p {output.pkg} -b {output.build_status} -a {output.artifact_hash} {input.artifact} &> {output.log} || echo "{input.artifact}, `date +%s.%N`, script_crash" > {output.build_status}
""" + \ """ + \
("'" if SYSTEM == "g5k" else "") ("'" if SYSTEM == "g5k" else "")
rule update_blacklist:
input:
build_status=expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv",
artifact=ARTIFACTS
)
output:
f"{BLACKLIST_FOLDER}/{{date}}.csv"
shell:
# We need to ignore lines where build is successful:
f"cat {{input}} | grep -v ',success' > {{output}} || true"
# Analysis: # Analysis:
rule softenv_analysis: #rule softenv_analysis:
wildcard_constraints: # wildcard_constraints:
date="\d+" # date="\d+"
input: # input:
expand(f"{PREFIX}/pkgs/{{artifact}}/{{{{date}}}}.csv", # expand(f"{PREFIX}{{conference}}/pkgs/{{artifact}}/{{{{date}}}}.csv",
artifact = ARTIFACTS # artifact = ARTIFACTS
) # )
output: # output:
sources_stats = f"{ANALYSIS_DIR}/sources_stats/{{date}}.csv", # sources_stats = f"{ANALYSIS_DIR}/sources_stats/{{date}}.csv",
pkgs_changes = f"{ANALYSIS_DIR}/pkgs_changes/{{date}}.csv" # pkgs_changes = f"{ANALYSIS_DIR}/pkgs_changes/{{date}}.csv"
shell: # shell:
f""" # f"""
{ANALYSIS_WRAPPER} files {ANALYSIS_SCRIPTS_DIR}/softenv_analysis.py -t sources-stats {{output.sources_stats}} {{input}} # {ANALYSIS_WRAPPER} files {ANALYSIS_SCRIPTS_DIR}/softenv_analysis.py -t sources-stats {{output.sources_stats}} {{input}}
{ANALYSIS_WRAPPER} dirs {ANALYSIS_SCRIPTS_DIR}/softenv_analysis.py -t pkgs-changes {{output.pkgs_changes}} {SOFTENV_ANALYSIS_DIRS} # {ANALYSIS_WRAPPER} dirs {ANALYSIS_SCRIPTS_DIR}/softenv_analysis.py -t pkgs-changes {{output.pkgs_changes}} {SOFTENV_ANALYSIS_DIRS}
""" # """
#
rule buildstatus_analysis: #rule buildstatus_analysis:
wildcard_constraints: # wildcard_constraints:
date="\d+" # date="\d+"
input: # input:
expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv", # expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv",
artifact = ARTIFACTS # artifact = ARTIFACTS
), # ),
output: # output:
f"{ANALYSIS_DIR}/build_status/{{date}}.csv" # f"{ANALYSIS_DIR}/build_status/{{date}}.csv"
shell: # shell:
f""" # f"""
{ANALYSIS_WRAPPER} files {ANALYSIS_SCRIPTS_DIR}/buildstatus_analysis.py {{output}} {{input}} # {ANALYSIS_WRAPPER} files {ANALYSIS_SCRIPTS_DIR}/buildstatus_analysis.py {{output}} {{input}}
""" # """
#
rule artifact_analysis: #rule artifact_analysis:
wildcard_constraints: # wildcard_constraints:
date="\d+" # date="\d+"
input: # input:
expand(f"{PREFIX}/artifact_hash/{{artifact}}/{{{{date}}}}.csv", # expand(f"{PREFIX}/artifact_hash/{{artifact}}/{{{{date}}}}.csv",
artifact = ARTIFACTS # artifact = ARTIFACTS
) # )
output: # output:
f"{ANALYSIS_DIR}/artifact/{{date}}.csv" # f"{ANALYSIS_DIR}/artifact/{{date}}.csv"
shell: # shell:
f""" # f"""
{ANALYSIS_WRAPPER} dirs {ANALYSIS_SCRIPTS_DIR}/artifact_analysis.py {{output}} {ARTIFACT_ANALYSIS_DIRS} # {ANALYSIS_WRAPPER} dirs {ANALYSIS_SCRIPTS_DIR}/artifact_analysis.py {{output}} {ARTIFACT_ANALYSIS_DIRS}
""" # """
#
# Analysis aggregate: ## Analysis aggregate:
#
rule analysis_aggregate: #rule analysis_aggregate:
input: # input:
expand(f"{ANALYSIS_DIR}/{{cat}}/aggregated/{{date}}.csv", # expand(f"{ANALYSIS_DIR}/{{cat}}/aggregated/{{date}}.csv",
cat = ANALYSIS_CATS, # cat = ANALYSIS_CATS,
date = DATE # date = DATE
) # )
#
rule aggregate_by_type: #rule aggregate_by_type:
input: # input:
data=f"{ANALYSIS_DIR}/{{type}}/{{date}}.csv", # data=f"{ANALYSIS_DIR}/{{type}}/{{date}}.csv",
script="workflow/scripts/aggregate_wrapper.sh" # script="workflow/scripts/aggregate_wrapper.sh"
output: # output:
f"{ANALYSIS_DIR}/{{type}}/aggregated/{{date}}.csv" # f"{ANALYSIS_DIR}/{{type}}/aggregated/{{date}}.csv"
shell: # shell:
f"{input.script} {ANALYSIS_DIR}/{{type}} {{output}}" # f"{{input.script}} {ANALYSIS_DIR}/{{type}} {{output}}"
#
# Plot: ## Plot:
#
rule plot: #rule plot:
input: # input:
script = "plot/plot.r", # script = "plot/plot.r",
data = f"{ANALYSIS_DIR}/{{type}}/aggregated/{{date}}.csv", # data = f"{ANALYSIS_DIR}/{{type}}/aggregated/{{date}}.csv",
output: # output:
f"{ANALYSIS_DIR}/{{type}}/{{plot}}/{{date}}.pdf" # f"{ANALYSIS_DIR}/{{type}}/{{plot}}/{{date}}.pdf"
params: # params:
header = lambda w: PLOT_HEADERS[w.type] # header = lambda w: PLOT_HEADERS[w.type]
shell: # shell:
"Rscript {input.script} {wildcards.plot} {input.data} {output} {params.header} timestamp" # "Rscript {input.script} {wildcards.plot} {input.data} {output} {params.header} timestamp"

19
workflow/utils.smk
View File

@ -13,11 +13,14 @@ def get_blacklisted(blacklist_dir_path):
blacklisted.add(row[0]) blacklisted.add(row[0])
return blacklisted return blacklisted
def get_artifacts_to_build(artifacts_folder, blacklist_dir_path): #def get_artifacts_to_build(artifacts_folder, blacklist_dir_path):
blacklisted = get_blacklisted(blacklist_dir_path) # blacklisted = get_blacklisted(blacklist_dir_path)
all_artifacts = set([os.path.splitext(a)[0] for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))]) # all_artifacts = set([os.path.splitext(a)[0] for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))])
artifacts_to_build = list(all_artifacts.difference(blacklisted)) # artifacts_to_build = list(all_artifacts.difference(blacklisted))
if artifacts_to_build != []: # if artifacts_to_build != []:
return list(all_artifacts.difference(blacklisted)) # return list(all_artifacts.difference(blacklisted))
else: # else:
raise(Exception(f"There is no artifact to build! Either no artifact configuration files have been found, or they have all been blacklisted.")) # raise(Exception(f"There is no artifact to build! Either no artifact configuration files have been found, or they have all been blacklisted."))
def get_artifacts_to_build(artifacts_folder):
return [os.path.splitext(a)[0] for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))]