moving some config around

2024-08-29 12:58:50 +02:00 · 2024-08-29 12:58:50 +02:00 · a62d338bd0
commit a62d338bd0
parent 31e2ff0ca5
8 changed files with 164 additions and 120 deletions
--- a/artifacts/nickel/europar24/canon_solving.ncl
+++ b/artifacts/nickel/europar24/canon_solving.ncl
@ -0,0 +1,13 @@
+{
+  version = "1.0",
+  artifact_url = "https://zenodo.org/records/11636529/files/artifact-pap130.zip",
+  type = "zip",
+  doi = "10.5281/zenodo.11636529",
+  conf_date = 2024,
+  virtualization = "docker",
+  buildfile_dir = ".docker",
+  package_managers = [ "dpkg" ],
+  misc_packages = [
+    { name = "gurobi10.0.1_linux64", url = "https://packages.gurobi.com/10.0/gurobi10.0.1_linux64.tar.gz" }
+  ]
+}
--- a/artifacts/nickel/europar24/geijer_how.ncl
+++ b/artifacts/nickel/europar24/geijer_how.ncl
@ -0,0 +1,10 @@
+{
+  version = "1.0",
+  artifact_url = "https://zenodo.org/records/11547063/files/artifact.zip",
+  type = "zip",
+  doi = "10.5281/zenodo.11547063",
+  conf_date = 2024,
+  virtualization = "docker",
+  buildfile_dir = "artifact",
+  package_managers = [ "dpkg", "pip" ]
+}
--- a/artifacts/nickel/europar24/hiraga_peanuts.ncl
+++ b/artifacts/nickel/europar24/hiraga_peanuts.ncl
@ -0,0 +1,17 @@
+{
+  version = "1.0",
+  artifact_url = "https://zenodo.org/records/11558678/files/peanuts-playground.zip",
+  type = "zip",
+  doi = "10.5281/zenodo.11558678",
+  conf_date = 2024,
+  comment = "Files in /var/lib/apt/lists/ are removed.",
+  virtualization = "docker",
+  buildfile_dir = "./",
+  package_managers = [ "dpkg" ],
+  git_packages = [
+    { name = "spack", location = "/home/vscode/.cache/spack" }
+  ],
+  misc_packages = [
+    { name = "cmake-3.22.2-linux", url = "https://github.com/Kitware/CMake/releases/download/v3.22.2/cmake-3.22.2-linux-x86_64.sh" }
+  ]
+}
--- a/artifacts/nickel/europar24/lee_accelerated.ncl
+++ b/artifacts/nickel/europar24/lee_accelerated.ncl
@ -0,0 +1,10 @@
+{
+  version = "1.0",
+  artifact_url = "https://zenodo.org/records/11579181/files/bsa_spmm.zip",
+  type = "zip",
+  doi = "10.5281/zenodo.11579181",
+  conf_date = 2024,
+  comment = "Are there really Dockerfiles for this artifact?",
+  virtualization = "docker",
+  package_managers = [ "dpkg" ],
+}
--- a/artifacts/nickel/europar24/wolff_fast.ncl
+++ b/artifacts/nickel/europar24/wolff_fast.ncl
@ -0,0 +1,11 @@
+{
+  version = "1.0",
+  artifact_url = "https://zenodo.org/records/11775182/files/Euro-PAR_2024_paper_432.zip",
+  type = "zip",
+  doi = "10.5281/zenodo.11775182",
+  conf_date = 2024,
+  comment = "Files in /var/lib/apt/lists/ are removed.",
+  virtualization = "docker",
+  buildfile_dir = "./",
+  package_managers = [ "dpkg", "pip" ]
+}
--- a/config/config.yaml
+++ b/config/config.yaml
@ -12,3 +12,5 @@ max_duration: 60
 checkpoint: 1
 besteffort: True
 sleep_time: 30
+
+conference: "europar24"
--- a/workflow/Snakefile
+++ b/workflow/Snakefile
@ -8,24 +8,18 @@ DATE = datetime.datetime.now().strftime("%Y%m%d")

 ARTIFACTS_FOLDER_NICKEL = config["folder_artifacts_nickel"]
 ARTIFACTS_FOLDER_JSON   = config["folder_artifacts_json"]
-BLACKLIST_FOLDER = config["folder_blacklists"]
-EXTENSION = "json"
 SYSTEM = config["system"]
+CONFERENCE = config["conference"]

-ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL, BLACKLIST_FOLDER)
+ARTIFACTS = get_artifacts_to_build(ARTIFACTS_FOLDER_NICKEL + "/" + CONFERENCE)

 PREFIX = config["prefix"]
 ECG_OUTPUTS = ["pkgs", "build_status", "artifact_hash"]
-SHELLS_ECG = {
-  "local": f"./{{input.ecg_wrapper}} {{input.ecg}} {ARTIFACTS_FOLDER_JSON}/{{wildcards.artifact}}.{EXTENSION} {{output.pkg}} {{output.build_status}} {{output.artifact_hash}} {{output.log}}",
-  "g5k": f"python3 {{input.execo_wrapper}} --path {os.getcwd()} --script {{input.oar_wrapper}} --site {config['site']} --cluster {config['cluster']} --max-duration {config['max_duration']} --checkpoint {config['checkpoint']} {'--besteffort' if config['besteffort'] else ''} --sleep_time {config['sleep_time']} --build_status_file {{output.build_status}} --artifact {{wildcards.artifact}} -- '"
-}

 ANALYSIS_DIR = config["analysis_dir"]
 ANALYSIS_CATS = ["sources_stats", "pkgs_changes", "build_status", "artifact"]
 ANALYSIS_SCRIPTS_DIR = "analysis"
 ANALYSIS_WRAPPER = "workflow/scripts/analysis_wrapper.sh"
-AGGREGATE_WRAPPER = "workflow/scripts/aggregate_wrapper.sh"
 ARTIFACT_ANALYSIS_DIRS = " ".join(expand(f"{PREFIX}/artifact_hash/{{artifact}}",
  artifact = ARTIFACTS
 ))
@ -44,33 +38,28 @@ PLOT_HEADERS = {

 rule all:
  input:
-    expand(f"{ANALYSIS_DIR}/{{analysis_cat}}/plot/line/{{date}}.pdf",
-      analysis_cat = ANALYSIS_CATS,
-      date = DATE
-    ),
-    expand(f"{ANALYSIS_DIR}/{{analysis_cat}}/plot/bar/{{date}}.pdf",
-      analysis_cat = ["sources_stats", "build_status", "artifact"],
-      date = DATE
-    ),
-    f"{BLACKLIST_FOLDER}/{DATE}.csv"
+    expand(f"{PREFIX}/{{conference}}/build_status/{{artifact}}/{{date}}.csv",\
+           conference=config['conference'],\
+           artifact=ARTIFACTS,\
+           date = DATE)

 # Artifacts configuration files:

 rule check_all:
  input:
-    expand(f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json", artifact=ARTIFACTS)
+    expand(f"{ARTIFACTS_FOLDER_JSON}/{{conference}}/{{artifact}}.json", artifact=ARTIFACTS, conference=config['conference'])

 rule check_artifact:
  input:
    "flake.nix",
    "flake.lock",
    contract="workflow/nickel/artifact_contract.ncl",
-    artifact=f"{ARTIFACTS_FOLDER_NICKEL}/{{artifact}}.ncl"
+    artifact=f"{ARTIFACTS_FOLDER_NICKEL}/{{conference}}/{{artifact}}.ncl"
  output:
-    f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json"
+    f"{ARTIFACTS_FOLDER_JSON}/{{conference}}/{{artifact}}.json"
  shell:
    """
-    nickel export --format json --output {output} <<< 'let {{Artifact, ..}} = import "{input.contract}" in ((import "{input.artifact}") | Artifact)'
+    nix develop .#nickel --command nickel export --format json --output {output} <<< 'let {{Artifact, ..}} = import "{input.contract}" in ((import "{input.artifact}") | Artifact)'
    """

 # ECG:
@ -82,12 +71,12 @@ rule run_ecg:
    ecg="ecg/app/ecg.py",
    execo_wrapper="workflow/scripts/submission_g5k.py",
    oar_wrapper="workflow/scripts/ecg_oar_wrapper.oar.bash",
-    artifact=f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.{EXTENSION}"
+    artifact=f"{ARTIFACTS_FOLDER_JSON}/{{conference}}/{{artifact}}.json"
  output:
-    log           = f"{PREFIX}/logs/{{artifact}}/{{date}}.txt",
-    pkg           = f"{PREFIX}/pkgs/{{artifact}}/{{date}}.csv",
-    build_status  = f"{PREFIX}/build_status/{{artifact}}/{{date}}.csv",
-    artifact_hash = f"{PREFIX}/artifact_hash/{{artifact}}/{{date}}.csv",
+    log           = f"{PREFIX}/{{conference}}/logs/{{artifact}}/{{date}}.txt",
+    pkg           = f"{PREFIX}/{{conference}}/pkgs/{{artifact}}/{{date}}.csv",
+    build_status  = f"{PREFIX}/{{conference}}/build_status/{{artifact}}/{{date}}.csv",
+    artifact_hash = f"{PREFIX}/{{conference}}/artifact_hash/{{artifact}}/{{date}}.csv",
  shell:
    (f"python3 {{input.execo_wrapper}} --path {os.getcwd()} \
                                       --script {{input.oar_wrapper}} \
@ -100,94 +89,83 @@ rule run_ecg:
                                       --build_status_file {{output.build_status}} \
                                       --artifact {{wildcards.artifact}} -- '" if SYSTEM == "g5k" else "") + \
    """
-    nix develop .#ecg-shell --command ecg -p {output.pkg} -b {output.build_status} -a {output.artifact_hash} {output.artifact} &> {output.log} || echo "{input.artifact}, `date +%s.%N`, script_crash" > {output.build_status}
+    nix shell .#ecg --command ecg -p {output.pkg} -b {output.build_status} -a {output.artifact_hash} {input.artifact} &> {output.log} || echo "{input.artifact}, `date +%s.%N`, script_crash" > {output.build_status}
    """ + \
    ("'" if SYSTEM == "g5k" else "")

-rule update_blacklist:
-  input:
-    build_status=expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv",
-      artifact=ARTIFACTS
-    )
-  output:
-    f"{BLACKLIST_FOLDER}/{{date}}.csv"
-  shell:
-    # We need to ignore lines where build is successful:
-    f"cat {{input}} | grep -v ',success' > {{output}} || true"
-
 # Analysis:

-rule softenv_analysis:
-  wildcard_constraints:
-    date="\d+"
-  input:
-    expand(f"{PREFIX}/pkgs/{{artifact}}/{{{{date}}}}.csv",
-      artifact = ARTIFACTS
-    )
-  output:
-    sources_stats = f"{ANALYSIS_DIR}/sources_stats/{{date}}.csv",
-    pkgs_changes = f"{ANALYSIS_DIR}/pkgs_changes/{{date}}.csv"
-  shell:
-    f"""
-    {ANALYSIS_WRAPPER} files {ANALYSIS_SCRIPTS_DIR}/softenv_analysis.py -t sources-stats {{output.sources_stats}} {{input}}
-    {ANALYSIS_WRAPPER} dirs {ANALYSIS_SCRIPTS_DIR}/softenv_analysis.py -t pkgs-changes {{output.pkgs_changes}} {SOFTENV_ANALYSIS_DIRS}
-    """
-
-rule buildstatus_analysis:
-  wildcard_constraints:
-    date="\d+"
-  input:
-    expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv",
-      artifact = ARTIFACTS
-    ),
-  output:
-    f"{ANALYSIS_DIR}/build_status/{{date}}.csv"
-  shell:
-    f"""
-    {ANALYSIS_WRAPPER} files {ANALYSIS_SCRIPTS_DIR}/buildstatus_analysis.py {{output}} {{input}}
-    """
-
-rule artifact_analysis:
-  wildcard_constraints:
-    date="\d+"
-  input:
-    expand(f"{PREFIX}/artifact_hash/{{artifact}}/{{{{date}}}}.csv",
-      artifact = ARTIFACTS
-    )
-  output:
-    f"{ANALYSIS_DIR}/artifact/{{date}}.csv"
-  shell:
-    f"""
-    {ANALYSIS_WRAPPER} dirs {ANALYSIS_SCRIPTS_DIR}/artifact_analysis.py {{output}} {ARTIFACT_ANALYSIS_DIRS}
-    """
-
-# Analysis aggregate:
-
-rule analysis_aggregate:
-  input:
-    expand(f"{ANALYSIS_DIR}/{{cat}}/aggregated/{{date}}.csv",
-      cat = ANALYSIS_CATS,
-      date = DATE
-    )
-
-rule aggregate_by_type:
-  input:
-    data=f"{ANALYSIS_DIR}/{{type}}/{{date}}.csv",
-    script="workflow/scripts/aggregate_wrapper.sh"
-  output:
-    f"{ANALYSIS_DIR}/{{type}}/aggregated/{{date}}.csv"
-  shell:
-    f"{input.script} {ANALYSIS_DIR}/{{type}} {{output}}"
-
-# Plot:
-
-rule plot:
-  input:
-    script = "plot/plot.r",
-    data = f"{ANALYSIS_DIR}/{{type}}/aggregated/{{date}}.csv",
-  output:
-    f"{ANALYSIS_DIR}/{{type}}/{{plot}}/{{date}}.pdf"
-  params:
-    header = lambda w: PLOT_HEADERS[w.type]
-  shell:
-    "Rscript {input.script} {wildcards.plot} {input.data} {output} {params.header} timestamp"
+#rule softenv_analysis:
+#  wildcard_constraints:
+#    date="\d+"
+#  input:
+#    expand(f"{PREFIX}{{conference}}/pkgs/{{artifact}}/{{{{date}}}}.csv",
+#      artifact = ARTIFACTS
+#    )
+#  output:
+#    sources_stats = f"{ANALYSIS_DIR}/sources_stats/{{date}}.csv",
+#    pkgs_changes = f"{ANALYSIS_DIR}/pkgs_changes/{{date}}.csv"
+#  shell:
+#    f"""
+#    {ANALYSIS_WRAPPER} files {ANALYSIS_SCRIPTS_DIR}/softenv_analysis.py -t sources-stats {{output.sources_stats}} {{input}}
+#    {ANALYSIS_WRAPPER} dirs {ANALYSIS_SCRIPTS_DIR}/softenv_analysis.py -t pkgs-changes {{output.pkgs_changes}} {SOFTENV_ANALYSIS_DIRS}
+#    """
+#
+#rule buildstatus_analysis:
+#  wildcard_constraints:
+#    date="\d+"
+#  input:
+#    expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv",
+#      artifact = ARTIFACTS
+#    ),
+#  output:
+#    f"{ANALYSIS_DIR}/build_status/{{date}}.csv"
+#  shell:
+#    f"""
+#    {ANALYSIS_WRAPPER} files {ANALYSIS_SCRIPTS_DIR}/buildstatus_analysis.py {{output}} {{input}}
+#    """
+#
+#rule artifact_analysis:
+#  wildcard_constraints:
+#    date="\d+"
+#  input:
+#    expand(f"{PREFIX}/artifact_hash/{{artifact}}/{{{{date}}}}.csv",
+#      artifact = ARTIFACTS
+#    )
+#  output:
+#    f"{ANALYSIS_DIR}/artifact/{{date}}.csv"
+#  shell:
+#    f"""
+#    {ANALYSIS_WRAPPER} dirs {ANALYSIS_SCRIPTS_DIR}/artifact_analysis.py {{output}} {ARTIFACT_ANALYSIS_DIRS}
+#    """
+#
+## Analysis aggregate:
+#
+#rule analysis_aggregate:
+#  input:
+#    expand(f"{ANALYSIS_DIR}/{{cat}}/aggregated/{{date}}.csv",
+#      cat = ANALYSIS_CATS,
+#      date = DATE
+#    )
+#
+#rule aggregate_by_type:
+#  input:
+#    data=f"{ANALYSIS_DIR}/{{type}}/{{date}}.csv",
+#    script="workflow/scripts/aggregate_wrapper.sh"
+#  output:
+#    f"{ANALYSIS_DIR}/{{type}}/aggregated/{{date}}.csv"
+#  shell:
+#    f"{{input.script}} {ANALYSIS_DIR}/{{type}} {{output}}"
+#
+## Plot:
+#
+#rule plot:
+#  input:
+#    script = "plot/plot.r",
+#    data = f"{ANALYSIS_DIR}/{{type}}/aggregated/{{date}}.csv",
+#  output:
+#    f"{ANALYSIS_DIR}/{{type}}/{{plot}}/{{date}}.pdf"
+#  params:
+#    header = lambda w: PLOT_HEADERS[w.type]
+#  shell:
+#    "Rscript {input.script} {wildcards.plot} {input.data} {output} {params.header} timestamp"
--- a/workflow/utils.smk
+++ b/workflow/utils.smk
@ -13,11 +13,14 @@ def get_blacklisted(blacklist_dir_path):
                        blacklisted.add(row[0])
    return blacklisted

-def get_artifacts_to_build(artifacts_folder, blacklist_dir_path):
-    blacklisted = get_blacklisted(blacklist_dir_path)
-    all_artifacts = set([os.path.splitext(a)[0] for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))])
-    artifacts_to_build = list(all_artifacts.difference(blacklisted))
-    if artifacts_to_build != []:
-        return list(all_artifacts.difference(blacklisted))
-    else:
-        raise(Exception(f"There is no artifact to build! Either no artifact configuration files have been found, or they have all been blacklisted."))
+#def get_artifacts_to_build(artifacts_folder, blacklist_dir_path):
+#    blacklisted = get_blacklisted(blacklist_dir_path)
+#    all_artifacts = set([os.path.splitext(a)[0] for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))])
+#    artifacts_to_build = list(all_artifacts.difference(blacklisted))
+#    if artifacts_to_build != []:
+#        return list(all_artifacts.difference(blacklisted))
+#    else:
+#        raise(Exception(f"There is no artifact to build! Either no artifact configuration files have been found, or they have all been blacklisted."))
+
+def get_artifacts_to_build(artifacts_folder):
+    return [os.path.splitext(a)[0] for a in os.listdir(artifacts_folder) if not os.path.isdir(os.path.join(artifacts_folder, a))]