Fixed analysis part in Snakemake again, needs testing. Moved test and template artifacts back to parent folder. Changes analysis output dir.
This commit is contained in:
parent
ece5749b7a
commit
0a8b0c85c6
1
.gitignore
vendored
1
.gitignore
vendored
@ -9,3 +9,4 @@ pkglist.csv
|
|||||||
log.txt
|
log.txt
|
||||||
build_status.csv
|
build_status.csv
|
||||||
*.pdf
|
*.pdf
|
||||||
|
blacklists/*
|
||||||
|
@ -0,0 +1,10 @@
|
|||||||
|
albab_k9db,0,unknown_error
|
||||||
|
breitweiser_high,0,unknown_error
|
||||||
|
isakov_taxonomy,0,unknown_error
|
||||||
|
moses_high,0,unknown_error
|
||||||
|
parasyris_approximate,0,unknown_error
|
||||||
|
shi_welder,0,unknown_error
|
||||||
|
wang_tgopt,0,unknown_error
|
||||||
|
ziogas_deinsum-cpu,0,unknown_error
|
||||||
|
ziogas_deinsum-gpu,0,unknown_error
|
||||||
|
template,0,unknown_error
|
|
@ -4,7 +4,7 @@ folder_blacklists: "blacklists"
|
|||||||
symlink_blacklist: "blacklist.csv"
|
symlink_blacklist: "blacklist.csv"
|
||||||
system: "g5k" # can be "local" for local execution
|
system: "g5k" # can be "local" for local execution
|
||||||
prefix: "outputs"
|
prefix: "outputs"
|
||||||
analysis_dir: "analysis"
|
analysis_dir: "outputs/analysis"
|
||||||
|
|
||||||
site: "grenoble"
|
site: "grenoble"
|
||||||
cluster: "dahu"
|
cluster: "dahu"
|
||||||
|
2
run.sh
2
run.sh
@ -13,4 +13,4 @@ then
|
|||||||
mkdir $CACHE_DIR
|
mkdir $CACHE_DIR
|
||||||
fi
|
fi
|
||||||
|
|
||||||
./ecg.py $TESTFILE -v -p $OUTPUT_PATH/pkglist.csv -b $OUTPUT_PATH/build_status.csv -a $OUTPUT_PATH/artifact_hash.csv -c $CACHE_DIR --docker-cache
|
./ecg.py $TESTFILE -p $OUTPUT_PATH/pkglist.csv -b $OUTPUT_PATH/build_status.csv -a $OUTPUT_PATH/artifact_hash.csv -c $CACHE_DIR --docker-cache
|
@ -82,75 +82,93 @@ rule update_blacklist:
|
|||||||
# We need to ignore lines where build is successful:
|
# We need to ignore lines where build is successful:
|
||||||
f"cat {{input}} | grep -v ',success' > {{output}} && rm -rf {BLACKLIST} && ln -s {{output}} {BLACKLIST}"
|
f"cat {{input}} | grep -v ',success' > {{output}} && rm -rf {BLACKLIST} && ln -s {{output}} {BLACKLIST}"
|
||||||
|
|
||||||
ANALYSIS_SCRIPTS = {
|
|
||||||
"sources_stats": f"python3 {ANALYSIS_DIR}/softenv_analysis.py -t sources-stats",
|
|
||||||
"pkgs_changes": f"python3 {ANALYSIS_DIR}/softenv_analysis.py -t pkgs-changes",
|
|
||||||
"build_status": f"python3 {ANALYSIS_DIR}/buildstatus_analysis.py",
|
|
||||||
"artifact": f"python3 {ANALYSIS_DIR}/artifact_analysis.py"
|
|
||||||
}
|
|
||||||
|
|
||||||
rule analysis:
|
rule analysis:
|
||||||
input:
|
input:
|
||||||
{
|
expand(f"{PREFIX}/{{folder}}/{{artifact}}/{{{{date}}}}.csv",\
|
||||||
"sources_stats": expand(f"{PREFIX}/pkgs/{{artifact}}/{{date}}.csv",\
|
folder = ["pkgs", "build_status", "artifact_hash"],\
|
||||||
artifact = ARTIFACTS,\
|
artifact = ARTIFACTS
|
||||||
date = DATE
|
)
|
||||||
),
|
|
||||||
"pkgs_changes": expand(f"{PREFIX}/pkgs/{{artifact}}/{{date}}.csv",\
|
|
||||||
artifact = ARTIFACTS,\
|
|
||||||
date = DATE
|
|
||||||
),
|
|
||||||
"build_status": expand(f"{PREFIX}/build_status/{{artifact}}/{{date}}.csv",\
|
|
||||||
artifact = ARTIFACTS,\
|
|
||||||
date = DATE
|
|
||||||
),
|
|
||||||
"artifact_hash": expand(f"{PREFIX}/artifact_hash/{{artifact}}/{{date}}.csv",\
|
|
||||||
artifact = ARTIFACTS,\
|
|
||||||
date = DATE
|
|
||||||
),
|
|
||||||
}
|
|
||||||
output:
|
output:
|
||||||
{
|
expand(f"{ANALYSIS_DIR}/{{folder}}/{{date}}.csv",\
|
||||||
"sources_stats": expand(f"{ANALYSIS_DIR}/sources_stats/{{artifact}}/{{date}}.csv",\
|
folder = ["sources_stats", "pkgs_changes", "build_status", "artifact_hash"],\
|
||||||
artifact = ARTIFACTS,\
|
|
||||||
date = DATE
|
date = DATE
|
||||||
),
|
)
|
||||||
"pkgs_changes": expand(f"{ANALYSIS_DIR}/pkgs_changes/{{artifact}}/{{date}}.csv",\
|
|
||||||
artifact = ARTIFACTS,\
|
|
||||||
date = DATE
|
|
||||||
),
|
|
||||||
"build_status": expand(f"{ANALYSIS_DIR}/build_status/{{artifact}}/{{date}}.csv",\
|
|
||||||
artifact = ARTIFACTS,\
|
|
||||||
date = DATE
|
|
||||||
),
|
|
||||||
"artifact_hash": expand(f"{ANALYSIS_DIR}/artifact_hash/{{artifact}}/{{date}}.csv",\
|
|
||||||
artifact = ARTIFACTS,\
|
|
||||||
date = DATE
|
|
||||||
),
|
|
||||||
}
|
|
||||||
shell:
|
|
||||||
f"{{{{ANALYSIS_SCRIPTS[wildcards.folder]}}}} -i {{input[wildcards.folder]}} -o {{output[wildcards.folder]}}"
|
|
||||||
|
|
||||||
PLOT_HEADERS = {
|
rule softenv_analysis:
|
||||||
"sources_stats": ["dpkg", "rpm", "pacman", "pip", "conda", "git", "misc"],
|
|
||||||
"pkgs_changes": ["dpkg", "rpm", "pacman", "pip", "conda", "git", "misc"],
|
|
||||||
"build_status": ["success", "package_unavailable", "baseimage_unavailable", "artifact_unavailable", "dockerfile_not_found", "script_crash", "job_time_exceeded", "unknown_error"],
|
|
||||||
"artifact": ["available", "unavailable", "changed"]
|
|
||||||
}
|
|
||||||
|
|
||||||
rule plot:
|
|
||||||
input:
|
input:
|
||||||
expand(f"{ANALYSIS_DIR}/{{folder}}/{{artifact}}/{{date}}.csv",\
|
expand(f"{PREFIX}/pkgs/{{artifact}}/{{{{date}}}}.csv",\
|
||||||
folder = ["sources_stats", "pkgs_changes", "build_status", "artifact"],\
|
artifact = ARTIFACTS
|
||||||
artifact = ARTIFACTS,\
|
)
|
||||||
date = DATE
|
|
||||||
),
|
|
||||||
output:
|
output:
|
||||||
expand(f"{ANALYSIS_DIR}/{{folder}}/{{plot_type}}.pdf",\
|
sources_stats = f"{ANALYSIS_DIR}/sources_stats/{DATE}.csv",\
|
||||||
folder = ["sources_stats", "pkgs_changes", "build_status", "artifact"],\
|
pkgs_changes = f"{ANALYSIS_DIR}/pkgs_changes/{DATE}.csv"
|
||||||
plot_type = ["line", "bar"],\
|
|
||||||
artifact = ARTIFACTS,\
|
|
||||||
date = DATE
|
|
||||||
),
|
|
||||||
shell:
|
shell:
|
||||||
f"Rscript plot.r {{{{wildcards.plot_type}}}} {{{{PLOT_HEADERS[wildcards.folder]}}}}"
|
f"python3 analysis/softenv_analysis.py -t sources-stats -i {{input}} -o {{output.sources_stats}}"
|
||||||
|
f"python3 analysis/softenv_analysis.py -t pkgs-changes -i {{input}} -o {{output.pkgs_changes}}"
|
||||||
|
|
||||||
|
rule buildstatus_analysis:
|
||||||
|
input:
|
||||||
|
expand(f"{PREFIX}/build_status/{{artifact}}/{{{{date}}}}.csv",\
|
||||||
|
artifact = ARTIFACTS
|
||||||
|
)
|
||||||
|
output:
|
||||||
|
f"{ANALYSIS_DIR}/build_status/{DATE}.csv"
|
||||||
|
shell:
|
||||||
|
f"python3 analysis/buildstatus_analysis.py -i {{input}} -o {{output}}"
|
||||||
|
|
||||||
|
rule artifact_analysis:
|
||||||
|
input:
|
||||||
|
expand(f"{PREFIX}/artifact_hash/{{artifact}}/{{{{date}}}}.csv",\
|
||||||
|
artifact = ARTIFACTS
|
||||||
|
)
|
||||||
|
output:
|
||||||
|
f"{ANALYSIS_DIR}/artifact/{DATE}.csv"
|
||||||
|
shell:
|
||||||
|
f"python3 analysis/artifact_analysis.py -i {{input}} -o {{output}}"
|
||||||
|
|
||||||
|
rule analysis_aggregate:
|
||||||
|
input:
|
||||||
|
sources_stats = expand(f"{ANALYSIS_DIR}/sources_stats/{{{{date}}}}.csv"),
|
||||||
|
pkgs_changes = expand(f"{ANALYSIS_DIR}/pkgs_changes/{{{{date}}}}.csv"),
|
||||||
|
build_status = expand(f"{ANALYSIS_DIR}/build_status/{{{{date}}}}.csv"),
|
||||||
|
artifact = expand(f"{ANALYSIS_DIR}/artifact/{{{{date}}}}.csv")
|
||||||
|
output:
|
||||||
|
sources_stats = f"{ANALYSIS_DIR}/sources_stats/aggregated/{DATE}.csv",
|
||||||
|
pkgs_changes = f"{ANALYSIS_DIR}/pkgs_changes/aggregated/{DATE}.csv",
|
||||||
|
build_status = f"{ANALYSIS_DIR}/build_status/aggregated/{DATE}.csv",
|
||||||
|
artifact = f"{ANALYSIS_DIR}/artifact/aggregated/{DATE}.csv"
|
||||||
|
shell:
|
||||||
|
f"cat {{input.sources_stats}} > {{output.sources_stats}}"
|
||||||
|
f"cat {{input.pkgs_changes}} > {{output.pkgs_changes}}"
|
||||||
|
f"cat {{input.build_status}} > {{output.build_status}}"
|
||||||
|
f"cat {{input.artifact}} > {{output.artifact}}"
|
||||||
|
|
||||||
|
# PLOT_HEADERS = {
|
||||||
|
# "sources_stats": ["dpkg", "rpm", "pacman", "pip", "conda", "git", "misc"],
|
||||||
|
# "pkgs_changes": ["dpkg", "rpm", "pacman", "pip", "conda", "git", "misc"],
|
||||||
|
# "build_status": ["success", "package_unavailable", "baseimage_unavailable", "artifact_unavailable", "dockerfile_not_found", "script_crash", "job_time_exceeded", "unknown_error"],
|
||||||
|
# "artifact": ["available", "unavailable", "changed"]
|
||||||
|
# }
|
||||||
|
|
||||||
|
# rule plot_all:
|
||||||
|
# input:
|
||||||
|
# expand(f"{ANALYSIS_DIR}/{{folder}}/aggregated/{{date}}.csv",\
|
||||||
|
# folder = ["sources_stats", "pkgs_changes", "build_status", "artifact"],\
|
||||||
|
# date = DATE
|
||||||
|
# )
|
||||||
|
|
||||||
|
# rule line_plot:
|
||||||
|
# input:
|
||||||
|
# expand(f"{ANALYSIS_DIR}/{{folder}}/{{artifact}}/{{date}}.csv",\
|
||||||
|
# folder = ["sources_stats", "pkgs_changes", "build_status", "artifact"],\
|
||||||
|
# artifact = ARTIFACTS,\
|
||||||
|
# date = DATE
|
||||||
|
# ),
|
||||||
|
# output:
|
||||||
|
# expand(f"{ANALYSIS_DIR}/{{folder}}/line.pdf",\
|
||||||
|
# folder = ["sources_stats", "pkgs_changes", "build_status", "artifact"],\
|
||||||
|
# artifact = ARTIFACTS,\
|
||||||
|
# date = DATE
|
||||||
|
# ),
|
||||||
|
# shell:
|
||||||
|
# f"Rscript plot.r line {{{{PLOT_HEADERS[wildcards.folder]}}}}"
|
Loading…
Reference in New Issue
Block a user