From e943be4efe5ed9c646541c9795adfb9c5695fe21 Mon Sep 17 00:00:00 2001 From: antux18 Date: Mon, 19 Aug 2024 14:59:08 +0200 Subject: [PATCH] Integrated softenv analysis to workflow. Changed input and output cmd options in analysis scripts to take multiple files at once. Moved test and template artifacts in an excluded folder. --- analysis/artifact_analysis.py | 14 ++++++++------ analysis/buildstatus_analysis.py | 14 ++++++++------ analysis/softenv_analysis.py | 14 ++++++++------ artifacts/nickel/{ => excluded}/template.ncl | 0 artifacts/nickel/{ => excluded}/test.ncl | 0 blacklists/blacklist.csv | 2 -- workflow/Snakefile | 5 ++--- 7 files changed, 26 insertions(+), 23 deletions(-) rename artifacts/nickel/{ => excluded}/template.ncl (100%) rename artifacts/nickel/{ => excluded}/test.ncl (100%) diff --git a/analysis/artifact_analysis.py b/analysis/artifact_analysis.py index e42e18d..87bd703 100755 --- a/analysis/artifact_analysis.py +++ b/analysis/artifact_analysis.py @@ -122,10 +122,11 @@ def main(): parser.add_argument( "-i", "--input", action = "append", + nargs = "+", help = """ The CSV file used as input for the analysis function. Multiple files - can be specified by repeating this argument with different paths. + can be specified at once by separating them with a space. All the input files must be artifact hash logs generated by ECG. """, required = True @@ -139,15 +140,16 @@ def main(): required = True ) args = parser.parse_args() - input_paths = args.input + inputs = args.input output_path = args.output # Parsing the input files: input_table = [] - for path in input_paths: - input_file = open(path) - input_table += list(csv.reader(input_file)) - input_file.close() + for i in inputs: + for path in i: + input_file = open(path) + input_table += list(csv.reader(input_file)) + input_file.close() # Analyzing the inputs: output_dict = analysis(input_table) diff --git a/analysis/buildstatus_analysis.py b/analysis/buildstatus_analysis.py index 2c082ea..89c8866 100755 --- a/analysis/buildstatus_analysis.py +++ b/analysis/buildstatus_analysis.py @@ -55,10 +55,11 @@ def main(): parser.add_argument( "-i", "--input", action = "append", + nargs = "+", help = """ The CSV file used as input for the analysis function. Multiple files - can be specified by repeating this argument with different paths. + can be specified at once by separating them with a space. All the input files must be build status logs generated by ECG. """, required = True @@ -72,15 +73,16 @@ def main(): required = True ) args = parser.parse_args() - input_paths = args.input + inputs = args.input output_path = args.output # Parsing the input files: input_table = [] - for path in input_paths: - input_file = open(path) - input_table += list(csv.reader(input_file)) - input_file.close() + for i in inputs: + for path in i: + input_file = open(path) + input_table += list(csv.reader(input_file)) + input_file.close() # Analyzing the inputs: output_dict = analysis(input_table) diff --git a/analysis/softenv_analysis.py b/analysis/softenv_analysis.py index 0bda71e..2a55941 100755 --- a/analysis/softenv_analysis.py +++ b/analysis/softenv_analysis.py @@ -160,10 +160,11 @@ def main(): parser.add_argument( "-i", "--input", action = "append", + nargs = "+", help = """ The CSV file used as input for the analysis function. Multiple files - can be specified by repeating this argument with different paths. + can be specified at once by separating them with a space. All the input files must be package lists generated by ECG. """, required = True @@ -177,16 +178,17 @@ def main(): required = True ) args = parser.parse_args() - input_paths = args.input + inputs = args.input output_path = args.output analysis_type = args.analysis_type # Parsing the input files: input_table = [] - for path in input_paths: - input_file = open(path) - input_table += list(csv.reader(input_file)) - input_file.close() + for i in inputs: + for path in i: + input_file = open(path) + input_table += list(csv.reader(input_file)) + input_file.close() # Analyzing the inputs: if analysis_type == "sources-stats": diff --git a/artifacts/nickel/template.ncl b/artifacts/nickel/excluded/template.ncl similarity index 100% rename from artifacts/nickel/template.ncl rename to artifacts/nickel/excluded/template.ncl diff --git a/artifacts/nickel/test.ncl b/artifacts/nickel/excluded/test.ncl similarity index 100% rename from artifacts/nickel/test.ncl rename to artifacts/nickel/excluded/test.ncl diff --git a/blacklists/blacklist.csv b/blacklists/blacklist.csv index d213eb2..e69de29 100644 --- a/blacklists/blacklist.csv +++ b/blacklists/blacklist.csv @@ -1,2 +0,0 @@ -template, unknown_error, 0 -test, unknown_error, 0 \ No newline at end of file diff --git a/workflow/Snakefile b/workflow/Snakefile index 7865c5a..478dc15 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -35,7 +35,6 @@ rule check_all: input: expand(f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json", artifact=ARTIFACTS) - rule check_artifact: input: "flake.nix", @@ -99,7 +98,7 @@ rule all_analysis: rule softenv_analysis: input: - pkg = expand(f"-i {PREFIX}/pkgs/{{artifact}}/{{date}}.csv", + expand(f"{PREFIX}/pkgs/{{artifact}}/{{date}}.csv", artifact=ARTIFACTS,\ date=DATE ), @@ -109,6 +108,6 @@ rule softenv_analysis: date=DATE ), shell: - expand(f"python3 {ANALYSIS_DIR}/softenv_analysis.py -t {{analysis_type}} {{input.pkg}}",\ + expand(f"python3 {ANALYSIS_DIR}/softenv_analysis.py -t {{analysis_type}} -i {{{{input}}}} -o {{{{output}}}}",\ analysis_type = ["sources-stats", "pkgs-changes"] )