Integrated softenv analysis to workflow. Changed input and output cmd options in analysis scripts to take multiple files at once. Moved test and template artifacts in an excluded folder.

analysis/artifact_analysis.py

@@ -122,10 +122,11 @@ def main():
     parser.add_argument(
         "-i", "--input",
         action = "append",
+        nargs = "+",
         help =
         """
         The CSV file used as input for the analysis function. Multiple files
-        can be specified by repeating this argument with different paths.
+        can be specified at once by separating them with a space.
         All the input files must be artifact hash logs generated by ECG.
         """,
         required = True
@@ -139,12 +140,13 @@ def main():
         required = True
     )
     args = parser.parse_args()
-    input_paths = args.input
+    inputs = args.input
     output_path = args.output
 
     # Parsing the input files:
     input_table = []
-    for path in input_paths:
+    for i in inputs:
+        for path in i:
             input_file = open(path)
             input_table += list(csv.reader(input_file))
             input_file.close()

analysis/buildstatus_analysis.py

@@ -55,10 +55,11 @@ def main():
     parser.add_argument(
         "-i", "--input",
         action = "append",
+        nargs = "+",
         help =
         """
         The CSV file used as input for the analysis function. Multiple files
-        can be specified by repeating this argument with different paths.
+        can be specified at once by separating them with a space.
         All the input files must be build status logs generated by ECG.
         """,
         required = True
@@ -72,12 +73,13 @@ def main():
         required = True
     )
     args = parser.parse_args()
-    input_paths = args.input
+    inputs = args.input
     output_path = args.output
 
     # Parsing the input files:
     input_table = []
-    for path in input_paths:
+    for i in inputs:
+        for path in i:
             input_file = open(path)
             input_table += list(csv.reader(input_file))
             input_file.close()

analysis/softenv_analysis.py

@@ -160,10 +160,11 @@ def main():
     parser.add_argument(
         "-i", "--input",
         action = "append",
+        nargs = "+",
         help =
         """
         The CSV file used as input for the analysis function. Multiple files
-        can be specified by repeating this argument with different paths.
+        can be specified at once by separating them with a space.
         All the input files must be package lists generated by ECG.
         """,
         required = True
@@ -177,13 +178,14 @@ def main():
         required = True
     )
     args = parser.parse_args()
-    input_paths = args.input
+    inputs = args.input
     output_path = args.output
     analysis_type = args.analysis_type
 
     # Parsing the input files:
     input_table = []
-    for path in input_paths:
+    for i in inputs:
+        for path in i:
             input_file = open(path)
             input_table += list(csv.reader(input_file))
             input_file.close()

blacklists/blacklist.csv

@@ -1,2 +0,0 @@
-template, unknown_error, 0
-test, unknown_error, 0

workflow/Snakefile

@@ -35,7 +35,6 @@ rule check_all:
   input:
     expand(f"{ARTIFACTS_FOLDER_JSON}/{{artifact}}.json", artifact=ARTIFACTS)
 
-
 rule check_artifact:
   input:
     "flake.nix",
@@ -99,7 +98,7 @@ rule all_analysis:
 
 rule softenv_analysis:
   input:
-    pkg = expand(f"-i {PREFIX}/pkgs/{{artifact}}/{{date}}.csv",
+    expand(f"{PREFIX}/pkgs/{{artifact}}/{{date}}.csv",
                  artifact=ARTIFACTS,\
                  date=DATE
     ),
@@ -109,6 +108,6 @@ rule softenv_analysis:
            date=DATE
     ),
   shell:
-    expand(f"python3 {ANALYSIS_DIR}/softenv_analysis.py -t {{analysis_type}} {{input.pkg}}",\
+    expand(f"python3 {ANALYSIS_DIR}/softenv_analysis.py -t {{analysis_type}} -i {{{{input}}}} -o {{{{output}}}}",\
            analysis_type = ["sources-stats", "pkgs-changes"]
     )