Autorun multiple benchmarks

.gitignore

 venv/
 logs/
 *.stdout
-*.stderr
+*.stderr
\ No newline at end of file
+__pycache__
+.vscode
\ No newline at end of file

run_benchmark.py

@@ -2,6 +2,7 @@ import subprocess
 import argparse
 import os
 import sys
+import re
 
 parser = argparse.ArgumentParser(description='')
 parser.add_argument('--gpu',dest='gpu',action='store_true',help='Whether to use GPU for inference')
@@ -12,6 +13,9 @@ parser.add_argument('--parallelruns',dest='parallelruns',type=int,default=1, hel
 
 args=parser.parse_args()
 
+if not (args.gpu or args.dla1 or args.dla2):
+    raise ValueError("No device chosen. Where do you want the benchmark to run?")
+
 def gen_command(onnx_path, engine_path, useDLACore):
     onnx_path_arg = "--onnx=" + onnx_path
     engine_path_arg = "--loadEngine=" + engine_path
@@ -39,6 +43,23 @@ def gen_command(onnx_path, engine_path, useDLACore):
 
     return run_command
 
+def parse_avg_latency(fd):
+    fd.seek(0, 0)
+
+    sum_latency = 0
+    nlines = 0
+    for line in fd:
+        matches = re.search(r"Average\s+on\s+(\d+)\s+runs.*?"
+                            r"GPU\s+latency:\s+((?<!\d|\.)\d+(?:\.\d+)?)\s+.*?"
+                            r"end\s+to\s+end\s+((?<!\d|\.)\d+(?:\.\d+)?)\s+ms", line)
+        if matches==None:
+            continue
+        GPU_latency = float(matches.group(2))
+        sum_latency += GPU_latency
+        nlines += 1
+    fd.seek(0, 0)
+    return sum_latency / nlines
+
 processes = {
     'gpu':[],
     'dla1':[],
@@ -128,6 +149,7 @@ for process_list in processes.values():
     for p in process_list:
         p.wait()
 
+#Print log
 for key,file_pair_list in logfiles.items():
     if file_pair_list == []:
         continue
@@ -142,6 +164,17 @@ for key,file_pair_list in logfiles.items():
         print(pair[1].read(), file=sys.stderr, end='\n')
         print('-'*40+'------end------'+'-'*40, file=sys.stderr, end='\n')
 
+#Get FPS
+sum_FPS = 0
+for key,file_pair_list in logfiles.items():
+    if file_pair_list == []:
+        continue
+    for f_stdout, f_stderr in file_pair_list:
+        sum_FPS += args.bs / (parse_avg_latency(f_stdout)/1000)
+
+print("Total FPS: ",sum_FPS)
+        
+
 for key,file_pair_list in logfiles.items():
     if file_pair_list == []:
         continue

run_multiple_benchmarks.py

+import sys
+import subprocess
+import re
+
+def run_once(gpu, dla1, dla2, bs, nprocess):
+    interpreter = sys.executable
+    run_command = [interpreter]
+    run_command += ['run_benchmark.py']
+    if gpu:
+        run_command += ["--gpu"]
+    if dla1:
+        run_command += ["--dla1"]
+    if dla2:
+        run_command += ["--dla2"]
+    
+    run_command += ["--bs="+str(bs)]
+    run_command += ["--parallelruns="+str(nprocess)]
+
+    completed_status = subprocess.run(run_command, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    stdout = completed_status.stdout.decode(encoding='utf-8')
+    stderr = completed_status.stderr.decode(encoding='utf-8')
+    # if any(char.isalnum() for char in stderr):
+    #     print('-'*20+'STDERR of run \"'+' '.join(run_command)+"\" is not empty!!!"+'-'*20,file=sys.stderr)
+    #     print(stderr,file=sys.stderr)
+    #     print('-'*20+'-'*20,file=sys.stderr)
+
+    for line in stdout.split('\n'):
+        if not ('FPS' in line):
+            continue
+        pattern = r'Total\s+FPS:\s+((?<!\d|\.)\d+(?:\.\d+)?).*?'
+        FPS = float(re.search(pattern,line).group(1))
+
+    
+    return FPS
+
+
+# s = run_once(gpu=True,dla1=False,dla2=False,bs=16,nprocess=2)
+# print(s)
+
+print('Devices,bs,process,fps')
+for bs in (1,16):
+    for nprocess in (1,2,4,8,10):
+        FPS = run_once(gpu=True,dla1=False,dla2=False,bs=bs,nprocess=nprocess)
+        print('GPU,{:d},{:d},{:.4f}'.format(bs,nprocess,FPS))
+for device in ['GPU','GPU+2DLA']:
+    for bs in (1,2,4,16):
+        FPS = run_once(gpu=True, dla1=bool('2DLA' in device), dla2=bool('2DLA' in device), bs=bs, nprocess=1)
+        print(device+',{:d},{:d},{:.4f}'.format(bs,1,FPS))
+
+
+
+