fix a fifo problem and add scripts to test/time the coprocessor

DanieleParravicini · DanieleParravicini · commit be91c1494ba4 · 2020-07-13T21:12:33.000+02:00
diff --git a/re2compiler b/re2compiler
@@ -1 +1 @@
-Subproject commit 76f031f7550f6eee4d8b83b3d10bdcb2f11450e2
+Subproject commit 21f92759e045b0f008a0e4dd425041fa72ba05e6
diff --git a/rtl_src/fifo.sv b/rtl_src/fifo.sv
@@ -31,7 +31,7 @@ module fifo #(
 logic [2:0]             state_cur, state_next;
 logic [COUNT_WIDTH-1:0] head     , head_next ,  head_incremented;
 logic [COUNT_WIDTH-1:0] tail     , tail_next ,  tail_incremented;
-
+logic [COUNT_WIDTH-1:0] where_to_read;   
 
 (* ram_style = "block" *) logic [DWIDTH-1:0]      content [(2**COUNT_WIDTH)-1:0];
 logic write_enable, read_enable;
@@ -50,6 +50,7 @@ always_ff @( posedge clk ) begin
         head      <= head_next;
         tail      <= tail_next;
         state_cur <= state_next;
+        middle    <= middle_next;
     end
 end
 
@@ -60,9 +61,9 @@ always_ff @(posedge clk)begin
         content[tail] <= din;
     end
 
-    from_memory  <= content[head_incremented];
+    from_memory  <= content[where_to_read];
 
-    middle       <= middle_next;
+   
 end
 assign from_din = din;
 
@@ -89,9 +90,9 @@ assign from_din = din;
 //                                                   v |                             
 //                                                  W,R&W,_                 
 always_comb begin //create full empty signals
-    data_count       = tail - head;
-    head_incremented = head + 1 ;
-    tail_incremented = tail + 1 ;
+    data_count            = tail - head;
+    head_incremented      = head + 1 ;
+    tail_incremented      = tail + 1 ;
 
     //empty
     if( head == tail )            empty = 1'b1;
@@ -123,9 +124,11 @@ always_comb begin //create full empty signals
     //middle register and memory
     //bit states represent: register, memory_output, memory_content 
     // validity.
-    state_next  = state_cur;
-    middle_next = middle;
-    dout        = middle;
+    state_next    = state_cur;
+    middle_next   = middle;
+    dout          = middle;
+    //all read from h+1 but not state 111 with an incoming read
+    where_to_read = head_incremented;
     case(state_cur)
     3'b000:
     begin
@@ -159,11 +162,12 @@ always_comb begin //create full empty signals
     3'b101:
     begin
         dout = middle;
+       
         case({write_enable, read_enable})
             2'b11:
             begin
                 state_next = 3'b011;
-                //middle can't be updated (memory invalid)
+                //middle can't be updated (memory output invalid)
             end
             2'b10:
             begin
@@ -173,7 +177,7 @@ always_comb begin //create full empty signals
             2'b01:
             begin
                 state_next = 3'b010;
-                //middle can't be updated (memory invalid)
+                //middle can't be updated (memory output invalid)
             end
             default:
             begin
@@ -199,7 +203,7 @@ always_comb begin //create full empty signals
             2'b01:
             begin
                 state_next = 3'b000;
-                //queue is void
+                //queue would be void
             end
             default:
             begin
@@ -216,7 +220,8 @@ always_comb begin //create full empty signals
             begin
                 state_next = state_cur;
                 //data_in is written in memory and 
-                //middle  is not used
+                //from_memory, which was intended for middle is redirected toward output
+                //middle  can't be updated
             end
             2'b10:
             begin
@@ -226,6 +231,7 @@ always_comb begin //create full empty signals
             2'b01:
             begin
                 state_next = 3'b010;
+                
             end
             default:
             begin
@@ -238,13 +244,18 @@ always_comb begin //create full empty signals
     begin
         dout = middle;
         
-        if(read_enable )
+        if(read_enable)
         begin 
-            middle_next= from_memory;
-            if(data_count == 2 )
+            middle_next   = from_memory;
+            where_to_read = head_incremented +1;
+            if(data_count == 2 && ~ write_enable)
             begin
                 state_next = 3'b100;
             end
+            else if(data_count == 2 && write_enable)
+            begin
+                state_next = 3'b101;
+            end
             
         end
     end
diff --git a/scripts/comparison.py b/scripts/comparison.py
@@ -1,6 +1,6 @@
-import re2_driver
 import pynq
 from pynq import Overlay
+import re2_driver
 from test_execution import time_full_match, time_match, time_ignore_prefix_match
 import argparse
 
@@ -9,21 +9,23 @@
 arg_parser.add_argument('-ignore_prefix'    , 			help='do not pose any constraint on where the regular expression starts matching the string. Corresponds to .*<your_regex>'	    , default=False, action='store_true')	
 args = arg_parser.parse_args()
 
-re2_coprocessor = Overlay('re2_coprocessor.bit')
-regex_string        = 'a?a?bb'
-string              = 'b'+'a'*8
+re2_coprocessor = Overlay('../bitstreams/recopro2BBP100.bit')
+regex_string        = '((R|K|X)(...?)?(D|B|E|Z|X)(...?)?(Y|X))|(.(G|X)(R|K|X)(R|K|X))|((S|T|X).(R|K|X))'
+string              = 'MQNPLPEVMSPEHDKRTTTPMSKEANKFIRELDKKPGDLAVVSDFVKRNTGKRLPIGKRSNLYVRICDLSGTIYMGETFILESWEELYLPEPTKMEVLGTLESCCGIPPFPEWIVMVGEDQCVYAYGDEEILLFAYSVKQLVEEGIQETGISYKYPDDISDVDEEVLQQDEEIQKIRKKTREFVDKDAQEFQDFLNSLDASLLS'
 print('regex', regex_string, 'string', string)
 freq                = 100_000_000
-has_accepted = re2_coprocessor.re2_copro_0.compile_and_run(regex_string, string, full_match=args.full_match,ignore_prefix=args.ignore_prefix )
+has_accepted = re2_coprocessor.re2_copro_0.compile_and_run(regex_string, string, full_match=args.full_match,ignore_prefix=args.ignore_prefix , O1=False)
 cc_number =  re2_coprocessor.re2_copro_0.read_elapsed_clock_cycles()
 print('clock cycles taken:', cc_number)
 print('status:', re2_coprocessor.re2_copro_0.get_status())
 time = cc_number*1/freq*1_000_000_000
 print('time re2coprocessor', time, 'ns')
+'''
 if args.full_match:
     min_time,avg_time = time_full_match(regex_string, string)
 elif args.ignore_prefix:
     min_time,avg_time = time_ignore_prefix_match(regex_string, string)
 else:
     min_time,avg_time = time_match(regex_string, string)
-print('time python re module min:', min_time, 'avg:',avg_time, 'ns')
+print('time python re module min:', min_time, 'avg:',avg_time, 'ns')
+'''
diff --git a/scripts/re2_driver.py b/scripts/re2_driver.py
@@ -175,13 +175,13 @@ def reset(self):
         self.write_cmd(RE2_COPROCESSOR_COMMANDS.NOP)
         return True
 
-    def compile_and_run(self, regex_string, string, double_check =True,ignore_prefix=True, full_match=False ):
+    def compile_and_run(self, regex_string, string, double_check =True,ignore_prefix=True, full_match=False, O1=True ):
         
         try:
             
             import os.path
-            code_output_file = regex_string+'.out'
-
+            code_output_file = regex_string+('_ignore_' if ignore_prefix else '')+('_full_match_' if full_match else '')+('_O1_' if O1 else '')+'.out'
+            code_output_file = code_output_file.replace('/', u'\u2215')
             if os.path.exists(code_output_file):
                 print('reusing previously compilation')
                 code = ''
@@ -192,7 +192,7 @@ def compile_and_run(self, regex_string, string, double_check =True,ignore_prefix
                 sys.path.append('../re2compiler')
                 import re2compiler
                 print('start compilation')
-                code = re2compiler.compile(data=regex_string,o=code_output_file, O1=True,ignore_prefix=ignore_prefix, full_match=full_match)
+                code = re2compiler.compile(data=regex_string,o=code_output_file, O1=O1,ignore_prefix=ignore_prefix, full_match=full_match)
                 print('end compilation')
             code = code.split('\n')
             res  = self.load_and_run( code , string)
@@ -244,18 +244,18 @@ def load_and_run(self, code, string):
     debug = False
     #IP_BASE_ADDRESS = 0x43C00000 or equivalently 1136656384
     #ADDRESS_RANGE   = 6*4
-    re2_coprocessor = Overlay('re2_coprocessor.bit')
+    re2_coprocessor = Overlay('../bitstreams/re2_coprocessor16bbP85.bit')
     if debug :
         print('test:',re2_coprocessor.ip_dict)
 
     cc_number =  re2_coprocessor.re2_copro_0.read_elapsed_clock_cycles()
     print('status:', re2_coprocessor.re2_copro_0.get_status())
     time.sleep(1)
     
-    regex_string        = 'a?a?bb'
+    regex_string        = '(R|K|X)(...?)?(D|B|E|Z|X)(...?)?(Y|X)'
     string              = 'b'+'a'*8
     
-    string_to_accept    = "a"*16+'bb'
+    string_to_accept    = "MSIIGATRLQNDKSDTYSAGPCYAGGCSAFTPRGTCGKDWDLGEQTCASGFCTSQPLCARIKKTQVCGLRYSSKGKDPLVSAEWDSRGAPYVRCTYDADLIDTQAQVDQFVSMFGESPSLAERYCMRGVKNTAGELVSRVSSDADPAGGWCRKWYSAHRGPDQDAALGSFCIKNPGAADCKCINRASDPVYQKVKTLHAYPDQCWYVPCAADVGELKMGTQRDTPTNCPTQVCQIVFNMLDDGSVTMDDVKNTINCDFSKYVPPPPPPKPTPPTPPTPPTPPTPPTPPTPPTPRPVHNRKVMFFVAGAVLVAILISTVRW"
     string_to_reject    = "a"*15+"b"
     #test to accept
 
diff --git a/scripts/test.py b/scripts/test.py
@@ -0,0 +1,63 @@
+import test_execution
+import csv
+import os 
+import argparse
+
+arg_parser = argparse.ArgumentParser(description='test regular expression matching')
+arg_parser.add_argument('-startstr'		    , type=int , help='index first str.'	                                    , default=0   )
+arg_parser.add_argument('-endstr'		    , type=int , help='index end string.'	                                    , default=None)
+arg_parser.add_argument('-startreg'		    , type=int , help='index first reg.'	                                    , default=0   )
+arg_parser.add_argument('-endreg'		    , type=int , help='index end reg.'		                                    , default=None)
+arg_parser.add_argument('-testpy'		               , help='measure time taken by python', action='store_true'		, default=False)
+arg_parser.add_argument('-testcopro'		           , help='measure time taken by copro.', action='store_true'		, default=False)
+arg_parser.add_argument('-strfile'		, type=str , help='file containing test input'  ,	                          default='test2.input')
+arg_parser.add_argument('-regfile'		    , type=str , help='file containing test reg'    ,	                          default='test.reg')
+arg_parser.add_argument('-bitstream'		, type=str , help='bitstream file of the coprocessor'    ,	                          default='re2_coprocessor4bbP110.bit')
+arg_parser.add_argument('-do_not_optimize'	, help='do not optimize recopro code'    ,action='store_true', default=False)
+
+args = arg_parser.parse_args()
+optimize_str = 'O1' if not args.do_not_optimize else ''
+bitstream_filename = os.path.basename(args.bitstream)[:-4]
+with open(f'log_{bitstream_filename}_{optimize_str}.csv', 'w', newline='') as csvfile:
+    fout = csv.writer(csvfile, delimiter=';', quoting=csv.QUOTE_MINIMAL)
+    with open(args.strfile, 'r', errors='ignore') as f:
+        lines = f.readlines()[args.startstr:args.endstr]
+        for line in lines:
+            print('len', len(line),'->', bytes(line,'utf-8'))
+            line = line[:-1]
+            fout.writerow(['string', line, '', ''])
+            fout.writerow(['regex', 'result','copro',  'python re'])
+            with open(args.regfile) as f:
+                for r in f.readlines()[args.startreg:args.endreg]:
+                    r = r[:-1]
+                    print('regex', r,'\nstring', line)
+                    try:
+                        min_time_copro = 0
+                        has_accepted   = 'UNK'
+                        if args.testcopro:
+                            import re2_driver
+                            import pynq
+                            from pynq import Overlay
+                            re2_coprocessor = Overlay(args.bitstream)
+                            
+                            
+                            #freq                = 90_000_000
+                            has_accepted = re2_coprocessor.re2_copro_0.compile_and_run(r, line, ignore_prefix=True, O1=(not args.do_not_optimize) )
+                            cc_number =  re2_coprocessor.re2_copro_0.read_elapsed_clock_cycles()
+                            print('clock cycles taken:', cc_number)
+                            print('status:', re2_coprocessor.re2_copro_0.get_status())
+                            min_time_copro = cc_number#*1/freq*1_000_000_000
+                            print('time re2coprocessor', min_time_copro, 'cc')
+                        min_time_re = 0
+                        if args.testpy:
+                            min_time_re = test_execution.time_ignore_prefix_match(r, line, perf_counter=True)
+                            print('minimum time', min_time_re, 'ns')
+
+                        fout.writerow([r,has_accepted,  min_time_copro,  min_time_re,' '])
+                    except Exception as exc:
+                        raise exc
+                        print('error',r,exc)
+                    
+
+
+
diff --git a/scripts/test2.reg b/scripts/test2.reg
@@ -0,0 +1,12 @@
+(((R|K|X)(R|K|X))?.(S|T|X))
+(S|T|X)(..)?(D|B|E|Z|X)
+(C|X).(D|B|N|B|X)(....)?(F|Y|X).(C|X).(C|X)
+(C|X)(.....?)?(C|X)(C|X)(S|X)(..)?(G|X).(C|X)(G|X)(....?)?(F|Y|W|X)(C|X)
+(C|X)(..(..)?)?(C|X)(...)?(L|I|V|M|F|Y|W|C|X)(........)?(H|X)(...(..)?)?(H|X)
+((L|I|V|M|F|E|Z|X)(F|Y|X)(P|X)(W|X)(M|X)(K|R|Q|Z|T|A|X))
+((L|X)(M|X)(A|X)(E|Z|Q|Z|X)(G|X)(L|X)(Y|X)(N|B|X))
+((R|X)(P|X)(C|X)(..........)?(C|X)(V|X)(S|X))
+(R|K|X)(...?)?(D|B|E|Z|X)(...?)?(Y|X)
+.(G|X)(R|K|X)(R|K|X)
+(S|T|X).(R|K|X)
+((R|K|X)(...?)?(D|B|E|Z|X)(...?)?(Y|X))|(.(G|X)(R|K|X)(R|K|X))|((S|T|X).(R|K|X))
diff --git a/scripts/test5.input b/scripts/test5.input
@@ -0,0 +1,10 @@
+MAFSAEDVLKEYDRRRRMEALLLSLYYPNDRKLLDYKEWSPPRVQVECPKAPVEWNNPPSEKGLIVGHFSGIKYKGEKAQASEVDVNKMCCWVSKFKDAMRRYQGIQTCKIPGKVLSDLDAKIKAYNLTVEGVEGFVRYSRVTKQHVAAFLKELRHSKQYENVNLIHYILTDKRVDIQHLEKDLVKDFKALVESAHRMRQGHMINVKYILYQLLKKHGHGPDGPDILTVKTGSKGVLYDDSFRKIYTDLGWKFTPL
+MSIIGATRLQNDKSDTYSAGPCYAGGCSAFTPRGTCGKDWDLGEQTCASGFCTSQPLCARIKKTQVCGLRYSSKGKDPLVSAEWDSRGAPYVRCTYDADLIDTQAQVDQFVSMFGESPSLAERYCMRGVKNTAGELVSRVSSDADPAGGWCRKWYSAHRGPDQDAALGSFCIKNPGAADCKCINRASDPVYQKVKTLHAYPDQCWYVPCAADVGELKMGTQRDTPTNCPTQVCQIVFNMLDDGSVTMDDVKNTINCDFSKYVPPPPPPKPTPPTPPTPPTPPTPPTPPTPPTPRPVHNRKVMFFVAGAVLVAILISTVRW
+MASNTVSAQGGSNRPVRDFSNIQDVAQFLLFDPIWNEQPGSIVPWKMNREQALAERYPELQTSEPSEDYSGPVESLELLPLEIKLDIMQYLSWEQISWCKHPWLWTRWYKDNVVRVSAITFEDFQREYAFPEKIQEIHFTDTRAEEIKAILETTPNVTRLVIRRIDDMNYNTHGDLGLDDLEFLTHLMVEDACGFTDFWAPSLTHLTIKNLDMHPRWFGPVMDGIKSMQSTLKYLYIFETYGVNKPFVQWCTDNIETFYCTNSYRYENVPRPIYVWVLFQEDEWHGYRVEDNKFHRRYMYSTILHKRDTDWVENNPLKTPAQVEMYKFLLRISQLNRDGTGYESDSDPENEHFDDESFSSGEEDSSDEDDPTWAPDSDDSDWETETEEEPSVAARILEKGKLTITNLMKSLGFKPKPKKIQSIDRYFCSLDSNYNSEDEDFEYDSDSEDDDSDSEDDC
+MYQAINPCPQSWYGSPQLEREIVCKMSGAPHYPNYYPVHPNALGGAWFDTSLNARSLTTTPSLTTCTPPSLAACTPPTSLGMVDSPPHINPPRRIGTLCFDFGSAKSPQRCECVASDRPSTTSNTAPDTYRLLITNSKTRKNNYGTCRLEPLTYGI
+MARPLLGKTSSVRRRLESLSACSIFFFLRKFCQKMASLVFLNSPVYQMSNILLTERRQVDRAMGGSDDDGVMVVALSPSDFKTVLGSALLAVERDMVHVVPKYLQTPGILHDMLVLLTPIFGEALSVDMSGATDVMVQQIATAGFVDVDPLHSSVSWKDNVSCPVALLAVSNAVRTMMGQPCQVTLIIDVGTQNILRDLVNLPVEMSGDLQVMAYTKDPLGKVPAVGVSVFDSGSVQKGDAHSVGAPDGLVSFHTHPVSSAVELNYHAGWPSNVDMSSLLTMKNLMHVVVAEEGLWTMARTLSMQRLTKVLTDAEKDVMRAAAFNLFLPLNELRVMGTKDSNNKSLKTYFEVFETFTIGALMKHSGVTPTAFVDRRWLDNTIYHMGFIPWGRDMRFVVEYDLDGTNPFLNTVPTLMSVKRKAKIQEMFDNMVSRMVTS
+MNAKYDTDQGVGRMLFLGTIGLAVVVGGLMAYGYYYDGKTPSSGTSFHTASPSFSSRYRY
+MRYTVLIALQGALLLLLLIDDGQGQSPYPYPGMPCNSSRQCGLGTCVHSRCAHCSSDGTLCSPEDPTMVWPCCPESSCQLVVGLPSLVNHYNCLPNQCTDSSQCPGGFGCMTRRSKCELCKADGEACNSPYLDWRKDKECCSGYCHTEARGLEGVCIDPKKIFCTPKNPWQLAPYPPSYHQPTTLRPPTSLYDSWLMSGFLVKSTTAPSTQEEEDDY
+MQNPLPEVMSPEHDKRTTTPMSKEANKFIRELDKKPGDLAVVSDFVKRNTGKRLPIGKRSNLYVRICDLSGTIYMGETFILESWEELYLPEPTKMEVLGTLESCCGIPPFPEWIVMVGEDQCVYAYGDEEILLFAYSVKQLVEEGIQETGISYKYPDDISDVDEEVLQQDEEIQKIRKKTREFVDKDAQEFQDFLNSLDASLLS
+MDSLNEVCYEQIKGTFYKGLFGDFPLIVDKKTGCFNATKLCVLGGKRFVDWNKTLRSKKLIQYYETRCDIKTESLLYEIKGDNNDEITKQITGTYLPKEFILDIASWISVEFYDKCNNIIINYFVNEYKTMDKKTLQSKINEVEEKMQKLLNEKEEELQEKNDKIDELILFSKRMEEDRKKDREMMIKQEKMLRELGIHLEDVSSQNNELIEKVDEQVEQNAVLNFKIDNIQNKLEIAVEDRAPQPKQNLKRERFILLKRNDDYYPYYTIRAQDINARSALKRQKNLYNEVSVLLDLTCHPNSKTLYVRVKDELKQKGVVFNLCKVSISNSKINEEELIKAMETINDEKRDV
+MYKMYFLKDQKFSLSGTIRINDKTQSEYGSVWCPGLSITGLHHDAIDHNMFEEMETEIIEYLGPWVQAEYRRIKG
diff --git a/scripts/test_arbiter_creation.py b/scripts/test_arbiter_creation.py
@@ -1,4 +1,4 @@
-BB_N = 10
+BB_N = 16
 res = ""
 res = 'digraph{\n'
 for i in range(BB_N):
diff --git a/scripts/test_execution.py b/scripts/test_execution.py
diff --git a/sim/AXI_top_tb_from_file.sv b/sim/AXI_top_tb_from_file.sv

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-BB_N = 10`
	`1`	`+BB_N = 16`
`2`	`2`	`res = ""`
`3`	`3`	`res = 'digraph{\n'`
`4`	`4`	`for i in range(BB_N):`