From 4db6d400747ce76c5f687572e1226a2a23a0edb5 Mon Sep 17 00:00:00 2001 From: mikep996 <54072448+mikep996@users.noreply.github.com> Date: Mon, 4 Nov 2024 15:36:38 +0100 Subject: [PATCH 1/5] Update run.py coverage support added --- vunit/aes-encryption/run.py | 187 ++++++++++++++++++++++++------------ 1 file changed, 125 insertions(+), 62 deletions(-) diff --git a/vunit/aes-encryption/run.py b/vunit/aes-encryption/run.py index 3881447..21ea7d5 100644 --- a/vunit/aes-encryption/run.py +++ b/vunit/aes-encryption/run.py @@ -1,5 +1,8 @@ #!/usr/bin/env python3 +import os +import sys +import string # Importing main vunit library from vunit import VUnit @@ -7,6 +10,9 @@ import rand_enc from rand_enc import prepare_data +#os.environ['VUNIT_SIMULATOR'] = 'rivierapro' +#os.environ['VUNIT_RIVIERAPRO_PATH'] = '/edatools/Aldec/Riviera-PRO-2024.04-x64/bin' + # Create VUnit instance by parsing command line arguments vu = VUnit.from_argv() @@ -23,73 +29,130 @@ # Add all files ending in .vhd in tb directory to library lib.add_source_files("tb/*.vhd") -# Hardcoded test -tb = lib.test_bench("tb_enc_hardcoded") -test = tb.test("hardcoded encryption test") - -# Create configuration of the encryption test using golden references from different sources -tb = lib.test_bench("tb_enc_generics") -test = tb.test("generic encryption test") - -for source, key, plaintext, ciphertext in [ - ( - "aes_specification", - 'x"3c4fcf098815f7aba6d2ae2816157e2b"', - 'x"340737e0a29831318d305a88a8f64332"', - 'x"320b6a19978511dcfb09dc021d842539"', - ), - ( - "zero_inputs", - 'x"00000000000000000000000000000000"', - 'x"00000000000000000000000000000000"', - 'x"2e2b34ca59fa4c883b2c8aefd44be966"', - ), - ( - "cryptographic_standard_doc", - 'x"3c4fcf098815f7aba6d2ae2816157e2b"', - 'x"2a179373117e3de9969f402ee2bec16b"', - 'x"97ef6624f3ca9ea860367a0db47bd73a"', - ), - ( - "one_two_three", - 'x"32211332211332211332211332211332"', - 'x"12233112233112233112233112233112"', - 'x"4fdfcdf5481f204df7dc282d8f645119"', - ), - ( - "test_vector_1", - 'x"f74eb5c67f8ead89ce6fb4edac7b8392"', - 'x"320b8d7f6e2bcfad36d8bc8529837ead"', - 'x"6dc7fe4482c891d38faf915cbed856bf"', - ), - ( - "test_vector_2", - 'x"74c0563e4daa164875eda570cf29bb46"', - 'x"12d72ab5ad3f0972fd7e93cf9a8d6eb3"', - 'x"1eacbe7973a224fffba1f8cf42d77f99"', - ), -]: - test.add_config( - name=source, - generics=dict(key=key, plaintext=plaintext, expected_ciphertext=ciphertext), - ) - -# Create configuration of the encryption test using random input vectors -tb = lib.test_bench("tb_enc_generics") -test = tb.test("random generic encryption test") +rand_data_test = 3 # Invoke a function to prepare randomized data test cases # Provide another argument to make different number of test cases -data_list = rand_enc.prepare_data(20) +data_list = rand_enc.prepare_data(rand_data_test) + +encryption_tests_data = [ + ( + "aes_specification", + 'x"3c4fcf098815f7aba6d2ae2816157e2b"', + 'x"340737e0a29831318d305a88a8f64332"', + 'x"320b6a19978511dcfb09dc021d842539"', + ), + ( + "zero_inputs", + 'x"00000000000000000000000000000000"', + 'x"00000000000000000000000000000000"', + 'x"2e2b34ca59fa4c883b2c8aefd44be966"', + ), + ( + "cryptographic_standard_doc", + 'x"3c4fcf098815f7aba6d2ae2816157e2b"', + 'x"2a179373117e3de9969f402ee2bec16b"', + 'x"97ef6624f3ca9ea860367a0db47bd73a"', + ), + ( + "one_two_three", + 'x"32211332211332211332211332211332"', + 'x"12233112233112233112233112233112"', + 'x"4fdfcdf5481f204df7dc282d8f645119"', + ), + ( + "test_vector_1", + 'x"f74eb5c67f8ead89ce6fb4edac7b8392"', + 'x"320b8d7f6e2bcfad36d8bc8529837ead"', + 'x"6dc7fe4482c891d38faf915cbed856bf"', + ), + ( + "test_vector_2", + 'x"74c0563e4daa164875eda570cf29bb46"', + 'x"12d72ab5ad3f0972fd7e93cf9a8d6eb3"', + 'x"1eacbe7973a224fffba1f8cf42d77f99"', + ) + ] + +# Ask for contribution tests or all tests +CONTRIBUTION='0' +if "CONTRIBUTION" in os.environ: + CONTRIBUTION=os.environ["CONTRIBUTION"] + +if CONTRIBUTION=='1': + print("Following contribution tests will be conducted:") + test_list = [] + with open("contribution.txt", "r") as f: + for x in f: + print(x) + # Read contribution.txt file and add tests to the list + test_list.append(x.strip()) + + + + # Tests lists declaration + encryption_tests_list = ['aes_specification', 'zero_inputs', "cryptographic_standard_doc", "one_two_three", "test_vector_1", "test_vector_2" ] + random_encryption_tests_list = [f"random_test_number_{x}" for x in range(rand_data_test)] + + # If x is one of the tests declared in contribution.txt, a specific test case will be conducted + for x in test_list: + if x=='hardcoded encryption test': + tb = lib.test_bench("tb_enc_hardcoded") + test = tb.test("hardcoded encryption test") + elif x in encryption_tests_list: + tb = lib.test_bench("tb_enc_generics") + test = tb.test("generic encryption test") + + generic_encryption_test_data = [k for k in encryption_tests_data if k[0]==x][0] + + test.add_config( + name=generic_encryption_test_data[0], + generics=dict(key=generic_encryption_test_data[1], plaintext=generic_encryption_test_data[2], expected_ciphertext=generic_encryption_test_data[3]), + ) + elif x in random_encryption_tests_list: + tb = lib.test_bench("tb_enc_generics") + test = tb.test("random generic encryption test") + + for k in data_list: + if k[0]==x: + random_generic_encryption_test_data = k + break + + test.add_config( + name=random_generic_encryption_test_data[0], + generics=dict(key=random_generic_encryption_test_data[1], plaintext=random_generic_encryption_test_data[2], expected_ciphertext=random_generic_encryption_test_data[3]), + ) + +# Run all tests +else: + # Hardcoded test + tb = lib.test_bench("tb_enc_hardcoded") + test = tb.test("hardcoded encryption test") + + # Create configuration of the encryption test using golden references from different sources + tb = lib.test_bench("tb_enc_generics") + test = tb.test("generic encryption test") + + for source, key, plaintext, ciphertext in encryption_tests_data: + test.add_config( + name=source, + generics=dict(key=key, plaintext=plaintext, expected_ciphertext=ciphertext), + ) + + # Create configuration of the encryption test using random input vectors + tb = lib.test_bench("tb_enc_generics") + test = tb.test("random generic encryption test") + + for source, key, plaintext, ciphertext in data_list: + test.add_config( + name=source, + generics=dict(key=key, plaintext=plaintext, expected_ciphertext=ciphertext), + ) -for source, key, plaintext, ciphertext in data_list: - test.add_config( - name=source, - generics=dict(key=key, plaintext=plaintext, expected_ciphertext=ciphertext), - ) +lib.set_sim_option("enable_coverage", True) -#Enable design profiler for Aldec's simulators -#lib.set_sim_option("enable_profiler", True) +lib.set_compile_option("rivierapro.vcom_flags", ["-coverage","sb"]) +lib.set_sim_option("rivierapro.vsim_flags", ["-acdb_cov sbaecmtf"]) # Run vunit main function vu.main() From e65452b80695b11ba54817e66ad68dc94120c5ba Mon Sep 17 00:00:00 2001 From: mikep996 <54072448+mikep996@users.noreply.github.com> Date: Mon, 4 Nov 2024 15:37:34 +0100 Subject: [PATCH 2/5] Add files via upload --- vunit/aes-encryption/acdb.do | 65 ++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 vunit/aes-encryption/acdb.do diff --git a/vunit/aes-encryption/acdb.do b/vunit/aes-encryption/acdb.do new file mode 100644 index 0000000..3781921 --- /dev/null +++ b/vunit/aes-encryption/acdb.do @@ -0,0 +1,65 @@ +#------------------------------------------------------------------------------ +#-- Subject : summary VUNIT coverage +#-- File name : acdb.do +#-- Date : 10/16/2024 +#-- Developer : Michal Pacula +#-- Copyright : (C) 2024 ALDEC Inc. +#------------------------------------------------------------------------------ + +package require fileutil +set fileList {} + +proc testnameUpdt { FileName testname} { +set data ""; +set dataout ""; +set line ""; +if {![file exists "$FileName"]} { + puts "Error: Cannot find $FileName" + } else { + acdb2xml -i $FileName -o $FileName.xml + set fi [open "$FileName.xml" r]; + set data [read $fi]; + close $fi; + set fi [open "$FileName.converted" w]; + foreach line [split $data "\n"] { + if [string match "*hnode logical_name=*" $line] { + if [string match "*hnode logical_name=*" $line] { + regexp {(.*)(logical_name=\")(coverage)(.*)} $line m0 m1 m2 m3 m4 m5; + puts $fi "$m1$m2$testname$m4"; + } else { + set firstline $line + } + } else { + puts $fi $line; + } + }; + }; + close $fi; + puts "Converted File: $FileName.converted"; + xml2acdb -i $FileName.converted -o $FileName +}; +# main() + +foreach file [fileutil::findByPattern "./vunit_out/test_output" *.acdb] { + regexp {(./vunit_out/test_output/lib.)(.*)(\.)(.*)(_)(.*)(/rivierapro/coverage.acdb)} $file m0 m1 m2 m3 m4 m5 m6; + puts $file + puts $m2 + if [string match "*.*" $m2] { + regexp {(.*)(\.)(.*)} $m2 l0 l1 l2 l3 ; + set testbench $l1 + set testname $l3 + } else { + set testbench $m2 + set testname $m4 + } + puts $testbench + puts $testname + lappend fileList -i $file -path "\/$testbench/enc_inst" + #testnameUpdt $file $testname +} +#xml2acdb -dataorder id,feature,description,link,type,weight,user,goal -i testplan.xml -o acdb/plan.acdb + +acdb merge -associative $fileList -o acdb/results.acdb +acdb report -html -o acdb/results.html -i acdb/results.acdb +acdb rank -goal 25 -html -i acdb/results.acdb -o acdb/rank.html + From 3e517cf27242dd46d5298cfc96af10be1ad5fbda Mon Sep 17 00:00:00 2001 From: mikep996 <54072448+mikep996@users.noreply.github.com> Date: Mon, 4 Nov 2024 15:49:51 +0100 Subject: [PATCH 3/5] Update README.md coverage support --- vunit/aes-encryption/README.md | 95 ++++++---------------------------- 1 file changed, 16 insertions(+), 79 deletions(-) diff --git a/vunit/aes-encryption/README.md b/vunit/aes-encryption/README.md index 1eaa77f..b8a3857 100644 --- a/vunit/aes-encryption/README.md +++ b/vunit/aes-encryption/README.md @@ -79,90 +79,27 @@ Replace the 'activehdl.py' in /vunit_sim_if with the file ae **Figure 5:** Console view after running test with Active-HDL using five threads. -## 4. Logic Data Profiling +## 4. Coveraging -### 4.1. Using Riviera-PRO Logic Data Profiler +### 4.1. Using Active-HDL/Riviera-PRO Coverage -To use logic data profiler with Riviera-PRO apply the necessary patch and generate profiler report. Following steps will guide you through the whole process starting from localize vunit_hdl package and replacing necessary files. +To use coverage (statement and branch) with Active-HDL/Riviera-PRO following arguments added to the run.py: -**4.1.1.** ```pip show vunit_hdl``` +lib.set_compile_option("activehdl.vcom_flags", ["-coverage","sb"]) +lib.set_sim_option("activehdl.vsim_flags", ["-acdb_cov sb"]) +or +lib.set_compile_option("rivierapro.vcom_flags", ["-coverage","sb"]) +lib.set_sim_option("rivierapro.vsim_flags", ["-acdb_cov sb"]) -![Figure 6: Vunit Package location in Ubuntu OS](img/VUnit_Package_location_in_Ubuntu_OS.png) +For more info about other coverages like assertion, expression, conditional or FSM please check Active-HDL/Riviera-PRO documentation. -**Figure 6:** Vunit Package location in Unix OS. +### 4.2 Collecting and Merging Coverage data -**4.1.2.** Replace files named 'factory.py' and 'rivierapro.py' in /vunit/sim_if with the files 'factory.py' and 'rivierapro.py' from aes-encryption/patch directory. +To summariuse coverage data please execute enclosed acdb.do script after VUnit: -**4.1.3.** Uncomment line 92nd in run.py script to enable logic profiler +Active-HDL: +vsim -c -tcl acdb.do; quit +Riviera-PRO +vsim -c -do acdb.do; quit -![Figure 7: Uncommented line with enabled Data Profiling](img/Uncommented_line_with_enabled_Data_Profiling.png) - -**Figure 7:** Uncommented line with enabled Data Profiling. - -**4.1.4.** Run the run.py script to generate profiler output files - -```python3 run.py``` - -**4.1.5.** Run the Riviera-PRO simulator in aes-encryption directory - -**4.1.6.** In the Riviera-PRO console type below command to generate profiler report - -```profiler report -tbp $curdir/vunit_out/test_output//rivierapro/Profiler/profiler.tbp -html .html``` - -Profiler Report will be stored in the aes-encryption directory. - -**4.1.7.** Open Profiler Report file for review the profiling results - -![Figure 8: Example Profiler Report in Riviera-PRO](img/Example_Profiler_Report_in_Riviera-PRO.png) - -**Figure 8:** Example Profiler Report in Riviera-PRO. - -### 4.2 Using Active-HDL Logic Data Profiler - -To use logic data profiler with Active-HDL apply the necessary patch and generate profiler report. Following steps will guide you through the whole process starting from localize vunit_hdl package and replacing necessary files. - -**4.2.1.** pip show vunit_hdl - -![Figure 9: Vunit Package location in Windows OS](img/VUnit_Package_location_in_Windows_OS.png) - -**Figure 9:** Vunit Package location in Windows OS. - -**4.2.2.** Replace files named 'factory.py' and 'activehdl.py' in /vunit/sim_if with the files 'factory.py' and 'activehdl.py' from aes-encryption/patch directory. - -**4.2.3.** Uncomment line 92nd in run.py script to enable logic profiler - -![Figure 10: Uncommented line with enabled Data Profiling](img/Uncommented_line_with_enabled_Data_Profiling.png) - -**Figure 10:** Uncommented line with enabled Data Profiling. - -**4.2.4.** Run the run.py script in GUI mode - -```python run.py -g``` - -**4.2.5.** Run the VUnit simulation via Active-HDL console - -```run -all``` - -![Figure 11: Running Vunit simulation in Active-HDL](img/Running_VUnit_simulation_in_Active-HDL.png) - -**Figure 11:** Running Vunit simulation in Active-HDL. - -**4.2.6** Stop the simulation to obtain Profiler results - -```endsim``` - -![Figure 12: Stopping VUnit sinulation in Active-HDL](img/Stopping_VUnit_simulation_in_Active-HDL.png) - -**Figure 12:** Stopping VUnit sinulation in Active-HDL. - -**4.2.7.** In the Active-HDL console type the following command to generate profiler report: - -```profiler report -tbp $dsn/../../../Profiler/profiler.tbp -html $dsn/../../../Profiler/.html``` - -Profiler Report will be stored in the aes-encryption/vunit_out/test_output//Profiler directory. - -**4.2.8.** Open Profiler Report file for review the profiling results - -![Figure 13: Example Profiler Report in Active-HDL](img/Example_Profiler_Report_in_Active-HDL.png) - -**Figure 13:** Example Profiler Report in Active-HDL. +As a result all acdb files will be merged and html reports will be generated withion /acdb subfolder. From b30d24fc28fce2ea95e43bc6d74d3ac02ed343a3 Mon Sep 17 00:00:00 2001 From: mikep996 <54072448+mikep996@users.noreply.github.com> Date: Mon, 4 Nov 2024 16:05:57 +0100 Subject: [PATCH 4/5] Update README.md --- vunit/aes-encryption/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vunit/aes-encryption/README.md b/vunit/aes-encryption/README.md index b8a3857..c9475d6 100644 --- a/vunit/aes-encryption/README.md +++ b/vunit/aes-encryption/README.md @@ -91,8 +91,6 @@ or lib.set_compile_option("rivierapro.vcom_flags", ["-coverage","sb"]) lib.set_sim_option("rivierapro.vsim_flags", ["-acdb_cov sb"]) -For more info about other coverages like assertion, expression, conditional or FSM please check Active-HDL/Riviera-PRO documentation. - ### 4.2 Collecting and Merging Coverage data To summariuse coverage data please execute enclosed acdb.do script after VUnit: @@ -103,3 +101,5 @@ Riviera-PRO vsim -c -do acdb.do; quit As a result all acdb files will be merged and html reports will be generated withion /acdb subfolder. + +For more info about other coverages like assertion, expression, conditional or FSM, verification plan etc. please refer directly to Active-HDL/Riviera-PRO documentation. From d9279ade5b75e3687313ccdd58682dc3f163f07d Mon Sep 17 00:00:00 2001 From: mikep996 <54072448+mikep996@users.noreply.github.com> Date: Mon, 4 Nov 2024 16:12:26 +0100 Subject: [PATCH 5/5] Update README.md --- vunit/aes-encryption/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vunit/aes-encryption/README.md b/vunit/aes-encryption/README.md index c9475d6..c9de7f3 100644 --- a/vunit/aes-encryption/README.md +++ b/vunit/aes-encryption/README.md @@ -102,4 +102,12 @@ vsim -c -do acdb.do; quit As a result all acdb files will be merged and html reports will be generated withion /acdb subfolder. +## 5. Contribution tests + +To run only necessary tests please define environment variable: +CONTRIBUTRION + +If variable is equal to 1 the list of tests form contribution.txt file will be only executed. +Previously generated /acdb/rank.html report can be used as a source of contributed test list. + For more info about other coverages like assertion, expression, conditional or FSM, verification plan etc. please refer directly to Active-HDL/Riviera-PRO documentation.