Skip to content

Commit 418c33d

Browse files
arcuri82arcuri82
authored
Merge pull request #1364 from WebFuzzing/vw-test-summary-example
Vw test summary example
2 parents 3eb5fcb + 482520c commit 418c33d

6 files changed

Lines changed: 124 additions & 33 deletions

File tree

‎core/src/main/kotlin/org/evomaster/core/output/service/RestTestCaseWriter.kt‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import org.evomaster.core.output.Lines
66
import org.evomaster.core.output.SqlWriter
77
import org.evomaster.core.output.TestCase
88
import org.evomaster.core.output.TestWriterUtils
9+
import org.evomaster.core.problem.api.param.Param
910
import org.evomaster.core.problem.enterprise.EnterpriseActionResult
1011
import org.evomaster.core.problem.httpws.HttpWsAction
1112
import org.evomaster.core.problem.httpws.HttpWsCallResult
@@ -15,12 +16,18 @@ import org.evomaster.core.problem.rest.data.RestCallResult
1516
import org.evomaster.core.problem.rest.data.RestIndividual
1617
import org.evomaster.core.problem.rest.link.RestLinkParameter
1718
import org.evomaster.core.problem.rest.param.BodyParam
19+
import org.evomaster.core.problem.rest.param.HeaderParam
20+
import org.evomaster.core.problem.rest.param.PathParam
21+
import org.evomaster.core.problem.rest.param.QueryParam
1822
import org.evomaster.core.problem.rest.service.CallGraphService
1923
import org.evomaster.core.search.action.Action
2024
import org.evomaster.core.search.action.ActionResult
2125
import org.evomaster.core.search.EvaluatedIndividual
2226
import org.evomaster.core.search.Individual
27+
import org.evomaster.core.search.gene.collection.EnumGene
28+
import org.evomaster.core.search.gene.interfaces.NamedExamplesGene
2329
import org.evomaster.core.search.gene.utils.GeneUtils
30+
import org.evomaster.core.search.gene.wrapper.ChoiceGene
2431
import org.evomaster.core.utils.StringUtils
2532
import org.slf4j.LoggerFactory
2633
import java.nio.file.Path
@@ -542,6 +549,25 @@ class RestTestCaseWriter : HttpWsTestCaseWriter {
542549
return ind.seeFullTreeGenes()
543550
.filter { it.name == RestActionBuilderV3.EXAMPLES_NAME }
544551
.filter { it.staticCheckIfImpactPhenotype() }
545-
.map { it.getValueAsRawString() }
552+
.map {
553+
val name = if(it is NamedExamplesGene){
554+
"(${it.getValueName()?: "-"}) "
555+
} else {
556+
""
557+
}
558+
559+
val param = it.getFirstParent { p -> p is Param }
560+
val pName = when(param) {
561+
is QueryParam -> "QUERY: ${param.name}"
562+
is HeaderParam -> "HEADER: ${param.name}"
563+
is PathParam -> "PATH: ${param.name}"
564+
is BodyParam -> "BODY"
565+
else -> ""
566+
}
567+
568+
val value = it.getValueAsRawString()
569+
570+
"$name$pName -> $value"
571+
}
546572
}
547573
}

‎core/src/main/kotlin/org/evomaster/core/problem/rest/builder/RestActionBuilderV3.kt‎

Lines changed: 48 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -559,21 +559,26 @@ object RestActionBuilderV3 {
559559
example: Any?,
560560
examples: Map<String, Example>?,
561561
messages: MutableList<String>
562-
) : List<Any>{
562+
) : List<Pair<Any,String?>>{
563+
564+
/**
565+
* List of pairs value/name.
566+
* the name if optional, as only defined for "examples"
567+
*/
568+
val data = mutableListOf<Pair<Any, String?>>()
563569

564-
val data = mutableListOf<Any>()
565570
if(example != null){
566-
data.add(example)
571+
data.add(Pair(example,null))
567572
}
568573
if(!examples.isNullOrEmpty()){
569-
examples.values.forEach {
570-
val exm = if(it.`$ref` != null){
571-
SchemaUtils.getReferenceExample(schemaHolder, currentSchema, it.`$ref`, messages)
574+
examples.entries.forEach {
575+
val exm = if(it.value.`$ref` != null){
576+
SchemaUtils.getReferenceExample(schemaHolder, currentSchema, it.value.`$ref`, messages)
572577
} else {
573-
it
578+
it.value
574579
}
575580
if(exm != null) {
576-
data.add(exm.value)
581+
data.add(Pair(exm.value, it.key))
577582
}
578583
}
579584
}
@@ -780,7 +785,7 @@ object RestActionBuilderV3 {
780785
referenceClassDef: String?,
781786
options: Options,
782787
isInPath: Boolean = false,
783-
examples: List<Any> = listOf(),
788+
examples: List<Pair<Any,String?>> = listOf(),
784789
messages: MutableList<String>
785790
): Gene {
786791

@@ -974,7 +979,7 @@ object RestActionBuilderV3 {
974979
history: Deque<String>,
975980
referenceTypeName: String?,
976981
options: Options,
977-
examples: List<Any>,
982+
examples: List<Pair<Any,String?>>,
978983
messages: MutableList<String>
979984
): Gene {
980985

@@ -1131,7 +1136,7 @@ object RestActionBuilderV3 {
11311136
history: Deque<String>,
11321137
referenceTypeName: String?,
11331138
options: Options,
1134-
examples: List<Any>,
1139+
examples: List<Pair<Any,String?>>,
11351140
messages: MutableList<String>
11361141
) : Gene{
11371142
/*
@@ -1237,7 +1242,7 @@ object RestActionBuilderV3 {
12371242
fields: List<Gene>,
12381243
additionalFieldTemplate: PairGene<StringGene, Gene>?,
12391244
referenceTypeName: String?,
1240-
otherExampleValues: List<Any>,
1245+
otherExampleValues: List<Pair<Any,String?>>,
12411246
messages: MutableList<String>
12421247
) : Gene{
12431248
if (fields.isEmpty()) {
@@ -1269,19 +1274,29 @@ object RestActionBuilderV3 {
12691274
val exampleValue = if(options.probUseExamples > 0) schema.example else null
12701275
val multiExampleValues = if(options.probUseExamples > 0) schema.examples else null
12711276

1272-
val examples = mutableListOf<ObjectGene>()
1277+
val examples = mutableListOf<Pair<ObjectGene,String?>>()
12731278
if(exampleValue != null){
12741279
duplicateObjectWithExampleFields(name,mainGene, exampleValue)?.let {
1275-
examples.add(it)
1280+
examples.add(Pair(it,null))
12761281
}
12771282
}
12781283
if(multiExampleValues != null ){
1279-
examples.addAll(multiExampleValues.mapNotNull { duplicateObjectWithExampleFields(name,mainGene, it) })
1284+
examples.addAll(multiExampleValues
1285+
.mapNotNull { duplicateObjectWithExampleFields(name,mainGene, it) }
1286+
.map { Pair(it, null) }
1287+
)
12801288
}
1281-
examples.addAll(otherExampleValues.mapNotNull { duplicateObjectWithExampleFields(name,mainGene, it) })
1289+
examples.addAll(otherExampleValues
1290+
.mapNotNull { duplicateObjectWithExampleFields(name,mainGene, it.first)
1291+
?.let { obj -> Pair(obj,it.second) }
1292+
}
1293+
)
1294+
1295+
val v = examples.map { it.first } //values
1296+
val n = examples.map{it.second} // names
12821297

12831298
val exampleGene = if(examples.isNotEmpty()){
1284-
ChoiceGene(EXAMPLES_NAME, examples)
1299+
ChoiceGene(EXAMPLES_NAME, v, valueNames = n)
12851300
} else null
12861301
val defaultGene = if(defaultValue != null){
12871302
duplicateObjectWithExampleFields("default", mainGene, defaultValue)
@@ -1372,7 +1387,7 @@ object RestActionBuilderV3 {
13721387
options: Options,
13731388
collectionTemplate: Gene?,
13741389
isInPath: Boolean,
1375-
examples: List<Any>,
1390+
examples: List<Pair<Any,String?>>,
13761391
messages: MutableList<String>
13771392
) : Gene{
13781393

@@ -1412,7 +1427,7 @@ object RestActionBuilderV3 {
14121427
collectionTemplate: Gene? = null,
14131428
//might need to add extra constraints if in path
14141429
isInPath: Boolean,
1415-
exampleObjects: List<Any>,
1430+
exampleObjects: List<Pair<Any,String?>>,
14161431
format: String? = null,
14171432
messages: MutableList<String>
14181433
) : Gene{
@@ -1544,10 +1559,12 @@ object RestActionBuilderV3 {
15441559
val exampleValue = if(options.probUseExamples > 0) schema.example else null
15451560
val multiExampleValues = if(options.probUseExamples > 0) schema.examples else null
15461561

1547-
val examples = mutableListOf<String>()
1562+
//value and optional name
1563+
val examples = mutableListOf<Pair<String,String?>>()
1564+
15481565
if(exampleValue != null) {
15491566
val raw = asRawString(exampleValue)
1550-
examples.add(raw)
1567+
examples.add(Pair(raw,null))
15511568
val arrayM = if(raw.startsWith("[")) "If you are wrongly passing to it an array of values, " +
15521569
"the parser would read it as an array string or simply ignore it. "
15531570
else ""
@@ -1558,9 +1575,9 @@ object RestActionBuilderV3 {
15581575
}
15591576
if(multiExampleValues != null && multiExampleValues.isNotEmpty()){
15601577
//possibly bug in parser, but it was reading strings values double-quoted in this case
1561-
examples.addAll(multiExampleValues.map { asRawString(it) })
1578+
examples.addAll(multiExampleValues.map { Pair(asRawString(it), null) })
15621579
}
1563-
examples.addAll( exampleObjects.map { asRawString(it) })
1580+
examples.addAll( exampleObjects.map { Pair(asRawString(it.first), it.second) })
15641581

15651582

15661583
val defaultGene = if(defaultValue != null){
@@ -1581,15 +1598,20 @@ object RestActionBuilderV3 {
15811598
}
15821599
} else null
15831600

1601+
//values
1602+
val v = examples.map { it.first }
1603+
//optional names
1604+
val n = examples.map { it.second }
1605+
15841606
val exampleGene = if(examples.isNotEmpty()){
15851607
when{
15861608
NumberGene::class.java.isAssignableFrom(geneClass)
1587-
-> EnumGene(EXAMPLES_NAME, examples,0,true)
1609+
-> EnumGene(EXAMPLES_NAME, v,0,true, n)
15881610

15891611
geneClass == StringGene::class.java
15901612
|| geneClass == Base64StringGene::class.java
15911613
|| geneClass == RegexGene::class.java
1592-
-> EnumGene<String>(EXAMPLES_NAME, examples,0,false)
1614+
-> EnumGene<String>(EXAMPLES_NAME, v,0,false, n)
15931615

15941616
//TODO Arrays
15951617
else -> {
@@ -1692,7 +1714,7 @@ object RestActionBuilderV3 {
16921714
currentSchema: SchemaOpenAPI,
16931715
history: Deque<String> = ArrayDeque(),
16941716
options: Options,
1695-
examples: List<Any>,
1717+
examples: List<Pair<Any,String?>>,
16961718
messages: MutableList<String>
16971719
): Gene {
16981720

‎core/src/main/kotlin/org/evomaster/core/search/gene/collection/EnumGene.kt‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package org.evomaster.core.search.gene.collection
22

33
import org.evomaster.core.output.OutputFormat
44
import org.evomaster.core.search.gene.Gene
5+
import org.evomaster.core.search.gene.interfaces.NamedExamplesGene
56
import org.evomaster.core.search.gene.string.StringGene
67
import org.evomaster.core.search.gene.root.SimpleGene
78
import org.evomaster.core.search.gene.utils.GeneUtils
@@ -30,8 +31,13 @@ class EnumGene<T : Comparable<T>>(
3031
* to avoid specifying exact types. Still, should not be printed out as string.
3132
* Recall that an enum is just a group of constants that cannot be mutated
3233
*/
33-
private val treatAsNotString : Boolean = false
34-
) : SimpleGene(name) {
34+
private val treatAsNotString : Boolean = false,
35+
/**
36+
* An optional list of 'names' for each/some of the values in this enumeration.
37+
* This is usually just extra information, eg, to recognize named "examples" in OpenAPI schemas
38+
*/
39+
private val valueNames: List<String?>? = null
40+
) : SimpleGene(name), NamedExamplesGene {
3541

3642
companion object {
3743

@@ -81,6 +87,10 @@ class EnumGene<T : Comparable<T>>(
8187
if (index < 0 || index >= values.size) {
8288
throw IllegalArgumentException("Invalid index: $index")
8389
}
90+
91+
if(valueNames != null && valueNames.size != values.size) {
92+
throw IllegalArgumentException("Invalid valueNames size: ${valueNames.size}!=${values.size}")
93+
}
8494
}
8595

8696
if(treatAsNotString && values.isNotEmpty() && values[0] !is String){
@@ -98,7 +108,7 @@ class EnumGene<T : Comparable<T>>(
98108

99109
override fun copyContent(): Gene {
100110
//recall: "values" is immutable
101-
return EnumGene<T>(name, values, index, treatAsNotString)
111+
return EnumGene<T>(name, values, index, treatAsNotString, valueNames)
102112
}
103113

104114
override fun setValueWithRawString(value: String) {
@@ -183,6 +193,10 @@ class EnumGene<T : Comparable<T>>(
183193
return values[index].toString()
184194
}
185195

196+
override fun getValueName(): String?{
197+
return valueNames?.get(index)
198+
}
199+
186200
override fun copyValueFrom(other: Gene): Boolean {
187201
if (other !is EnumGene<*>) {
188202
throw IllegalArgumentException("Invalid gene type ${other.javaClass}")

‎core/src/main/kotlin/org/evomaster/core/search/gene/interfaces/NamedExamplesGene.kt‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package org.evomaster.core.search.gene.interfaces
2+
3+
/**
4+
* A gene representing possible different examples provided by the user.
5+
* Such examples might have unique names/ids used to easily identify them
6+
*/
7+
interface NamedExamplesGene {
8+
9+
fun getValueName(): String?
10+
}

‎core/src/main/kotlin/org/evomaster/core/search/gene/wrapper/ChoiceGene.kt‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package org.evomaster.core.search.gene.wrapper
33
import org.evomaster.core.logging.LoggingUtil
44
import org.evomaster.core.output.OutputFormat
55
import org.evomaster.core.search.gene.Gene
6+
import org.evomaster.core.search.gene.interfaces.NamedExamplesGene
67
import org.evomaster.core.search.gene.root.CompositeFixedGene
78
import org.evomaster.core.search.gene.utils.GeneUtils
89
import org.evomaster.core.search.service.AdaptiveParameterControl
@@ -26,9 +27,14 @@ class ChoiceGene<T>(
2627
/**
2728
* Potentially, associate different probabilities for the different choices
2829
*/
29-
probabilities: List<Double>? = null
30+
probabilities: List<Double>? = null,
31+
/**
32+
* Optional list of name values for each of choices.
33+
* This is usually just extra information, eg, to recognize named "examples" in OpenAPI schemas
34+
*/
35+
valueNames: List<String?>? = null,
3036

31-
) : CompositeFixedGene(name, geneChoices), WrapperGene where T : Gene {
37+
) : CompositeFixedGene(name, geneChoices), NamedExamplesGene, WrapperGene where T : Gene {
3238

3339
companion object {
3440
private val log: Logger = LoggerFactory.getLogger(ChoiceGene::class.java)
@@ -39,6 +45,8 @@ class ChoiceGene<T>(
3945

4046
private val probabilities = probabilities?.toList() //make a copy
4147

48+
private val valueNames = valueNames?.toList()
49+
4250
init {
4351
if (geneChoices.isEmpty()) {
4452
throw IllegalArgumentException("The list of gene choices cannot be empty")
@@ -50,6 +58,9 @@ class ChoiceGene<T>(
5058
if(probabilities != null && probabilities.size != geneChoices.size){
5159
throw IllegalArgumentException("If probabilities are defined, then they must be same number as the genes")
5260
}
61+
if(valueNames != null && valueNames.size != geneChoices.size) {
62+
throw IllegalArgumentException("If value names are defined, then they must be same number as the genes")
63+
}
5364
}
5465

5566

@@ -167,6 +178,10 @@ class ChoiceGene<T>(
167178
.getValueAsRawString()
168179
}
169180

181+
override fun getValueName(): String?{
182+
return valueNames?.get(activeGeneIndex)
183+
}
184+
170185
/**
171186
* Copies the value of the other gene. The other gene
172187
* does not have to be [ChoiceGene].
@@ -260,7 +275,8 @@ class ChoiceGene<T>(
260275
name,
261276
activeChoice = this.activeGeneIndex,
262277
geneChoices = this.geneChoices.map { it.copy() }.toList(),
263-
probabilities = probabilities // immutable
278+
probabilities = probabilities, // immutable
279+
valueNames = valueNames // immutable
264280
)
265281

266282
/**

‎release_notes.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ Under development in `master` branch.
1111
- Option _--disabledOracleCodes_ to disable the checking of specific fault types based on their WFC fault codes. By default, all fault types are checked for.
1212
- Option _--endpointExclude_ to exclude specific REST endpoints from the fuzzing.
1313

14+
### Improvements
15+
16+
- Generated tests for REST APIs now do have a better summaries for the use of "examples" entries.
1417

1518
### Addressed GitHub Issues
1619

0 commit comments

Comments
 (0)