Merge pull request #1392 from WebFuzzing/refactor-copyfrom-v2

arcuri82 · web-flow · commit be3d9e64f5e2 · 2025-11-28T13:09:56.000+01:00
Refactor copyfrom v2
diff --git a/core/src/main/kotlin/org/evomaster/core/problem/enterprise/EnterpriseIndividual.kt b/core/src/main/kotlin/org/evomaster/core/problem/enterprise/EnterpriseIndividual.kt
@@ -180,6 +180,8 @@ abstract class EnterpriseIndividual(
      */
     fun ensureFlattenedStructure() : Boolean{
 
+        Lazy.assert { verifyValidity(); true }
+
         val before = seeAllActions().size
 
         val issues = doFlattenStructure()
@@ -189,14 +191,7 @@ abstract class EnterpriseIndividual(
         //no base action should have been lost
         Lazy.assert { seeAllActions().size == before }
 
-        /*
-            FIXME There is some major bugs in Gene regarding
-            copyValueFrom() and setFromDifferentGene()
-            until fixed, this check will fail.
-            but the fix will require some major refactoring and testing... it will take time
-            TODO put back once fixed
-         */
-        //verifyValidity()
+        Lazy.assert { verifyValidity(); true }
 
         return issues
     }
diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/data/RestIndividual.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/data/RestIndividual.kt
@@ -154,6 +154,11 @@ class RestIndividual(
                 .sortBy { if ((it as SqlAction).representExistingData) 0 else 1 }
         }
 
+        /*
+            Can't do this, as it can change how the test behave when there are bounded tainted values
+         */
+        //seeAllActions().forEach { it.forceNewTaints() }
+
         /*
             if we move any environment action to the beginning of the individual, it might impact the fitness
          */
diff --git a/core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestIndividualBuilder.kt b/core/src/main/kotlin/org/evomaster/core/problem/rest/service/RestIndividualBuilder.kt
@@ -103,6 +103,7 @@ class RestIndividualBuilder {
             other.resetLocalIdRecursively()
 
             //avoid possible taint id conflicts
+            base.seeAllActions().forEach { it.forceNewTaints() }
             other.seeAllActions().forEach { it.forceNewTaints() }
 
             val duplicates = base.addInitializingActions(other.seeInitializingActions())
@@ -122,7 +123,7 @@ class RestIndividualBuilder {
             //merge shouldn't lose any actions
             assert(before == (after+duplicates)) { "$after+$duplicates!=$before" }
 
-            base.verifyValidity()
+            base.verifyValidity(true)
 
             return base
         }
diff --git a/core/src/main/kotlin/org/evomaster/core/search/Individual.kt b/core/src/main/kotlin/org/evomaster/core/search/Individual.kt
@@ -180,7 +180,7 @@ abstract class Individual(
      * All invariants should always be satisfied after any modification of the individual.
      * If not, this is a bug.
      */
-    fun verifyValidity(){
+    fun verifyValidity(checkForTaints: Boolean = false){
 
         groupsView()?.verifyGroups()
 
@@ -202,9 +202,18 @@ abstract class Individual(
             throw IllegalStateException("There are invalid local ids:\n" + localIdErrors.joinToString("\n"))
         }
 
-        val taintIdErrors = verifyTaintIds()
-        if(taintIdErrors.isNotEmpty()){
-            throw IllegalStateException("There are invalid taint ids:\n" + taintIdErrors.joinToString("\n"))
+        /*
+            We cannot really verify it all the time.
+            Duplicates might exist due to bounded genes.
+            But flattening (done at minimization, for example) removes the binding, leading
+            this check to fail.
+            further problem, many phases (eg security) are done _after_ minimization...
+         */
+        if(checkForTaints) {
+            val taintIdErrors = verifyTaintIds()
+            if (taintIdErrors.isNotEmpty()) {
+                throw IllegalStateException("There are invalid taint ids:\n" + taintIdErrors.joinToString("\n"))
+            }
         }
     }
 
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/Gene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/Gene.kt
@@ -876,12 +876,16 @@ abstract class Gene(
     /**
      * Genes might contain a value that is also stored
      * in another gene of the same type.
+     *
+     * If the type of [other] is different, this method might throw an [IllegalArgumentException]
+     *
+     * TODO refactor, in which type-check is done here
      */
     abstract fun containsSameValueAs(other: Gene): Boolean
 
 
     /**
-     * evaluate whether [this] and [gene] belong to one evolution during search
+     * evaluate whether `this` and [gene] belong to one evolution during search
      */
     open fun possiblySame(gene: Gene): Boolean = gene.name == name && gene::class == this::class
 
@@ -1210,8 +1214,7 @@ abstract class Gene(
             //revert back
             val success = unsafeCopyValueFrom(current)
             //reversion should always work... if fails, it is a bug
-            // FIXME put back once all are implemented, eg, TaintedMapGene currently missing
-            //assert(success)
+            assert(success)
             return false
         }
         return true
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/ObjectGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/ObjectGene.kt
@@ -112,11 +112,10 @@ class ObjectGene(
                 .forEach { it.randomize(randomness, tryToForceNewValue) }
 
         if (!isFixed){
-            Lazy.assert {
-                template != null && additionalFields != null
-            }
-            if (additionalFields!!.isNotEmpty())
+            Lazy.assert { template != null && additionalFields != null }
+            if (additionalFields!!.isNotEmpty()) {
                 killChildren(additionalFields!!)
+            }
             val num = randomness.nextInt(MAX_SIZE_ADDITIONAL_FIELDS)
             repeat(num){
                 val added = sampleElementToAdd(randomness)
@@ -265,9 +264,12 @@ class ObjectGene(
         }
 
         if(!isFixed){
-            //TODO what if there is a mismatch here? semantic of this function is unclear
-            for (i in additionalFields!!.indices){
-                ok = ok && this.additionalFields!![i].unsafeCopyValueFrom(other.additionalFields!![i])
+            if (additionalFields!!.isNotEmpty()) {
+                killChildren(additionalFields!!)
+            }
+            val otherAdditionalFields = other.additionalFields
+            otherAdditionalFields?.forEach {
+                addChild(it.copy())
             }
         }
 
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/SeededGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/SeededGene.kt
@@ -114,15 +114,14 @@ class SeededGene<T>(
         if (other !is SeededGene<*>)
            return false
 
+        this.employSeeded = other.employSeeded
+        this.isEmploySeededMutable = other.isEmploySeededMutable
+
         val ok = if (employSeeded)
             this.seeded.unsafeCopyValueFrom(other.seeded)
         else
             this.gene.unsafeCopyValueFrom(other.gene as Gene)
 
-        if (ok){
-            this.employSeeded = other.employSeeded
-            this.isEmploySeededMutable = other.isEmploySeededMutable
-        }
         return ok
     }
 
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/UUIDGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/UUIDGene.kt
@@ -92,7 +92,7 @@ class UUIDGene(
         val gene = other.getPhenotype()
 
         if (gene !is UUIDGene) {
-            throw IllegalArgumentException("Invalid gene type ${gene.javaClass}")
+            return false
         }
         return this.mostSigBits.unsafeCopyValueFrom(gene.mostSigBits)
                 && this.leastSigBits.unsafeCopyValueFrom(gene.leastSigBits)
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/collection/FlexibleMapGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/collection/FlexibleMapGene.kt
@@ -58,10 +58,18 @@ where T : Gene {
     }
 
     override fun unsafeCopyValueFrom(other: Gene): Boolean {
-        //TODO
-        LoggingUtil.uniqueWarn(log,"unsafeCopyValueFrom() not implemented for FlexibleMapGene")
 
-        return false
+        if(other !is FlexibleMapGene<*>) {
+            return false
+        }
+
+        killAllChildren()
+        val elements = other.elements
+            .mapNotNull { it.copy() as? PairGene<T, FlexibleGene> }
+            .toMutableList()
+        elements.forEach { it.resetLocalIdRecursively() }
+        addChildren(elements)
+        return true
     }
 
     override fun isPrintable(): Boolean {
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/collection/MapGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/collection/MapGene.kt
@@ -243,7 +243,14 @@ abstract class MapGene<K, V>(
             TODO support other types if needed
          */
         if (isElementApplicableToUniqueCheck(geneValue)){
-            return elements.filter { ParamUtil.getValueGene(it.first).containsSameValueAs(ParamUtil.getValueGene(geneValue)) }
+            return elements.filter {
+                try {
+                    //TODO need refactoring/cleanup... should avoid try/catch
+                    ParamUtil.getValueGene(it.first).containsSameValueAs(ParamUtil.getValueGene(geneValue))
+                }catch (e: Exception){
+                    false
+                }
+            }
         }
         return listOf()
     }
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/collection/TaintedMapGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/collection/TaintedMapGene.kt
@@ -220,16 +220,44 @@ class TaintedMapGene(
 
 
     override fun containsSameValueAs(other: Gene): Boolean {
-        //TODO
 
-        return false
+        if(other !is TaintedMapGene){
+            return false
+        }
+
+        if(this.elements.size != other.elements.size){
+            return false
+        }
+
+        for(e in this.elements){
+            val x = other.elements.find { it.first.name == e.first.name }
+                ?: return false
+            if(! e.second.containsSameValueAs(x.second)){
+                return false
+            }
+        }
+
+        return true
     }
 
     override fun unsafeCopyValueFrom(other: Gene): Boolean {
-        //TODO
-        LoggingUtil.uniqueWarn(log,"unsafeCopyValueFrom() not implemented for TaintedMapGene")
 
-        return false
+        if(other !is TaintedMapGene){
+            return false
+        }
+
+        killAllChildren()
+        learnedKeys.clear()
+        learnedKeys.addAll(other.learnedKeys)
+        learnedTypes.clear()
+        learnedTypes.putAll(other.learnedTypes)
+
+        other.elements.forEach {
+            addElement(it.copy() as PairGene<StringGene, Gene>)
+        }
+        taintId = other.taintId
+
+        return true
     }
 
 
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/PatternCharacterBlockGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/PatternCharacterBlockGene.kt
@@ -50,7 +50,7 @@ class PatternCharacterBlockGene(
 
     override fun containsSameValueAs(other: Gene): Boolean {
         if (other !is PatternCharacterBlockGene) {
-            throw IllegalArgumentException("Invalid gene type ${other.javaClass}")
+            return false
         }
         return this.stringBlock == other.stringBlock
     }
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlAutoIncrementGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlAutoIncrementGene.kt
@@ -58,7 +58,13 @@ class SqlAutoIncrementGene(name: String) : SimpleGene(name) {
         if (other !is SqlAutoIncrementGene) {
             return false
         }
-        return this === other
+
+        /*
+            auto-increment are always unique... but need to satisfy invariants on copyFrom...
+            TODO  might check for side-effects, and see if need more complex handling
+         */
+        //return this === other
+        return true
     }
 
     override fun isMutable() = false
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlForeignKeyGene.kt
@@ -142,6 +142,11 @@ class SqlForeignKeyGene(
         return true
     }
 
+    override fun isPrintable(): Boolean {
+        //TODO what about pk? anyway, likely PK/FK will need a major overhaul...
+        return isBound() || nullable
+    }
+
     override fun getValueAsPrintableString(previousGenes: List<Gene>, mode: GeneUtils.EscapeMode?, targetFormat: OutputFormat?, extraCheck: Boolean): String {
 
         if (!isBound()) {
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlRangeGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/SqlRangeGene.kt
@@ -92,6 +92,11 @@ class SqlRangeGene<T>(
         if (other !is SqlRangeGene<*>) {
             throw IllegalArgumentException("Invalid gene type ${other.javaClass}")
         }
+
+        if(this.template.javaClass != other.template.javaClass) {
+            return false
+        }
+
         return isLeftClosed.containsSameValueAs(other.isLeftClosed)
                 && left.containsSameValueAs(other.left as Gene)
                 && right.containsSameValueAs(other.right as Gene)
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/geometric/SqlPolygonGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/geometric/SqlPolygonGene.kt
@@ -46,14 +46,15 @@ class SqlPolygonGene(
         val pointList = mutableListOf<SqlPointGene>()
         repeat(minLengthOfPolygonRing) {
             val newGene = points.template.copy() as SqlPointGene
+            newGene.doInitialize(randomness)
             pointList.add(newGene)
             do {
                 newGene.randomize(randomness, tryToForceNewValue)
             } while (onlyNonIntersectingPolygons && !noCrossOvers(pointList))
         }
         points.randomize(randomness, tryToForceNewValue)
         points.killAllChildren()
-        pointList.map { points.addChild(it) }
+        pointList.forEach { points.addChild(it) }
         assert(isLocallyValid())
     }
 
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/sql/textsearch/SqlTextSearchQueryGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/sql/textsearch/SqlTextSearchQueryGene.kt
@@ -78,6 +78,7 @@ class SqlTextSearchQueryGene(
          */
         if (queryLexemes.getViewOfElements().isEmpty()) {
             val stringGene = StringGene("lexeme")
+            stringGene.doInitialize(randomness)
             stringGene.randomize(randomness, tryToForceNewValue)
             queryLexemes.addElement(stringGene)
         }
diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/wrapper/ChoiceGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/wrapper/ChoiceGene.kt
@@ -199,6 +199,20 @@ class ChoiceGene<T>(
             TODO this is bit limited... what about if there are wrapper genes involved?
         */
 
+        //first check if can find gene of same type
+        for(i in geneChoices.indices){
+            val g = geneChoices[i]
+
+            if(g.javaClass == x.javaClass) {
+                val updated = g.copyValueFrom(x)
+                if (updated) {
+                    this.activeGeneIndex = i
+                    return true
+                }
+            }
+        }
+
+        //if none found, check at any other possibly compatible types
         for(i in geneChoices.indices){
             val g = geneChoices[i]
 
@@ -208,11 +222,12 @@ class ChoiceGene<T>(
                 of them would be fine. also if one works, we should not break validity of the other
                 options even if not selected
              */
-            val updated = g.copyValueFrom(x)
-            g.forceNewTaints()
-            if(updated){
-                this.activeGeneIndex = i
-                return true
+            if(g.javaClass != x.javaClass) {
+                val updated = g.copyValueFrom(x)
+                if (updated) {
+                    this.activeGeneIndex = i
+                    return true
+                }
             }
         }
 
@@ -240,12 +255,9 @@ class ChoiceGene<T>(
         if (other !is ChoiceGene<*>) {
             throw IllegalArgumentException("Invalid gene type ${other.javaClass}")
         }
-        if (this.activeGeneIndex != other.activeGeneIndex) {
-            return false
-        }
 
-        return this.geneChoices[activeGeneIndex]
-            .containsSameValueAs(other.geneChoices[activeGeneIndex])
+        return this.activeGene().javaClass == other.activeGene().javaClass
+                && this.activeGene().containsSameValueAs(other.activeGene())
     }
 
     /**
diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt b/core/src/main/kotlin/org/evomaster/core/search/service/Sampler.kt
@@ -99,7 +99,7 @@ abstract class Sampler<T> : TrackOperator where T : Individual {
 
         samplePostProcessing(ind)
 
-        org.evomaster.core.Lazy.assert { ind.verifyValidity(); true }
+        org.evomaster.core.Lazy.assert { ind.verifyValidity(true); true }
         return ind
     }
 
diff --git a/core/src/main/kotlin/org/evomaster/core/search/service/mutator/Mutator.kt b/core/src/main/kotlin/org/evomaster/core/search/service/mutator/Mutator.kt
diff --git a/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTest.kt b/core/src/test/kotlin/org/evomaster/core/problem/rest/individual/RestIndividualTest.kt
diff --git a/core/src/test/kotlin/org/evomaster/core/search/gene/GeneRandomizedTest.kt b/core/src/test/kotlin/org/evomaster/core/search/gene/GeneRandomizedTest.kt
diff --git a/core/src/test/kotlin/org/evomaster/core/search/gene/GeneSamplerForTests.kt b/core/src/test/kotlin/org/evomaster/core/search/gene/GeneSamplerForTests.kt
diff --git a/core/src/test/kotlin/org/evomaster/core/search/gene/collection/TaintedMapGeneTest.kt b/core/src/test/kotlin/org/evomaster/core/search/gene/collection/TaintedMapGeneTest.kt

Original file line number	Diff line number	Diff line change
`@@ -92,7 +92,7 @@ class UUIDGene(`
`92`	`92`	`val gene = other.getPhenotype()`
`93`	`93`
`94`	`94`	`if (gene !is UUIDGene) {`
`95`		`- throw IllegalArgumentException("Invalid gene type ${gene.javaClass}")`
	`95`	`+ return false`
`96`	`96`	`}`
`97`	`97`	`return this.mostSigBits.unsafeCopyValueFrom(gene.mostSigBits)`
`98`	`98`	`&& this.leastSigBits.unsafeCopyValueFrom(gene.leastSigBits)`