From 00670ec7786bb889f53d63d0cb7ffb45b98812ec Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sat, 18 Jul 2026 09:51:30 +0530 Subject: [PATCH 01/21] chore: add sdkman env for java --- .sdkmanrc | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .sdkmanrc diff --git a/.sdkmanrc b/.sdkmanrc new file mode 100644 index 0000000..91d8367 --- /dev/null +++ b/.sdkmanrc @@ -0,0 +1,3 @@ +# Enable auto-env through the sdkman_auto_env config +# Add key=value pairs of SDKs to use below +java=11.0.31-tem From 30cd32ab08b5da62f3c5d8a27c15f385a4464516 Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sat, 18 Jul 2026 09:58:54 +0530 Subject: [PATCH 02/21] style: format code --- pom.xml | 780 ++++++++++++++++++++++++++------------------------------ 1 file changed, 359 insertions(+), 421 deletions(-) diff --git a/pom.xml b/pom.xml index dae0bac..c59aa56 100644 --- a/pom.xml +++ b/pom.xml @@ -1,429 +1,367 @@ - - 4.0.0 - - com.societegenerale.github-crawler - github-crawler-parent - 2.2.1-SNAPSHOT - pom - - github-crawler - - - org.springframework.boot - spring-boot-starter-parent - 2.7.5 - - - - - github-crawler-core - github-crawler-autoconfigure - github-crawler-starter - - - - - - Apache 2 - http://www.apache.org/licenses/LICENSE-2.0.html - repo - - - - - https://github.com/societe-generale/github-crawler/issues - GitHub - - - - https://github.com/societe-generale/github-crawler - scm:git:git@github.com:societe-generale/github-crawler.git - scm:git:git@github.com:societe-generale/github-crawler.git - HEAD - - - - - ossrh - https://s01.oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - UTF-8 - UTF-8 - 11 - - - 1.8.10 - 2.10 - 0.8.8 - - - - - - - - jcenter - JCenter - https://jcenter.bintray.com/ - - - - - - - - org.springframework.cloud - spring-cloud-openfeign-dependencies - 3.1.5 - pom - import - - - - - - - + + 4.0.0 + com.societegenerale.github-crawler + github-crawler-parent + 2.2.1-SNAPSHOT + pom + github-crawler + + org.springframework.boot + spring-boot-starter-parent + 2.7.5 + + + + github-crawler-core + github-crawler-autoconfigure + github-crawler-starter + + + + Apache 2 + http://www.apache.org/licenses/LICENSE-2.0.html + repo + + + + https://github.com/societe-generale/github-crawler/issues + GitHub + + + https://github.com/societe-generale/github-crawler + scm:git:git@github.com:societe-generale/github-crawler.git + scm:git:git@github.com:societe-generale/github-crawler.git + + HEAD + + + + ossrh + https://s01.oss.sonatype.org/content/repositories/snapshots + + + ossrh + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ + + + + UTF-8 + UTF-8 + 11 + 1.8.10 + 2.10 + 0.8.8 + + + + jcenter + JCenter + https://jcenter.bintray.com/ + + + - - - com.google.guava - guava - 31.1-jre - - - - - org.springframework.boot - spring-boot-starter-test - test - - - com.vaadin.external.google - android-json - - - - ch.qos.logback - logback-classic - - - - - - - org.jetbrains.kotlin - kotlin-stdlib-jdk8 - ${kotlin.version} - - - - org.jetbrains.kotlin - kotlin-reflect - ${kotlin.version} - - - - org.jetbrains.kotlin - kotlin-annotations-jvm - ${kotlin.version} - - - - org.jetbrains.kotlin - kotlin-test - ${kotlin.version} - test - - - - org.dom4j - dom4j - 2.1.3 - - - - - jaxen - jaxen - 1.1.6 - - - + + org.springframework.cloud + spring-cloud-openfeign-dependencies + 3.1.5 + pom + import + - - - - ${project.basedir}/src/main/kotlin - + + + + com.google.guava + guava + 31.1-jre + + + org.springframework.boot + spring-boot-starter-test + test + + + com.vaadin.external.google + android-json + + + ch.qos.logback + logback-classic + + + + + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + ${kotlin.version} + + + org.jetbrains.kotlin + kotlin-reflect + ${kotlin.version} + + + org.jetbrains.kotlin + kotlin-annotations-jvm + ${kotlin.version} + + + org.jetbrains.kotlin + kotlin-test + ${kotlin.version} + test + + + org.dom4j + dom4j + 2.1.3 + + + jaxen + jaxen + 1.1.6 + + + + ${project.basedir}/src/main/kotlin + + + kotlin-maven-plugin + org.jetbrains.kotlin + ${kotlin.version} + + + compile + compile + + compile + + + + ${project.basedir}/src/main/kotlin + + + + + test-compile + test-compile + + test-compile + + + + ${project.basedir}/src/test/kotlin + ${project.basedir}/src/test/java + + + + + + 1.8 + + -Xjvm-default=all-compatibility + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.10.1 + + + default-compile + none + + + default-testCompile + none + + + java-compile + compile + + compile + + + + java-test-compile + test-compile + + testCompile + + + + + + org.apache.maven.plugins + maven-release-plugin + 2.5.3 + + https://s01.oss.sonatype.org/service/local/staging/deploy/maven2 + + github_crawler_@{project.version} + true + true + false + false + makeRelease + forked-path + + + + org.apache.maven.plugins + maven-surefire-plugin + + ${surefireArgLine} + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + + integration-test + verify + + + ${failsafeArgLine} + + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + + + pre-unit-tests + + prepare-agent + + + surefireArgLine + + + + pre-integration-tests + + prepare-agent-integration + + + failsafeArgLine + + + + report + test + + report + + + + + + org.eluder.coveralls + coveralls-maven-plugin + 4.3.0 + + + javax.xml.bind + jaxb-api + 2.3.1 + + + + + org.sonatype.plugins + nexus-staging-maven-plugin + 1.6.8 + true + + ossrh + https://s01.oss.sonatype.org/ + true + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.2 + + + + + + makeRelease + - - - kotlin-maven-plugin - org.jetbrains.kotlin - ${kotlin.version} - - - compile - compile - - compile - - - - ${project.basedir}/src/main/kotlin - - - - - test-compile - test-compile - - test-compile - - - - ${project.basedir}/src/test/kotlin - ${project.basedir}/src/test/java - - - - - - 1.8 - - -Xjvm-default=all-compatibility - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.10.1 - - - - default-compile - none - - - - default-testCompile - none - - - java-compile - compile - - compile - - - - java-test-compile - test-compile - - testCompile - - - - - - - org.apache.maven.plugins - maven-release-plugin - 2.5.3 - - https://s01.oss.sonatype.org/service/local/staging/deploy/maven2 - github_crawler_@{project.version} - true - true - false - false - makeRelease - forked-path - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - ${surefireArgLine} - - - - - org.apache.maven.plugins - maven-failsafe-plugin - - - - integration-test - verify - - - ${failsafeArgLine} - - - - - - - org.jacoco - jacoco-maven-plugin - ${jacoco.version} - - - pre-unit-tests - - prepare-agent - - - surefireArgLine - - - - pre-integration-tests - - prepare-agent-integration - - - failsafeArgLine - - - - report - test - - report - - - - - - - org.eluder.coveralls - coveralls-maven-plugin - 4.3.0 - - - javax.xml.bind - jaxb-api - 2.3.1 - - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.8 - true + + org.codehaus.mojo + build-helper-maven-plugin + 3.0.0 + + + generate-sources + + add-source + - ossrh - https://s01.oss.sonatype.org/ - true + + ${project.basedir}/src/main/kotlin + - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.2 - - + + + + + org.apache.maven.plugins + maven-source-plugin + 3.0.1 + + + attach-sources + + jar-no-fork + + + + + + org.jetbrains.dokka + dokka-maven-plugin + 1.6.10 + + + attach-javadocs + package + + javadocJar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + 3.0.1 + + + sign-artifacts + verify + + sign + + + + - - - - - makeRelease - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.0.0 - - - - generate-sources - - add-source - - - - ${project.basedir}/src/main/kotlin - - - - - - - - org.apache.maven.plugins - maven-source-plugin - 3.0.1 - - - attach-sources - - jar-no-fork - - - - - - - org.jetbrains.dokka - dokka-maven-plugin - 1.6.10 - - - attach-javadocs - package - - javadocJar - - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 3.0.1 - - - sign-artifacts - verify - - sign - - - - - - - - - + + + From 05adb33b0e78514a40b7943f6f7cef42396e09b3 Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sat, 18 Jul 2026 10:30:34 +0530 Subject: [PATCH 03/21] style: format code --- github-crawler-autoconfigure/pom.xml | 95 ++++----- github-crawler-core/pom.xml | 284 ++++++++++++--------------- github-crawler-starter/pom.xml | 78 ++++---- 3 files changed, 199 insertions(+), 258 deletions(-) diff --git a/github-crawler-autoconfigure/pom.xml b/github-crawler-autoconfigure/pom.xml index 6fbcfef..9cd0f6d 100644 --- a/github-crawler-autoconfigure/pom.xml +++ b/github-crawler-autoconfigure/pom.xml @@ -1,58 +1,45 @@ - - 4.0.0 - + + 4.0.0 + github-crawler-autoconfigure + jar + github-crawler-autoconfigure + com.societegenerale.github-crawler - github-crawler-autoconfigure + github-crawler-parent 2.2.1-SNAPSHOT - jar - - github-crawler-autoconfigure - - - com.societegenerale.github-crawler - github-crawler-parent - 2.2.1-SNAPSHOT - - - - - - com.societegenerale.github-crawler - github-crawler-core - ${project.parent.version} - - - - net.code-story - http - 2.104 - test - - - - org.slf4j - slf4j-simple - - - - - - com.github.tomakehurst - wiremock-jre8 - 2.32.0 - test - - - - - com.jayway.awaitility - awaitility - 1.7.0 - test - - - - - + + + + com.societegenerale.github-crawler + github-crawler-core + ${project.parent.version} + + + net.code-story + http + 2.104 + test + + + org.slf4j + slf4j-simple + + + + + com.github.tomakehurst + wiremock-jre8 + 2.32.0 + test + + + com.jayway.awaitility + awaitility + 1.7.0 + test + + diff --git a/github-crawler-core/pom.xml b/github-crawler-core/pom.xml index 5fa9e70..f9ed5bf 100644 --- a/github-crawler-core/pom.xml +++ b/github-crawler-core/pom.xml @@ -1,163 +1,129 @@ - - 4.0.0 - + + 4.0.0 + github-crawler-core + jar + github-crawler-core + com.societegenerale.github-crawler - github-crawler-core + github-crawler-parent 2.2.1-SNAPSHOT - jar - - github-crawler-core - - - com.societegenerale.github-crawler - github-crawler-parent - 2.2.1-SNAPSHOT - - - - - - - - com.google.code.gson - gson - ${gson.version} - - - - org.springframework.boot - spring-boot-starter-logging - - - - - org.slf4j - slf4j-simple - - - - - - - org.springframework.cloud - spring-cloud-starter-openfeign - - - - io.github.openfeign - feign-gson - - - - io.github.openfeign - feign-httpclient - - - - com.fasterxml.jackson.module - jackson-module-kotlin - - - - org.jetbrains.kotlin - kotlin-reflect - - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - commons-io - commons-io - 2.11.0 - - - - org.apache.commons - commons-lang3 - 3.7 - test - - - - com.squareup.okhttp3 - okhttp - - - - - com.squareup.okhttp3 - logging-interceptor - - - - - com.jayway.awaitility - awaitility - 1.7.0 - test - - - - com.jayway.jsonpath - json-path - 2.7.0 - - - - org.springframework.boot - spring-boot-configuration-processor - true - - - - - - - - - kotlin-maven-plugin - org.jetbrains.kotlin - ${kotlin.version} - - - - kapt - - kapt - - - - src/main/kotlin - - - - - - - - com.google.dagger - dagger-compiler - 2.9 - - - - - - - - - + + + + + com.google.code.gson + gson + ${gson.version} + + + org.springframework.boot + spring-boot-starter-logging + + + org.slf4j + slf4j-simple + + + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + io.github.openfeign + feign-gson + + + io.github.openfeign + feign-httpclient + + + com.fasterxml.jackson.module + jackson-module-kotlin + + + org.jetbrains.kotlin + kotlin-reflect + + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + + + commons-io + commons-io + 2.11.0 + + + org.apache.commons + commons-lang3 + 3.7 + test + + + com.squareup.okhttp3 + okhttp + + + com.squareup.okhttp3 + logging-interceptor + + + com.jayway.awaitility + awaitility + 1.7.0 + test + + + com.jayway.jsonpath + json-path + 2.7.0 + + + org.springframework.boot + spring-boot-configuration-processor + true + + + + + + kotlin-maven-plugin + org.jetbrains.kotlin + ${kotlin.version} + + + kapt + + kapt + + + + src/main/kotlin + + + + + com.google.dagger + dagger-compiler + 2.9 + + + + + + + + diff --git a/github-crawler-starter/pom.xml b/github-crawler-starter/pom.xml index 6acc814..90f0328 100644 --- a/github-crawler-starter/pom.xml +++ b/github-crawler-starter/pom.xml @@ -1,49 +1,37 @@ - - 4.0.0 - + + 4.0.0 + github-crawler-starter + jar + github-crawler-starter + com.societegenerale.github-crawler - github-crawler-starter + github-crawler-parent 2.2.1-SNAPSHOT - jar - - github-crawler-starter - - - com.societegenerale.github-crawler - github-crawler-parent - 2.2.1-SNAPSHOT - - - - - - com.societegenerale.github-crawler - github-crawler-autoconfigure - ${project.parent.version} - - - - - - - - - - - org.springframework.boot - spring-boot-maven-plugin - - com.societegenerale.githubcrawler.GitHubCrawlerApplication - exec - - - - - + + + + com.societegenerale.github-crawler + github-crawler-autoconfigure + ${project.parent.version} + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + com.societegenerale.githubcrawler.GitHubCrawlerApplication + exec + + + + From c3fbcf30668d037e60e4b8ca818f85e4ca82c82c Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sat, 18 Jul 2026 10:33:11 +0530 Subject: [PATCH 04/21] build: replace parent with bom and additional bom bom added for kotlin, gauva and spring-boot --- pom.xml | 70 ++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 25 deletions(-) diff --git a/pom.xml b/pom.xml index c59aa56..5466729 100644 --- a/pom.xml +++ b/pom.xml @@ -8,12 +8,6 @@ 2.2.1-SNAPSHOT pom github-crawler - - org.springframework.boot - spring-boot-starter-parent - 2.7.5 - - github-crawler-core github-crawler-autoconfigure @@ -22,7 +16,7 @@ Apache 2 - http://www.apache.org/licenses/LICENSE-2.0.html + https://www.apache.org/licenses/LICENSE-2.0.html repo @@ -52,8 +46,16 @@ UTF-8 11 1.8.10 + + 2.7.5 + 3.1.5 + 31.1-jre + 2.1.3 2.10 + 1.1.6 + 0.8.8 + 3.10.1 @@ -64,10 +66,31 @@ + + org.springframework.boot + spring-boot-dependencies + ${spring-boot-dependencies.version} + pom + import + org.springframework.cloud spring-cloud-openfeign-dependencies - 3.1.5 + ${spring-cloud-openfeign-dependencies.version} + pom + import + + + com.google.guava + guava-bom + ${guava-bom.version} + pom + import + + + org.jetbrains.kotlin + kotlin-bom + ${kotlin.version} pom import @@ -77,32 +100,19 @@ com.google.guava guava - 31.1-jre org.springframework.boot spring-boot-starter-test test - - - com.vaadin.external.google - android-json - - - ch.qos.logback - logback-classic - - org.jetbrains.kotlin kotlin-stdlib-jdk8 - ${kotlin.version} org.jetbrains.kotlin kotlin-reflect - ${kotlin.version} org.jetbrains.kotlin @@ -112,18 +122,17 @@ org.jetbrains.kotlin kotlin-test - ${kotlin.version} test org.dom4j dom4j - 2.1.3 + ${dom4j.version} jaxen jaxen - 1.1.6 + ${jaxen.version} @@ -170,7 +179,18 @@ org.apache.maven.plugins maven-compiler-plugin - 3.10.1 + ${maven-compiler-plugin.version} + + ${java.version} + ${java.version} + + + org.projectlombok + lombok + 1.18.24 + + + default-compile From bbdaac2c536b663edd2c8a053ec420c262931476 Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 10:46:55 +0530 Subject: [PATCH 05/21] feat: bump jdk 25 and all dependency versions --- .github/workflows/build_workflow.yml | 2 +- .github/workflows/release_workflow.yml | 2 +- .sdkmanrc | 2 +- github-crawler-autoconfigure/pom.xml | 24 +++--- .../githubcrawler/AzureDevopsCrawlerIT.java | 2 +- .../githubcrawler/BitbucketCrawlerIT.java | 42 +++++---- .../githubcrawler/GitHubCrawlerIT.java | 30 +++---- .../githubcrawler/RemoteGitHubImplTest.kt | 7 +- .../githubcrawler/RemoteGitLabImplTest.kt | 13 +-- .../githubcrawler/config/TestConfig.java | 4 +- .../githubcrawler/mocks/BitbucketMock.java | 20 ++--- .../githubcrawler/mocks/GitHubMock.java | 35 ++++---- .../githubcrawler/mocks/GitLabMock.java | 8 +- github-crawler-core/pom.xml | 44 ++++------ .../githubcrawler/FileToParse.kt | 2 +- .../model/bitbucket/Repositories.kt | 4 +- .../output/CIdroidReadyJsonFileOutput.kt | 4 +- .../githubcrawler/output/CsvFileOutput.kt | 6 +- .../githubcrawler/output/FileOutput.kt | 6 +- .../githubcrawler/output/HttpOutput.kt | 4 +- .../githubcrawler/parsers/JsonPathParser.kt | 5 +- .../CodeSearchRequestDetailsSerializer.kt | 8 +- .../remote/RemoteAzureDevopsImpl.kt | 29 +++---- .../remote/RemoteBitBucketImpl.kt | 44 ++++------ .../githubcrawler/remote/RemoteGitHubImpl.kt | 54 ++++++------ .../githubcrawler/remote/RemoteGitLabImpl.kt | 45 +++++----- .../githubcrawler/ConfigParserTest.java | 17 ++-- .../githubcrawler/GitHubCrawlerTest.kt | 6 +- .../githubcrawler/output/FileOutputTest.kt | 6 +- .../resources/sample_invalid_yamlfile.yml | 2 +- .../src/test/resources/sample_yamlfile.yml | 2 +- github-crawler-starter/pom.xml | 1 - ...ot.autoconfigure.AutoConfiguration.imports | 1 + pom.xml | 86 ++++++++++++++----- 34 files changed, 289 insertions(+), 278 deletions(-) create mode 100644 github-crawler-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index 8d40611..c7140a9 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -32,7 +32,7 @@ jobs: uses: actions/setup-java@v5 with: distribution: temurin - java-version: 11 + java-version: 25 - name: Cache Maven dependencies uses: actions/cache@v5 with: diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml index d20d3b6..cfdb049 100644 --- a/.github/workflows/release_workflow.yml +++ b/.github/workflows/release_workflow.yml @@ -26,7 +26,7 @@ jobs: uses: actions/setup-java@v5 with: distribution: temurin - java-version: 11 + java-version: 25 gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} gpg-passphrase: MAVEN_GPG_PASSPHRASE server-id: central diff --git a/.sdkmanrc b/.sdkmanrc index 91d8367..b5e2fc1 100644 --- a/.sdkmanrc +++ b/.sdkmanrc @@ -1,3 +1,3 @@ # Enable auto-env through the sdkman_auto_env config # Add key=value pairs of SDKs to use below -java=11.0.31-tem +java=25.0.3-tem diff --git a/github-crawler-autoconfigure/pom.xml b/github-crawler-autoconfigure/pom.xml index 9cd0f6d..448501a 100644 --- a/github-crawler-autoconfigure/pom.xml +++ b/github-crawler-autoconfigure/pom.xml @@ -15,31 +15,27 @@ com.societegenerale.github-crawler github-crawler-core - ${project.parent.version} + + org.springframework.boot + spring-boot-starter-restclient + + net.code-story http - 2.104 + ${http.version} test - - - org.slf4j - slf4j-simple - - - com.github.tomakehurst - wiremock-jre8 - 2.32.0 + org.wiremock + wiremock-standalone + ${wiremock-standalone.version} test - com.jayway.awaitility + org.awaitility awaitility - 1.7.0 - test diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/AzureDevopsCrawlerIT.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/AzureDevopsCrawlerIT.java index 114f921..4834b3c 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/AzureDevopsCrawlerIT.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/AzureDevopsCrawlerIT.java @@ -102,7 +102,7 @@ void shouldCrawl() throws IOException { assertThat(branchWhereResultWasFound.getName()).isEqualTo("refs/heads/main"); List valueForIndicator=(ArrayList)firstTaskResult.getValue().get("pipelineTemplateLocation"); - assertThat(valueForIndicator.get(0)).isEqualTo("azure-pipelines.yml"); + assertThat(valueForIndicator.getFirst()).isEqualTo("azure-pipelines.yml"); } diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/BitbucketCrawlerIT.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/BitbucketCrawlerIT.java index 4c240ae..25cbb1f 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/BitbucketCrawlerIT.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/BitbucketCrawlerIT.java @@ -5,21 +5,20 @@ import com.societegenerale.githubcrawler.mocks.BitbucketMock; import com.societegenerale.githubcrawler.model.Repository; import org.assertj.core.api.SoftAssertions; +import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.annotation.DirtiesContext.ClassMode; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit.jupiter.SpringExtension; import java.io.IOException; import java.util.*; -import static com.jayway.awaitility.Awaitility.await; + import static java.util.Map.entry; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; @@ -27,7 +26,6 @@ import static org.assertj.core.api.Assertions.assertThat; -@ExtendWith(SpringExtension.class) @SpringBootTest(classes = {TestConfig.class, GitHubCrawlerAutoConfiguration.class}) @ActiveProfiles(profiles = {"bitBucketTest"}) @DirtiesContext(classMode = ClassMode.BEFORE_CLASS) @@ -58,8 +56,8 @@ public void mockSetUp() { bitbucketMockServer.start(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(BitbucketMock.hasStarted())); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(BitbucketMock.hasStarted()).isTrue()); hasBitbucketMockServerStarted = true; } @@ -94,8 +92,8 @@ void shouldHitOrgaNextPageOfRepositoriesIfMorethanOne() throws IOException { Collection processedRepositories = output.getAnalyzedRepositories().values(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); assertThat(bitbucketMockServer.isHasCalledNextPage()).as("next page wasn't called").isTrue(); @@ -125,7 +123,7 @@ void shouldLoadIndicatorsConfig() { .containsKeys(pomXmlFile, new FileToParse("pom.xml", null), new FileToParse("Jenkinsfile", null), new FileToParse("Dockerfile", null)); - IndicatorDefinition pomXmlIndicatorDefinition1 = crawler.getGitHubCrawlerProperties().getIndicatorsToFetchByFile().get(pomXmlFile).get(0); + IndicatorDefinition pomXmlIndicatorDefinition1 = crawler.getGitHubCrawlerProperties().getIndicatorsToFetchByFile().get(pomXmlFile).getFirst(); assertThat(pomXmlIndicatorDefinition1).isNotNull(); assertThat(pomXmlIndicatorDefinition1.getName()).isEqualTo("spring_boot_starter_parent_version"); assertThat(pomXmlIndicatorDefinition1.getType()).isEqualTo("findDependencyVersionInXml"); @@ -140,8 +138,8 @@ void shouldTryToFetchFilesWithIndicators() throws IOException { crawler.crawl(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(bitbucketMockServer.getPomXmlHits()).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(bitbucketMockServer.getPomXmlHits()).hasSize(nbRepositoriesInOrga)); } @@ -152,10 +150,10 @@ void shouldCopyActiveProfilesAsGroupsOnRepo() throws IOException { Collection processedRepositories = output.getAnalyzedRepositories().values(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); - processedRepositories.stream().forEach(repo -> { + processedRepositories.forEach(repo -> { assertThat(repo.getGroups()).containsExactlyInAnyOrder("bitBucketTest"); }); @@ -182,8 +180,8 @@ void shouldParseFileWithIndicatorsWhenItExists() throws IOException { crawler.crawl(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(output.getAnalyzedRepositories().values()).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(output.getAnalyzedRepositories().values()).hasSize(nbRepositoriesInOrga)); Collection processedRepositories = output.getAnalyzedRepositories().values(); @@ -252,8 +250,8 @@ void shouldParseAllBranchesWhenConfiguredAccordingly() throws IOException { Collection processedRepositories = output.getAnalyzedRepositories().values(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); for (Repository processedRepository : processedRepositories) { @@ -284,8 +282,8 @@ void processNormallyIfNoConfigFileOnRepoSide() throws IOException { crawler.crawl(); Collection actualRepositories = output.getAnalyzedRepositories().values(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); assertThat(bitbucketMockServer.getRepoConfigHits()).hasSize(nbRepositoriesInOrga); @@ -303,8 +301,8 @@ void outputResultsShouldHaveACrawlerRunId() throws IOException { crawler.crawl(); Collection actualRepositories = output.getAnalyzedRepositories().values(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); assertThat(actualRepositories.stream().filter(r -> !r.getCrawlerRunId().equals(GitHubCrawler.NO_CRAWLER_RUN_ID_DEFINED)) .count()) diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java index d898ac1..db589f2 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java @@ -1,10 +1,11 @@ package com.societegenerale.githubcrawler; -import static com.jayway.awaitility.Awaitility.await; + import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import com.societegenerale.githubcrawler.config.GitHubCrawlerAutoConfiguration; import com.societegenerale.githubcrawler.config.TestConfig; @@ -23,16 +24,13 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.annotation.DirtiesContext.ClassMode; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@ExtendWith(SpringExtension.class) @SpringBootTest(classes = {TestConfig.class, GitHubCrawlerAutoConfiguration.class}) @ActiveProfiles(profiles = {"gitHubTest", "profilesAreAWayOfGrouping"}) @DirtiesContext(classMode = ClassMode.BEFORE_CLASS) @@ -64,7 +62,7 @@ public void mockSetUp() { githubMockServer.start(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(GitHubMock.hasStarted())); + .untilAsserted(() -> assertThat(GitHubMock.hasStarted())); hasGitHubMockServerStarted = true; } @@ -100,7 +98,7 @@ void shouldHitOrgaNextPageOfRepositoriesIfMorethanOne() throws IOException { Collection processedRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); assertThat(githubMockServer.isHasCalledNextPage()).as("next page wasn't called").isTrue(); @@ -129,7 +127,7 @@ void shouldLoadIndicatorsConfig() { assertThat(crawler.getGitHubCrawlerProperties().getIndicatorsToFetchByFile()).containsKeys(pomXmlFile, new FileToParse("pom.xml", null), new FileToParse("Jenkinsfile", null), new FileToParse("Dockerfile", null)); - IndicatorDefinition pomXmlIndicatorDefinition1 = crawler.getGitHubCrawlerProperties().getIndicatorsToFetchByFile().get(pomXmlFile).get(0); + IndicatorDefinition pomXmlIndicatorDefinition1 = crawler.getGitHubCrawlerProperties().getIndicatorsToFetchByFile().get(pomXmlFile).getFirst(); assertThat(pomXmlIndicatorDefinition1).isNotNull(); assertThat(pomXmlIndicatorDefinition1.getName()).isEqualTo("spring_boot_starter_parent_version"); assertThat(pomXmlIndicatorDefinition1.getType()).isEqualTo("findDependencyVersionInXml"); @@ -147,7 +145,7 @@ void shouldTryToFetchFilesWithIndicators() throws IOException { crawler.crawl(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(githubMockServer.getPomXmlHits()).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(githubMockServer.getPomXmlHits()).hasSize(nbRepositoriesInOrga)); } @@ -159,9 +157,9 @@ void shouldCopyActiveProfilesAsGroupsOnRepo() throws IOException { Collection processedRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); - processedRepositories.stream().forEach(repo -> { + processedRepositories.forEach(repo -> { assertThat(repo.getGroups()).containsExactlyInAnyOrder("gitHubTest", "profilesAreAWayOfGrouping"); }); @@ -189,7 +187,7 @@ void shouldParseFileWithIndicatorsWhenItExists() throws IOException { crawler.crawl(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(output.getAnalyzedRepositories().values()).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(output.getAnalyzedRepositories().values()).hasSize(nbRepositoriesInOrga)); Collection processedRepositories = output.getAnalyzedRepositories().values(); @@ -259,7 +257,7 @@ void shouldParseAllBranchesWhenConfiguredAccordingly() throws IOException { Collection processedRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); for (Repository processedRepository : processedRepositories) { @@ -291,7 +289,7 @@ void processNormallyIfNoConfigFileOnRepoSide() throws IOException { Collection actualRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); assertThat(githubMockServer.getRepoConfigHits()).hasSize(nbRepositoriesInOrga); @@ -314,11 +312,11 @@ void shouldPerformSearchOnAllRepos() throws IOException { Collection actualRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); assertThat(githubMockServer.getSearchHitsCount()).isEqualTo(nbRepositoriesInOrga); - actualRepositories.stream().forEach(repo -> { + actualRepositories.forEach(repo -> { assertThat(repo.getMiscTasksResults()).hasSize(1); @@ -335,7 +333,7 @@ void outputResultsShouldHaveACrawlerRunId() throws IOException { Collection actualRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); assertThat(actualRepositories.stream().filter(r -> !r.getCrawlerRunId().equals(GitHubCrawler.NO_CRAWLER_RUN_ID_DEFINED)) .count()) diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitHubImplTest.kt b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitHubImplTest.kt index 381c916..4145bba 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitHubImplTest.kt +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitHubImplTest.kt @@ -1,9 +1,10 @@ package com.societegenerale.githubcrawler -import com.jayway.awaitility.Awaitility.await + import com.societegenerale.githubcrawler.mocks.GitHubMock import com.societegenerale.githubcrawler.remote.RemoteGitHubImpl import org.assertj.core.api.Assertions.assertThat +import org.awaitility.Awaitility import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach @@ -30,8 +31,8 @@ class RemoteGitHubImplTest { githubMockServer.start() - await().atMost(5, SECONDS) - .until{ assertThat(GitHubMock.hasStarted()) } + Awaitility.await().atMost(5, SECONDS) + .untilAsserted { assertThat(GitHubMock.hasStarted()).isTrue() } } @AfterAll diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitLabImplTest.kt b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitLabImplTest.kt index 4474b04..63195fe 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitLabImplTest.kt +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitLabImplTest.kt @@ -1,10 +1,11 @@ package com.societegenerale.githubcrawler -import com.jayway.awaitility.Awaitility.await + import com.societegenerale.githubcrawler.mocks.GitLabMock import com.societegenerale.githubcrawler.mocks.RemoteServiceMock.GITLAB_MOCK_PORT import com.societegenerale.githubcrawler.remote.RemoteGitLabImpl import org.assertj.core.api.Assertions.assertThat +import org.awaitility.Awaitility import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach @@ -24,7 +25,7 @@ class RemoteGitLabImplTest { companion object { - val gitlabMockServer: GitLabMock = GitLabMock(); + val gitlabMockServer: GitLabMock = GitLabMock() @BeforeAll @JvmStatic @@ -32,8 +33,8 @@ class RemoteGitLabImplTest { gitlabMockServer.start() - await().atMost(5, SECONDS) - .until{ assertThat(GitLabMock.hasStarted()) } + Awaitility.await().atMost(5, SECONDS) + .untilAsserted { assertThat(GitLabMock.hasStarted()).isTrue() } } @AfterAll @@ -49,7 +50,7 @@ class RemoteGitLabImplTest { gitlabMockServer.reset() } - val remoteGitLab = RemoteGitLabImpl("http://localhost:"+GITLAB_MOCK_PORT+"/api/v4", "someToken"); + val remoteGitLab = RemoteGitLabImpl("http://localhost:"+GITLAB_MOCK_PORT+"/api/v4", "someToken") @Test fun shouldGetRepositoriesForAgroup() { @@ -90,7 +91,7 @@ class RemoteGitLabImplTest { val searchResults=remoteGitLab.fetchCodeSearchResult("h5bp/html5-boilerplate","blabla") - assertThat(searchResults.totalCount).isEqualTo(2); + assertThat(searchResults.totalCount).isEqualTo(2) } diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/config/TestConfig.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/config/TestConfig.java index 6e02484..b7d482b 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/config/TestConfig.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/config/TestConfig.java @@ -3,8 +3,8 @@ import com.societegenerale.githubcrawler.GitHubCrawlerProperties; import com.societegenerale.githubcrawler.model.Repository; import com.societegenerale.githubcrawler.output.GitHubCrawlerOutput; -import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -21,7 +21,7 @@ public GitHubCrawlerOutput output() { public static class InMemoryGitHubCrawlerOutput implements GitHubCrawlerOutput { - private final Map analyzedRepositories = new HashMap<>(); + private final Map analyzedRepositories = new ConcurrentHashMap<>(); public Map getAnalyzedRepositories() { return analyzedRepositories; diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/BitbucketMock.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/BitbucketMock.java index eb754f8..27d73af 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/BitbucketMock.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/BitbucketMock.java @@ -86,27 +86,27 @@ public boolean start() { bitbucketWebServer = new WebServer(); bitbucketWebServer.configure( routes -> { - routes.get("/projects/myProject/repos/:repo/raw/.BitBucketCrawler?at=master", (context, repo) -> getRepoConfigFileOnRepo(repo)); - routes.get("/projects/myProject/repos/:repo/raw/pom.xml?at=master", (context, repo) -> getPomXmlFileOnRepo(repo)); - routes.get("/projects/myProject/repos?start=:start", (context, start) -> getOrganisationContent(Integer.parseInt(start))); + routes.get("/projects/myProject/repos/:repo/raw/.BitBucketCrawler?at=master", (_, repo) -> getRepoConfigFileOnRepo(repo)); + routes.get("/projects/myProject/repos/:repo/raw/pom.xml?at=master", (_, repo) -> getPomXmlFileOnRepo(repo)); + routes.get("/projects/myProject/repos?start=:start", (_, start) -> getOrganisationContent(Integer.parseInt(start))); //for other resources than pom.xml.. //hack for resources that are not at the root of the repository, so that we don't have to hardcode too many things routes.get("/projects/myProject/repos/:repo/raw/:aSubDirectory/:resource?at=:branchName", - (context, repo, aSubDirectory, resource, branchName) -> getResource(repo, aSubDirectory, resource, branchName)); + (_, repo, aSubDirectory, resource, branchName) -> getResource(repo, aSubDirectory, resource, branchName)); routes.get("/projects/myProject/repos/:repo/contents/:resource?ref=:branchName", - (context, repo, resource, branchName) -> getResourceFileOnRepo(repo, resource, null, branchName)); + (_, repo, resource, branchName) -> getResourceFileOnRepo(repo, resource, null, branchName)); routes.get("/raw/myProject/:repo/:branchName/:resource", - (context, repo, branchName, resource) -> getResource(repo, branchName, null, resource)); + (_, repo, branchName, resource) -> getResource(repo, branchName, null, resource)); - routes.get("/projects/myProject/repos/:repo/branches", (context, repo) -> getBranches(repo)); + routes.get("/projects/myProject/repos/:repo/branches", (_, repo) -> getBranches(repo)); routes.get("/admin/groups", this::getTeams); - routes.get("/api/v3/teams/:team/members", (context, team) -> getTeamsMembers(team)); + routes.get("/api/v3/teams/:team/members", (_, team) -> getTeamsMembers(team)); - routes.get("/projects/myProject/repos/:repo/commits?limit=1", (context, repo) -> getCommits(repo)); - routes.get("/projects/myProject/repos/:repo/commits/:commit", (context, repo, commit) -> getCommit(repo, commit)); + routes.get("/projects/myProject/repos/:repo/commits?limit=1", (_, repo) -> getCommits(repo)); + routes.get("/projects/myProject/repos/:repo/commits/:commit", (_, repo, commit) -> getCommit(repo, commit)); } ).start(BITBUCKET_MOCK_PORT); diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitHubMock.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitHubMock.java index 2bf4239..05a0928 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitHubMock.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitHubMock.java @@ -11,6 +11,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; import net.codestory.http.Context; import net.codestory.http.WebServer; import net.codestory.http.constants.HttpStatus; @@ -57,9 +58,9 @@ public boolean isHasCalledNextPage() { return hasCalledNextPage; } - private List repoConfigHits = new ArrayList<>(); + private List repoConfigHits = new CopyOnWriteArrayList<>(); - private List pomXmlHits = new ArrayList<>(); + private List pomXmlHits = new CopyOnWriteArrayList<>(); private int nbPages = 1; @@ -91,38 +92,38 @@ public boolean start() { gitHubWebServer.configure( routes -> { - routes.get("/api/v3/repos/MyOrganization/:repo/contents/.githubCrawler", (context, repo) -> getRepoConfigFileOnRepo(repo)); - routes.get("/raw/MyOrganization/:repo/master/.githubCrawler", (context, repo) -> getActualRepoConfig(repo)); + routes.get("/api/v3/repos/MyOrganization/:repo/contents/.githubCrawler", (_, repo) -> getRepoConfigFileOnRepo(repo)); + routes.get("/raw/MyOrganization/:repo/master/.githubCrawler", (_, repo) -> getActualRepoConfig(repo)); routes.get("/api/v3/orgs/MyOrganization/repos", context -> getOrganisationContent(context)); - routes.get("/api/v3/organizations/1114/repos", context -> getOrganisationContentForNextPage()); + routes.get("/api/v3/organizations/1114/repos", _ -> getOrganisationContentForNextPage()); - routes.get("/api/v3/users/someUser/repos", context -> geUserReposContent()); + routes.get("/api/v3/users/someUser/repos", _ -> geUserReposContent()); - routes.get("/api/v3/repos/MyOrganization/:repo/contents/pom.xml?ref=master", (context, repo) -> getPomXmlFileOnRepo(repo)); - routes.get("/raw/MyOrganization/:repo/:branchName/pom.xml", (context, repo, branchName) -> getActualPomXML(repo, branchName)); + routes.get("/api/v3/repos/MyOrganization/:repo/contents/pom.xml?ref=master", (_, repo) -> getPomXmlFileOnRepo(repo)); + routes.get("/raw/MyOrganization/:repo/:branchName/pom.xml", (_, repo, branchName) -> getActualPomXML(repo, branchName)); //for other resources than pom.xml.. //hack for resources that are not at the root of the repository, so that we don't have to hardcode too many things routes.get("/api/v3/repos/MyOrganization/:repo/contents/:aSubDirectory/:resource?ref=:branchName", - (context, repo, aSubDirectory, resource, branchName) -> getResourceFileOnRepo(repo, resource, aSubDirectory, branchName)); + (_, repo, aSubDirectory, resource, branchName) -> getResourceFileOnRepo(repo, resource, aSubDirectory, branchName)); routes.get("/raw/MyOrganization/:repo/:branchName/:aSubDirectory/:resource", - (context, repo, branchName, aSubDirectory, resource) -> getResource(repo, branchName, aSubDirectory, resource)); + (_, repo, branchName, aSubDirectory, resource) -> getResource(repo, branchName, aSubDirectory, resource)); routes.get("/api/v3/repos/MyOrganization/:repo/contents/:resource?ref=:branchName", - (context, repo, resource, branchName) -> getResourceFileOnRepo(repo, resource, null, branchName)); + (_, repo, resource, branchName) -> getResourceFileOnRepo(repo, resource, null, branchName)); routes.get("/raw/MyOrganization/:repo/:branchName/:resource", - (context, repo, branchName, resource) -> getResource(repo, branchName, null, resource)); + (_, repo, branchName, resource) -> getResource(repo, branchName, null, resource)); - routes.get("/api/v3/repos/MyOrganization/:repo/branches", (context, repo) -> getBranches(repo)); + routes.get("/api/v3/repos/MyOrganization/:repo/branches", (_, repo) -> getBranches(repo)); - routes.get("/api/v3/search/code?q=:searchQuery", (context, searchQuery) -> getSearchResult(searchQuery)); + routes.get("/api/v3/search/code?q=:searchQuery", (_, searchQuery) -> getSearchResult(searchQuery)); routes.get("/api/v3/orgs/MyOrganization/teams", this::getTeams); - routes.get("/api/v3/teams/:team/members", (context, team) -> getTeamsMembers(team)); + routes.get("/api/v3/teams/:team/members", (_, team) -> getTeamsMembers(team)); - routes.get("/api/v3/repos/MyOrganization/:repo/commits?per_page=150", (context, repo) -> getCommits(repo)); - routes.get("/api/v3/repos/MyOrganization/:repo/commits/:commit", (context, repo, commit) -> getCommit(repo, commit)); + routes.get("/api/v3/repos/MyOrganization/:repo/commits?per_page=150", (_, repo) -> getCommits(repo)); + routes.get("/api/v3/repos/MyOrganization/:repo/commits/:commit", (_, repo, commit) -> getCommit(repo, commit)); } ).start(GITHUB_MOCK_PORT); diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitLabMock.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitLabMock.java index 43b08aa..eeb8637 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitLabMock.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitLabMock.java @@ -36,12 +36,12 @@ public boolean start() { gitLabWebServer.configure( routes -> { - routes.get("/api/v4/groups?search=:groupName", (context,groupName) -> getGroups(groupName)); + routes.get("/api/v4/groups?search=:groupName", (_,groupName) -> getGroups(groupName)); - routes.get("/api/v4/groups/:groupId/projects", (context, groupId) -> getRepositories(groupId)); - routes.get("/api/v4/projects/:repoId/repository/files/:filePath/raw?ref=:branchName", (context, repoId, filePath,branchName) -> getFileContent(repoId, filePath, branchName)); + routes.get("/api/v4/groups/:groupId/projects", (_, groupId) -> getRepositories(groupId)); + routes.get("/api/v4/projects/:repoId/repository/files/:filePath/raw?ref=:branchName", (_, repoId, filePath,branchName) -> getFileContent(repoId, filePath, branchName)); - routes.get("/api/v4/projects/:repoId/search?scope=blobs&:searchString", (context, repoId, searchString) -> getRepoSearchResults(repoId, searchString)); + routes.get("/api/v4/projects/:repoId/search?scope=blobs&:searchString", (_, repoId, searchString) -> getRepoSearchResults(repoId, searchString)); } diff --git a/github-crawler-core/pom.xml b/github-crawler-core/pom.xml index f9ed5bf..9f6dc6b 100644 --- a/github-crawler-core/pom.xml +++ b/github-crawler-core/pom.xml @@ -27,12 +27,6 @@ org.springframework.boot spring-boot-starter-logging - - - org.slf4j - slf4j-simple - - org.springframework.cloud @@ -47,29 +41,17 @@ feign-httpclient - com.fasterxml.jackson.module + tools.jackson.module jackson-module-kotlin - - - org.jetbrains.kotlin - kotlin-reflect - - - com.fasterxml.jackson.dataformat + tools.jackson.dataformat jackson-dataformat-yaml commons-io commons-io - 2.11.0 - - - org.apache.commons - commons-lang3 - 3.7 - test + ${commons-io.version} com.squareup.okhttp3 @@ -79,22 +61,26 @@ com.squareup.okhttp3 logging-interceptor - - com.jayway.awaitility - awaitility - 1.7.0 - test - com.jayway.jsonpath json-path - 2.7.0 + ${json-path.version} org.springframework.boot spring-boot-configuration-processor true + + + org.apache.commons + commons-lang3 + test + + + org.awaitility + awaitility + @@ -117,7 +103,7 @@ com.google.dagger dagger-compiler - 2.9 + 2.60.1 diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/FileToParse.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/FileToParse.kt index 860a134..31c165b 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/FileToParse.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/FileToParse.kt @@ -18,7 +18,7 @@ class FileToParseConversionService : ConversionService { return targetType.name == FileToParse::class.java.name } - override fun convert(@Nullable o: Any?, targetType: Class): T? { + override fun convert(source: Any?, targetType: Class): T? { return null } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/model/bitbucket/Repositories.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/model/bitbucket/Repositories.kt index d2365ea..eac942f 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/model/bitbucket/Repositories.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/model/bitbucket/Repositories.kt @@ -5,6 +5,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties @JsonIgnoreProperties(ignoreUnknown = true) data class Repositories(val values: Set, val isLastPage: Boolean, - val start: Int, - val nextPageStart: Int, + val start: Int = 0, + val nextPageStart: Int = 0, ) diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CIdroidReadyJsonFileOutput.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CIdroidReadyJsonFileOutput.kt index 30e321b..58655f3 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CIdroidReadyJsonFileOutput.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CIdroidReadyJsonFileOutput.kt @@ -8,7 +8,7 @@ import java.io.IOException import java.nio.charset.StandardCharsets import java.nio.file.Files import java.nio.file.OpenOption -import java.nio.file.Paths +import java.nio.file.Path import java.nio.file.StandardOpenOption import java.time.LocalDateTime import java.time.format.DateTimeFormatter @@ -113,7 +113,7 @@ class CIdroidReadyJsonFileOutput (val indicatorsToOutput: List, val with @Suppress("SpreadOperator") //no performance impact given the number of values (a couple, max).. private fun openFileWithOptions(vararg options : OpenOption) : BufferedWriter { - return Files.newBufferedWriter(Paths.get(finalOutputFileName), + return Files.newBufferedWriter(Path.of(finalOutputFileName), StandardCharsets.UTF_8, *options) } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CsvFileOutput.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CsvFileOutput.kt index 32b1302..39e1787 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CsvFileOutput.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CsvFileOutput.kt @@ -5,7 +5,7 @@ import org.slf4j.Logger import java.io.IOException import java.nio.charset.StandardCharsets import java.nio.file.Files -import java.nio.file.Paths +import java.nio.file.Path import java.nio.file.StandardOpenOption import java.time.LocalDateTime import java.time.format.DateTimeFormatter @@ -34,7 +34,7 @@ abstract class CsvFileOutput(initParam : Any) : GitHubCrawlerOutput{ finalOutputFileName = getPrefix() + now.format(formatter) + ".csv" - val writer = Files.newBufferedWriter(Paths.get(finalOutputFileName), + val writer = Files.newBufferedWriter(Path.of(finalOutputFileName), StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW) @@ -46,7 +46,7 @@ abstract class CsvFileOutput(initParam : Any) : GitHubCrawlerOutput{ @Throws(IOException::class) override fun output(analyzedRepository: Repository) { - val writer = Files.newBufferedWriter(Paths.get(finalOutputFileName), + val writer = Files.newBufferedWriter(Path.of(finalOutputFileName), StandardCharsets.UTF_8, StandardOpenOption.WRITE, StandardOpenOption.APPEND) diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/FileOutput.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/FileOutput.kt index dd9bf68..6df254e 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/FileOutput.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/FileOutput.kt @@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory import java.io.IOException import java.nio.charset.StandardCharsets import java.nio.file.Files -import java.nio.file.Paths +import java.nio.file.Path import java.nio.file.StandardOpenOption import java.time.LocalDateTime import java.time.format.DateTimeFormatter @@ -27,7 +27,7 @@ class FileOutput (filenamePrefix: String) : GitHubCrawlerOutput { finalOutputFileName = filenamePrefix + "_" + now.format(formatter) + ".txt" - Files.newBufferedWriter(Paths.get(finalOutputFileName), + Files.newBufferedWriter(Path.of(finalOutputFileName), StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW).use{ @@ -40,7 +40,7 @@ class FileOutput (filenamePrefix: String) : GitHubCrawlerOutput { @Throws(IOException::class) override fun output(analyzedRepository: Repository) { - Files.newBufferedWriter(Paths.get(finalOutputFileName), + Files.newBufferedWriter(Path.of(finalOutputFileName), StandardCharsets.UTF_8, StandardOpenOption.WRITE, StandardOpenOption.APPEND).use{ diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/HttpOutput.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/HttpOutput.kt index 547a839..d7f175c 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/HttpOutput.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/HttpOutput.kt @@ -41,10 +41,10 @@ class HttpOutput(targetUrl: String, response = restTemplate.postForEntity(internalTargetUrl, output, String::class.java) if (!response.statusCode.is2xxSuccessful) { - logHttpError(analyzedRepository.name, response.statusCodeValue,response.body) + logHttpError(analyzedRepository.name, response.statusCode.value(),response.body) } } catch (e : HttpClientErrorException) { - logHttpError(analyzedRepository.name, e.rawStatusCode,e.message ?: "no additional message provided") + logHttpError(analyzedRepository.name, e.statusCode.value(),e.message ?: "no additional message provided") } } } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/parsers/JsonPathParser.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/parsers/JsonPathParser.kt index 81c32a3..fc264fd 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/parsers/JsonPathParser.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/parsers/JsonPathParser.kt @@ -3,7 +3,6 @@ package com.societegenerale.githubcrawler.parsers import com.jayway.jsonpath.JsonPath.read import com.jayway.jsonpath.PathNotFoundException import com.societegenerale.githubcrawler.IndicatorDefinition -import net.minidev.json.JSONArray import org.slf4j.LoggerFactory /** @@ -50,12 +49,12 @@ class JsonPathParser : FileContentParser { if (result is String) { result - } else if (result is JSONArray && !result.isEmpty()) { + } else if (result is List<*> && result.isNotEmpty()) { result[0] as String } else { NOT_FOUND } - } catch (e: PathNotFoundException) { + } catch (_: PathNotFoundException) { NOT_FOUND } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/CodeSearchRequestDetailsSerializer.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/CodeSearchRequestDetailsSerializer.kt index 47e95e6..3e6099a 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/CodeSearchRequestDetailsSerializer.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/CodeSearchRequestDetailsSerializer.kt @@ -1,13 +1,13 @@ package com.societegenerale.githubcrawler.remote -import com.fasterxml.jackson.core.JsonGenerator +import tools.jackson.core.JsonGenerator -import com.fasterxml.jackson.databind.SerializerProvider -import com.fasterxml.jackson.databind.ser.std.StdSerializer +import tools.jackson.databind.ser.std.StdSerializer +import tools.jackson.databind.SerializationContext class CodeSearchRequestDetailsSerializer() : StdSerializer(CodeSearchRequestDetails::class.java) { - override fun serialize(p0: CodeSearchRequestDetails?, p1: JsonGenerator?, p2: SerializerProvider?) { + override fun serialize(p0: CodeSearchRequestDetails?, p1: JsonGenerator?, p2: SerializationContext?) { TODO("Not yet implemented") } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteAzureDevopsImpl.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteAzureDevopsImpl.kt index 6a8d56d..7074e38 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteAzureDevopsImpl.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteAzureDevopsImpl.kt @@ -1,11 +1,8 @@ package com.societegenerale.githubcrawler.remote -import com.fasterxml.jackson.core.JsonParseException -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.kotlin.KotlinModule -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper + +import tools.jackson.module.kotlin.KotlinModule +import tools.jackson.module.kotlin.jacksonObjectMapper import com.societegenerale.githubcrawler.RepositoryConfig import com.societegenerale.githubcrawler.model.* import com.societegenerale.githubcrawler.model.azuredevops.Repositories @@ -22,7 +19,10 @@ import okhttp3.logging.HttpLoggingInterceptor import org.apache.commons.io.IOUtils import org.slf4j.LoggerFactory import org.springframework.http.HttpStatus -import java.io.IOException +import tools.jackson.core.JacksonException +import tools.jackson.core.exc.StreamReadException +import tools.jackson.dataformat.yaml.YAMLMapper + import java.io.StringWriter import java.util.* import java.util.stream.Collectors.toSet @@ -51,7 +51,7 @@ class RemoteAzureDevopsImpl @JvmOverloads constructor(private val azureDevopsUrl private var azureOrg=splitedOrgName.get(0) private var azureProject=splitedOrgName.get(1) - private val objectMapper = jacksonObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + private val objectMapper = jacksonObjectMapper() private val basicAuthentCredentials: String = Credentials.basic("", personalAccessToken) @@ -103,7 +103,7 @@ class RemoteAzureDevopsImpl @JvmOverloads constructor(private val azureDevopsUrl return objectMapper.readValue(bodyAsString, Repositories::class.java) } - catch( e : JsonParseException){ + catch( e : StreamReadException){ throw IllegalArgumentException("unable to read the repositories to crawl from the response we got $bodyAsString",e) } @@ -234,12 +234,9 @@ internal class CodeSearchResultItem(val path : String) internal class AzureDevopsResponseDecoder { val log = LoggerFactory.getLogger(this.javaClass) - val repoConfigMapper = ObjectMapper(YAMLFactory()) - - init { - repoConfigMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - repoConfigMapper.registerModule(KotlinModule.Builder().build()) - } + val repoConfigMapper: YAMLMapper = YAMLMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() fun decodeRepoConfig(response: okhttp3.Response): RepositoryConfig { @@ -261,7 +258,7 @@ internal class AzureDevopsResponseDecoder { try { return repoConfigMapper.readValue(responseAsString, RepositoryConfig::class.java) - } catch (e: IOException) { + } catch (e: JacksonException) { throw Repository.RepoConfigException(HttpStatus.BAD_REQUEST,"unable to parse config for repo - content : \"" + response.body + "\"", e) } } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteBitBucketImpl.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteBitBucketImpl.kt index bf6f384..e334b1b 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteBitBucketImpl.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteBitBucketImpl.kt @@ -1,10 +1,9 @@ package com.societegenerale.githubcrawler.remote -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.kotlin.KotlinModule -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import tools.jackson.databind.json.JsonMapper +import tools.jackson.dataformat.yaml.YAMLMapper +import tools.jackson.module.kotlin.KotlinModule +import tools.jackson.module.kotlin.jacksonObjectMapper import com.societegenerale.githubcrawler.RepositoryConfig import com.societegenerale.githubcrawler.model.* import com.societegenerale.githubcrawler.model.Author @@ -23,12 +22,9 @@ import feign.gson.GsonEncoder import feign.httpclient.ApacheHttpClient import feign.slf4j.Slf4jLogger import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.http.HttpMessageConverters -import org.springframework.cloud.openfeign.support.ResponseEntityDecoder -import org.springframework.cloud.openfeign.support.SpringDecoder import org.springframework.http.HttpStatus -import org.springframework.http.MediaType -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter +import tools.jackson.core.JacksonException + import java.io.IOException import java.lang.reflect.Type import java.util.* @@ -58,8 +54,6 @@ class RemoteBitBucketImpl @JvmOverloads constructor( val log = LoggerFactory.getLogger(this.javaClass) - private val objectMapper = jacksonObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - override fun validateRemoteConfig(organizationName: String) { //TODO("Not yet implemented") } @@ -236,12 +230,9 @@ private interface InternalBitBucketClient { internal class BitBucketResponseDecoder : Decoder { val log = LoggerFactory.getLogger(this.javaClass) - val repoConfigMapper = ObjectMapper(YAMLFactory()) - - init { - repoConfigMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - repoConfigMapper.registerModule(KotlinModule.Builder().build()) - } + val repoConfigMapper: YAMLMapper = YAMLMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() fun decodeRepoConfig(response: String): RepositoryConfig { @@ -263,19 +254,22 @@ internal class BitBucketResponseDecoder : Decoder { log.debug("Decoding a successful response...") - if (type.typeName == MediaType.TEXT_PLAIN_VALUE) { + if (type == String::class.java) { log.debug("\t ... as a String") - return response.body().toString() + return response.body().asReader(Charsets.UTF_8).use { it.readText() } } log.debug("\t ... as a " + type.typeName) - val jacksonConverter = - MappingJackson2HttpMessageConverter(ObjectMapper().registerModule(KotlinModule.Builder().build())) - val objectFactory = { HttpMessageConverters(jacksonConverter) } - return ResponseEntityDecoder(SpringDecoder(objectFactory)).decode(response, type) + val jsonMapper = JsonMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() + + response.body().asInputStream().use { input -> + return jsonMapper.readValue(input, jsonMapper.constructType(type)) + } } } @@ -287,7 +281,7 @@ internal class BitBucketResponseDecoder : Decoder { try { return repoConfigMapper.readValue(responseAsString, RepositoryConfig::class.java) - } catch (e: IOException) { + } catch (e: JacksonException) { throw Repository.RepoConfigException( HttpStatus.BAD_REQUEST, "unable to parse config for repo - content : \"" + responseAsString + "\"", diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitHubImpl.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitHubImpl.kt index 63c55b4..b9429e0 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitHubImpl.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitHubImpl.kt @@ -1,13 +1,10 @@ package com.societegenerale.githubcrawler.remote -import com.fasterxml.jackson.core.JsonParseException -import com.fasterxml.jackson.core.JsonProcessingException -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.kotlin.KotlinModule -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.fasterxml.jackson.module.kotlin.readValue +import tools.jackson.databind.json.JsonMapper +import tools.jackson.dataformat.yaml.YAMLMapper +import tools.jackson.module.kotlin.KotlinModule +import tools.jackson.module.kotlin.jacksonObjectMapper +import tools.jackson.module.kotlin.readValue import com.societegenerale.githubcrawler.RepositoryConfig import com.societegenerale.githubcrawler.model.* import com.societegenerale.githubcrawler.model.commit.Commit @@ -26,12 +23,10 @@ import okhttp3.OkHttpClient import okhttp3.Response import org.apache.commons.io.IOUtils import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.http.HttpMessageConverters -import org.springframework.cloud.openfeign.support.ResponseEntityDecoder -import org.springframework.cloud.openfeign.support.SpringDecoder import org.springframework.http.HttpStatus -import org.springframework.http.MediaType -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter +import tools.jackson.core.JacksonException +import tools.jackson.core.exc.StreamReadException + import java.io.IOException import java.io.StringWriter import java.lang.reflect.Type @@ -72,7 +67,7 @@ class RemoteGitHubImpl @JvmOverloads constructor( val log = LoggerFactory.getLogger(this.javaClass) - private val objectMapper = jacksonObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + private val objectMapper = jacksonObjectMapper() @Throws(NoReachableRepositories::class) override fun validateRemoteConfig(organizationName: String) { @@ -81,7 +76,7 @@ class RemoteGitHubImpl @JvmOverloads constructor( try { extractRepositories(response) - } catch (e: JsonProcessingException) { + } catch (e: JacksonException) { throw NoReachableRepositories("not able to parse response : ${response.body}", e) } @@ -184,7 +179,7 @@ class RemoteGitHubImpl @JvmOverloads constructor( log.warn("response is null : {}", response) return emptySet() } - } catch (e: JsonProcessingException) { + } catch (e: JacksonException) { throw NoReachableRepositories("not able to parse response", e) } } @@ -244,7 +239,7 @@ class RemoteGitHubImpl @JvmOverloads constructor( return try { objectMapper.readValue(responseAsString, SearchResult::class.java) } - catch(e : JsonParseException){ + catch(e : StreamReadException){ log.warn("parsing error",e) SearchResult(0, emptyList()) } @@ -391,12 +386,9 @@ internal class GiHubErrorDecoder : ErrorDecoder { internal class GitHubResponseDecoder : Decoder { val log = LoggerFactory.getLogger(this.javaClass) - val repoConfigMapper = ObjectMapper(YAMLFactory()) - - init { - repoConfigMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - repoConfigMapper.registerModule(KotlinModule.Builder().build()) - } + val repoConfigMapper: YAMLMapper = YAMLMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() fun decodeRepoConfig(response: Response): RepositoryConfig { @@ -423,18 +415,22 @@ internal class GitHubResponseDecoder : Decoder { log.debug("Decoding a successful response...") - if (type.typeName == MediaType.TEXT_PLAIN_VALUE) { + if (type == String::class.java) { log.debug("\t ... as a String") - return response.body().toString() + return response.body().asReader(Charsets.UTF_8).use { it.readText() } } log.debug("\t ... as a " + type.typeName) - val jacksonConverter = MappingJackson2HttpMessageConverter(ObjectMapper().registerModule(KotlinModule.Builder().build())) - val objectFactory = { HttpMessageConverters(jacksonConverter) } - return ResponseEntityDecoder(SpringDecoder(objectFactory)).decode(response, type) + val jsonMapper = JsonMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() + + response.body().asInputStream().use { input -> + return jsonMapper.readValue(input, jsonMapper.constructType(type)) + } } } @@ -446,7 +442,7 @@ internal class GitHubResponseDecoder : Decoder { try { return repoConfigMapper.readValue(responseAsString, RepositoryConfig::class.java) - } catch (e: IOException) { + } catch (e: JacksonException) { throw Repository.RepoConfigException(HttpStatus.BAD_REQUEST,"unable to parse config for repo - content : \"" + response.body + "\"", e) } } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitLabImpl.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitLabImpl.kt index 55cc89a..5a401fb 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitLabImpl.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitLabImpl.kt @@ -1,12 +1,11 @@ package com.societegenerale.githubcrawler.remote import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.kotlin.KotlinModule -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.fasterxml.jackson.module.kotlin.readValue +import tools.jackson.databind.json.JsonMapper +import tools.jackson.dataformat.yaml.YAMLMapper +import tools.jackson.module.kotlin.KotlinModule +import tools.jackson.module.kotlin.jacksonObjectMapper +import tools.jackson.module.kotlin.readValue import com.societegenerale.githubcrawler.RepositoryConfig import com.societegenerale.githubcrawler.model.* import com.societegenerale.githubcrawler.model.commit.Commit @@ -23,12 +22,9 @@ import feign.httpclient.ApacheHttpClient import feign.slf4j.Slf4jLogger import okhttp3.OkHttpClient import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.http.HttpMessageConverters -import org.springframework.cloud.openfeign.support.ResponseEntityDecoder -import org.springframework.cloud.openfeign.support.SpringDecoder import org.springframework.http.HttpStatus -import org.springframework.http.MediaType -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter +import tools.jackson.core.JacksonException + import java.io.IOException import java.lang.reflect.Type import java.util.* @@ -71,7 +67,7 @@ class RemoteGitLabImpl constructor( val log = LoggerFactory.getLogger(this.javaClass) - private val objectMapper = jacksonObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + private val objectMapper = jacksonObjectMapper() override fun fetchRepoConfig(repositoryFullName: String, defaultBranch: String): RepositoryConfig { @@ -268,12 +264,9 @@ data class GitLabRepository (val id : Int,val web_url : String, val path : Strin internal class GitLabResponseDecoder : Decoder { val log = LoggerFactory.getLogger(this.javaClass) - val repoConfigMapper = ObjectMapper(YAMLFactory()) - - init { - repoConfigMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - repoConfigMapper.registerModule(KotlinModule()) - } + val repoConfigMapper: YAMLMapper = YAMLMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() @Throws(IOException::class) override fun decode(response: Response, type: Type): Any { @@ -290,18 +283,22 @@ internal class GitLabResponseDecoder : Decoder { log.debug("Decoding a successful response...") - if (type.typeName == MediaType.TEXT_PLAIN_VALUE) { + if (type == String::class.java) { log.debug("\t ... as a String") - return response.body().toString() + return response.body().asReader(Charsets.UTF_8).use { it.readText() } } log.debug("\t ... as a " + type.typeName) - val jacksonConverter = MappingJackson2HttpMessageConverter(ObjectMapper().registerModule(KotlinModule.Builder().build())) - val objectFactory = { HttpMessageConverters(jacksonConverter) } - return ResponseEntityDecoder(SpringDecoder(objectFactory)).decode(response, type) + val jsonMapper = JsonMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() + + response.body().asInputStream().use { input -> + return jsonMapper.readValue(input, jsonMapper.constructType(type)) + } } } @@ -313,7 +310,7 @@ internal class GitLabResponseDecoder : Decoder { try { return repoConfigMapper.readValue(responseAsString, RepositoryConfig::class.java) - } catch (e: IOException) { + } catch (e: JacksonException) { throw Repository.RepoConfigException(HttpStatus.BAD_REQUEST,"unable to parse config for repo - content : \"$responseAsString\"", e) } } diff --git a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/ConfigParserTest.java b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/ConfigParserTest.java index 566a6b8..73aa4da 100644 --- a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/ConfigParserTest.java +++ b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/ConfigParserTest.java @@ -2,9 +2,10 @@ import static org.assertj.core.api.Assertions.assertThat; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.module.kotlin.KotlinModule; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; +import tools.jackson.dataformat.yaml.YAMLMapper; +import tools.jackson.module.kotlin.KotlinModule; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; @@ -14,7 +15,9 @@ class ConfigParserTest { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + ObjectMapper mapper = YAMLMapper.builder() + .addModule(new KotlinModule.Builder().build()) + .build(); @Test void canParseSimpleYamlConfig() throws IOException { @@ -32,15 +35,13 @@ void canParseMoreComplexYamlConfig() throws IOException { InputStream is = getClass().getClassLoader().getResourceAsStream("sampleRepoConfig.yaml"); String configToParse = StreamUtils.copyToString(is, Charset.forName("UTF-8")); - mapper.registerModule(new KotlinModule()); - RepositoryConfig parsedRepositoryConfig = mapper.readValue(configToParse, RepositoryConfig.class); assertThat(parsedRepositoryConfig).isNotNull(); assertThat(parsedRepositoryConfig.getExcluded()).isFalse(); assertThat(parsedRepositoryConfig.getFilesToParse()).hasSize(1); - FileToParse firstFile = parsedRepositoryConfig.getFilesToParse().get(0); + FileToParse firstFile = parsedRepositoryConfig.getFilesToParse().getFirst(); assertThat(firstFile.getRedirectTo()).isEqualTo("moduleWhereDockerFileIs/Dockerfile"); assertThat(firstFile.getName()).isEqualTo("Dockerfile"); } @@ -50,7 +51,7 @@ void canParseSimpleJsonConfig() throws IOException { String configToParse = "{\"excluded\": true}"; - ObjectMapper mapper = new ObjectMapper(); + ObjectMapper mapper = new JsonMapper(); RepositoryConfig parsedRepositoryConfig = mapper.readValue(configToParse, RepositoryConfig.class); diff --git a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerTest.kt b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerTest.kt index 72e85e5..7c85d45 100644 --- a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerTest.kt +++ b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerTest.kt @@ -1,13 +1,13 @@ package com.societegenerale.githubcrawler import com.google.common.collect.ImmutableList -import com.jayway.awaitility.Awaitility.await import com.societegenerale.githubcrawler.model.Repository import com.societegenerale.githubcrawler.output.GitHubCrawlerOutput import com.societegenerale.githubcrawler.parsers.SimpleFilePathParser import com.societegenerale.githubcrawler.remote.RemoteSourceControl import org.assertj.core.api.Assertions.assertThat +import org.awaitility.Awaitility import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -142,8 +142,8 @@ class GitHubCrawlerTest { gitHubCrawler.crawl() - await().atMost(2, java.util.concurrent.TimeUnit.SECONDS) - .until({ assertThat(output.analyzedRepositories.values).hasSize(nbExpectedRecords) }) + Awaitility.await().atMost(2, java.util.concurrent.TimeUnit.SECONDS) + .untilAsserted { assertThat(output.analyzedRepositories.values).hasSize(nbExpectedRecords) } return output.analyzedRepositories diff --git a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/FileOutputTest.kt b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/FileOutputTest.kt index 9d874cf..a78e83a 100644 --- a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/FileOutputTest.kt +++ b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/FileOutputTest.kt @@ -8,7 +8,7 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.core.io.support.PathMatchingResourcePatternResolver import java.nio.file.Files -import java.nio.file.Paths +import java.nio.file.Path import java.util.* @@ -22,7 +22,7 @@ class FileOutputTest{ val resources = resolver.getResources("file:target/somePrfix*.txt") for(existingFile in resources){ - val fileToDeletePath = Paths.get(existingFile.uri) + val fileToDeletePath = Path.of(existingFile.uri) Files.delete(fileToDeletePath) } @@ -61,7 +61,7 @@ class FileOutputTest{ assertThat(resources).hasSize(1) - val lines=Files.readAllLines(Paths.get(resources[0].uri)) + val lines=Files.readAllLines(Path.of(resources[0].uri)) assertThat(lines).hasSize(3) diff --git a/github-crawler-core/src/test/resources/sample_invalid_yamlfile.yml b/github-crawler-core/src/test/resources/sample_invalid_yamlfile.yml index c667d36..693822f 100644 --- a/github-crawler-core/src/test/resources/sample_invalid_yamlfile.yml +++ b/github-crawler-core/src/test/resources/sample_invalid_yamlfile.yml @@ -19,7 +19,7 @@ boolean.property: true --- spring: - profiles: aProfile + config.activate.on-profile: aProfile server: ssl: key-store: hello diff --git a/github-crawler-core/src/test/resources/sample_yamlfile.yml b/github-crawler-core/src/test/resources/sample_yamlfile.yml index 3f8f2ea..fa0aaef 100644 --- a/github-crawler-core/src/test/resources/sample_yamlfile.yml +++ b/github-crawler-core/src/test/resources/sample_yamlfile.yml @@ -24,7 +24,7 @@ boolean.property: true --- spring: - profiles: aProfile + config.activate.on-profile: aProfile server: ssl: key-store: hello diff --git a/github-crawler-starter/pom.xml b/github-crawler-starter/pom.xml index 90f0328..02f1101 100644 --- a/github-crawler-starter/pom.xml +++ b/github-crawler-starter/pom.xml @@ -15,7 +15,6 @@ com.societegenerale.github-crawler github-crawler-autoconfigure - ${project.parent.version} diff --git a/github-crawler-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/github-crawler-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 0000000..239b8f0 --- /dev/null +++ b/github-crawler-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +com.societegenerale.githubcrawler.config.GitHubCrawlerAutoConfiguration \ No newline at end of file diff --git a/pom.xml b/pom.xml index 5466729..9d5b650 100644 --- a/pom.xml +++ b/pom.xml @@ -44,18 +44,25 @@ UTF-8 UTF-8 - 11 - 1.8.10 + 25 + 2.4.10 - 2.7.5 - 3.1.5 + 4.0.7 + 5.0.2 31.1-jre 2.1.3 2.10 1.1.6 + 2.105 + 3.13.2 + 4.3.0 + 5.4.0 + 2.22.0 + 3.0.0 - 0.8.8 - 3.10.1 + 0.8.15 + 3.15.0 + 3.5.4 @@ -66,6 +73,18 @@ + + + com.societegenerale.github-crawler + github-crawler-core + ${project.version} + + + com.societegenerale.github-crawler + github-crawler-autoconfigure + ${project.version} + + org.springframework.boot spring-boot-dependencies @@ -94,6 +113,19 @@ pom import + + com.squareup.okhttp3 + okhttp-bom + ${okhttp-bom.version} + pom + import + + + org.awaitility + awaitility + ${awaitility.version} + test + @@ -101,11 +133,6 @@ com.google.guava guava - - org.springframework.boot - spring-boot-starter-test - test - org.jetbrains.kotlin kotlin-stdlib-jdk8 @@ -124,6 +151,11 @@ kotlin-test test + + org.springframework.boot + spring-boot-starter-test + test + org.dom4j dom4j @@ -140,6 +172,13 @@ kotlin-maven-plugin + + + org.jetbrains.kotlin + kotlin-maven-allopen + ${kotlin.version} + + org.jetbrains.kotlin ${kotlin.version} @@ -170,26 +209,28 @@ - 1.8 + ${java.version} -Xjvm-default=all-compatibility + + spring + org.apache.maven.plugins maven-compiler-plugin ${maven-compiler-plugin.version} - - ${java.version} - ${java.version} + org.projectlombok lombok - 1.18.24 + 1.18.46 + ${java.version} @@ -235,13 +276,15 @@ org.apache.maven.plugins maven-surefire-plugin + ${maven-surefire-plugin.version} - ${surefireArgLine} + ${surefireArgLine} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true org.apache.maven.plugins maven-failsafe-plugin + ${maven-surefire-plugin.version} @@ -253,6 +296,9 @@ + + --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true + org.jacoco @@ -292,9 +338,9 @@ 4.3.0 - javax.xml.bind - jaxb-api - 2.3.1 + jakarta.xml.bind + jakarta.xml.bind-api + 4.0.5 From a524b19f0e42f6e72e8af31f6e9bcc806f389793 Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 10:53:10 +0530 Subject: [PATCH 06/21] build: bump version to 3.0.0-SNAPSHOT in pom.xml --- github-crawler-autoconfigure/pom.xml | 2 +- github-crawler-core/pom.xml | 2 +- github-crawler-starter/pom.xml | 2 +- pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/github-crawler-autoconfigure/pom.xml b/github-crawler-autoconfigure/pom.xml index 448501a..7c7f31f 100644 --- a/github-crawler-autoconfigure/pom.xml +++ b/github-crawler-autoconfigure/pom.xml @@ -9,7 +9,7 @@ com.societegenerale.github-crawler github-crawler-parent - 2.2.1-SNAPSHOT + 3.0.0-SNAPSHOT diff --git a/github-crawler-core/pom.xml b/github-crawler-core/pom.xml index 9f6dc6b..545ada2 100644 --- a/github-crawler-core/pom.xml +++ b/github-crawler-core/pom.xml @@ -9,7 +9,7 @@ com.societegenerale.github-crawler github-crawler-parent - 2.2.1-SNAPSHOT + 3.0.0-SNAPSHOT org.springframework.boot spring-boot-maven-plugin + ${spring-boot-maven-plugin.version} com.societegenerale.githubcrawler.GitHubCrawlerApplication exec diff --git a/pom.xml b/pom.xml index b656035..b217370 100644 --- a/pom.xml +++ b/pom.xml @@ -63,6 +63,7 @@ 0.8.15 3.15.0 3.5.4 + 4.0.7 From 2943f9d4a1225dd3730d96a64a2a31763a27a2ce Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 11:04:19 +0530 Subject: [PATCH 08/21] build: move spring-boot-configuration-processor to annotationProcessorPath --- github-crawler-core/pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/github-crawler-core/pom.xml b/github-crawler-core/pom.xml index 545ada2..87dce45 100644 --- a/github-crawler-core/pom.xml +++ b/github-crawler-core/pom.xml @@ -66,11 +66,6 @@ json-path ${json-path.version} - - org.springframework.boot - spring-boot-configuration-processor - true - org.apache.commons @@ -105,6 +100,11 @@ dagger-compiler 2.60.1 + + org.springframework.boot + spring-boot-configuration-processor + ${spring-boot-dependencies.version} + From 12b2542d81bf9060209521d3f8544c9fede0f2ba Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 11:05:54 +0530 Subject: [PATCH 09/21] build: update jvm-default argument for compatibility --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index b217370..6ca98db 100644 --- a/pom.xml +++ b/pom.xml @@ -212,7 +212,7 @@ ${java.version} - -Xjvm-default=all-compatibility + -jvm-default=enable spring From c4aa182613ab4c52062152fa62b28c2d8b94ee3c Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 11:11:12 +0530 Subject: [PATCH 10/21] refactor: fix deprecated code --- .../githubcrawler/FileToParse.kt | 7 +++---- .../remote/RemoteAzureDevopsImpl.kt | 6 +++--- .../remote/RemoteBitBucketImpl.kt | 2 +- .../githubcrawler/remote/RemoteGitHubImpl.kt | 18 +++++------------- .../githubcrawler/remote/RemoteGitLabImpl.kt | 6 +++--- .../githubcrawler/output/HttpOutputTest.kt | 1 + .../PathsForHitsOnRepoSearchTest.kt | 1 + 7 files changed, 17 insertions(+), 24 deletions(-) diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/FileToParse.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/FileToParse.kt index 31c165b..6ee39c3 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/FileToParse.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/FileToParse.kt @@ -2,7 +2,6 @@ package com.societegenerale.githubcrawler import org.springframework.core.convert.ConversionService import org.springframework.core.convert.TypeDescriptor -import org.springframework.lang.Nullable data class FileToParse(val name: String, @@ -10,11 +9,11 @@ data class FileToParse(val name: String, class FileToParseConversionService : ConversionService { - override fun canConvert(@Nullable aClass: Class<*>?, aClass1: Class<*>): Boolean { + override fun canConvert(aClass: Class<*>?, aClass1: Class<*>): Boolean { return false } - override fun canConvert(@Nullable sourcetype: TypeDescriptor?, targetType: TypeDescriptor): Boolean { + override fun canConvert(sourcetype: TypeDescriptor?, targetType: TypeDescriptor): Boolean { return targetType.name == FileToParse::class.java.name } @@ -22,7 +21,7 @@ class FileToParseConversionService : ConversionService { return null } - override fun convert(@Nullable value: Any?, @Nullable sourceType: TypeDescriptor?, targetType: TypeDescriptor): Any? { + override fun convert(value: Any?, sourceType: TypeDescriptor?, targetType: TypeDescriptor): Any? { return if (targetType.name == FileToParse::class.java.name) { FileToParse(value as String, null) diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteAzureDevopsImpl.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteAzureDevopsImpl.kt index 7074e38..9c53cbd 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteAzureDevopsImpl.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteAzureDevopsImpl.kt @@ -146,7 +146,7 @@ class RemoteAzureDevopsImpl @JvmOverloads constructor(private val azureDevopsUrl val responseBody= httpClient.newCall(request).execute().body - val repoSearchResult = objectMapper.readValue(responseBody?.string(), CodeSearchResult::class.java) + val repoSearchResult = objectMapper.readValue(responseBody.string(), CodeSearchResult::class.java) return repoSearchResult.toStandardSearchResult() @@ -164,7 +164,7 @@ class RemoteAzureDevopsImpl @JvmOverloads constructor(private val azureDevopsUrl val response=httpClient.newCall(request).execute() if(response.isSuccessful){ - return response.body?.string() ?: "" + return response.body.string() } else{ throw NoFileFoundException("can't find $fileToFetch in repo $repositoryFullName, in branch $branchName") @@ -245,7 +245,7 @@ internal class AzureDevopsResponseDecoder { } val writer = StringWriter() - IOUtils.copy(response.body?.byteStream(), writer, "UTF-8") + IOUtils.copy(response.body.byteStream(), writer, "UTF-8") val responseAsString = writer.toString() return parseRepositoryConfigResponse(responseAsString, response) diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteBitBucketImpl.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteBitBucketImpl.kt index e334b1b..2c7633e 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteBitBucketImpl.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteBitBucketImpl.kt @@ -45,7 +45,7 @@ class RemoteBitBucketImpl @JvmOverloads constructor( .client(ApacheHttpClient()) .encoder(GsonEncoder()) .decoder(BitBucketResponseDecoder()) - .decode404() + .dismiss404() .requestInterceptor(BitBucketOauthTokenSetter(apiKey)) .logger(Slf4jLogger(RemoteBitBucketImpl::class.java)) .logLevel(Logger.Level.FULL) diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitHubImpl.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitHubImpl.kt index b9429e0..c1e94c8 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitHubImpl.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitHubImpl.kt @@ -56,7 +56,7 @@ class RemoteGitHubImpl @JvmOverloads constructor( .encoder(GsonEncoder()) .decoder(GitHubResponseDecoder()) .errorDecoder(GiHubErrorDecoder()) - .decode404() + .dismiss404() .requestInterceptor(GitHubOauthTokenSetter(apiKey)) .logger(Slf4jLogger(RemoteGitHubImpl::class.java)) .logLevel(Logger.Level.FULL) @@ -170,15 +170,7 @@ class RemoteGitHubImpl @JvmOverloads constructor( private fun extractRepositories(response: Response): Set { try { - - val body = response.body - - if (body != null) { - return objectMapper.readValue(body.string()) - } else { - log.warn("response is null : {}", response) - return emptySet() - } + return objectMapper.readValue(response.body.string()) } catch (e: JacksonException) { throw NoReachableRepositories("not able to parse response", e) } @@ -233,7 +225,7 @@ class RemoteGitHubImpl @JvmOverloads constructor( val response = httpClient.newCall(request).execute() - val responseAsString=response.body?.string() + val responseAsString=response.body.string() log.info("response : "+responseAsString) return try { @@ -271,7 +263,7 @@ class RemoteGitHubImpl @JvmOverloads constructor( val response = httpClient.newCall(request).execute() - return response.body?.string() ?: "" + return response.body.string() } @@ -393,7 +385,7 @@ internal class GitHubResponseDecoder : Decoder { fun decodeRepoConfig(response: Response): RepositoryConfig { val writer = StringWriter() - IOUtils.copy(response.body?.byteStream(), writer, "UTF-8") + IOUtils.copy(response.body.byteStream(), writer, "UTF-8") val responseAsString = writer.toString() return parseRepositoryConfigResponse(responseAsString, response) diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitLabImpl.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitLabImpl.kt index 5a401fb..9596e9d 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitLabImpl.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitLabImpl.kt @@ -53,7 +53,7 @@ class RemoteGitLabImpl constructor( .encoder(GsonEncoder()) .decoder(GitLabResponseDecoder()) .errorDecoder(GiLabErrorDecoder()) - .decode404() + .dismiss404() .requestInterceptor(GitLabPrivateTokenSetter(privateToken)) .logger(Slf4jLogger(RemoteGitLabImpl::class.java)) .logLevel(Logger.Level.FULL) @@ -102,7 +102,7 @@ class RemoteGitLabImpl constructor( val httpResponse=httpClient.newCall(request).execute() - val gitlabSearchResult : List = objectMapper.readValue(httpResponse.body!!.string()) + val gitlabSearchResult : List = objectMapper.readValue(httpResponse.body.string()) return SearchResult(gitlabSearchResult.size,gitlabSearchResult.map{ it -> it.toSearchResultItem()}) @@ -181,7 +181,7 @@ class RemoteGitLabImpl constructor( throw NoReachableRepositories("GET call to ${fetchFileUrl} wasn't successful. Code : ${response.code}, Message : ${response.message}") } - return response.body!!.string() + return response.body.string() } @Throws(NoReachableRepositories::class) diff --git a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/HttpOutputTest.kt b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/HttpOutputTest.kt index 1a0f7f5..8f83b62 100644 --- a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/HttpOutputTest.kt +++ b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/HttpOutputTest.kt @@ -37,6 +37,7 @@ class HttpOutputTest { inline fun argumentCaptor() = ArgumentCaptor.forClass(T::class.java) @Test + @Suppress("UNCHECKED_CAST") fun shouldLogResponseBodyWhenErrorDuringPost() { //Mock logging infra val root = LoggerFactory.getLogger(ch.qos.logback.classic.Logger.ROOT_LOGGER_NAME) as ch.qos.logback.classic.Logger diff --git a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/repoTaskToPerform/PathsForHitsOnRepoSearchTest.kt b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/repoTaskToPerform/PathsForHitsOnRepoSearchTest.kt index c47a211..52044b8 100644 --- a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/repoTaskToPerform/PathsForHitsOnRepoSearchTest.kt +++ b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/repoTaskToPerform/PathsForHitsOnRepoSearchTest.kt @@ -49,6 +49,7 @@ class PathsForHitsOnRepoSearchTest { @Test + @Suppress("UNCHECKED_CAST") fun shouldYield_NotFound_WhenNoMatch() { `when`(mockRemoteGithub.fetchCodeSearchResult(repoToSearch.fullName, "someSearch")) From b1d39b7b4d60c518431c0e9919d569d59e20753f Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 11:19:58 +0530 Subject: [PATCH 11/21] build: add --sun-misc-unsafe-memory-access=allow argument to JVM configuration --- .mvn/jvm.config | 1 + pom.xml | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 .mvn/jvm.config diff --git a/.mvn/jvm.config b/.mvn/jvm.config new file mode 100644 index 0000000..fecd9cc --- /dev/null +++ b/.mvn/jvm.config @@ -0,0 +1 @@ +--sun-misc-unsafe-memory-access=allow diff --git a/pom.xml b/pom.xml index 6ca98db..d7ba704 100644 --- a/pom.xml +++ b/pom.xml @@ -279,7 +279,7 @@ maven-surefire-plugin ${maven-surefire-plugin.version} - ${surefireArgLine} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true + ${surefireArgLine} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true --sun-misc-unsafe-memory-access=allow @@ -293,12 +293,12 @@ verify - ${failsafeArgLine} + ${failsafeArgLine} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true --sun-misc-unsafe-memory-access=allow - --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true + --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true --sun-misc-unsafe-memory-access=allow From 12fe52e1f3b6b00d86f1029641544abdbb9e5189 Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 11:20:10 +0530 Subject: [PATCH 12/21] test: replace ArrayList with CopyOnWriteArrayList for thread safety --- .../societegenerale/githubcrawler/mocks/BitbucketMock.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/BitbucketMock.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/BitbucketMock.java index 27d73af..e1e322b 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/BitbucketMock.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/BitbucketMock.java @@ -19,6 +19,7 @@ import java.io.InputStream; import java.nio.charset.Charset; import java.util.*; +import java.util.concurrent.CopyOnWriteArrayList; @Component @@ -53,9 +54,9 @@ public boolean isHasCalledNextPage() { return hasCalledNextPage; } - private List repoConfigHits = new ArrayList<>(); + private List repoConfigHits = new CopyOnWriteArrayList<>(); - private List pomXmlHits = new ArrayList<>(); + private List pomXmlHits = new CopyOnWriteArrayList<>(); private int nbPages = 1; From 2d168dc0865408b3ebbcee8803578e71659cb304 Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 11:23:22 +0530 Subject: [PATCH 13/21] build: add Mockito agent path to JVM arguments for test execution --- pom.xml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index d7ba704..c87bbe6 100644 --- a/pom.xml +++ b/pom.xml @@ -274,12 +274,24 @@ forked-path + + org.apache.maven.plugins + maven-dependency-plugin + + + get-mockito-agent-path + + properties + + + + org.apache.maven.plugins maven-surefire-plugin ${maven-surefire-plugin.version} - ${surefireArgLine} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true --sun-misc-unsafe-memory-access=allow + -javaagent:${org.mockito:mockito-core:jar} ${surefireArgLine} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true --sun-misc-unsafe-memory-access=allow @@ -293,7 +305,7 @@ verify - ${failsafeArgLine} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true --sun-misc-unsafe-memory-access=allow + -javaagent:${org.mockito:mockito-core:jar} ${failsafeArgLine} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true --sun-misc-unsafe-memory-access=allow From 8855f599fe875ba3fc3520b4b8dc96da33ac4708 Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 15:51:01 +0530 Subject: [PATCH 14/21] ci(build): update the step name --- .github/workflows/build_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index c7140a9..5a4140c 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - - name: Set up JDK 11 + - name: Set up JDK 25 uses: actions/setup-java@v5 with: distribution: temurin From 99614ba4228b4e65bc624848632fd3b3ec993fa1 Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 15:51:47 +0530 Subject: [PATCH 15/21] ci(release): configure git user for release --- .github/workflows/release_workflow.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml index cfdb049..cf1fc79 100644 --- a/.github/workflows/release_workflow.yml +++ b/.github/workflows/release_workflow.yml @@ -22,7 +22,7 @@ jobs: version=$(echo "$raw_tag_name" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+$') echo "VERSION=$version" >> $GITHUB_ENV - - name: Set up JDK 11 with GPG + - name: Set up JDK 25 with GPG uses: actions/setup-java@v5 with: distribution: temurin @@ -33,6 +33,11 @@ jobs: server-username: CENTRAL_USERNAME server-password: CENTRAL_PASSWORD + - name: Configure git user for release + run: | + git config user.name "github-actions" + git config user.email "github-actions@github.com" + - name: Build & Deploy to Staging env: CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }} From c499271cfbd4d02888dfbda9bc170c93c7b299ba Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 15:52:35 +0530 Subject: [PATCH 16/21] docs: update version and maintainers --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 8417da1..ccbcafd 100644 --- a/README.md +++ b/README.md @@ -362,3 +362,14 @@ public class PersonalGitHubCrawlerApplication implements CommandLineRunner { see [here](https://github.com/vincent-fuchs/my-custom-github-crawler/blob/0e48dd7961b6b625802a2e1eb6b2fc4f8c4d5cdb/src/main/java/com/github/vincent_fuchs/output/CustomOutput.java) or [here](https://github.com/vincent-fuchs/my-custom-github-crawler/blob/ec7ed9a74f91b31794b8a0afb1196553434b1567/src/main/java/com/github/vincent_fuchs/parsers/MyOwnParser.java) for examples - see the javadoc in [FileContentParser](./github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/parsers/FileContentParser.kt) , [RepoTaskToPerform](./github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/repoTaskToPerform/RepoTaskToPerform.kt), [GitHubCrawlerOutput](./github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/GitHubCrawlerOutput.kt) which are the main extension points. + +## Versions + +- v3.x is using JDK25 +- v2.x is using JDK11 + +## Official maintainers + +- [Paul Williams](https://github.com/paul58914080) +- [Vincent Fuchs](https://github.com/vincent-fuchs) + From d1663fb3d13d51f213acdf051884b81634d51591 Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 15:53:08 +0530 Subject: [PATCH 17/21] build: update for releasing to sonatype --- pom.xml | 54 ++++++++++++++++++++++++------------------------------ 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/pom.xml b/pom.xml index c87bbe6..57085fe 100644 --- a/pom.xml +++ b/pom.xml @@ -31,16 +31,6 @@ HEAD - - - ossrh - https://s01.oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ - - UTF-8 UTF-8 @@ -64,6 +54,9 @@ 3.15.0 3.5.4 4.0.7 + 3.4.0 + 3.2.8 + 0.9.0 @@ -263,8 +256,6 @@ maven-release-plugin 2.5.3 - https://s01.oss.sonatype.org/service/local/staging/deploy/maven2 - github_crawler_@{project.version} true true @@ -357,22 +348,6 @@ - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.8 - true - - ossrh - https://s01.oss.sonatype.org/ - true - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.2 - @@ -401,7 +376,7 @@ org.apache.maven.plugins maven-source-plugin - 3.0.1 + ${maven-source-plugin.version} attach-sources @@ -428,7 +403,7 @@ org.apache.maven.plugins maven-gpg-plugin - 3.0.1 + ${maven-gpg-plugin.version} sign-artifacts @@ -438,6 +413,25 @@ + + + --pinentry-mode + loopback + + + + + + + org.sonatype.central + central-publishing-maven-plugin + ${central-publishing-maven-plugin.version} + true + + central + true + published + From aef3a9746a9e57f9703d7f83644703f4c601cff4 Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 15:55:17 +0530 Subject: [PATCH 18/21] build: bump spring-boot to 4.1.0 --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 57085fe..71df147 100644 --- a/pom.xml +++ b/pom.xml @@ -37,7 +37,7 @@ 25 2.4.10 - 4.0.7 + 4.1.0 5.0.2 31.1-jre 2.1.3 @@ -53,7 +53,7 @@ 0.8.15 3.15.0 3.5.4 - 4.0.7 + 4.1.0 3.4.0 3.2.8 0.9.0 From 5d78d3212e51f9a1e1bf9b94b8cdaf4a0c9a6719 Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 16:14:06 +0530 Subject: [PATCH 19/21] build: update plugin versions in pom.xml --- pom.xml | 60 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 31 deletions(-) diff --git a/pom.xml b/pom.xml index 71df147..d3daeee 100644 --- a/pom.xml +++ b/pom.xml @@ -50,13 +50,17 @@ 2.22.0 3.0.0 - 0.8.15 + 0.8.15 3.15.0 - 3.5.4 + 3.5.6 4.1.0 3.4.0 3.2.8 0.9.0 + 3.5.6 + 3.11.0 + 4.3.0 + 3.6.1 @@ -251,23 +255,10 @@ - - org.apache.maven.plugins - maven-release-plugin - 2.5.3 - - github_crawler_@{project.version} - true - true - false - false - makeRelease - forked-path - - org.apache.maven.plugins maven-dependency-plugin + ${maven-dependency-plugin.version} get-mockito-agent-path @@ -282,13 +273,18 @@ maven-surefire-plugin ${maven-surefire-plugin.version} - -javaagent:${org.mockito:mockito-core:jar} ${surefireArgLine} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true --sun-misc-unsafe-memory-access=allow + -javaagent:${org.mockito:mockito-core:jar} ${surefireArgLine} --add-opens + java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens + java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true + --sun-misc-unsafe-memory-access=allow + org.apache.maven.plugins maven-failsafe-plugin - ${maven-surefire-plugin.version} + ${maven-failsafe-plugin.version} @@ -296,18 +292,28 @@ verify - -javaagent:${org.mockito:mockito-core:jar} ${failsafeArgLine} --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true --sun-misc-unsafe-memory-access=allow + -javaagent:${org.mockito:mockito-core:jar} ${failsafeArgLine} --add-opens + java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens + java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true + --sun-misc-unsafe-memory-access=allow + - --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true --sun-misc-unsafe-memory-access=allow + --add-opens java.base/java.lang=ALL-UNNAMED --add-opens + java.base/java.lang.reflect=ALL-UNNAMED --add-opens + java.base/jdk.internal.misc=ALL-UNNAMED --add-opens + java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true + --sun-misc-unsafe-memory-access=allow + org.jacoco jacoco-maven-plugin - ${jacoco.version} + ${jacoco-plugin.version} pre-unit-tests @@ -339,14 +345,7 @@ org.eluder.coveralls coveralls-maven-plugin - 4.3.0 - - - jakarta.xml.bind - jakarta.xml.bind-api - 4.0.5 - - + ${coveralls-maven-plugin.version} @@ -358,7 +357,7 @@ org.codehaus.mojo build-helper-maven-plugin - 3.0.0 + ${build-helper-maven-plugin.version} generate-sources @@ -420,7 +419,6 @@ - org.sonatype.central From 02521462657435851d6b8ef64071a758bdefe487 Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 16:44:18 +0530 Subject: [PATCH 20/21] build: add repackage execution --- github-crawler-starter/pom.xml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/github-crawler-starter/pom.xml b/github-crawler-starter/pom.xml index 7592b21..153562a 100644 --- a/github-crawler-starter/pom.xml +++ b/github-crawler-starter/pom.xml @@ -31,6 +31,13 @@ com.societegenerale.githubcrawler.GitHubCrawlerApplication exec + + + + repackage + + + From 1632c60dc43ba500157ce27508ddd7b3eed048ed Mon Sep 17 00:00:00 2001 From: Paul Williams Date: Sun, 19 Jul 2026 16:51:45 +0530 Subject: [PATCH 21/21] test: correct the flanky assertions --- .../githubcrawler/GitHubCrawlerIT.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java index db589f2..f72e566 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java @@ -307,22 +307,21 @@ void shouldPerformSearchOnAllRepos() throws IOException { assertThat(output.getAnalyzedRepositories()).isEmpty(); assertThat(githubMockServer.getSearchHitsCount()).isEqualTo(0); - crawler.crawl(); - Collection actualRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .untilAsserted(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> { + assertThat(output.getAnalyzedRepositories().values()).hasSize(nbRepositoriesInOrga); + assertThat(githubMockServer.getSearchHitsCount()).isEqualTo(nbRepositoriesInOrga); + }); - assertThat(githubMockServer.getSearchHitsCount()).isEqualTo(nbRepositoriesInOrga); + Collection actualRepositories = new ArrayList<>(output.getAnalyzedRepositories().values()); actualRepositories.forEach(repo -> { - assertThat(repo.getMiscTasksResults()).hasSize(1); - Map masterMiscTaskResults=repo.getMiscTasksResults().get(new Branch("master")); - assertThat(masterMiscTaskResults).containsEntry("nbOfMetricsInPomXml","2"); - + Map masterMiscTaskResults = repo.getMiscTasksResults().get(new Branch("master")); + assertThat(masterMiscTaskResults).containsEntry("nbOfMetricsInPomXml", "2"); }); }