diff --git a/.github/workflows/build_workflow.yml b/.github/workflows/build_workflow.yml index 8d40611c..5a4140c2 100644 --- a/.github/workflows/build_workflow.yml +++ b/.github/workflows/build_workflow.yml @@ -28,11 +28,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - - name: Set up JDK 11 + - name: Set up JDK 25 uses: actions/setup-java@v5 with: distribution: temurin - java-version: 11 + java-version: 25 - name: Cache Maven dependencies uses: actions/cache@v5 with: diff --git a/.github/workflows/release_workflow.yml b/.github/workflows/release_workflow.yml index d20d3b60..cf1fc79d 100644 --- a/.github/workflows/release_workflow.yml +++ b/.github/workflows/release_workflow.yml @@ -22,17 +22,22 @@ jobs: version=$(echo "$raw_tag_name" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+$') echo "VERSION=$version" >> $GITHUB_ENV - - name: Set up JDK 11 with GPG + - name: Set up JDK 25 with GPG uses: actions/setup-java@v5 with: distribution: temurin - java-version: 11 + java-version: 25 gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }} gpg-passphrase: MAVEN_GPG_PASSPHRASE server-id: central server-username: CENTRAL_USERNAME server-password: CENTRAL_PASSWORD + - name: Configure git user for release + run: | + git config user.name "github-actions" + git config user.email "github-actions@github.com" + - name: Build & Deploy to Staging env: CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }} diff --git a/.mvn/jvm.config b/.mvn/jvm.config new file mode 100644 index 00000000..fecd9cc7 --- /dev/null +++ b/.mvn/jvm.config @@ -0,0 +1 @@ +--sun-misc-unsafe-memory-access=allow diff --git a/.sdkmanrc b/.sdkmanrc new file mode 100644 index 00000000..b5e2fc10 --- /dev/null +++ b/.sdkmanrc @@ -0,0 +1,3 @@ +# Enable auto-env through the sdkman_auto_env config +# Add key=value pairs of SDKs to use below +java=25.0.3-tem diff --git a/README.md b/README.md index 8417da17..ccbcafd8 100644 --- a/README.md +++ b/README.md @@ -362,3 +362,14 @@ public class PersonalGitHubCrawlerApplication implements CommandLineRunner { see [here](https://github.com/vincent-fuchs/my-custom-github-crawler/blob/0e48dd7961b6b625802a2e1eb6b2fc4f8c4d5cdb/src/main/java/com/github/vincent_fuchs/output/CustomOutput.java) or [here](https://github.com/vincent-fuchs/my-custom-github-crawler/blob/ec7ed9a74f91b31794b8a0afb1196553434b1567/src/main/java/com/github/vincent_fuchs/parsers/MyOwnParser.java) for examples - see the javadoc in [FileContentParser](./github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/parsers/FileContentParser.kt) , [RepoTaskToPerform](./github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/repoTaskToPerform/RepoTaskToPerform.kt), [GitHubCrawlerOutput](./github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/GitHubCrawlerOutput.kt) which are the main extension points. + +## Versions + +- v3.x is using JDK25 +- v2.x is using JDK11 + +## Official maintainers + +- [Paul Williams](https://github.com/paul58914080) +- [Vincent Fuchs](https://github.com/vincent-fuchs) + diff --git a/github-crawler-autoconfigure/pom.xml b/github-crawler-autoconfigure/pom.xml index 6fbcfef8..7c7f31f9 100644 --- a/github-crawler-autoconfigure/pom.xml +++ b/github-crawler-autoconfigure/pom.xml @@ -1,58 +1,41 @@ - - 4.0.0 - + + 4.0.0 + github-crawler-autoconfigure + jar + github-crawler-autoconfigure + com.societegenerale.github-crawler - github-crawler-autoconfigure - 2.2.1-SNAPSHOT - jar - - github-crawler-autoconfigure - - - com.societegenerale.github-crawler - github-crawler-parent - 2.2.1-SNAPSHOT - - - - - - com.societegenerale.github-crawler - github-crawler-core - ${project.parent.version} - - - - net.code-story - http - 2.104 - test - - - - org.slf4j - slf4j-simple - - - - - - com.github.tomakehurst - wiremock-jre8 - 2.32.0 - test - - - - - com.jayway.awaitility - awaitility - 1.7.0 - test - - - - - + github-crawler-parent + 3.0.0-SNAPSHOT + + + + com.societegenerale.github-crawler + github-crawler-core + + + org.springframework.boot + spring-boot-starter-restclient + + + + net.code-story + http + ${http.version} + test + + + org.wiremock + wiremock-standalone + ${wiremock-standalone.version} + test + + + org.awaitility + awaitility + + diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/AzureDevopsCrawlerIT.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/AzureDevopsCrawlerIT.java index 114f921b..4834b3ca 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/AzureDevopsCrawlerIT.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/AzureDevopsCrawlerIT.java @@ -102,7 +102,7 @@ void shouldCrawl() throws IOException { assertThat(branchWhereResultWasFound.getName()).isEqualTo("refs/heads/main"); List valueForIndicator=(ArrayList)firstTaskResult.getValue().get("pipelineTemplateLocation"); - assertThat(valueForIndicator.get(0)).isEqualTo("azure-pipelines.yml"); + assertThat(valueForIndicator.getFirst()).isEqualTo("azure-pipelines.yml"); } diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/BitbucketCrawlerIT.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/BitbucketCrawlerIT.java index 4c240aee..25cbb1fa 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/BitbucketCrawlerIT.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/BitbucketCrawlerIT.java @@ -5,21 +5,20 @@ import com.societegenerale.githubcrawler.mocks.BitbucketMock; import com.societegenerale.githubcrawler.model.Repository; import org.assertj.core.api.SoftAssertions; +import org.awaitility.Awaitility; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.annotation.DirtiesContext.ClassMode; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit.jupiter.SpringExtension; import java.io.IOException; import java.util.*; -import static com.jayway.awaitility.Awaitility.await; + import static java.util.Map.entry; import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; @@ -27,7 +26,6 @@ import static org.assertj.core.api.Assertions.assertThat; -@ExtendWith(SpringExtension.class) @SpringBootTest(classes = {TestConfig.class, GitHubCrawlerAutoConfiguration.class}) @ActiveProfiles(profiles = {"bitBucketTest"}) @DirtiesContext(classMode = ClassMode.BEFORE_CLASS) @@ -58,8 +56,8 @@ public void mockSetUp() { bitbucketMockServer.start(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(BitbucketMock.hasStarted())); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(BitbucketMock.hasStarted()).isTrue()); hasBitbucketMockServerStarted = true; } @@ -94,8 +92,8 @@ void shouldHitOrgaNextPageOfRepositoriesIfMorethanOne() throws IOException { Collection processedRepositories = output.getAnalyzedRepositories().values(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); assertThat(bitbucketMockServer.isHasCalledNextPage()).as("next page wasn't called").isTrue(); @@ -125,7 +123,7 @@ void shouldLoadIndicatorsConfig() { .containsKeys(pomXmlFile, new FileToParse("pom.xml", null), new FileToParse("Jenkinsfile", null), new FileToParse("Dockerfile", null)); - IndicatorDefinition pomXmlIndicatorDefinition1 = crawler.getGitHubCrawlerProperties().getIndicatorsToFetchByFile().get(pomXmlFile).get(0); + IndicatorDefinition pomXmlIndicatorDefinition1 = crawler.getGitHubCrawlerProperties().getIndicatorsToFetchByFile().get(pomXmlFile).getFirst(); assertThat(pomXmlIndicatorDefinition1).isNotNull(); assertThat(pomXmlIndicatorDefinition1.getName()).isEqualTo("spring_boot_starter_parent_version"); assertThat(pomXmlIndicatorDefinition1.getType()).isEqualTo("findDependencyVersionInXml"); @@ -140,8 +138,8 @@ void shouldTryToFetchFilesWithIndicators() throws IOException { crawler.crawl(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(bitbucketMockServer.getPomXmlHits()).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(bitbucketMockServer.getPomXmlHits()).hasSize(nbRepositoriesInOrga)); } @@ -152,10 +150,10 @@ void shouldCopyActiveProfilesAsGroupsOnRepo() throws IOException { Collection processedRepositories = output.getAnalyzedRepositories().values(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); - processedRepositories.stream().forEach(repo -> { + processedRepositories.forEach(repo -> { assertThat(repo.getGroups()).containsExactlyInAnyOrder("bitBucketTest"); }); @@ -182,8 +180,8 @@ void shouldParseFileWithIndicatorsWhenItExists() throws IOException { crawler.crawl(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(output.getAnalyzedRepositories().values()).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(output.getAnalyzedRepositories().values()).hasSize(nbRepositoriesInOrga)); Collection processedRepositories = output.getAnalyzedRepositories().values(); @@ -252,8 +250,8 @@ void shouldParseAllBranchesWhenConfiguredAccordingly() throws IOException { Collection processedRepositories = output.getAnalyzedRepositories().values(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); for (Repository processedRepository : processedRepositories) { @@ -284,8 +282,8 @@ void processNormallyIfNoConfigFileOnRepoSide() throws IOException { crawler.crawl(); Collection actualRepositories = output.getAnalyzedRepositories().values(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); assertThat(bitbucketMockServer.getRepoConfigHits()).hasSize(nbRepositoriesInOrga); @@ -303,8 +301,8 @@ void outputResultsShouldHaveACrawlerRunId() throws IOException { crawler.crawl(); Collection actualRepositories = output.getAnalyzedRepositories().values(); - await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); + Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) + .untilAsserted(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); assertThat(actualRepositories.stream().filter(r -> !r.getCrawlerRunId().equals(GitHubCrawler.NO_CRAWLER_RUN_ID_DEFINED)) .count()) diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java index d898ac19..f72e566c 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerIT.java @@ -1,10 +1,11 @@ package com.societegenerale.githubcrawler; -import static com.jayway.awaitility.Awaitility.await; + import static java.util.concurrent.TimeUnit.SECONDS; import static java.util.stream.Collectors.toList; import static java.util.stream.Collectors.toSet; import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import com.societegenerale.githubcrawler.config.GitHubCrawlerAutoConfiguration; import com.societegenerale.githubcrawler.config.TestConfig; @@ -23,16 +24,13 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.annotation.DirtiesContext.ClassMode; import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.junit.jupiter.SpringExtension; -@ExtendWith(SpringExtension.class) @SpringBootTest(classes = {TestConfig.class, GitHubCrawlerAutoConfiguration.class}) @ActiveProfiles(profiles = {"gitHubTest", "profilesAreAWayOfGrouping"}) @DirtiesContext(classMode = ClassMode.BEFORE_CLASS) @@ -64,7 +62,7 @@ public void mockSetUp() { githubMockServer.start(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(GitHubMock.hasStarted())); + .untilAsserted(() -> assertThat(GitHubMock.hasStarted())); hasGitHubMockServerStarted = true; } @@ -100,7 +98,7 @@ void shouldHitOrgaNextPageOfRepositoriesIfMorethanOne() throws IOException { Collection processedRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); assertThat(githubMockServer.isHasCalledNextPage()).as("next page wasn't called").isTrue(); @@ -129,7 +127,7 @@ void shouldLoadIndicatorsConfig() { assertThat(crawler.getGitHubCrawlerProperties().getIndicatorsToFetchByFile()).containsKeys(pomXmlFile, new FileToParse("pom.xml", null), new FileToParse("Jenkinsfile", null), new FileToParse("Dockerfile", null)); - IndicatorDefinition pomXmlIndicatorDefinition1 = crawler.getGitHubCrawlerProperties().getIndicatorsToFetchByFile().get(pomXmlFile).get(0); + IndicatorDefinition pomXmlIndicatorDefinition1 = crawler.getGitHubCrawlerProperties().getIndicatorsToFetchByFile().get(pomXmlFile).getFirst(); assertThat(pomXmlIndicatorDefinition1).isNotNull(); assertThat(pomXmlIndicatorDefinition1.getName()).isEqualTo("spring_boot_starter_parent_version"); assertThat(pomXmlIndicatorDefinition1.getType()).isEqualTo("findDependencyVersionInXml"); @@ -147,7 +145,7 @@ void shouldTryToFetchFilesWithIndicators() throws IOException { crawler.crawl(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(githubMockServer.getPomXmlHits()).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(githubMockServer.getPomXmlHits()).hasSize(nbRepositoriesInOrga)); } @@ -159,9 +157,9 @@ void shouldCopyActiveProfilesAsGroupsOnRepo() throws IOException { Collection processedRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); - processedRepositories.stream().forEach(repo -> { + processedRepositories.forEach(repo -> { assertThat(repo.getGroups()).containsExactlyInAnyOrder("gitHubTest", "profilesAreAWayOfGrouping"); }); @@ -189,7 +187,7 @@ void shouldParseFileWithIndicatorsWhenItExists() throws IOException { crawler.crawl(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(output.getAnalyzedRepositories().values()).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(output.getAnalyzedRepositories().values()).hasSize(nbRepositoriesInOrga)); Collection processedRepositories = output.getAnalyzedRepositories().values(); @@ -259,7 +257,7 @@ void shouldParseAllBranchesWhenConfiguredAccordingly() throws IOException { Collection processedRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga)); for (Repository processedRepository : processedRepositories) { @@ -291,7 +289,7 @@ void processNormallyIfNoConfigFileOnRepoSide() throws IOException { Collection actualRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); assertThat(githubMockServer.getRepoConfigHits()).hasSize(nbRepositoriesInOrga); @@ -309,22 +307,21 @@ void shouldPerformSearchOnAllRepos() throws IOException { assertThat(output.getAnalyzedRepositories()).isEmpty(); assertThat(githubMockServer.getSearchHitsCount()).isEqualTo(0); - crawler.crawl(); - Collection actualRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> { + assertThat(output.getAnalyzedRepositories().values()).hasSize(nbRepositoriesInOrga); + assertThat(githubMockServer.getSearchHitsCount()).isEqualTo(nbRepositoriesInOrga); + }); - assertThat(githubMockServer.getSearchHitsCount()).isEqualTo(nbRepositoriesInOrga); - - actualRepositories.stream().forEach(repo -> { + Collection actualRepositories = new ArrayList<>(output.getAnalyzedRepositories().values()); + actualRepositories.forEach(repo -> { assertThat(repo.getMiscTasksResults()).hasSize(1); - Map masterMiscTaskResults=repo.getMiscTasksResults().get(new Branch("master")); - assertThat(masterMiscTaskResults).containsEntry("nbOfMetricsInPomXml","2"); - + Map masterMiscTaskResults = repo.getMiscTasksResults().get(new Branch("master")); + assertThat(masterMiscTaskResults).containsEntry("nbOfMetricsInPomXml", "2"); }); } @@ -335,7 +332,7 @@ void outputResultsShouldHaveACrawlerRunId() throws IOException { Collection actualRepositories = output.getAnalyzedRepositories().values(); await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS) - .until(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); + .untilAsserted(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga)); assertThat(actualRepositories.stream().filter(r -> !r.getCrawlerRunId().equals(GitHubCrawler.NO_CRAWLER_RUN_ID_DEFINED)) .count()) diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitHubImplTest.kt b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitHubImplTest.kt index 381c9169..4145bba8 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitHubImplTest.kt +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitHubImplTest.kt @@ -1,9 +1,10 @@ package com.societegenerale.githubcrawler -import com.jayway.awaitility.Awaitility.await + import com.societegenerale.githubcrawler.mocks.GitHubMock import com.societegenerale.githubcrawler.remote.RemoteGitHubImpl import org.assertj.core.api.Assertions.assertThat +import org.awaitility.Awaitility import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach @@ -30,8 +31,8 @@ class RemoteGitHubImplTest { githubMockServer.start() - await().atMost(5, SECONDS) - .until{ assertThat(GitHubMock.hasStarted()) } + Awaitility.await().atMost(5, SECONDS) + .untilAsserted { assertThat(GitHubMock.hasStarted()).isTrue() } } @AfterAll diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitLabImplTest.kt b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitLabImplTest.kt index 4474b044..63195fe2 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitLabImplTest.kt +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/RemoteGitLabImplTest.kt @@ -1,10 +1,11 @@ package com.societegenerale.githubcrawler -import com.jayway.awaitility.Awaitility.await + import com.societegenerale.githubcrawler.mocks.GitLabMock import com.societegenerale.githubcrawler.mocks.RemoteServiceMock.GITLAB_MOCK_PORT import com.societegenerale.githubcrawler.remote.RemoteGitLabImpl import org.assertj.core.api.Assertions.assertThat +import org.awaitility.Awaitility import org.junit.jupiter.api.AfterAll import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.BeforeEach @@ -24,7 +25,7 @@ class RemoteGitLabImplTest { companion object { - val gitlabMockServer: GitLabMock = GitLabMock(); + val gitlabMockServer: GitLabMock = GitLabMock() @BeforeAll @JvmStatic @@ -32,8 +33,8 @@ class RemoteGitLabImplTest { gitlabMockServer.start() - await().atMost(5, SECONDS) - .until{ assertThat(GitLabMock.hasStarted()) } + Awaitility.await().atMost(5, SECONDS) + .untilAsserted { assertThat(GitLabMock.hasStarted()).isTrue() } } @AfterAll @@ -49,7 +50,7 @@ class RemoteGitLabImplTest { gitlabMockServer.reset() } - val remoteGitLab = RemoteGitLabImpl("http://localhost:"+GITLAB_MOCK_PORT+"/api/v4", "someToken"); + val remoteGitLab = RemoteGitLabImpl("http://localhost:"+GITLAB_MOCK_PORT+"/api/v4", "someToken") @Test fun shouldGetRepositoriesForAgroup() { @@ -90,7 +91,7 @@ class RemoteGitLabImplTest { val searchResults=remoteGitLab.fetchCodeSearchResult("h5bp/html5-boilerplate","blabla") - assertThat(searchResults.totalCount).isEqualTo(2); + assertThat(searchResults.totalCount).isEqualTo(2) } diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/config/TestConfig.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/config/TestConfig.java index 6e024845..b7d482bb 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/config/TestConfig.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/config/TestConfig.java @@ -3,8 +3,8 @@ import com.societegenerale.githubcrawler.GitHubCrawlerProperties; import com.societegenerale.githubcrawler.model.Repository; import com.societegenerale.githubcrawler.output.GitHubCrawlerOutput; -import java.util.HashMap; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -21,7 +21,7 @@ public GitHubCrawlerOutput output() { public static class InMemoryGitHubCrawlerOutput implements GitHubCrawlerOutput { - private final Map analyzedRepositories = new HashMap<>(); + private final Map analyzedRepositories = new ConcurrentHashMap<>(); public Map getAnalyzedRepositories() { return analyzedRepositories; diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/BitbucketMock.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/BitbucketMock.java index eb754f8a..e1e322b4 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/BitbucketMock.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/BitbucketMock.java @@ -19,6 +19,7 @@ import java.io.InputStream; import java.nio.charset.Charset; import java.util.*; +import java.util.concurrent.CopyOnWriteArrayList; @Component @@ -53,9 +54,9 @@ public boolean isHasCalledNextPage() { return hasCalledNextPage; } - private List repoConfigHits = new ArrayList<>(); + private List repoConfigHits = new CopyOnWriteArrayList<>(); - private List pomXmlHits = new ArrayList<>(); + private List pomXmlHits = new CopyOnWriteArrayList<>(); private int nbPages = 1; @@ -86,27 +87,27 @@ public boolean start() { bitbucketWebServer = new WebServer(); bitbucketWebServer.configure( routes -> { - routes.get("/projects/myProject/repos/:repo/raw/.BitBucketCrawler?at=master", (context, repo) -> getRepoConfigFileOnRepo(repo)); - routes.get("/projects/myProject/repos/:repo/raw/pom.xml?at=master", (context, repo) -> getPomXmlFileOnRepo(repo)); - routes.get("/projects/myProject/repos?start=:start", (context, start) -> getOrganisationContent(Integer.parseInt(start))); + routes.get("/projects/myProject/repos/:repo/raw/.BitBucketCrawler?at=master", (_, repo) -> getRepoConfigFileOnRepo(repo)); + routes.get("/projects/myProject/repos/:repo/raw/pom.xml?at=master", (_, repo) -> getPomXmlFileOnRepo(repo)); + routes.get("/projects/myProject/repos?start=:start", (_, start) -> getOrganisationContent(Integer.parseInt(start))); //for other resources than pom.xml.. //hack for resources that are not at the root of the repository, so that we don't have to hardcode too many things routes.get("/projects/myProject/repos/:repo/raw/:aSubDirectory/:resource?at=:branchName", - (context, repo, aSubDirectory, resource, branchName) -> getResource(repo, aSubDirectory, resource, branchName)); + (_, repo, aSubDirectory, resource, branchName) -> getResource(repo, aSubDirectory, resource, branchName)); routes.get("/projects/myProject/repos/:repo/contents/:resource?ref=:branchName", - (context, repo, resource, branchName) -> getResourceFileOnRepo(repo, resource, null, branchName)); + (_, repo, resource, branchName) -> getResourceFileOnRepo(repo, resource, null, branchName)); routes.get("/raw/myProject/:repo/:branchName/:resource", - (context, repo, branchName, resource) -> getResource(repo, branchName, null, resource)); + (_, repo, branchName, resource) -> getResource(repo, branchName, null, resource)); - routes.get("/projects/myProject/repos/:repo/branches", (context, repo) -> getBranches(repo)); + routes.get("/projects/myProject/repos/:repo/branches", (_, repo) -> getBranches(repo)); routes.get("/admin/groups", this::getTeams); - routes.get("/api/v3/teams/:team/members", (context, team) -> getTeamsMembers(team)); + routes.get("/api/v3/teams/:team/members", (_, team) -> getTeamsMembers(team)); - routes.get("/projects/myProject/repos/:repo/commits?limit=1", (context, repo) -> getCommits(repo)); - routes.get("/projects/myProject/repos/:repo/commits/:commit", (context, repo, commit) -> getCommit(repo, commit)); + routes.get("/projects/myProject/repos/:repo/commits?limit=1", (_, repo) -> getCommits(repo)); + routes.get("/projects/myProject/repos/:repo/commits/:commit", (_, repo, commit) -> getCommit(repo, commit)); } ).start(BITBUCKET_MOCK_PORT); diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitHubMock.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitHubMock.java index 2bf42391..05a09285 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitHubMock.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitHubMock.java @@ -11,6 +11,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; import net.codestory.http.Context; import net.codestory.http.WebServer; import net.codestory.http.constants.HttpStatus; @@ -57,9 +58,9 @@ public boolean isHasCalledNextPage() { return hasCalledNextPage; } - private List repoConfigHits = new ArrayList<>(); + private List repoConfigHits = new CopyOnWriteArrayList<>(); - private List pomXmlHits = new ArrayList<>(); + private List pomXmlHits = new CopyOnWriteArrayList<>(); private int nbPages = 1; @@ -91,38 +92,38 @@ public boolean start() { gitHubWebServer.configure( routes -> { - routes.get("/api/v3/repos/MyOrganization/:repo/contents/.githubCrawler", (context, repo) -> getRepoConfigFileOnRepo(repo)); - routes.get("/raw/MyOrganization/:repo/master/.githubCrawler", (context, repo) -> getActualRepoConfig(repo)); + routes.get("/api/v3/repos/MyOrganization/:repo/contents/.githubCrawler", (_, repo) -> getRepoConfigFileOnRepo(repo)); + routes.get("/raw/MyOrganization/:repo/master/.githubCrawler", (_, repo) -> getActualRepoConfig(repo)); routes.get("/api/v3/orgs/MyOrganization/repos", context -> getOrganisationContent(context)); - routes.get("/api/v3/organizations/1114/repos", context -> getOrganisationContentForNextPage()); + routes.get("/api/v3/organizations/1114/repos", _ -> getOrganisationContentForNextPage()); - routes.get("/api/v3/users/someUser/repos", context -> geUserReposContent()); + routes.get("/api/v3/users/someUser/repos", _ -> geUserReposContent()); - routes.get("/api/v3/repos/MyOrganization/:repo/contents/pom.xml?ref=master", (context, repo) -> getPomXmlFileOnRepo(repo)); - routes.get("/raw/MyOrganization/:repo/:branchName/pom.xml", (context, repo, branchName) -> getActualPomXML(repo, branchName)); + routes.get("/api/v3/repos/MyOrganization/:repo/contents/pom.xml?ref=master", (_, repo) -> getPomXmlFileOnRepo(repo)); + routes.get("/raw/MyOrganization/:repo/:branchName/pom.xml", (_, repo, branchName) -> getActualPomXML(repo, branchName)); //for other resources than pom.xml.. //hack for resources that are not at the root of the repository, so that we don't have to hardcode too many things routes.get("/api/v3/repos/MyOrganization/:repo/contents/:aSubDirectory/:resource?ref=:branchName", - (context, repo, aSubDirectory, resource, branchName) -> getResourceFileOnRepo(repo, resource, aSubDirectory, branchName)); + (_, repo, aSubDirectory, resource, branchName) -> getResourceFileOnRepo(repo, resource, aSubDirectory, branchName)); routes.get("/raw/MyOrganization/:repo/:branchName/:aSubDirectory/:resource", - (context, repo, branchName, aSubDirectory, resource) -> getResource(repo, branchName, aSubDirectory, resource)); + (_, repo, branchName, aSubDirectory, resource) -> getResource(repo, branchName, aSubDirectory, resource)); routes.get("/api/v3/repos/MyOrganization/:repo/contents/:resource?ref=:branchName", - (context, repo, resource, branchName) -> getResourceFileOnRepo(repo, resource, null, branchName)); + (_, repo, resource, branchName) -> getResourceFileOnRepo(repo, resource, null, branchName)); routes.get("/raw/MyOrganization/:repo/:branchName/:resource", - (context, repo, branchName, resource) -> getResource(repo, branchName, null, resource)); + (_, repo, branchName, resource) -> getResource(repo, branchName, null, resource)); - routes.get("/api/v3/repos/MyOrganization/:repo/branches", (context, repo) -> getBranches(repo)); + routes.get("/api/v3/repos/MyOrganization/:repo/branches", (_, repo) -> getBranches(repo)); - routes.get("/api/v3/search/code?q=:searchQuery", (context, searchQuery) -> getSearchResult(searchQuery)); + routes.get("/api/v3/search/code?q=:searchQuery", (_, searchQuery) -> getSearchResult(searchQuery)); routes.get("/api/v3/orgs/MyOrganization/teams", this::getTeams); - routes.get("/api/v3/teams/:team/members", (context, team) -> getTeamsMembers(team)); + routes.get("/api/v3/teams/:team/members", (_, team) -> getTeamsMembers(team)); - routes.get("/api/v3/repos/MyOrganization/:repo/commits?per_page=150", (context, repo) -> getCommits(repo)); - routes.get("/api/v3/repos/MyOrganization/:repo/commits/:commit", (context, repo, commit) -> getCommit(repo, commit)); + routes.get("/api/v3/repos/MyOrganization/:repo/commits?per_page=150", (_, repo) -> getCommits(repo)); + routes.get("/api/v3/repos/MyOrganization/:repo/commits/:commit", (_, repo, commit) -> getCommit(repo, commit)); } ).start(GITHUB_MOCK_PORT); diff --git a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitLabMock.java b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitLabMock.java index 43b08aa1..eeb86370 100644 --- a/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitLabMock.java +++ b/github-crawler-autoconfigure/src/test/java/com/societegenerale/githubcrawler/mocks/GitLabMock.java @@ -36,12 +36,12 @@ public boolean start() { gitLabWebServer.configure( routes -> { - routes.get("/api/v4/groups?search=:groupName", (context,groupName) -> getGroups(groupName)); + routes.get("/api/v4/groups?search=:groupName", (_,groupName) -> getGroups(groupName)); - routes.get("/api/v4/groups/:groupId/projects", (context, groupId) -> getRepositories(groupId)); - routes.get("/api/v4/projects/:repoId/repository/files/:filePath/raw?ref=:branchName", (context, repoId, filePath,branchName) -> getFileContent(repoId, filePath, branchName)); + routes.get("/api/v4/groups/:groupId/projects", (_, groupId) -> getRepositories(groupId)); + routes.get("/api/v4/projects/:repoId/repository/files/:filePath/raw?ref=:branchName", (_, repoId, filePath,branchName) -> getFileContent(repoId, filePath, branchName)); - routes.get("/api/v4/projects/:repoId/search?scope=blobs&:searchString", (context, repoId, searchString) -> getRepoSearchResults(repoId, searchString)); + routes.get("/api/v4/projects/:repoId/search?scope=blobs&:searchString", (_, repoId, searchString) -> getRepoSearchResults(repoId, searchString)); } diff --git a/github-crawler-core/pom.xml b/github-crawler-core/pom.xml index 5fa9e70c..87dce454 100644 --- a/github-crawler-core/pom.xml +++ b/github-crawler-core/pom.xml @@ -1,163 +1,115 @@ - - 4.0.0 - + + 4.0.0 + github-crawler-core + jar + github-crawler-core + com.societegenerale.github-crawler - github-crawler-core - 2.2.1-SNAPSHOT - jar - - github-crawler-core - - - com.societegenerale.github-crawler - github-crawler-parent - 2.2.1-SNAPSHOT - - - - - - - - com.google.code.gson - gson - ${gson.version} - - - - org.springframework.boot - spring-boot-starter-logging - - - - - org.slf4j - slf4j-simple - - - - - - - org.springframework.cloud - spring-cloud-starter-openfeign - - - - io.github.openfeign - feign-gson - - - - io.github.openfeign - feign-httpclient - - - - com.fasterxml.jackson.module - jackson-module-kotlin - - - - org.jetbrains.kotlin - kotlin-reflect - - - - - - com.fasterxml.jackson.dataformat - jackson-dataformat-yaml - - - - commons-io - commons-io - 2.11.0 - - - - org.apache.commons - commons-lang3 - 3.7 - test - - - - com.squareup.okhttp3 - okhttp - - - - - com.squareup.okhttp3 - logging-interceptor - - - - - com.jayway.awaitility - awaitility - 1.7.0 - test - - - - com.jayway.jsonpath - json-path - 2.7.0 - - - - org.springframework.boot - spring-boot-configuration-processor - true - - - - - - - - - kotlin-maven-plugin - org.jetbrains.kotlin - ${kotlin.version} - - - - kapt - - kapt - - - - src/main/kotlin - - - - - - - - com.google.dagger - dagger-compiler - 2.9 - - - - - - - - - + github-crawler-parent + 3.0.0-SNAPSHOT + + + + + com.google.code.gson + gson + ${gson.version} + + + org.springframework.boot + spring-boot-starter-logging + + + org.springframework.cloud + spring-cloud-starter-openfeign + + + io.github.openfeign + feign-gson + + + io.github.openfeign + feign-httpclient + + + tools.jackson.module + jackson-module-kotlin + + + tools.jackson.dataformat + jackson-dataformat-yaml + + + commons-io + commons-io + ${commons-io.version} + + + com.squareup.okhttp3 + okhttp + + + com.squareup.okhttp3 + logging-interceptor + + + com.jayway.jsonpath + json-path + ${json-path.version} + + + + org.apache.commons + commons-lang3 + test + + + org.awaitility + awaitility + + + + + + kotlin-maven-plugin + org.jetbrains.kotlin + ${kotlin.version} + + + kapt + + kapt + + + + src/main/kotlin + + + + + com.google.dagger + dagger-compiler + 2.60.1 + + + org.springframework.boot + spring-boot-configuration-processor + ${spring-boot-dependencies.version} + + + + + + + + diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/FileToParse.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/FileToParse.kt index 860a134f..6ee39c34 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/FileToParse.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/FileToParse.kt @@ -2,7 +2,6 @@ package com.societegenerale.githubcrawler import org.springframework.core.convert.ConversionService import org.springframework.core.convert.TypeDescriptor -import org.springframework.lang.Nullable data class FileToParse(val name: String, @@ -10,19 +9,19 @@ data class FileToParse(val name: String, class FileToParseConversionService : ConversionService { - override fun canConvert(@Nullable aClass: Class<*>?, aClass1: Class<*>): Boolean { + override fun canConvert(aClass: Class<*>?, aClass1: Class<*>): Boolean { return false } - override fun canConvert(@Nullable sourcetype: TypeDescriptor?, targetType: TypeDescriptor): Boolean { + override fun canConvert(sourcetype: TypeDescriptor?, targetType: TypeDescriptor): Boolean { return targetType.name == FileToParse::class.java.name } - override fun convert(@Nullable o: Any?, targetType: Class): T? { + override fun convert(source: Any?, targetType: Class): T? { return null } - override fun convert(@Nullable value: Any?, @Nullable sourceType: TypeDescriptor?, targetType: TypeDescriptor): Any? { + override fun convert(value: Any?, sourceType: TypeDescriptor?, targetType: TypeDescriptor): Any? { return if (targetType.name == FileToParse::class.java.name) { FileToParse(value as String, null) diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/model/bitbucket/Repositories.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/model/bitbucket/Repositories.kt index d2365ead..eac942f6 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/model/bitbucket/Repositories.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/model/bitbucket/Repositories.kt @@ -5,6 +5,6 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties @JsonIgnoreProperties(ignoreUnknown = true) data class Repositories(val values: Set, val isLastPage: Boolean, - val start: Int, - val nextPageStart: Int, + val start: Int = 0, + val nextPageStart: Int = 0, ) diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CIdroidReadyJsonFileOutput.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CIdroidReadyJsonFileOutput.kt index 30e321b0..58655f34 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CIdroidReadyJsonFileOutput.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CIdroidReadyJsonFileOutput.kt @@ -8,7 +8,7 @@ import java.io.IOException import java.nio.charset.StandardCharsets import java.nio.file.Files import java.nio.file.OpenOption -import java.nio.file.Paths +import java.nio.file.Path import java.nio.file.StandardOpenOption import java.time.LocalDateTime import java.time.format.DateTimeFormatter @@ -113,7 +113,7 @@ class CIdroidReadyJsonFileOutput (val indicatorsToOutput: List, val with @Suppress("SpreadOperator") //no performance impact given the number of values (a couple, max).. private fun openFileWithOptions(vararg options : OpenOption) : BufferedWriter { - return Files.newBufferedWriter(Paths.get(finalOutputFileName), + return Files.newBufferedWriter(Path.of(finalOutputFileName), StandardCharsets.UTF_8, *options) } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CsvFileOutput.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CsvFileOutput.kt index 32b13028..39e1787d 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CsvFileOutput.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/CsvFileOutput.kt @@ -5,7 +5,7 @@ import org.slf4j.Logger import java.io.IOException import java.nio.charset.StandardCharsets import java.nio.file.Files -import java.nio.file.Paths +import java.nio.file.Path import java.nio.file.StandardOpenOption import java.time.LocalDateTime import java.time.format.DateTimeFormatter @@ -34,7 +34,7 @@ abstract class CsvFileOutput(initParam : Any) : GitHubCrawlerOutput{ finalOutputFileName = getPrefix() + now.format(formatter) + ".csv" - val writer = Files.newBufferedWriter(Paths.get(finalOutputFileName), + val writer = Files.newBufferedWriter(Path.of(finalOutputFileName), StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW) @@ -46,7 +46,7 @@ abstract class CsvFileOutput(initParam : Any) : GitHubCrawlerOutput{ @Throws(IOException::class) override fun output(analyzedRepository: Repository) { - val writer = Files.newBufferedWriter(Paths.get(finalOutputFileName), + val writer = Files.newBufferedWriter(Path.of(finalOutputFileName), StandardCharsets.UTF_8, StandardOpenOption.WRITE, StandardOpenOption.APPEND) diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/FileOutput.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/FileOutput.kt index dd9bf683..6df254e0 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/FileOutput.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/FileOutput.kt @@ -5,7 +5,7 @@ import org.slf4j.LoggerFactory import java.io.IOException import java.nio.charset.StandardCharsets import java.nio.file.Files -import java.nio.file.Paths +import java.nio.file.Path import java.nio.file.StandardOpenOption import java.time.LocalDateTime import java.time.format.DateTimeFormatter @@ -27,7 +27,7 @@ class FileOutput (filenamePrefix: String) : GitHubCrawlerOutput { finalOutputFileName = filenamePrefix + "_" + now.format(formatter) + ".txt" - Files.newBufferedWriter(Paths.get(finalOutputFileName), + Files.newBufferedWriter(Path.of(finalOutputFileName), StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW).use{ @@ -40,7 +40,7 @@ class FileOutput (filenamePrefix: String) : GitHubCrawlerOutput { @Throws(IOException::class) override fun output(analyzedRepository: Repository) { - Files.newBufferedWriter(Paths.get(finalOutputFileName), + Files.newBufferedWriter(Path.of(finalOutputFileName), StandardCharsets.UTF_8, StandardOpenOption.WRITE, StandardOpenOption.APPEND).use{ diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/HttpOutput.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/HttpOutput.kt index 547a839b..d7f175c9 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/HttpOutput.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/HttpOutput.kt @@ -41,10 +41,10 @@ class HttpOutput(targetUrl: String, response = restTemplate.postForEntity(internalTargetUrl, output, String::class.java) if (!response.statusCode.is2xxSuccessful) { - logHttpError(analyzedRepository.name, response.statusCodeValue,response.body) + logHttpError(analyzedRepository.name, response.statusCode.value(),response.body) } } catch (e : HttpClientErrorException) { - logHttpError(analyzedRepository.name, e.rawStatusCode,e.message ?: "no additional message provided") + logHttpError(analyzedRepository.name, e.statusCode.value(),e.message ?: "no additional message provided") } } } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/parsers/JsonPathParser.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/parsers/JsonPathParser.kt index 81c32a3c..fc264fd6 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/parsers/JsonPathParser.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/parsers/JsonPathParser.kt @@ -3,7 +3,6 @@ package com.societegenerale.githubcrawler.parsers import com.jayway.jsonpath.JsonPath.read import com.jayway.jsonpath.PathNotFoundException import com.societegenerale.githubcrawler.IndicatorDefinition -import net.minidev.json.JSONArray import org.slf4j.LoggerFactory /** @@ -50,12 +49,12 @@ class JsonPathParser : FileContentParser { if (result is String) { result - } else if (result is JSONArray && !result.isEmpty()) { + } else if (result is List<*> && result.isNotEmpty()) { result[0] as String } else { NOT_FOUND } - } catch (e: PathNotFoundException) { + } catch (_: PathNotFoundException) { NOT_FOUND } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/CodeSearchRequestDetailsSerializer.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/CodeSearchRequestDetailsSerializer.kt index 47e95e68..3e6099aa 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/CodeSearchRequestDetailsSerializer.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/CodeSearchRequestDetailsSerializer.kt @@ -1,13 +1,13 @@ package com.societegenerale.githubcrawler.remote -import com.fasterxml.jackson.core.JsonGenerator +import tools.jackson.core.JsonGenerator -import com.fasterxml.jackson.databind.SerializerProvider -import com.fasterxml.jackson.databind.ser.std.StdSerializer +import tools.jackson.databind.ser.std.StdSerializer +import tools.jackson.databind.SerializationContext class CodeSearchRequestDetailsSerializer() : StdSerializer(CodeSearchRequestDetails::class.java) { - override fun serialize(p0: CodeSearchRequestDetails?, p1: JsonGenerator?, p2: SerializerProvider?) { + override fun serialize(p0: CodeSearchRequestDetails?, p1: JsonGenerator?, p2: SerializationContext?) { TODO("Not yet implemented") } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteAzureDevopsImpl.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteAzureDevopsImpl.kt index 6a8d56d0..9c53cbd4 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteAzureDevopsImpl.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteAzureDevopsImpl.kt @@ -1,11 +1,8 @@ package com.societegenerale.githubcrawler.remote -import com.fasterxml.jackson.core.JsonParseException -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.kotlin.KotlinModule -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper + +import tools.jackson.module.kotlin.KotlinModule +import tools.jackson.module.kotlin.jacksonObjectMapper import com.societegenerale.githubcrawler.RepositoryConfig import com.societegenerale.githubcrawler.model.* import com.societegenerale.githubcrawler.model.azuredevops.Repositories @@ -22,7 +19,10 @@ import okhttp3.logging.HttpLoggingInterceptor import org.apache.commons.io.IOUtils import org.slf4j.LoggerFactory import org.springframework.http.HttpStatus -import java.io.IOException +import tools.jackson.core.JacksonException +import tools.jackson.core.exc.StreamReadException +import tools.jackson.dataformat.yaml.YAMLMapper + import java.io.StringWriter import java.util.* import java.util.stream.Collectors.toSet @@ -51,7 +51,7 @@ class RemoteAzureDevopsImpl @JvmOverloads constructor(private val azureDevopsUrl private var azureOrg=splitedOrgName.get(0) private var azureProject=splitedOrgName.get(1) - private val objectMapper = jacksonObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + private val objectMapper = jacksonObjectMapper() private val basicAuthentCredentials: String = Credentials.basic("", personalAccessToken) @@ -103,7 +103,7 @@ class RemoteAzureDevopsImpl @JvmOverloads constructor(private val azureDevopsUrl return objectMapper.readValue(bodyAsString, Repositories::class.java) } - catch( e : JsonParseException){ + catch( e : StreamReadException){ throw IllegalArgumentException("unable to read the repositories to crawl from the response we got $bodyAsString",e) } @@ -146,7 +146,7 @@ class RemoteAzureDevopsImpl @JvmOverloads constructor(private val azureDevopsUrl val responseBody= httpClient.newCall(request).execute().body - val repoSearchResult = objectMapper.readValue(responseBody?.string(), CodeSearchResult::class.java) + val repoSearchResult = objectMapper.readValue(responseBody.string(), CodeSearchResult::class.java) return repoSearchResult.toStandardSearchResult() @@ -164,7 +164,7 @@ class RemoteAzureDevopsImpl @JvmOverloads constructor(private val azureDevopsUrl val response=httpClient.newCall(request).execute() if(response.isSuccessful){ - return response.body?.string() ?: "" + return response.body.string() } else{ throw NoFileFoundException("can't find $fileToFetch in repo $repositoryFullName, in branch $branchName") @@ -234,12 +234,9 @@ internal class CodeSearchResultItem(val path : String) internal class AzureDevopsResponseDecoder { val log = LoggerFactory.getLogger(this.javaClass) - val repoConfigMapper = ObjectMapper(YAMLFactory()) - - init { - repoConfigMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - repoConfigMapper.registerModule(KotlinModule.Builder().build()) - } + val repoConfigMapper: YAMLMapper = YAMLMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() fun decodeRepoConfig(response: okhttp3.Response): RepositoryConfig { @@ -248,7 +245,7 @@ internal class AzureDevopsResponseDecoder { } val writer = StringWriter() - IOUtils.copy(response.body?.byteStream(), writer, "UTF-8") + IOUtils.copy(response.body.byteStream(), writer, "UTF-8") val responseAsString = writer.toString() return parseRepositoryConfigResponse(responseAsString, response) @@ -261,7 +258,7 @@ internal class AzureDevopsResponseDecoder { try { return repoConfigMapper.readValue(responseAsString, RepositoryConfig::class.java) - } catch (e: IOException) { + } catch (e: JacksonException) { throw Repository.RepoConfigException(HttpStatus.BAD_REQUEST,"unable to parse config for repo - content : \"" + response.body + "\"", e) } } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteBitBucketImpl.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteBitBucketImpl.kt index bf6f384f..2c7633ee 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteBitBucketImpl.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteBitBucketImpl.kt @@ -1,10 +1,9 @@ package com.societegenerale.githubcrawler.remote -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.kotlin.KotlinModule -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper +import tools.jackson.databind.json.JsonMapper +import tools.jackson.dataformat.yaml.YAMLMapper +import tools.jackson.module.kotlin.KotlinModule +import tools.jackson.module.kotlin.jacksonObjectMapper import com.societegenerale.githubcrawler.RepositoryConfig import com.societegenerale.githubcrawler.model.* import com.societegenerale.githubcrawler.model.Author @@ -23,12 +22,9 @@ import feign.gson.GsonEncoder import feign.httpclient.ApacheHttpClient import feign.slf4j.Slf4jLogger import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.http.HttpMessageConverters -import org.springframework.cloud.openfeign.support.ResponseEntityDecoder -import org.springframework.cloud.openfeign.support.SpringDecoder import org.springframework.http.HttpStatus -import org.springframework.http.MediaType -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter +import tools.jackson.core.JacksonException + import java.io.IOException import java.lang.reflect.Type import java.util.* @@ -49,7 +45,7 @@ class RemoteBitBucketImpl @JvmOverloads constructor( .client(ApacheHttpClient()) .encoder(GsonEncoder()) .decoder(BitBucketResponseDecoder()) - .decode404() + .dismiss404() .requestInterceptor(BitBucketOauthTokenSetter(apiKey)) .logger(Slf4jLogger(RemoteBitBucketImpl::class.java)) .logLevel(Logger.Level.FULL) @@ -58,8 +54,6 @@ class RemoteBitBucketImpl @JvmOverloads constructor( val log = LoggerFactory.getLogger(this.javaClass) - private val objectMapper = jacksonObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - override fun validateRemoteConfig(organizationName: String) { //TODO("Not yet implemented") } @@ -236,12 +230,9 @@ private interface InternalBitBucketClient { internal class BitBucketResponseDecoder : Decoder { val log = LoggerFactory.getLogger(this.javaClass) - val repoConfigMapper = ObjectMapper(YAMLFactory()) - - init { - repoConfigMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - repoConfigMapper.registerModule(KotlinModule.Builder().build()) - } + val repoConfigMapper: YAMLMapper = YAMLMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() fun decodeRepoConfig(response: String): RepositoryConfig { @@ -263,19 +254,22 @@ internal class BitBucketResponseDecoder : Decoder { log.debug("Decoding a successful response...") - if (type.typeName == MediaType.TEXT_PLAIN_VALUE) { + if (type == String::class.java) { log.debug("\t ... as a String") - return response.body().toString() + return response.body().asReader(Charsets.UTF_8).use { it.readText() } } log.debug("\t ... as a " + type.typeName) - val jacksonConverter = - MappingJackson2HttpMessageConverter(ObjectMapper().registerModule(KotlinModule.Builder().build())) - val objectFactory = { HttpMessageConverters(jacksonConverter) } - return ResponseEntityDecoder(SpringDecoder(objectFactory)).decode(response, type) + val jsonMapper = JsonMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() + + response.body().asInputStream().use { input -> + return jsonMapper.readValue(input, jsonMapper.constructType(type)) + } } } @@ -287,7 +281,7 @@ internal class BitBucketResponseDecoder : Decoder { try { return repoConfigMapper.readValue(responseAsString, RepositoryConfig::class.java) - } catch (e: IOException) { + } catch (e: JacksonException) { throw Repository.RepoConfigException( HttpStatus.BAD_REQUEST, "unable to parse config for repo - content : \"" + responseAsString + "\"", diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitHubImpl.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitHubImpl.kt index 63c55b41..c1e94c89 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitHubImpl.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitHubImpl.kt @@ -1,13 +1,10 @@ package com.societegenerale.githubcrawler.remote -import com.fasterxml.jackson.core.JsonParseException -import com.fasterxml.jackson.core.JsonProcessingException -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.kotlin.KotlinModule -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.fasterxml.jackson.module.kotlin.readValue +import tools.jackson.databind.json.JsonMapper +import tools.jackson.dataformat.yaml.YAMLMapper +import tools.jackson.module.kotlin.KotlinModule +import tools.jackson.module.kotlin.jacksonObjectMapper +import tools.jackson.module.kotlin.readValue import com.societegenerale.githubcrawler.RepositoryConfig import com.societegenerale.githubcrawler.model.* import com.societegenerale.githubcrawler.model.commit.Commit @@ -26,12 +23,10 @@ import okhttp3.OkHttpClient import okhttp3.Response import org.apache.commons.io.IOUtils import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.http.HttpMessageConverters -import org.springframework.cloud.openfeign.support.ResponseEntityDecoder -import org.springframework.cloud.openfeign.support.SpringDecoder import org.springframework.http.HttpStatus -import org.springframework.http.MediaType -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter +import tools.jackson.core.JacksonException +import tools.jackson.core.exc.StreamReadException + import java.io.IOException import java.io.StringWriter import java.lang.reflect.Type @@ -61,7 +56,7 @@ class RemoteGitHubImpl @JvmOverloads constructor( .encoder(GsonEncoder()) .decoder(GitHubResponseDecoder()) .errorDecoder(GiHubErrorDecoder()) - .decode404() + .dismiss404() .requestInterceptor(GitHubOauthTokenSetter(apiKey)) .logger(Slf4jLogger(RemoteGitHubImpl::class.java)) .logLevel(Logger.Level.FULL) @@ -72,7 +67,7 @@ class RemoteGitHubImpl @JvmOverloads constructor( val log = LoggerFactory.getLogger(this.javaClass) - private val objectMapper = jacksonObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + private val objectMapper = jacksonObjectMapper() @Throws(NoReachableRepositories::class) override fun validateRemoteConfig(organizationName: String) { @@ -81,7 +76,7 @@ class RemoteGitHubImpl @JvmOverloads constructor( try { extractRepositories(response) - } catch (e: JsonProcessingException) { + } catch (e: JacksonException) { throw NoReachableRepositories("not able to parse response : ${response.body}", e) } @@ -175,16 +170,8 @@ class RemoteGitHubImpl @JvmOverloads constructor( private fun extractRepositories(response: Response): Set { try { - - val body = response.body - - if (body != null) { - return objectMapper.readValue(body.string()) - } else { - log.warn("response is null : {}", response) - return emptySet() - } - } catch (e: JsonProcessingException) { + return objectMapper.readValue(response.body.string()) + } catch (e: JacksonException) { throw NoReachableRepositories("not able to parse response", e) } } @@ -238,13 +225,13 @@ class RemoteGitHubImpl @JvmOverloads constructor( val response = httpClient.newCall(request).execute() - val responseAsString=response.body?.string() + val responseAsString=response.body.string() log.info("response : "+responseAsString) return try { objectMapper.readValue(responseAsString, SearchResult::class.java) } - catch(e : JsonParseException){ + catch(e : StreamReadException){ log.warn("parsing error",e) SearchResult(0, emptyList()) } @@ -276,7 +263,7 @@ class RemoteGitHubImpl @JvmOverloads constructor( val response = httpClient.newCall(request).execute() - return response.body?.string() ?: "" + return response.body.string() } @@ -391,17 +378,14 @@ internal class GiHubErrorDecoder : ErrorDecoder { internal class GitHubResponseDecoder : Decoder { val log = LoggerFactory.getLogger(this.javaClass) - val repoConfigMapper = ObjectMapper(YAMLFactory()) - - init { - repoConfigMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - repoConfigMapper.registerModule(KotlinModule.Builder().build()) - } + val repoConfigMapper: YAMLMapper = YAMLMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() fun decodeRepoConfig(response: Response): RepositoryConfig { val writer = StringWriter() - IOUtils.copy(response.body?.byteStream(), writer, "UTF-8") + IOUtils.copy(response.body.byteStream(), writer, "UTF-8") val responseAsString = writer.toString() return parseRepositoryConfigResponse(responseAsString, response) @@ -423,18 +407,22 @@ internal class GitHubResponseDecoder : Decoder { log.debug("Decoding a successful response...") - if (type.typeName == MediaType.TEXT_PLAIN_VALUE) { + if (type == String::class.java) { log.debug("\t ... as a String") - return response.body().toString() + return response.body().asReader(Charsets.UTF_8).use { it.readText() } } log.debug("\t ... as a " + type.typeName) - val jacksonConverter = MappingJackson2HttpMessageConverter(ObjectMapper().registerModule(KotlinModule.Builder().build())) - val objectFactory = { HttpMessageConverters(jacksonConverter) } - return ResponseEntityDecoder(SpringDecoder(objectFactory)).decode(response, type) + val jsonMapper = JsonMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() + + response.body().asInputStream().use { input -> + return jsonMapper.readValue(input, jsonMapper.constructType(type)) + } } } @@ -446,7 +434,7 @@ internal class GitHubResponseDecoder : Decoder { try { return repoConfigMapper.readValue(responseAsString, RepositoryConfig::class.java) - } catch (e: IOException) { + } catch (e: JacksonException) { throw Repository.RepoConfigException(HttpStatus.BAD_REQUEST,"unable to parse config for repo - content : \"" + response.body + "\"", e) } } diff --git a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitLabImpl.kt b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitLabImpl.kt index 55cc89af..9596e9df 100644 --- a/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitLabImpl.kt +++ b/github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/remote/RemoteGitLabImpl.kt @@ -1,12 +1,11 @@ package com.societegenerale.githubcrawler.remote import com.fasterxml.jackson.annotation.JsonIgnoreProperties -import com.fasterxml.jackson.databind.DeserializationFeature -import com.fasterxml.jackson.databind.ObjectMapper -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory -import com.fasterxml.jackson.module.kotlin.KotlinModule -import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper -import com.fasterxml.jackson.module.kotlin.readValue +import tools.jackson.databind.json.JsonMapper +import tools.jackson.dataformat.yaml.YAMLMapper +import tools.jackson.module.kotlin.KotlinModule +import tools.jackson.module.kotlin.jacksonObjectMapper +import tools.jackson.module.kotlin.readValue import com.societegenerale.githubcrawler.RepositoryConfig import com.societegenerale.githubcrawler.model.* import com.societegenerale.githubcrawler.model.commit.Commit @@ -23,12 +22,9 @@ import feign.httpclient.ApacheHttpClient import feign.slf4j.Slf4jLogger import okhttp3.OkHttpClient import org.slf4j.LoggerFactory -import org.springframework.boot.autoconfigure.http.HttpMessageConverters -import org.springframework.cloud.openfeign.support.ResponseEntityDecoder -import org.springframework.cloud.openfeign.support.SpringDecoder import org.springframework.http.HttpStatus -import org.springframework.http.MediaType -import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter +import tools.jackson.core.JacksonException + import java.io.IOException import java.lang.reflect.Type import java.util.* @@ -57,7 +53,7 @@ class RemoteGitLabImpl constructor( .encoder(GsonEncoder()) .decoder(GitLabResponseDecoder()) .errorDecoder(GiLabErrorDecoder()) - .decode404() + .dismiss404() .requestInterceptor(GitLabPrivateTokenSetter(privateToken)) .logger(Slf4jLogger(RemoteGitLabImpl::class.java)) .logLevel(Logger.Level.FULL) @@ -71,7 +67,7 @@ class RemoteGitLabImpl constructor( val log = LoggerFactory.getLogger(this.javaClass) - private val objectMapper = jacksonObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) + private val objectMapper = jacksonObjectMapper() override fun fetchRepoConfig(repositoryFullName: String, defaultBranch: String): RepositoryConfig { @@ -106,7 +102,7 @@ class RemoteGitLabImpl constructor( val httpResponse=httpClient.newCall(request).execute() - val gitlabSearchResult : List = objectMapper.readValue(httpResponse.body!!.string()) + val gitlabSearchResult : List = objectMapper.readValue(httpResponse.body.string()) return SearchResult(gitlabSearchResult.size,gitlabSearchResult.map{ it -> it.toSearchResultItem()}) @@ -185,7 +181,7 @@ class RemoteGitLabImpl constructor( throw NoReachableRepositories("GET call to ${fetchFileUrl} wasn't successful. Code : ${response.code}, Message : ${response.message}") } - return response.body!!.string() + return response.body.string() } @Throws(NoReachableRepositories::class) @@ -268,12 +264,9 @@ data class GitLabRepository (val id : Int,val web_url : String, val path : Strin internal class GitLabResponseDecoder : Decoder { val log = LoggerFactory.getLogger(this.javaClass) - val repoConfigMapper = ObjectMapper(YAMLFactory()) - - init { - repoConfigMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - repoConfigMapper.registerModule(KotlinModule()) - } + val repoConfigMapper: YAMLMapper = YAMLMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() @Throws(IOException::class) override fun decode(response: Response, type: Type): Any { @@ -290,18 +283,22 @@ internal class GitLabResponseDecoder : Decoder { log.debug("Decoding a successful response...") - if (type.typeName == MediaType.TEXT_PLAIN_VALUE) { + if (type == String::class.java) { log.debug("\t ... as a String") - return response.body().toString() + return response.body().asReader(Charsets.UTF_8).use { it.readText() } } log.debug("\t ... as a " + type.typeName) - val jacksonConverter = MappingJackson2HttpMessageConverter(ObjectMapper().registerModule(KotlinModule.Builder().build())) - val objectFactory = { HttpMessageConverters(jacksonConverter) } - return ResponseEntityDecoder(SpringDecoder(objectFactory)).decode(response, type) + val jsonMapper = JsonMapper.builder() + .addModule(KotlinModule.Builder().build()) + .build() + + response.body().asInputStream().use { input -> + return jsonMapper.readValue(input, jsonMapper.constructType(type)) + } } } @@ -313,7 +310,7 @@ internal class GitLabResponseDecoder : Decoder { try { return repoConfigMapper.readValue(responseAsString, RepositoryConfig::class.java) - } catch (e: IOException) { + } catch (e: JacksonException) { throw Repository.RepoConfigException(HttpStatus.BAD_REQUEST,"unable to parse config for repo - content : \"$responseAsString\"", e) } } diff --git a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/ConfigParserTest.java b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/ConfigParserTest.java index 566a6b89..73aa4da4 100644 --- a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/ConfigParserTest.java +++ b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/ConfigParserTest.java @@ -2,9 +2,10 @@ import static org.assertj.core.api.Assertions.assertThat; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.module.kotlin.KotlinModule; +import tools.jackson.databind.ObjectMapper; +import tools.jackson.databind.json.JsonMapper; +import tools.jackson.dataformat.yaml.YAMLMapper; +import tools.jackson.module.kotlin.KotlinModule; import java.io.IOException; import java.io.InputStream; import java.nio.charset.Charset; @@ -14,7 +15,9 @@ class ConfigParserTest { - ObjectMapper mapper = new ObjectMapper(new YAMLFactory()); + ObjectMapper mapper = YAMLMapper.builder() + .addModule(new KotlinModule.Builder().build()) + .build(); @Test void canParseSimpleYamlConfig() throws IOException { @@ -32,15 +35,13 @@ void canParseMoreComplexYamlConfig() throws IOException { InputStream is = getClass().getClassLoader().getResourceAsStream("sampleRepoConfig.yaml"); String configToParse = StreamUtils.copyToString(is, Charset.forName("UTF-8")); - mapper.registerModule(new KotlinModule()); - RepositoryConfig parsedRepositoryConfig = mapper.readValue(configToParse, RepositoryConfig.class); assertThat(parsedRepositoryConfig).isNotNull(); assertThat(parsedRepositoryConfig.getExcluded()).isFalse(); assertThat(parsedRepositoryConfig.getFilesToParse()).hasSize(1); - FileToParse firstFile = parsedRepositoryConfig.getFilesToParse().get(0); + FileToParse firstFile = parsedRepositoryConfig.getFilesToParse().getFirst(); assertThat(firstFile.getRedirectTo()).isEqualTo("moduleWhereDockerFileIs/Dockerfile"); assertThat(firstFile.getName()).isEqualTo("Dockerfile"); } @@ -50,7 +51,7 @@ void canParseSimpleJsonConfig() throws IOException { String configToParse = "{\"excluded\": true}"; - ObjectMapper mapper = new ObjectMapper(); + ObjectMapper mapper = new JsonMapper(); RepositoryConfig parsedRepositoryConfig = mapper.readValue(configToParse, RepositoryConfig.class); diff --git a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerTest.kt b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerTest.kt index 72e85e59..7c85d45c 100644 --- a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerTest.kt +++ b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/GitHubCrawlerTest.kt @@ -1,13 +1,13 @@ package com.societegenerale.githubcrawler import com.google.common.collect.ImmutableList -import com.jayway.awaitility.Awaitility.await import com.societegenerale.githubcrawler.model.Repository import com.societegenerale.githubcrawler.output.GitHubCrawlerOutput import com.societegenerale.githubcrawler.parsers.SimpleFilePathParser import com.societegenerale.githubcrawler.remote.RemoteSourceControl import org.assertj.core.api.Assertions.assertThat +import org.awaitility.Awaitility import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test @@ -142,8 +142,8 @@ class GitHubCrawlerTest { gitHubCrawler.crawl() - await().atMost(2, java.util.concurrent.TimeUnit.SECONDS) - .until({ assertThat(output.analyzedRepositories.values).hasSize(nbExpectedRecords) }) + Awaitility.await().atMost(2, java.util.concurrent.TimeUnit.SECONDS) + .untilAsserted { assertThat(output.analyzedRepositories.values).hasSize(nbExpectedRecords) } return output.analyzedRepositories diff --git a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/FileOutputTest.kt b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/FileOutputTest.kt index 9d874cfa..a78e83aa 100644 --- a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/FileOutputTest.kt +++ b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/FileOutputTest.kt @@ -8,7 +8,7 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import org.springframework.core.io.support.PathMatchingResourcePatternResolver import java.nio.file.Files -import java.nio.file.Paths +import java.nio.file.Path import java.util.* @@ -22,7 +22,7 @@ class FileOutputTest{ val resources = resolver.getResources("file:target/somePrfix*.txt") for(existingFile in resources){ - val fileToDeletePath = Paths.get(existingFile.uri) + val fileToDeletePath = Path.of(existingFile.uri) Files.delete(fileToDeletePath) } @@ -61,7 +61,7 @@ class FileOutputTest{ assertThat(resources).hasSize(1) - val lines=Files.readAllLines(Paths.get(resources[0].uri)) + val lines=Files.readAllLines(Path.of(resources[0].uri)) assertThat(lines).hasSize(3) diff --git a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/HttpOutputTest.kt b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/HttpOutputTest.kt index 1a0f7f53..8f83b622 100644 --- a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/HttpOutputTest.kt +++ b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/output/HttpOutputTest.kt @@ -37,6 +37,7 @@ class HttpOutputTest { inline fun argumentCaptor() = ArgumentCaptor.forClass(T::class.java) @Test + @Suppress("UNCHECKED_CAST") fun shouldLogResponseBodyWhenErrorDuringPost() { //Mock logging infra val root = LoggerFactory.getLogger(ch.qos.logback.classic.Logger.ROOT_LOGGER_NAME) as ch.qos.logback.classic.Logger diff --git a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/repoTaskToPerform/PathsForHitsOnRepoSearchTest.kt b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/repoTaskToPerform/PathsForHitsOnRepoSearchTest.kt index c47a211a..52044b86 100644 --- a/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/repoTaskToPerform/PathsForHitsOnRepoSearchTest.kt +++ b/github-crawler-core/src/test/java/com/societegenerale/githubcrawler/repoTaskToPerform/PathsForHitsOnRepoSearchTest.kt @@ -49,6 +49,7 @@ class PathsForHitsOnRepoSearchTest { @Test + @Suppress("UNCHECKED_CAST") fun shouldYield_NotFound_WhenNoMatch() { `when`(mockRemoteGithub.fetchCodeSearchResult(repoToSearch.fullName, "someSearch")) diff --git a/github-crawler-core/src/test/resources/sample_invalid_yamlfile.yml b/github-crawler-core/src/test/resources/sample_invalid_yamlfile.yml index c667d36f..693822f1 100644 --- a/github-crawler-core/src/test/resources/sample_invalid_yamlfile.yml +++ b/github-crawler-core/src/test/resources/sample_invalid_yamlfile.yml @@ -19,7 +19,7 @@ boolean.property: true --- spring: - profiles: aProfile + config.activate.on-profile: aProfile server: ssl: key-store: hello diff --git a/github-crawler-core/src/test/resources/sample_yamlfile.yml b/github-crawler-core/src/test/resources/sample_yamlfile.yml index 3f8f2ea9..fa0aaefa 100644 --- a/github-crawler-core/src/test/resources/sample_yamlfile.yml +++ b/github-crawler-core/src/test/resources/sample_yamlfile.yml @@ -24,7 +24,7 @@ boolean.property: true --- spring: - profiles: aProfile + config.activate.on-profile: aProfile server: ssl: key-store: hello diff --git a/github-crawler-starter/pom.xml b/github-crawler-starter/pom.xml index 6acc8140..153562a3 100644 --- a/github-crawler-starter/pom.xml +++ b/github-crawler-starter/pom.xml @@ -1,49 +1,44 @@ - - 4.0.0 - + + 4.0.0 + github-crawler-starter + jar + github-crawler-starter + com.societegenerale.github-crawler - github-crawler-starter - 2.2.1-SNAPSHOT - jar - - github-crawler-starter - - - com.societegenerale.github-crawler - github-crawler-parent - 2.2.1-SNAPSHOT - - - - - - com.societegenerale.github-crawler - github-crawler-autoconfigure - ${project.parent.version} - - - - - - - - - - - org.springframework.boot - spring-boot-maven-plugin - - com.societegenerale.githubcrawler.GitHubCrawlerApplication - exec - - - - - + github-crawler-parent + 3.0.0-SNAPSHOT + + + + com.societegenerale.github-crawler + github-crawler-autoconfigure + + + + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring-boot-maven-plugin.version} + + com.societegenerale.githubcrawler.GitHubCrawlerApplication + exec + + + + + repackage + + + + + + diff --git a/github-crawler-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/github-crawler-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 00000000..239b8f08 --- /dev/null +++ b/github-crawler-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +com.societegenerale.githubcrawler.config.GitHubCrawlerAutoConfiguration \ No newline at end of file diff --git a/pom.xml b/pom.xml index dae0bacc..d3daeeea 100644 --- a/pom.xml +++ b/pom.xml @@ -1,429 +1,438 @@ - - 4.0.0 - - com.societegenerale.github-crawler - github-crawler-parent - 2.2.1-SNAPSHOT - pom - - github-crawler - - + + 4.0.0 + com.societegenerale.github-crawler + github-crawler-parent + 3.0.0-SNAPSHOT + pom + github-crawler + + github-crawler-core + github-crawler-autoconfigure + github-crawler-starter + + + + Apache 2 + https://www.apache.org/licenses/LICENSE-2.0.html + repo + + + + https://github.com/societe-generale/github-crawler/issues + GitHub + + + https://github.com/societe-generale/github-crawler + scm:git:git@github.com:societe-generale/github-crawler.git + scm:git:git@github.com:societe-generale/github-crawler.git + + HEAD + + + UTF-8 + UTF-8 + 25 + 2.4.10 + + 4.1.0 + 5.0.2 + 31.1-jre + 2.1.3 + 2.10 + 1.1.6 + 2.105 + 3.13.2 + 4.3.0 + 5.4.0 + 2.22.0 + 3.0.0 + + 0.8.15 + 3.15.0 + 3.5.6 + 4.1.0 + 3.4.0 + 3.2.8 + 0.9.0 + 3.5.6 + 3.11.0 + 4.3.0 + 3.6.1 + + + + jcenter + JCenter + https://jcenter.bintray.com/ + + + + + + + com.societegenerale.github-crawler + github-crawler-core + ${project.version} + + + com.societegenerale.github-crawler + github-crawler-autoconfigure + ${project.version} + + + org.springframework.boot - spring-boot-starter-parent - 2.7.5 - - - - - github-crawler-core - github-crawler-autoconfigure - github-crawler-starter - - - - - - Apache 2 - http://www.apache.org/licenses/LICENSE-2.0.html - repo - - - - - https://github.com/societe-generale/github-crawler/issues - GitHub - - - - https://github.com/societe-generale/github-crawler - scm:git:git@github.com:societe-generale/github-crawler.git - scm:git:git@github.com:societe-generale/github-crawler.git - HEAD - - - - - ossrh - https://s01.oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - UTF-8 - UTF-8 - 11 - - - 1.8.10 - 2.10 - 0.8.8 - - - - - - - - jcenter - JCenter - https://jcenter.bintray.com/ - - - - + spring-boot-dependencies + ${spring-boot-dependencies.version} + pom + import + + + org.springframework.cloud + spring-cloud-openfeign-dependencies + ${spring-cloud-openfeign-dependencies.version} + pom + import + + + com.google.guava + guava-bom + ${guava-bom.version} + pom + import + + + org.jetbrains.kotlin + kotlin-bom + ${kotlin.version} + pom + import + + + com.squareup.okhttp3 + okhttp-bom + ${okhttp-bom.version} + pom + import + + + org.awaitility + awaitility + ${awaitility.version} + test + + + + + + com.google.guava + guava + + + org.jetbrains.kotlin + kotlin-stdlib-jdk8 + + + org.jetbrains.kotlin + kotlin-reflect + + + org.jetbrains.kotlin + kotlin-annotations-jvm + ${kotlin.version} + + + org.jetbrains.kotlin + kotlin-test + test + + + org.springframework.boot + spring-boot-starter-test + test + + + org.dom4j + dom4j + ${dom4j.version} + + + jaxen + jaxen + ${jaxen.version} + + + + ${project.basedir}/src/main/kotlin + + + kotlin-maven-plugin - - - org.springframework.cloud - spring-cloud-openfeign-dependencies - 3.1.5 - pom - import - - - - - - - - - - - com.google.guava - guava - 31.1-jre - - - - - org.springframework.boot - spring-boot-starter-test - test - - - com.vaadin.external.google - android-json - - - - ch.qos.logback - logback-classic - - - - - - - org.jetbrains.kotlin - kotlin-stdlib-jdk8 - ${kotlin.version} - - - - org.jetbrains.kotlin - kotlin-reflect - ${kotlin.version} - - - - org.jetbrains.kotlin - kotlin-annotations-jvm - ${kotlin.version} - - - + org.jetbrains.kotlin - kotlin-test + kotlin-maven-allopen ${kotlin.version} - test - - - - org.dom4j - dom4j - 2.1.3 - - - - - jaxen - jaxen - 1.1.6 - - - - - - - - ${project.basedir}/src/main/kotlin - + + + org.jetbrains.kotlin + ${kotlin.version} + + + compile + compile + + compile + + + + ${project.basedir}/src/main/kotlin + + + + + test-compile + test-compile + + test-compile + + + + ${project.basedir}/src/test/kotlin + ${project.basedir}/src/test/java + + + + + + ${java.version} + + -jvm-default=enable + + + spring + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${maven-compiler-plugin.version} + + + + org.projectlombok + lombok + 1.18.46 + + + ${java.version} + + + + default-compile + none + + + default-testCompile + none + + + java-compile + compile + + compile + + + + java-test-compile + test-compile + + testCompile + + + + + + org.apache.maven.plugins + maven-dependency-plugin + ${maven-dependency-plugin.version} + + + get-mockito-agent-path + + properties + + + + + + org.apache.maven.plugins + maven-surefire-plugin + ${maven-surefire-plugin.version} + + -javaagent:${org.mockito:mockito-core:jar} ${surefireArgLine} --add-opens + java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens + java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true + --sun-misc-unsafe-memory-access=allow + + + + + org.apache.maven.plugins + maven-failsafe-plugin + ${maven-failsafe-plugin.version} + + + + integration-test + verify + + + -javaagent:${org.mockito:mockito-core:jar} ${failsafeArgLine} --add-opens + java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.lang.reflect=ALL-UNNAMED + --add-opens java.base/jdk.internal.misc=ALL-UNNAMED --add-opens + java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true + --sun-misc-unsafe-memory-access=allow + + + + + + --add-opens java.base/java.lang=ALL-UNNAMED --add-opens + java.base/java.lang.reflect=ALL-UNNAMED --add-opens + java.base/jdk.internal.misc=ALL-UNNAMED --add-opens + java.base/jdk.internal.reflect=ALL-UNNAMED -Dnet.bytebuddy.experimental=true + --sun-misc-unsafe-memory-access=allow + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco-plugin.version} + + + pre-unit-tests + + prepare-agent + + + surefireArgLine + + + + pre-integration-tests + + prepare-agent-integration + + + failsafeArgLine + + + + report + test + + report + + + + + + org.eluder.coveralls + coveralls-maven-plugin + ${coveralls-maven-plugin.version} + + + + + + makeRelease + - - - kotlin-maven-plugin - org.jetbrains.kotlin - ${kotlin.version} - - - compile - compile - - compile - - - - ${project.basedir}/src/main/kotlin - - - - - test-compile - test-compile - - test-compile - - - - ${project.basedir}/src/test/kotlin - ${project.basedir}/src/test/java - - - - - - 1.8 - - -Xjvm-default=all-compatibility - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.10.1 - - - - default-compile - none - - - - default-testCompile - none - - - java-compile - compile - - compile - - - - java-test-compile - test-compile - - testCompile - - - - - - - org.apache.maven.plugins - maven-release-plugin - 2.5.3 - - https://s01.oss.sonatype.org/service/local/staging/deploy/maven2 - github_crawler_@{project.version} - true - true - false - false - makeRelease - forked-path - - - - org.apache.maven.plugins - maven-surefire-plugin - - - - ${surefireArgLine} - - - - - org.apache.maven.plugins - maven-failsafe-plugin - - - - integration-test - verify - - - ${failsafeArgLine} - - - - - - - org.jacoco - jacoco-maven-plugin - ${jacoco.version} - - - pre-unit-tests - - prepare-agent - - - surefireArgLine - - - - pre-integration-tests - - prepare-agent-integration - - - failsafeArgLine - - - - report - test - - report - - - - - - - org.eluder.coveralls - coveralls-maven-plugin - 4.3.0 - - - javax.xml.bind - jaxb-api - 2.3.1 - - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.8 - true + + org.codehaus.mojo + build-helper-maven-plugin + ${build-helper-maven-plugin.version} + + + generate-sources + + add-source + - ossrh - https://s01.oss.sonatype.org/ - true + + ${project.basedir}/src/main/kotlin + - - - - org.apache.maven.plugins - maven-deploy-plugin - 2.8.2 - - + + + + + org.apache.maven.plugins + maven-source-plugin + ${maven-source-plugin.version} + + + attach-sources + + jar-no-fork + + + + + + org.jetbrains.dokka + dokka-maven-plugin + 1.6.10 + + + attach-javadocs + package + + javadocJar + + + + + + org.apache.maven.plugins + maven-gpg-plugin + ${maven-gpg-plugin.version} + + + sign-artifacts + verify + + sign + + + + + + --pinentry-mode + loopback + + + + + + org.sonatype.central + central-publishing-maven-plugin + ${central-publishing-maven-plugin.version} + true + + central + true + published + + - - - - - makeRelease - - - - - - org.codehaus.mojo - build-helper-maven-plugin - 3.0.0 - - - - generate-sources - - add-source - - - - ${project.basedir}/src/main/kotlin - - - - - - - - org.apache.maven.plugins - maven-source-plugin - 3.0.1 - - - attach-sources - - jar-no-fork - - - - - - - org.jetbrains.dokka - dokka-maven-plugin - 1.6.10 - - - attach-javadocs - package - - javadocJar - - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 3.0.1 - - - sign-artifacts - verify - - sign - - - - - - - - - + + +