Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
00670ec
chore: add sdkman env for java
paul58914080 Jul 18, 2026
30cd32a
style: format code
paul58914080 Jul 18, 2026
05adb33
style: format code
paul58914080 Jul 18, 2026
c3fbcf3
build: replace parent with bom and additional bom
paul58914080 Jul 18, 2026
bbdaac2
feat: bump jdk 25 and all dependency versions
paul58914080 Jul 19, 2026
a524b19
build: bump version to 3.0.0-SNAPSHOT in pom.xml
paul58914080 Jul 19, 2026
1507491
build: add spring-boot-maven-plugin version
paul58914080 Jul 19, 2026
2943f9d
build: move spring-boot-configuration-processor to annotationProcesso…
paul58914080 Jul 19, 2026
12b2542
build: update jvm-default argument for compatibility
paul58914080 Jul 19, 2026
c4aa182
refactor: fix deprecated code
paul58914080 Jul 19, 2026
b1d39b7
build: add --sun-misc-unsafe-memory-access=allow argument to JVM conf…
paul58914080 Jul 19, 2026
12fe52e
test: replace ArrayList with CopyOnWriteArrayList for thread safety
paul58914080 Jul 19, 2026
2d168dc
build: add Mockito agent path to JVM arguments for test execution
paul58914080 Jul 19, 2026
8855f59
ci(build): update the step name
paul58914080 Jul 19, 2026
99614ba
ci(release): configure git user for release
paul58914080 Jul 19, 2026
c499271
docs: update version and maintainers
paul58914080 Jul 19, 2026
d1663fb
build: update for releasing to sonatype
paul58914080 Jul 19, 2026
aef3a97
build: bump spring-boot to 4.1.0
paul58914080 Jul 19, 2026
5d78d32
build: update plugin versions in pom.xml
paul58914080 Jul 19, 2026
0252146
build: add repackage execution
paul58914080 Jul 19, 2026
1632c60
test: correct the flanky assertions
paul58914080 Jul 19, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/build_workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,11 @@ jobs:
steps:
- name: Checkout
uses: actions/checkout@v6
- name: Set up JDK 11
- name: Set up JDK 25
uses: actions/setup-java@v5
with:
distribution: temurin
java-version: 11
java-version: 25
- name: Cache Maven dependencies
uses: actions/cache@v5
with:
Expand Down
9 changes: 7 additions & 2 deletions .github/workflows/release_workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,22 @@ jobs:
version=$(echo "$raw_tag_name" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+$')
echo "VERSION=$version" >> $GITHUB_ENV

- name: Set up JDK 11 with GPG
- name: Set up JDK 25 with GPG
uses: actions/setup-java@v5
with:
distribution: temurin
java-version: 11
java-version: 25
gpg-private-key: ${{ secrets.GPG_PRIVATE_KEY }}
gpg-passphrase: MAVEN_GPG_PASSPHRASE
server-id: central
server-username: CENTRAL_USERNAME
server-password: CENTRAL_PASSWORD

- name: Configure git user for release
run: |
git config user.name "github-actions"
git config user.email "github-actions@github.com"

- name: Build & Deploy to Staging
env:
CENTRAL_USERNAME: ${{ secrets.CENTRAL_USERNAME }}
Expand Down
1 change: 1 addition & 0 deletions .mvn/jvm.config
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
--sun-misc-unsafe-memory-access=allow
3 changes: 3 additions & 0 deletions .sdkmanrc
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Enable auto-env through the sdkman_auto_env config
# Add key=value pairs of SDKs to use below
java=25.0.3-tem
11 changes: 11 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -362,3 +362,14 @@ public class PersonalGitHubCrawlerApplication implements CommandLineRunner {
see [here](https://github.com/vincent-fuchs/my-custom-github-crawler/blob/0e48dd7961b6b625802a2e1eb6b2fc4f8c4d5cdb/src/main/java/com/github/vincent_fuchs/output/CustomOutput.java) or [here](https://github.com/vincent-fuchs/my-custom-github-crawler/blob/ec7ed9a74f91b31794b8a0afb1196553434b1567/src/main/java/com/github/vincent_fuchs/parsers/MyOwnParser.java) for examples

- see the javadoc in [FileContentParser](./github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/parsers/FileContentParser.kt) , [RepoTaskToPerform](./github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/repoTaskToPerform/RepoTaskToPerform.kt), [GitHubCrawlerOutput](./github-crawler-core/src/main/kotlin/com/societegenerale/githubcrawler/output/GitHubCrawlerOutput.kt) which are the main extension points.

## Versions

- v3.x is using JDK25
- v2.x is using JDK11

## Official maintainers

- [Paul Williams](https://github.com/paul58914080)
- [Vincent Fuchs](https://github.com/vincent-fuchs)

93 changes: 38 additions & 55 deletions github-crawler-autoconfigure/pom.xml
Original file line number Diff line number Diff line change
@@ -1,58 +1,41 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>github-crawler-autoconfigure</artifactId>
<packaging>jar</packaging>
<name>github-crawler-autoconfigure</name>
<parent>
<groupId>com.societegenerale.github-crawler</groupId>
<artifactId>github-crawler-autoconfigure</artifactId>
<version>2.2.1-SNAPSHOT</version>
<packaging>jar</packaging>

<name>github-crawler-autoconfigure</name>

<parent>
<groupId>com.societegenerale.github-crawler</groupId>
<artifactId>github-crawler-parent</artifactId>
<version>2.2.1-SNAPSHOT</version>
</parent>

<dependencies>

<dependency>
<groupId>com.societegenerale.github-crawler</groupId>
<artifactId>github-crawler-core</artifactId>
<version>${project.parent.version}</version>
</dependency>

<dependency>
<groupId>net.code-story</groupId>
<artifactId>http</artifactId>
<version>2.104</version>
<scope>test</scope>

<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-simple</artifactId>
</exclusion>
</exclusions>
</dependency>

<dependency>
<groupId>com.github.tomakehurst</groupId>
<artifactId>wiremock-jre8</artifactId>
<version>2.32.0</version>
<scope>test</scope>
</dependency>


<dependency>
<groupId>com.jayway.awaitility</groupId>
<artifactId>awaitility</artifactId>
<version>1.7.0</version>
<scope>test</scope>
</dependency>

</dependencies>


<artifactId>github-crawler-parent</artifactId>
<version>3.0.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>com.societegenerale.github-crawler</groupId>
<artifactId>github-crawler-core</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-restclient</artifactId>
</dependency>
<!-- Test dependencies -->
<dependency>
<groupId>net.code-story</groupId>
<artifactId>http</artifactId>
<version>${http.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.wiremock</groupId>
<artifactId>wiremock-standalone</artifactId>
<version>${wiremock-standalone.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.awaitility</groupId>
<artifactId>awaitility</artifactId>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ void shouldCrawl() throws IOException {
assertThat(branchWhereResultWasFound.getName()).isEqualTo("refs/heads/main");

List<String> valueForIndicator=(ArrayList)firstTaskResult.getValue().get("pipelineTemplateLocation");
assertThat(valueForIndicator.get(0)).isEqualTo("azure-pipelines.yml");
assertThat(valueForIndicator.getFirst()).isEqualTo("azure-pipelines.yml");
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,29 +5,27 @@
import com.societegenerale.githubcrawler.mocks.BitbucketMock;
import com.societegenerale.githubcrawler.model.Repository;
import org.assertj.core.api.SoftAssertions;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.annotation.DirtiesContext.ClassMode;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit.jupiter.SpringExtension;

import java.io.IOException;
import java.util.*;

import static com.jayway.awaitility.Awaitility.await;

import static java.util.Map.entry;
import static java.util.concurrent.TimeUnit.SECONDS;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toSet;
import static org.assertj.core.api.Assertions.assertThat;


@ExtendWith(SpringExtension.class)
@SpringBootTest(classes = {TestConfig.class, GitHubCrawlerAutoConfiguration.class})
@ActiveProfiles(profiles = {"bitBucketTest"})
@DirtiesContext(classMode = ClassMode.BEFORE_CLASS)
Expand Down Expand Up @@ -58,8 +56,8 @@ public void mockSetUp() {

bitbucketMockServer.start();

await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.until(() -> assertThat(BitbucketMock.hasStarted()));
Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.untilAsserted(() -> assertThat(BitbucketMock.hasStarted()).isTrue());

hasBitbucketMockServerStarted = true;
}
Expand Down Expand Up @@ -94,8 +92,8 @@ void shouldHitOrgaNextPageOfRepositoriesIfMorethanOne() throws IOException {

Collection<Repository> processedRepositories = output.getAnalyzedRepositories().values();

await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga));
Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga));

assertThat(bitbucketMockServer.isHasCalledNextPage()).as("next page wasn't called").isTrue();

Expand Down Expand Up @@ -125,7 +123,7 @@ void shouldLoadIndicatorsConfig() {
.containsKeys(pomXmlFile, new FileToParse("pom.xml", null),
new FileToParse("Jenkinsfile", null), new FileToParse("Dockerfile", null));

IndicatorDefinition pomXmlIndicatorDefinition1 = crawler.getGitHubCrawlerProperties().getIndicatorsToFetchByFile().get(pomXmlFile).get(0);
IndicatorDefinition pomXmlIndicatorDefinition1 = crawler.getGitHubCrawlerProperties().getIndicatorsToFetchByFile().get(pomXmlFile).getFirst();
assertThat(pomXmlIndicatorDefinition1).isNotNull();
assertThat(pomXmlIndicatorDefinition1.getName()).isEqualTo("spring_boot_starter_parent_version");
assertThat(pomXmlIndicatorDefinition1.getType()).isEqualTo("findDependencyVersionInXml");
Expand All @@ -140,8 +138,8 @@ void shouldTryToFetchFilesWithIndicators() throws IOException {

crawler.crawl();

await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.until(() -> assertThat(bitbucketMockServer.getPomXmlHits()).hasSize(nbRepositoriesInOrga));
Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.untilAsserted(() -> assertThat(bitbucketMockServer.getPomXmlHits()).hasSize(nbRepositoriesInOrga));

}

Expand All @@ -152,10 +150,10 @@ void shouldCopyActiveProfilesAsGroupsOnRepo() throws IOException {

Collection<Repository> processedRepositories = output.getAnalyzedRepositories().values();

await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga));
Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga));

processedRepositories.stream().forEach(repo -> {
processedRepositories.forEach(repo -> {
assertThat(repo.getGroups()).containsExactlyInAnyOrder("bitBucketTest");
});

Expand All @@ -182,8 +180,8 @@ void shouldParseFileWithIndicatorsWhenItExists() throws IOException {

crawler.crawl();

await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.until(() -> assertThat(output.getAnalyzedRepositories().values()).hasSize(nbRepositoriesInOrga));
Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.untilAsserted(() -> assertThat(output.getAnalyzedRepositories().values()).hasSize(nbRepositoriesInOrga));

Collection<Repository> processedRepositories = output.getAnalyzedRepositories().values();

Expand Down Expand Up @@ -252,8 +250,8 @@ void shouldParseAllBranchesWhenConfiguredAccordingly() throws IOException {

Collection<Repository> processedRepositories = output.getAnalyzedRepositories().values();

await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.until(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga));
Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.untilAsserted(() -> assertThat(processedRepositories).hasSize(nbRepositoriesInOrga));

for (Repository processedRepository : processedRepositories) {

Expand Down Expand Up @@ -284,8 +282,8 @@ void processNormallyIfNoConfigFileOnRepoSide() throws IOException {
crawler.crawl();

Collection<Repository> actualRepositories = output.getAnalyzedRepositories().values();
await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.until(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga));
Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.untilAsserted(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga));

assertThat(bitbucketMockServer.getRepoConfigHits()).hasSize(nbRepositoriesInOrga);

Expand All @@ -303,8 +301,8 @@ void outputResultsShouldHaveACrawlerRunId() throws IOException {
crawler.crawl();

Collection<Repository> actualRepositories = output.getAnalyzedRepositories().values();
await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.until(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga));
Awaitility.await().atMost(MAX_TIMEOUT_FOR_CRAWLER, SECONDS)
.untilAsserted(() -> assertThat(actualRepositories).hasSize(nbRepositoriesInOrga));

assertThat(actualRepositories.stream().filter(r -> !r.getCrawlerRunId().equals(GitHubCrawler.NO_CRAWLER_RUN_ID_DEFINED))
.count())
Expand Down
Loading
Loading