diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/api/coderepo/controller/CodeRepoController.java b/backend/src/main/java/io/mixeway/mixewayflowapi/api/coderepo/controller/CodeRepoController.java index bc1a28b5..00acb2ea 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/api/coderepo/controller/CodeRepoController.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/api/coderepo/controller/CodeRepoController.java @@ -4,10 +4,10 @@ import io.mixeway.mixewayflowapi.api.coderepo.service.CodeRepoApiService; import io.mixeway.mixewayflowapi.api.gitlabcicd.dto.GitLabCICDRequestDto; import io.mixeway.mixewayflowapi.db.entity.CodeRepo; -import io.mixeway.mixewayflowapi.db.entity.CodeRepoBranch; import io.mixeway.mixewayflowapi.domain.coderepo.CreateCodeRepoService; import io.mixeway.mixewayflowapi.domain.coderepo.DeleteCodeRepoService; import io.mixeway.mixewayflowapi.exceptions.CodeRepoNotFoundException; +import io.mixeway.mixewayflowapi.exceptions.ScanThrottledException; import io.mixeway.mixewayflowapi.exceptions.TeamNotFoundException; import io.mixeway.mixewayflowapi.exceptions.UnauthorizedException; import io.mixeway.mixewayflowapi.utils.StatusDTO; @@ -15,13 +15,16 @@ import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; import java.security.Principal; import java.util.List; +import java.util.NoSuchElementException; @RestController @Validated @@ -136,6 +139,30 @@ public ResponseEntity> getGitBranches(@PathVariable("id") Long id, } } + @PreAuthorize("hasAuthority('USER')") + @PostMapping(value = "/api/v1/coderepo/{id}/scan/sbom", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) + public ResponseEntity uploadSbomAndScan( + @PathVariable("id") Long id, + @RequestPart("file") MultipartFile file, + @RequestParam(value = "branch", required = false) String branch, + Principal principal) { + try { + codeRepoApiService.runScanFromUploadedSbom(id, file, branch, principal); + return new ResponseEntity<>(new StatusDTO("ok"), HttpStatus.ACCEPTED); + } catch (ScanThrottledException e) { + log.warn("[CodeRepo] SBOM upload throttled for repo {}: {}", id, e.getMessage()); + return new ResponseEntity<>(new StatusDTO(e.getMessage()), HttpStatus.TOO_MANY_REQUESTS); + } catch (IllegalArgumentException e) { + log.warn("[CodeRepo] Invalid SBOM upload for {}: {}", id, e.getMessage()); + return new ResponseEntity<>(new StatusDTO(e.getMessage()), HttpStatus.BAD_REQUEST); + } catch (NoSuchElementException e) { + return new ResponseEntity<>(new StatusDTO("Repository not found"), HttpStatus.NOT_FOUND); + } catch (Exception e) { + log.error("[CodeRepo] SBOM upload failed for {}: {}", id, e.getMessage(), e); + return new ResponseEntity<>(new StatusDTO("Not ok"), HttpStatus.BAD_REQUEST); + } + } + @PreAuthorize("hasAuthority('USER')") @PostMapping(value= "/api/v1/coderepo/{id}/run/branch") public ResponseEntity runScanForBranch(@PathVariable("id") Long id, @Valid @RequestBody RunScanBranchRequestDto request, Principal principal){ diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/api/coderepo/service/CodeRepoApiService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/api/coderepo/service/CodeRepoApiService.java index a28bbe2d..ce92553c 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/api/coderepo/service/CodeRepoApiService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/api/coderepo/service/CodeRepoApiService.java @@ -15,15 +15,22 @@ import io.mixeway.mixewayflowapi.domain.team.FindTeamService; import io.mixeway.mixewayflowapi.integrations.repo.service.GitService; import io.mixeway.mixewayflowapi.exceptions.CodeRepoNotFoundException; +import io.mixeway.mixewayflowapi.exceptions.ScanThrottledException; import io.mixeway.mixewayflowapi.exceptions.TeamNotFoundException; import io.mixeway.mixewayflowapi.scanmanager.service.ScanManagerService; import io.mixeway.mixewayflowapi.utils.PermissionFactory; +import com.fasterxml.jackson.core.JacksonException; import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; -import org.aspectj.apache.bcel.classfile.Code; import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.security.Principal; +import java.util.Comparator; import java.util.*; import java.util.stream.Collectors; @@ -31,6 +38,9 @@ @Log4j2 @RequiredArgsConstructor public class CodeRepoApiService { + + private static final long MAX_SBOM_UPLOAD_BYTES = 50L * 1024 * 1024; + private final FindCodeRepoService findCodeRepoService; private final ScanManagerService scanManagerService; private final PermissionFactory permissionFactory; @@ -40,6 +50,7 @@ public class CodeRepoApiService { private final FindFindingService findFindingService; private final GitService gitService; private final GetOrCreateCodeRepoBranchService getOrCreateCodeRepoBranchService; + private final ObjectMapper objectMapper; public List getRepos(Principal principal) { return findCodeRepoService.getCodeReposResponseDtos(principal); @@ -92,6 +103,63 @@ public void runScanForBranch(Long id, String branchName, Principal principal) { scanManagerService.scanRepository(repo, branch, null, null); } + /** + * Validates and stores an uploaded CycloneDX JSON SBOM as {@code sbom.json}, then queues an SCA-only Grype scan. + * + * @param branchName optional branch to associate findings with; defaults to the repository default branch + */ + public void runScanFromUploadedSbom(Long repoId, MultipartFile file, String branchName, Principal principal) + throws IOException { + if (file == null || file.isEmpty()) { + throw new IllegalArgumentException("SBOM file is required."); + } + if (file.getSize() > MAX_SBOM_UPLOAD_BYTES) { + throw new IllegalArgumentException("SBOM file exceeds maximum allowed size (50 MB)."); + } + + CodeRepo repo = findCodeRepoService.findById(repoId, principal); + CodeRepoBranch branch; + if (branchName != null && !branchName.isBlank()) { + branch = getOrCreateCodeRepoBranchService.getOrCreateCodeRepoBranch(branchName.trim(), repo); + } else { + branch = repo.getDefaultBranch(); + } + + Path workDir = Files.createTempDirectory("flow-sbom-upload-"); + Path target = workDir.resolve("sbom.json"); + try { + try (var in = file.getInputStream()) { + Files.copy(in, target, StandardCopyOption.REPLACE_EXISTING); + } + objectMapper.readTree(target.toFile()); + scanManagerService.scanRepositoryFromUploadedSbom(repo, branch, workDir); + } catch (ScanThrottledException e) { + deleteSbomWorkDirQuietly(workDir); + throw e; + } catch (JacksonException e) { + deleteSbomWorkDirQuietly(workDir); + throw new IllegalArgumentException("Invalid JSON SBOM: " + e.getMessage(), e); + } + } + + private void deleteSbomWorkDirQuietly(Path workDir) { + try { + if (Files.isDirectory(workDir)) { + try (var walk = Files.walk(workDir)) { + walk.sorted(Comparator.reverseOrder()).forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException ignored) { + // best effort + } + }); + } + } + } catch (IOException e) { + log.warn("[CodeRepo] Failed to delete SBOM work directory {}: {}", workDir, e.getMessage()); + } + } + public void changeTeam(Long codeRepoId, Long newTeamId, Principal principal) { // Find the code repo CodeRepo codeRepo = findCodeRepoService.findById(codeRepoId) diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/api/teamfindings/service/FindingsByTeamService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/api/teamfindings/service/FindingsByTeamService.java index 85af78aa..e63aca70 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/api/teamfindings/service/FindingsByTeamService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/api/teamfindings/service/FindingsByTeamService.java @@ -165,6 +165,10 @@ public Page getCloudAndRepoFindingsAndVulns(Str String severity = filters.getOrDefault("severity", null); String source = filters.getOrDefault("source", null); String status = filters.getOrDefault("status", null); + String name = filters.getOrDefault("name", null); + if (name != null && name.isBlank()) { + name = null; + } String epssString = filters.getOrDefault("epss", null); BigDecimal epss = (epssString != null) ? new BigDecimal(epssString) : null; String kevStr = filters.getOrDefault("kev", null); @@ -173,8 +177,8 @@ public Page getCloudAndRepoFindingsAndVulns(Str else if ("f".equalsIgnoreCase(kevStr) || "false".equalsIgnoreCase(kevStr)) kev = false; String urgencyFilter = filters.getOrDefault("urgency", null); // expected values: "urgent" | "notable" - Page codeRepoFindingsPage = findingRepository.findByCodeReposPageable(codeRepos, pageable, severity, source, status, epss, kev); - Page cloudSubscriptionFindingsPage = findingRepository.findByCloudSubscriptionsPageable(cloudSubscriptions, pageable, severity, source, status, epss, kev); + Page codeRepoFindingsPage = findingRepository.findByCodeReposPageable(codeRepos, pageable, severity, source, status, epss, kev, name); + Page cloudSubscriptionFindingsPage = findingRepository.findByCloudSubscriptionsPageable(cloudSubscriptions, pageable, severity, source, status, epss, kev, name); List combinedFindings = Stream.concat( codeRepoFindingsPage.getContent().stream(), diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/db/repository/CodeRepoRepository.java b/backend/src/main/java/io/mixeway/mixewayflowapi/db/repository/CodeRepoRepository.java index 31065f48..b296a777 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/db/repository/CodeRepoRepository.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/db/repository/CodeRepoRepository.java @@ -38,6 +38,13 @@ public interface CodeRepoRepository extends CrudRepository { List findByType(CodeRepo.RepoType type); + /** + * Remote IDs already imported for this Git host (same {@code apiUrl} prefix as stored on {@link CodeRepo#getRepourl()}), + * so multiple providers of the same {@link CodeRepo.RepoType} on different hosts do not share deduplication. + */ + @Query("SELECT c.remoteId FROM CodeRepo c WHERE c.type = :type AND c.repourl LIKE CONCAT(:prefix, '%')") + List findRemoteIdsByTypeAndRepourlPrefix(@Param("type") CodeRepo.RepoType type, @Param("prefix") String prefix); + @Query("SELECT count(c) FROM CodeRepo c WHERE c.repourl LIKE CONCAT(:gitHostUrl, '%')") long countByRepoUrlStartingWith(@Param("gitHostUrl") String gitHostUrl); diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/db/repository/FindingRepository.java b/backend/src/main/java/io/mixeway/mixewayflowapi/db/repository/FindingRepository.java index 9c1678ed..97255376 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/db/repository/FindingRepository.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/db/repository/FindingRepository.java @@ -113,8 +113,9 @@ List findByCodeRepoAndVulnerabilityNameAndBranchAndLocation( "AND (COALESCE(:status, f.status) = f.status) " + "AND (:epss IS NULL OR v.epss >= :epss)" + "AND (COALESCE(:kev, v.exploitExists) = v.exploitExists)" + + "AND (:name IS NULL OR LOWER(v.name) LIKE LOWER(CONCAT('%', :name, '%')))" + "AND b = cr.defaultBranch") - Page findByCodeReposPageable(@Param("codeRepos") List codeRepos, Pageable pageable, @Param("severity") String severity, @Param("source") String source, @Param("status") String status, @Param("epss") BigDecimal epss, @Param("kev") Boolean exploitExists); + Page findByCodeReposPageable(@Param("codeRepos") List codeRepos, Pageable pageable, @Param("severity") String severity, @Param("source") String source, @Param("status") String status, @Param("epss") BigDecimal epss, @Param("kev") Boolean exploitExists, @Param("name") String name); @Query("SELECT f FROM Finding f " + "JOIN f.vulnerability v " + @@ -123,8 +124,9 @@ List findByCodeRepoAndVulnerabilityNameAndBranchAndLocation( "AND (COALESCE(:source, f.source) = f.source) " + "AND (COALESCE(:status, f.status) = f.status) " + "AND (:epss IS NULL OR v.epss >= :epss)" + - "AND (COALESCE(:kev, v.exploitExists) = v.exploitExists)") - Page findByCloudSubscriptionsPageable(@Param("cloudSubscriptions") List cloudSubscriptions, Pageable pageable, @Param("severity") String severity, @Param("source") String source, @Param("status") String status, @Param("epss") BigDecimal epss, @Param("kev") Boolean exploitExists); + "AND (COALESCE(:kev, v.exploitExists) = v.exploitExists)" + + "AND (:name IS NULL OR LOWER(v.name) LIKE LOWER(CONCAT('%', :name, '%')))") + Page findByCloudSubscriptionsPageable(@Param("cloudSubscriptions") List cloudSubscriptions, Pageable pageable, @Param("severity") String severity, @Param("source") String source, @Param("status") String status, @Param("epss") BigDecimal epss, @Param("kev") Boolean exploitExists, @Param("name") String name); List findAllByCodeRepoAndVulnerabilityAndLocation(CodeRepo repo, Vulnerability vuln, diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/UpdateCodeRepoService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/UpdateCodeRepoService.java index e04e14ea..f1f4be0b 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/UpdateCodeRepoService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/domain/coderepo/UpdateCodeRepoService.java @@ -285,6 +285,16 @@ public void setScanRunning(CodeRepo codeRepo) { codeRepoRepository.save(codeRepo); } + /** + * Marks only the SCA scan as running (e.g. SBOM upload flow without a full repository clone). + */ + @Transactional + public void setScaScanRunning(CodeRepo codeRepo) { + CodeRepo managed = findCodeRepoService.findById(codeRepo.getId()).orElse(codeRepo); + managed.updateScaScanStatus(CodeRepo.ScanStatus.RUNNING); + codeRepoRepository.save(managed); + } + @Modifying @Transactional public void setScaPending(CodeRepo codeRepo) { diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/exceptions/ScanThrottledException.java b/backend/src/main/java/io/mixeway/mixewayflowapi/exceptions/ScanThrottledException.java new file mode 100644 index 00000000..8f4bc6df --- /dev/null +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/exceptions/ScanThrottledException.java @@ -0,0 +1,11 @@ +package io.mixeway.mixewayflowapi.exceptions; + +/** + * Raised when a new scan cannot start because a recent scan for the same repository is still in the throttle window. + */ +public class ScanThrottledException extends RuntimeException { + + public ScanThrottledException(String message) { + super(message); + } +} diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/repo/service/RepositorySyncService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/repo/service/RepositorySyncService.java index e87fd362..7ba1601c 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/repo/service/RepositorySyncService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/repo/service/RepositorySyncService.java @@ -3,6 +3,7 @@ import io.mixeway.mixewayflowapi.db.entity.CodeRepo; import io.mixeway.mixewayflowapi.db.entity.RepositoryProvider; import io.mixeway.mixewayflowapi.db.repository.CodeRepoRepository; +import io.mixeway.mixewayflowapi.db.repository.RepositoryProviderRepository; import io.mixeway.mixewayflowapi.domain.coderepo.CreateCodeRepoService; import io.mixeway.mixewayflowapi.integrations.repo.apiclient.BitbucketApiClientService; import io.mixeway.mixewayflowapi.integrations.repo.apiclient.GitHubApiClientService; @@ -17,7 +18,11 @@ import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; +import java.time.LocalDateTime; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -27,6 +32,7 @@ public class RepositorySyncService { private final CodeRepoRepository codeRepoRepository; + private final RepositoryProviderRepository repositoryProviderRepository; private final CreateCodeRepoService createCodeRepoService; private final GitLabApiClientService gitLabApiClientService; private final GitHubApiClientService gitHubApiClientService; @@ -38,9 +44,11 @@ public void syncProvider(RepositoryProvider provider) { log.info("Starting repository sync for provider: {}", provider.getApiUrl()); // String accessToken = encryptionUtil.decrypt(provider.getEncryptedAccessToken()); - Set existingRemoteIds = codeRepoRepository.findByType(provider.getProviderType()) + String repoUrlPrefix = normalizeProviderApiUrlPrefix(provider.getApiUrl()); + Set existingRemoteIds = codeRepoRepository + .findRemoteIdsByTypeAndRepourlPrefix(provider.getProviderType(), repoUrlPrefix) .stream() - .map(CodeRepo::getRemoteId) + .filter(Objects::nonNull) .collect(Collectors.toSet()); Flux repositoryFlux; @@ -53,6 +61,7 @@ public void syncProvider(RepositoryProvider provider) { } else if (provider.getProviderType().equals(CodeRepo.RepoType.BITBUCKET)) { repositoryFlux = bitbucketApiClientService.fetchAllRepositories(provider.getApiUrl(), provider.getEncryptedAccessToken()); } else { + log.warn("Repository sync skipped: unsupported provider type {} for {}", provider.getProviderType(), provider.getApiUrl()); return; } @@ -83,12 +92,38 @@ public void syncProvider(RepositoryProvider provider) { ) , 5) // The concurrency parameter limits how many imports run in parallel .doOnComplete(() -> { - long repoCount = codeRepoRepository.countByRepoUrlStartingWith(provider.getApiUrl()); - provider.setSyncedRepoCount((int) repoCount); - provider.setLastSyncDate(java.time.LocalDateTime.now()); - log.info("Finished repository sync for provider: {}. Total synced repos: {}", provider.getApiUrl(), repoCount); + long repoCount = codeRepoRepository.countByRepoUrlStartingWith(repoUrlPrefix); + log.info("Finished repository sync for provider: {}. Repositories on this host in DB: {}", provider.getApiUrl(), repoCount); + Long providerId = provider.getId(); + if (providerId == null) { + log.warn("Provider has no id; cannot persist sync metadata for {}", provider.getApiUrl()); + return; + } + Mono.fromRunnable(() -> repositoryProviderRepository.findById(providerId).ifPresent(p -> { + p.setSyncedRepoCount((int) repoCount); + p.setLastSyncDate(LocalDateTime.now()); + repositoryProviderRepository.save(p); + })) + .subscribeOn(Schedulers.boundedElastic()) + .subscribe( + unused -> {}, + e -> log.error("Failed to persist sync metadata for provider {}: {}", provider.getApiUrl(), e.getMessage(), e)); }) - .doOnError(error -> log.error("Synchronization failed for provider {}: {}", provider.getApiUrl(), error.getMessage())) + .doOnError(error -> log.error("Synchronization failed for provider {}: {}", provider.getApiUrl(), error.getMessage(), error)) .subscribe(); } + + /** + * Aligns {@code apiUrl} with how {@link CodeRepo#getRepourl()} is stored (web URL sharing the same origin prefix). + */ + private static String normalizeProviderApiUrlPrefix(String apiUrl) { + if (apiUrl == null) { + return ""; + } + String t = apiUrl.trim(); + while (t.endsWith("/")) { + t = t.substring(0, t.length() - 1); + } + return t; + } } diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/scanner/sca/service/CdxGenService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/scanner/sca/service/CdxGenService.java index 502d6b10..ab7c909b 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/scanner/sca/service/CdxGenService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/integrations/scanner/sca/service/CdxGenService.java @@ -3,14 +3,16 @@ import ch.qos.logback.core.spi.ScanException; import io.mixeway.mixewayflowapi.db.entity.CodeRepo; import io.mixeway.mixewayflowapi.db.entity.CodeRepoBranch; -import lombok.RequiredArgsConstructor; import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStreamReader; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -25,20 +27,44 @@ */ @Service @Log4j2 -@RequiredArgsConstructor public class CdxGenService { + /** + * Prepended to {@code PATH} for cdxgen and related subprocesses so the same JVM + * can resolve {@code mvn}, {@code npm}, {@code gradle}, etc. as in an interactive shell + * (e.g. {@code /opt/tools/bin:/usr/local/bin} in Docker). + */ + @Value("${scan.subprocess.path-extra:}") + private String pathExtra; + + /** + * If set, exported as {@code JAVA_HOME} for subprocesses when the environment + * does not already define it. Otherwise {@code java.home} of the running JVM is used + * when {@code JAVA_HOME} is missing (helps Maven/Gradle invoked by cdxgen). + */ + @Value("${scan.subprocess.java-home:}") + private String javaHomeOverride; + + /** + * When true (default), runs {@code pipreqs .} before cdxgen if {@code pipreqs} is on {@code PATH}. + * Set to false to match a manual {@code cdxgen} run and avoid overwriting {@code requirements.txt}. + */ + @Value("${scan.cdxgen.run-pipreqs:true}") + private boolean runPipreqs; + + @Value("${proxy.host:#{null}}") + private String proxyHost; + + @Value("${proxy.port:#{null}}") + private Integer proxyPort; + /** * Generates the SBOM (Software Bill of Materials) file using the cdxgen tool. * - *

This method executes the cdxgen command in the specified repository directory, - * redirecting both standard output and error streams to prevent blocking. - * It conditionally sets environment variables for proxy configuration if the - * system properties proxy.host and proxy.port are provided. - * The method waits for the process to complete, with a timeout of 2 minutes. - * If the process exceeds the timeout, it is forcibly terminated. - * After the process completes, the method validates the generated bom.json - * file by checking for its existence and ensuring it has content.

+ *

This method executes cdxgen in the specified repository directory with + * {@link ProcessBuilder#environment()} configured for CDXGEN/CDX_* variables and optional proxy. + * Optional {@code scan.subprocess.path-extra} widens {@code PATH} so language toolchains + * match non-interactive vs login-shell setups.

* * @param repoDir the directory of the repository where cdxgen will run * @param codeRepo the code repository entity @@ -51,70 +77,35 @@ public void generateBom(String repoDir, CodeRepo codeRepo, CodeRepoBranch codeRe throws IOException, InterruptedException { log.info("[CdxGen] Starting SBOM generation for: {} branch: {}", codeRepo.getName(), codeRepoBranch.getName()); - // Step 1: Verify if 'pipreqs' command is available - boolean isPipreqsAvailable = false; - try { - ProcessBuilder pbCheckPipreqs = new ProcessBuilder("sh", "-c", "command -v pipreqs"); - pbCheckPipreqs.redirectErrorStream(true); - Process pCheckPipreqs = pbCheckPipreqs.start(); - int exitCode = pCheckPipreqs.waitFor(); - if (exitCode == 0) { - isPipreqsAvailable = true; - log.debug("[CdxGen] 'pipreqs' is available."); - } else { - log.debug("[CdxGen] 'pipreqs' is not available."); - } - } catch (IOException e) { - // Command not found - log.debug("[CdxGen] Exception while checking for 'pipreqs': {}", e.getMessage()); - } - - // Step 2: If available, execute 'pipreqs .' in repoDir - if (isPipreqsAvailable) { - log.debug("[CdxGen] Executing 'pipreqs .' in {}", repoDir); - ProcessBuilder pbPipreqs = new ProcessBuilder("pipreqs", "."); - pbPipreqs.directory(new File(repoDir)); - pbPipreqs.redirectOutput(ProcessBuilder.Redirect.INHERIT); - pbPipreqs.redirectError(ProcessBuilder.Redirect.INHERIT); - Process pPipreqs = pbPipreqs.start(); - - // Wait for 'pipreqs' to finish - boolean finished = pPipreqs.waitFor(10, TimeUnit.MINUTES); - if (!finished) { - log.debug("[CdxGen] 'pipreqs' did not finish within 10 minutes. Terminating process."); - pPipreqs.destroyForcibly(); - } else { - int exitCode = pPipreqs.exitValue(); - if (exitCode != 0) { - log.debug("[CdxGen] 'pipreqs' exited with non-zero exit code: {}", exitCode); - } else { - log.debug("[CdxGen] 'pipreqs' executed successfully."); - } - } + if (runPipreqs) { + runPipreqsIfAvailable(repoDir); } - // Step 3: Proceed with executing 'cdxgen' - String proxyHost = System.getProperty("proxy.host"); - String proxyPort = System.getProperty("proxy.port"); - String command; - - if (proxyHost != null && proxyPort != null) { - command = "CDXGEN_DEBUG_MODE=debug " - + "CDX_MAVEN_INCLUDE_TEST_SCOPE=false " - + "HTTP_PROXY=http://" + proxyHost + ":" + proxyPort + " " - + "HTTPS_PROXY=http://" + proxyHost + ":" + proxyPort + " " - + "cdxgen --recurse --required-only --output sbom.json ."; - log.info("[CdxGen] Proxy settings applied: {}:{}", proxyHost, proxyPort); - } else { - command = "CDXGEN_DEBUG_MODE=debug CDX_MAVEN_INCLUDE_TEST_SCOPE=false cdxgen --recurse --required-only --output sbom.json ."; - } - - // Use 'sh -c' to execute the command in a shell - ProcessBuilder pb = new ProcessBuilder("sh", "-c", command); + ProcessBuilder pb = new ProcessBuilder( + "cdxgen", + "--recurse", + "--required-only", + "--output", + "sbom.json", + "." + ); pb.directory(new File(repoDir)); pb.redirectOutput(ProcessBuilder.Redirect.PIPE); pb.redirectError(ProcessBuilder.Redirect.PIPE); + applyScannerEnvironment(pb); + + Map env = pb.environment(); + env.put("CDXGEN_DEBUG_MODE", "debug"); + env.put("CDX_MAVEN_INCLUDE_TEST_SCOPE", "false"); + + if (StringUtils.hasText(proxyHost) && proxyPort != null) { + String proxyUrl = "http://" + proxyHost + ":" + proxyPort; + env.put("HTTP_PROXY", proxyUrl); + env.put("HTTPS_PROXY", proxyUrl); + log.info("[CdxGen] Proxy settings applied: {}:{}", proxyHost, proxyPort); + } + Process process = pb.start(); ExecutorService executorService = Executors.newFixedThreadPool(2); @@ -142,7 +133,6 @@ public void generateBom(String repoDir, CodeRepo codeRepo, CodeRepoBranch codeRe } }); - // Wait for 'cdxgen' to finish with a timeout of 30 minutes boolean finished = process.waitFor(30, TimeUnit.MINUTES); if (!finished) { log.warn("[CdxGen] SBOM generation did not finish within 30 minutes. Terminating process."); @@ -166,7 +156,6 @@ public void generateBom(String repoDir, CodeRepo codeRepo, CodeRepoBranch codeRe } } - // Validate the 'sbom.json' file File bomFile = new File(repoDir, "sbom.json"); if (bomFile.exists()) { if (bomFile.length() > 0) { @@ -179,6 +168,79 @@ public void generateBom(String repoDir, CodeRepo codeRepo, CodeRepoBranch codeRe } } + private void runPipreqsIfAvailable(String repoDir) throws IOException, InterruptedException { + boolean isPipreqsAvailable = false; + try { + ProcessBuilder pbCheckPipreqs = new ProcessBuilder("sh", "-c", "command -v pipreqs"); + applyScannerEnvironment(pbCheckPipreqs); + pbCheckPipreqs.redirectErrorStream(true); + Process pCheckPipreqs = pbCheckPipreqs.start(); + int exitCode = pCheckPipreqs.waitFor(); + if (exitCode == 0) { + isPipreqsAvailable = true; + log.debug("[CdxGen] 'pipreqs' is available."); + } else { + log.debug("[CdxGen] 'pipreqs' is not available."); + } + } catch (IOException e) { + log.debug("[CdxGen] Exception while checking for 'pipreqs': {}", e.getMessage()); + } + + if (!isPipreqsAvailable) { + return; + } + + log.debug("[CdxGen] Executing 'pipreqs .' in {}", repoDir); + ProcessBuilder pbPipreqs = new ProcessBuilder("pipreqs", "."); + pbPipreqs.directory(new File(repoDir)); + applyScannerEnvironment(pbPipreqs); + pbPipreqs.redirectOutput(ProcessBuilder.Redirect.INHERIT); + pbPipreqs.redirectError(ProcessBuilder.Redirect.INHERIT); + Process pPipreqs = pbPipreqs.start(); + + boolean finished = pPipreqs.waitFor(10, TimeUnit.MINUTES); + if (!finished) { + log.debug("[CdxGen] 'pipreqs' did not finish within 10 minutes. Terminating process."); + pPipreqs.destroyForcibly(); + } else { + int exitCode = pPipreqs.exitValue(); + if (exitCode != 0) { + log.debug("[CdxGen] 'pipreqs' exited with non-zero exit code: {}", exitCode); + } else { + log.debug("[CdxGen] 'pipreqs' executed successfully."); + } + } + } + + /** + * Ensures subprocesses see the same toolchains as typical CLI usage: optional {@code PATH} + * prefix, {@code JAVA_HOME}, and a defined {@code HOME} when missing. + */ + private void applyScannerEnvironment(ProcessBuilder pb) { + Map env = pb.environment(); + if (StringUtils.hasText(pathExtra)) { + String path = env.getOrDefault("PATH", ""); + env.put("PATH", pathExtra + File.pathSeparator + path); + log.debug("[CdxGen] PATH prefix applied (scan.subprocess.path-extra)"); + } + if (!StringUtils.hasText(env.get("JAVA_HOME"))) { + if (StringUtils.hasText(javaHomeOverride)) { + env.put("JAVA_HOME", javaHomeOverride.trim()); + } else { + String jvmHome = System.getProperty("java.home"); + if (StringUtils.hasText(jvmHome)) { + env.put("JAVA_HOME", jvmHome); + } + } + } + + if (!StringUtils.hasText(env.get("HOME"))) { + String userHome = System.getProperty("user.home"); + if (StringUtils.hasText(userHome)) { + env.put("HOME", userHome); + } + } + } } diff --git a/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/service/ScanManagerService.java b/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/service/ScanManagerService.java index 4e31be41..ca3d2b07 100644 --- a/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/service/ScanManagerService.java +++ b/backend/src/main/java/io/mixeway/mixewayflowapi/scanmanager/service/ScanManagerService.java @@ -13,6 +13,7 @@ import io.mixeway.mixewayflowapi.domain.vulnerability.FindVulnerabilityService; import io.mixeway.mixewayflowapi.domain.vulnerability.UpdateVulnerabilityService; import io.mixeway.mixewayflowapi.exceptions.GitException; +import io.mixeway.mixewayflowapi.exceptions.ScanThrottledException; import io.mixeway.mixewayflowapi.integrations.repo.service.BitbucketScanReportService; import io.mixeway.mixewayflowapi.integrations.repo.service.GitCommentService; import io.mixeway.mixewayflowapi.integrations.repo.service.GitService; @@ -805,6 +806,59 @@ public void processKEV(){ log.info("[KEV] Done processing KEV..."); } + /** + * Runs an SCA (Grype) scan using a CycloneDX JSON SBOM already stored as {@code sbom.json} under {@code workDir}. + * Does not clone the repository. Uses the same per-repo throttle as full scans. + * + * @param workDir directory containing {@code sbom.json}; deleted after the scan attempt + */ + public void scanRepositoryFromUploadedSbom(CodeRepo codeRepo, CodeRepoBranch codeRepoBranch, Path workDir) { + Object repoLock = repoLocks.computeIfAbsent(codeRepo.getId(), k -> new Object()); + + synchronized (repoLock) { + if (scanThrottler.getIfPresent(codeRepo.getId()) != null) { + throw new ScanThrottledException( + "A scan for this repository was started recently. Please wait before uploading again."); + } + scanThrottler.put(codeRepo.getId(), Boolean.TRUE); + } + + String workDirPath = workDir.toAbsolutePath().toString(); + + executorService.submit(() -> { + try { + updateCodeRepoService.setScaScanRunning(codeRepo); + validateInputs(codeRepo, codeRepoBranch); + try { + sCAGrypeService.runGrype(workDirPath, codeRepo, codeRepoBranch); + } catch (Exception e) { + if (Thread.currentThread().isInterrupted()) { + log.warn("[ScanManagerService] Uploaded SBOM SCA scan interrupted for {}.", codeRepo.getRepourl()); + Thread.currentThread().interrupt(); + } else { + log.error("[ScanManagerService] Uploaded SBOM SCA scan failed for {}: {}", + codeRepo.getRepourl(), e.getMessage(), e); + } + } finally { + try { + updateCodeRepoService.updateCodeRepoStatus(codeRepo, codeRepoBranch, ""); + } catch (Exception updateEx) { + log.error("[ScanManagerService] Failed to update CodeRepo status after SBOM upload for {}: {}", + codeRepo.getName(), updateEx.getMessage(), updateEx); + } + try { + cleanUp(workDirPath); + } catch (IOException cleanupEx) { + log.error("[ScanManagerService] Failed to clean up SBOM work directory {}: {}", + workDirPath, cleanupEx.getMessage()); + } + } + } finally { + repoLocks.remove(codeRepo.getId()); + } + }); + } + private Future runZAPScan(String repoDir, CodeRepo codeRepo, CodeRepoBranch codeRepoBranch) { Callable task = () -> { int currentZapScans = zapScansRunning.incrementAndGet(); diff --git a/backend/src/main/resources/application-dev.properties b/backend/src/main/resources/application-dev.properties index 69a8e5b2..0ed1bd21 100644 --- a/backend/src/main/resources/application-dev.properties +++ b/backend/src/main/resources/application-dev.properties @@ -22,6 +22,8 @@ spring.datasource.hikari.max-lifetime=1200000 spring.datasource.hikari.auto-commit=true #logging.level.root = DEBUG spring.codec.max-in-memory-size=30MB +spring.servlet.multipart.max-file-size=50MB +spring.servlet.multipart.max-request-size=55MB spring.liquibase.change-log=classpath:db/changelog/db.changelog-master.sql spring.liquibase.enabled=true diff --git a/backend/src/main/resources/application-prod.properties b/backend/src/main/resources/application-prod.properties index 2b802e1c..9dedfcb8 100644 --- a/backend/src/main/resources/application-prod.properties +++ b/backend/src/main/resources/application-prod.properties @@ -29,6 +29,8 @@ spring.datasource.hikari.max-lifetime=1200000 spring.datasource.hikari.auto-commit=true #logging.level.root = DEBUG spring.codec.max-in-memory-size=30MB +spring.servlet.multipart.max-file-size=50MB +spring.servlet.multipart.max-request-size=55MB spring.liquibase.change-log=classpath:db/changelog/db.changelog-master.sql frontend.url="localhost" diff --git a/frontend/nginx/nginx-http.conf b/frontend/nginx/nginx-http.conf index dc33ba1e..49631a9c 100644 --- a/frontend/nginx/nginx-http.conf +++ b/frontend/nginx/nginx-http.conf @@ -30,7 +30,7 @@ proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 600s; + proxy_read_timeout 900s; } location /oauth2/ { proxy_pass http://backend:8888; diff --git a/frontend/nginx/nginx.conf b/frontend/nginx/nginx.conf index fff3d900..976c38d7 100644 --- a/frontend/nginx/nginx.conf +++ b/frontend/nginx/nginx.conf @@ -32,7 +32,7 @@ proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; - proxy_read_timeout 600s; + proxy_read_timeout 900s; } location /oauth2/ { proxy_pass https://backend:8443; diff --git a/frontend/src/app/service/RepoService.ts b/frontend/src/app/service/RepoService.ts index d4f04e1d..2202c1c3 100644 --- a/frontend/src/app/service/RepoService.ts +++ b/frontend/src/app/service/RepoService.ts @@ -53,6 +53,19 @@ export class RepoService { return this.http.post(this.loginUrl + '/api/v1/coderepo/' + id + '/run/branch', { branchName }, { withCredentials: true }); } + /** + * Upload a CycloneDX JSON SBOM; backend runs SCA (Grype) only and associates results with the given branch (or default). + */ + uploadSbomScan(id: number, file: File, branch?: string): Observable { + const formData = new FormData(); + formData.append('file', file); + let url = `${this.loginUrl}/api/v1/coderepo/${id}/scan/sbom`; + if (branch && branch.trim()) { + url += `?branch=${encodeURIComponent(branch.trim())}`; + } + return this.http.post(url, formData, { withCredentials: true }); + } + suppressMultipleFindings(number: number, selectedFindings: number[]) { return this.http.post(this.loginUrl + '/api/v1/coderepo/' + number+ '/supress', selectedFindings,{ withCredentials: true }); diff --git a/frontend/src/app/views/show-repo/repository-info/repository-info.component.html b/frontend/src/app/views/show-repo/repository-info/repository-info.component.html index 7c715e53..3116f9d4 100644 --- a/frontend/src/app/views/show-repo/repository-info/repository-info.component.html +++ b/frontend/src/app/views/show-repo/repository-info/repository-info.component.html @@ -54,6 +54,11 @@

Scan Other Branch +
  • + +

    • + + +
    Upload SBOM for SCA
    + +     + +

    + CycloneDX JSON (e.g. from cdxgen). Findings are tied to the branch below (default: repository default branch). +

    + + + + +
    {{ sbomError }}
    +     + + + + +     +
    Rename repository
    diff --git a/frontend/src/app/views/show-repo/repository-info/repository-info.component.ts b/frontend/src/app/views/show-repo/repository-info/repository-info.component.ts index b7799377..fc4f7d6a 100644 --- a/frontend/src/app/views/show-repo/repository-info/repository-info.component.ts +++ b/frontend/src/app/views/show-repo/repository-info/repository-info.component.ts @@ -91,8 +91,14 @@ export class RepositoryInfoComponent implements OnInit { selectedBranch = ''; branchLoadError: string | null = null; + sbomModalVisible = false; + sbomBranch = ''; + sbomError: string | null = null; + sbomFile: File | null = null; + @Output() runScanEvent = new EventEmitter(); @Output() runScanBranchEvent = new EventEmitter(); + @Output() uploadSbomScanEvent = new EventEmitter<{ file: File; branch?: string }>(); @Output() openChangeTeamModalEvent = new EventEmitter(); @Output() deleteRepoEvent = new EventEmitter(); @@ -153,6 +159,39 @@ export class RepositoryInfoComponent implements OnInit { this.runScanBranchEvent.emit(this.selectedBranch); } + openSbomUploadModal(): void { + this.scanDropdownOpen = false; + this.sbomModalVisible = true; + this.sbomError = null; + this.sbomFile = null; + this.sbomBranch = this.repoData?.defaultBranch?.name ?? ''; + } + + onSbomFileChange(event: Event): void { + const input = event.target as HTMLInputElement; + const file = input.files?.[0] ?? null; + this.sbomFile = file; + this.sbomError = null; + } + + confirmSbomUpload(): void { + if (!this.sbomFile) { + this.sbomError = 'Choose a JSON SBOM file.'; + return; + } + const name = this.sbomFile.name.toLowerCase(); + if (!name.endsWith('.json')) { + this.sbomError = 'File must be JSON (e.g. CycloneDX .json).'; + return; + } + const branch = (this.sbomBranch || '').trim(); + this.sbomModalVisible = false; + this.uploadSbomScanEvent.emit({ + file: this.sbomFile, + branch: branch.length > 0 ? branch : undefined, + }); + } + constructor(private codeService: RepoService) {} diff --git a/frontend/src/app/views/show-repo/show-repo.component.html b/frontend/src/app/views/show-repo/show-repo.component.html index 53bf339d..e795d41a 100644 --- a/frontend/src/app/views/show-repo/show-repo.component.html +++ b/frontend/src/app/views/show-repo/show-repo.component.html @@ -7,6 +7,7 @@ [options]="options" (runScanEvent)="runScan()" (runScanBranchEvent)="runScanForBranch($event)" + (uploadSbomScanEvent)="uploadSbomScan($event)" (openChangeTeamModalEvent)="openChangeTeamModal()" (deleteRepoEvent)="openDeleteRepoModal()" > diff --git a/frontend/src/app/views/show-repo/show-repo.component.ts b/frontend/src/app/views/show-repo/show-repo.component.ts index 8fb3405f..2d8b58a6 100644 --- a/frontend/src/app/views/show-repo/show-repo.component.ts +++ b/frontend/src/app/views/show-repo/show-repo.component.ts @@ -1105,6 +1105,30 @@ export class ShowRepoComponent implements OnInit, AfterViewInit { }); } + uploadSbomScan(payload: { file: File; branch?: string }) { + this.repoService.uploadSbomScan(+this.repoId, payload.file, payload.branch).subscribe({ + next: () => { + this.toastStatus = 'success'; + this.toastMessage = 'SBOM uploaded; SCA scan has been queued'; + this.toggleToast(); + this.loadRepoInfo(); + }, + error: (err: any) => { + this.toastStatus = 'danger'; + const status = err?.status; + const msg = err?.error?.status ?? err?.error?.message ?? err?.message; + if (status === 429) { + this.toastMessage = typeof msg === 'string' ? msg : 'Too many scan requests; try again later'; + } else if (status === 400 && typeof msg === 'string') { + this.toastMessage = msg; + } else { + this.toastMessage = 'Failed to upload SBOM or start SCA scan'; + } + this.toggleToast(); + }, + }); + } + openDeleteRepoModal(): void { this.deleteConfirmationText = ''; this.deleteRepoConfirmationVisible = true;