Skip to content

Commit 33e42fb

Browse files
absurdlylongusernameabsurdlylongusername
committed
Merge branch 'dev' into soundcloud-validate-http-response
2 parents b2299dc + eb98890 commit 33e42fb

114 files changed

Lines changed: 5556 additions & 4945 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ plugins {
1010
allprojects {
1111
apply(plugin = "java-library")
1212

13-
version = "v0.26.0"
13+
version = "v0.26.1"
1414

1515
tasks.withType<JavaCompile> {
1616
options.encoding = Charsets.UTF_8.toString()

‎extractor/build.gradle.kts‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -191,10 +191,7 @@ publishing {
191191
}
192192

193193
signing {
194+
setRequired(shouldSignCIRelease)
194195
useInMemoryPgpKeys(ciSigningKey, ciSigningPassword)
195196
sign(publishing.publications["snapshot"])
196197
}
197-
198-
tasks.withType<Sign> {
199-
onlyIf("Signing credentials are present (only used for maven central)") { shouldSignCIRelease }
200-
}

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudCommentsExtractor.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public InfoItemsPage<CommentsInfoItem> getInitialPage() throws ExtractionExcepti
4141
public InfoItemsPage<CommentsInfoItem> getPage(final Page page) throws ExtractionException,
4242
IOException {
4343
if (page == null || isNullOrEmpty(page.getUrl())) {
44-
throw new IllegalArgumentException("Page doesn't contain an URL");
44+
return InfoItemsPage.emptyPage();
4545
}
4646
return getPage(page.getUrl());
4747
}
@@ -63,7 +63,8 @@ private InfoItemsPage<CommentsInfoItem> getPage(@Nonnull final String url)
6363
getServiceId());
6464

6565
collectStreamsFrom(collector, json.getArray("collection"));
66-
return new InfoItemsPage<>(collector, new Page(json.getString("next_href", null)));
66+
final String nextHref = json.getString("next_href");
67+
return new InfoItemsPage<>(collector, isNullOrEmpty(nextHref) ? null : new Page(nextHref));
6768
}
6869

6970
@Override

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,10 @@ public static boolean isY2ubeURL(@Nonnull final URL url) {
235235
*/
236236
public static int parseDurationString(@Nonnull final String input)
237237
throws ParsingException, NumberFormatException {
238+
if (!input.matches(".*\\d.*") && !input.equalsIgnoreCase("SHORTS")) {
239+
throw new ParsingException("Error duration string contains no digits: " + input);
240+
}
241+
238242
// If time separator : is not detected, try . instead
239243
final String[] splitInput = input.contains(":")
240244
? input.split(":")

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemExtractor.java‎

Lines changed: 29 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import java.time.format.DateTimeFormatter;
5050
import java.util.List;
5151
import java.util.regex.Pattern;
52+
import java.util.stream.Collectors;
5253

5354
public class YoutubeStreamInfoItemExtractor implements StreamInfoItemExtractor {
5455

@@ -155,19 +156,24 @@ public long getDuration() throws ParsingException {
155156
duration = videoInfo.getString("lengthSeconds");
156157

157158
if (isNullOrEmpty(duration)) {
158-
final JsonObject timeOverlay = videoInfo.getArray("thumbnailOverlays")
159+
final List<String> timeOverlays = videoInfo.getArray("thumbnailOverlays")
159160
.stream()
160161
.filter(JsonObject.class::isInstance)
161162
.map(JsonObject.class::cast)
162163
.filter(thumbnailOverlay ->
163164
thumbnailOverlay.has("thumbnailOverlayTimeStatusRenderer"))
164-
.findFirst()
165-
.orElse(null);
166-
167-
if (timeOverlay != null) {
168-
duration = getTextFromObject(
169-
timeOverlay.getObject("thumbnailOverlayTimeStatusRenderer")
170-
.getObject("text"));
165+
.map(thumbnailOverlay -> getTextFromObject(
166+
thumbnailOverlay.getObject("thumbnailOverlayTimeStatusRenderer")
167+
.getObject("text")))
168+
.filter(text -> !isNullOrEmpty(text))
169+
.collect(Collectors.toList());
170+
171+
for (final String timeOverlayText : timeOverlays) {
172+
try {
173+
return YoutubeParsingHelper.parseDurationString(timeOverlayText);
174+
} catch (final ParsingException ex) {
175+
// try next
176+
}
171177
}
172178
}
173179

@@ -452,24 +458,21 @@ public boolean isShortFormContent() throws ParsingException {
452458
}
453459

454460
if (!isShort) {
455-
final JsonObject thumbnailTimeOverlay = videoInfo.getArray("thumbnailOverlays")
456-
.stream()
457-
.filter(JsonObject.class::isInstance)
458-
.map(JsonObject.class::cast)
459-
.filter(thumbnailOverlay -> thumbnailOverlay.has(
460-
"thumbnailOverlayTimeStatusRenderer"))
461-
.map(thumbnailOverlay -> thumbnailOverlay.getObject(
462-
"thumbnailOverlayTimeStatusRenderer"))
463-
.findFirst()
464-
.orElse(null);
465-
466-
if (!isNullOrEmpty(thumbnailTimeOverlay)) {
467-
isShort = thumbnailTimeOverlay.getString("style", "")
468-
.equalsIgnoreCase("SHORTS")
469-
|| thumbnailTimeOverlay.getObject("icon")
470-
.getString("iconType", "")
471-
.toLowerCase()
472-
.contains("shorts");
461+
if (videoInfo.has("thumbnailOverlays")) {
462+
isShort = videoInfo.getArray("thumbnailOverlays")
463+
.stream()
464+
.filter(JsonObject.class::isInstance)
465+
.map(JsonObject.class::cast)
466+
.filter(thumbnailOverlay -> thumbnailOverlay.has(
467+
"thumbnailOverlayTimeStatusRenderer"))
468+
.map(thumbnailOverlay -> thumbnailOverlay.getObject(
469+
"thumbnailOverlayTimeStatusRenderer"))
470+
.anyMatch(timeOverlay -> timeOverlay.getString("style", "")
471+
.equalsIgnoreCase("SHORTS")
472+
|| timeOverlay.getObject("icon")
473+
.getString("iconType", "")
474+
.toLowerCase()
475+
.contains("shorts"));
473476
}
474477
}
475478

‎extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/extractors/YoutubeStreamInfoItemLockupExtractor.java‎

Lines changed: 32 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@
3131
/**
3232
* Note:
3333
* This extractor is currently (2025-07) only used to extract related video streams.<br>
34-
* The following features are currently not implemented because they have never been observed:
34+
* The following features are currently not implemented:
3535
* <ul>
36-
* <li>Shorts</li>
36+
* <li>Shorts: appear in related videos without a duration badge; getDuration() returns -1</li>
3737
* <li>Paid content (Premium, members first or only)</li>
3838
* </ul>
3939
*/
@@ -77,22 +77,22 @@ public StreamType getStreamType() throws ParsingException {
7777
}
7878

7979
private StreamType determineStreamType() throws ParsingException {
80-
if (JsonUtils.getArray(lockupViewModel, "contentImage.thumbnailViewModel.overlays")
81-
.streamAsJsonObjects()
80+
final JsonArray overlays = JsonUtils.getArray(lockupViewModel,
81+
"contentImage.thumbnailViewModel.overlays");
82+
83+
// thumbnailOverlayBadgeViewModel path (legacy/alternate overlay structure)
84+
if (overlays.streamAsJsonObjects()
8285
.flatMap(overlay -> overlay
8386
.getObject("thumbnailOverlayBadgeViewModel")
8487
.getArray("thumbnailBadges")
8588
.streamAsJsonObjects())
8689
.map(thumbnailBadge -> thumbnailBadge.getObject("thumbnailBadgeViewModel"))
87-
.anyMatch(thumbnailBadgeViewModel -> {
88-
if ("THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE".equals(
89-
thumbnailBadgeViewModel.getString("badgeStyle"))) {
90+
.anyMatch(vm -> {
91+
if ("THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE".equals(vm.getString("badgeStyle"))) {
9092
return true;
9193
}
92-
9394
// Fallback: Check if there is a live icon
94-
return thumbnailBadgeViewModel
95-
.getObject("icon")
95+
return vm.getObject("icon")
9696
.getArray("sources")
9797
.streamAsJsonObjects()
9898
.map(source -> source
@@ -103,6 +103,18 @@ private StreamType determineStreamType() throws ParsingException {
103103
return StreamType.LIVE_STREAM;
104104
}
105105

106+
// thumbnailBottomOverlayViewModel path (used in lockup format for both duration and live)
107+
if (overlays.streamAsJsonObjects()
108+
.flatMap(overlay -> overlay
109+
.getObject("thumbnailBottomOverlayViewModel")
110+
.getArray("badges")
111+
.streamAsJsonObjects())
112+
.map(badge -> badge.getObject("thumbnailBadgeViewModel"))
113+
.anyMatch(vm -> "THUMBNAIL_OVERLAY_BADGE_STYLE_LIVE".equals(
114+
vm.getString("badgeStyle")))) {
115+
return StreamType.LIVE_STREAM;
116+
}
117+
106118
return StreamType.VIDEO_STREAM;
107119
}
108120

@@ -155,27 +167,34 @@ public long getDuration() throws ParsingException {
155167
"contentImage.thumbnailViewModel.overlays")
156168
.streamAsJsonObjects()
157169
.flatMap(jsonObject -> jsonObject
158-
.getObject("thumbnailOverlayBadgeViewModel")
159-
.getArray("thumbnailBadges")
170+
.getObject("thumbnailBottomOverlayViewModel")
171+
.getArray("badges")
160172
.streamAsJsonObjects())
161173
.map(jsonObject -> jsonObject
162174
.getObject("thumbnailBadgeViewModel")
163175
.getString("text"))
164176
.collect(Collectors.toList());
165177

166178
if (potentialDurations.isEmpty()) {
167-
throw new ParsingException("Could not get duration: No parsable durations detected");
179+
return -1;
168180
}
169181

170182
ParsingException parsingException = null;
171183
for (final String potentialDuration : potentialDurations) {
184+
if (potentialDuration == null || !potentialDuration.matches(".*\\d.*")) {
185+
continue;
186+
}
172187
try {
173188
return YoutubeParsingHelper.parseDurationString(potentialDuration);
174189
} catch (final ParsingException ex) {
175190
parsingException = ex;
176191
}
177192
}
178193

194+
if (parsingException == null) {
195+
return -1; // e.g. only "SHORTS" or "CC" badge was present, no duration available
196+
}
197+
179198
throw new ParsingException("Could not get duration", parsingException);
180199
}
181200

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudCommentsExtractorTest.java‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
package org.schabi.newpipe.extractor.services.soundcloud;
2+
3+
import static org.junit.jupiter.api.Assertions.assertFalse;
4+
import static org.junit.jupiter.api.Assertions.assertTrue;
5+
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
6+
7+
import org.junit.jupiter.api.Nested;
8+
import org.junit.jupiter.api.Test;
9+
import org.schabi.newpipe.extractor.ListExtractor.InfoItemsPage;
10+
import org.schabi.newpipe.extractor.Page;
11+
import org.schabi.newpipe.extractor.comments.CommentsExtractor;
12+
import org.schabi.newpipe.extractor.comments.CommentsInfoItem;
13+
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
14+
import org.schabi.newpipe.extractor.services.DefaultSimpleExtractorTest;
15+
16+
import java.io.IOException;
17+
18+
public class SoundcloudCommentsExtractorTest {
19+
20+
/**
21+
* Regression test for <a href="https://github.com/TeamNewPipe/NewPipeExtractor/issues/1243">
22+
* issue #1243</a>: when the SoundCloud API returns {@code "next_href": null} (no more pages),
23+
* a subsequent call to {@link CommentsExtractor#getPage(Page)} with a null URL must not throw
24+
* {@link IllegalArgumentException} ("Page doesn't contain an URL"). Instead the extractor
25+
* must return {@link InfoItemsPage#emptyPage()}.
26+
*
27+
* <p>The crash manifests during pagination: the last page of comments stores
28+
* {@code new Page(null)} as the next page, and when Paging 3 tries to fetch it the
29+
* exception propagates and kills the app.</p>
30+
*/
31+
@Nested
32+
class TrackWithComments extends DefaultSimpleExtractorTest<CommentsExtractor> {
33+
// This track is known to reproduce issue #1243: it has comments, but when pagination
34+
// exhausts the pages the API returns next_href=null, which previously caused a crash.
35+
private static final String URL = "https://soundcloud.com/user-722618400/a-real-playa";
36+
37+
@Override
38+
protected CommentsExtractor createExtractor() throws Exception {
39+
return SoundCloud.getCommentsExtractor(URL);
40+
}
41+
42+
/**
43+
* The initial page must load successfully without throwing any exception.
44+
*/
45+
@Test
46+
void testGetInitialPageSucceeds() throws IOException, ExtractionException {
47+
final InfoItemsPage<CommentsInfoItem> page = extractor().getInitialPage();
48+
// The track has comments; we only assert the call itself does not throw
49+
// and that the result is a valid (non-null) page.
50+
assertTrue(page.getErrors().isEmpty(),
51+
"Expected no extractor errors on initial page");
52+
}
53+
54+
/**
55+
* Regression test for issue #1243: calling {@link CommentsExtractor#getPage(Page)} with a
56+
* {@link Page} whose URL is null (which is what gets stored when {@code next_href} is
57+
* absent in the API response) must return {@link InfoItemsPage#emptyPage()} rather than
58+
* throw {@link IllegalArgumentException}.
59+
*/
60+
@Test
61+
void testGetPageWithNullUrlReturnsEmptyPage() throws IOException, ExtractionException {
62+
final InfoItemsPage<CommentsInfoItem> page = extractor().getPage(new Page((String) null));
63+
assertTrue(page.getItems().isEmpty(),
64+
"Expected empty items when page URL is null");
65+
assertFalse(page.hasNextPage(),
66+
"Expected no next page when page URL is null");
67+
}
68+
}
69+
70+
/**
71+
* Tests a SoundCloud track that has no comments.
72+
*
73+
* <p>Verifies that the extractor handles an empty collection gracefully:
74+
* the initial page must load without error, return no items, and have no next page.</p>
75+
*/
76+
@Nested
77+
class TrackWithNoComments extends DefaultSimpleExtractorTest<CommentsExtractor> {
78+
private static final String URL = "https://soundcloud.com/user285130010/jdkskls";
79+
80+
@Override
81+
protected CommentsExtractor createExtractor() throws Exception {
82+
return SoundCloud.getCommentsExtractor(URL);
83+
}
84+
85+
/**
86+
* The initial page must load successfully, return an empty items list,
87+
* and report no next page.
88+
*/
89+
@Test
90+
void testGetInitialPageIsEmpty() throws IOException, ExtractionException {
91+
final InfoItemsPage<CommentsInfoItem> page = extractor().getInitialPage();
92+
assertTrue(page.getErrors().isEmpty(),
93+
"Expected no extractor errors on initial page");
94+
assertTrue(page.getItems().isEmpty(),
95+
"Expected no comments for a track with no comments");
96+
assertFalse(page.hasNextPage(),
97+
"Expected no next page for a track with no comments");
98+
}
99+
}
100+
}

‎extractor/src/test/java/org/schabi/newpipe/extractor/services/youtube/YoutubeStreamInfoItemTest.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,58 @@ void lockupViewModelPremiere()
8686
);
8787
}
8888

89+
@Test
90+
void lockupViewModelVideo()
91+
throws FileNotFoundException, JsonParserException {
92+
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
93+
YoutubeStreamInfoItemTest.class, "lockupViewModelVideo") + ".json"));
94+
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
95+
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser);
96+
assertAll(
97+
() -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()),
98+
() -> assertFalse(extractor.isAd()),
99+
() -> assertEquals("https://www.youtube.com/watch?v=dQw4w9WgXcQ", extractor.getUrl()),
100+
() -> assertEquals("VIDEO_TITLE", extractor.getName()),
101+
() -> assertEquals(974, extractor.getDuration()),
102+
() -> assertFalse(extractor.getThumbnails().isEmpty())
103+
);
104+
}
105+
106+
@Test
107+
void lockupViewModelLiveStream()
108+
throws FileNotFoundException, JsonParserException {
109+
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
110+
YoutubeStreamInfoItemTest.class, "lockupViewModelLiveStream") + ".json"));
111+
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
112+
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser);
113+
assertAll(
114+
() -> assertEquals(StreamType.LIVE_STREAM, extractor.getStreamType()),
115+
() -> assertFalse(extractor.isAd()),
116+
() -> assertEquals("https://www.youtube.com/watch?v=LIVE_VIDEO_ID", extractor.getUrl()),
117+
() -> assertEquals("LIVE_VIDEO_TITLE", extractor.getName()),
118+
() -> assertEquals(-1, extractor.getDuration()),
119+
() -> assertNull(extractor.getTextualUploadDate()),
120+
() -> assertNull(extractor.getUploadDate()),
121+
() -> assertEquals(0, extractor.getViewCount()),
122+
() -> assertFalse(extractor.getThumbnails().isEmpty())
123+
);
124+
}
125+
126+
@Test
127+
void lockupViewModelNoDuration()
128+
throws FileNotFoundException, JsonParserException {
129+
final var json = JsonParser.object().from(new FileInputStream(getMockPath(
130+
YoutubeStreamInfoItemTest.class, "lockupViewModelNoDuration") + ".json"));
131+
final var timeAgoParser = TimeAgoPatternsManager.getTimeAgoParserFor(Localization.DEFAULT);
132+
final var extractor = new YoutubeStreamInfoItemLockupExtractor(json, timeAgoParser);
133+
assertAll(
134+
() -> assertEquals(StreamType.VIDEO_STREAM, extractor.getStreamType()),
135+
() -> assertFalse(extractor.isAd()),
136+
() -> assertEquals(-1, extractor.getDuration()),
137+
() -> assertFalse(extractor.getThumbnails().isEmpty())
138+
);
139+
}
140+
89141
@Test
90142
void emptyTitle() throws FileNotFoundException, JsonParserException {
91143
final var json = JsonParser.object().from(new FileInputStream(getMockPath(

0 commit comments

Comments
 (0)