Skip to content

Commit 94bfbb0

Browse files
CloudExtremeCloudExtreme
authored
[Fix-18168] Fix OBS listStorageEntity not returning subdirectories (#18170)
1 parent 53a84b9 commit 94bfbb0

3 files changed

Lines changed: 223 additions & 4 deletions

File tree

‎dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/pom.xml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,30 @@
5353
<groupId>com.huaweicloud</groupId>
5454
<artifactId>esdk-obs-java-bundle</artifactId>
5555
</dependency>
56+
57+
<dependency>
58+
<groupId>org.mockito</groupId>
59+
<artifactId>mockito-core</artifactId>
60+
<scope>test</scope>
61+
</dependency>
62+
63+
<dependency>
64+
<groupId>org.mockito</groupId>
65+
<artifactId>mockito-inline</artifactId>
66+
<scope>test</scope>
67+
</dependency>
68+
69+
<dependency>
70+
<groupId>org.mockito</groupId>
71+
<artifactId>mockito-junit-jupiter</artifactId>
72+
<scope>test</scope>
73+
</dependency>
74+
75+
<dependency>
76+
<groupId>org.assertj</groupId>
77+
<artifactId>assertj-core</artifactId>
78+
<scope>test</scope>
79+
</dependency>
5680
</dependencies>
5781

5882
<build>

‎dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/main/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperator.java‎

Lines changed: 27 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -183,19 +183,27 @@ public List<String> fetchFileContent(String filePath, int skipLineNums, int limi
183183

184184
@Override
185185
public List<StorageEntity> listStorageEntity(String resourceAbsolutePath) {
186-
resourceAbsolutePath = transformObsKeyToAbsolutePath(resourceAbsolutePath);
186+
String obsResourceAbsolutePath = transformAbsolutePathToObsKey(resourceAbsolutePath);
187187

188188
ListObjectsRequest request = new ListObjectsRequest();
189189
request.setBucketName(bucketName);
190-
request.setPrefix(resourceAbsolutePath);
190+
request.setPrefix(obsResourceAbsolutePath);
191191
request.setDelimiter("/");
192192

193193
ObjectListing result = obsClient.listObjects(request);
194194

195-
return result.getObjects()
195+
List<StorageEntity> storageEntities = new ArrayList<>();
196+
storageEntities.addAll(result.getCommonPrefixes()
197+
.stream()
198+
.map(this::transformCommonPrefixToStorageEntity)
199+
.collect(Collectors.toList()));
200+
storageEntities.addAll(result.getObjects()
196201
.stream()
202+
.filter(object -> !object.getObjectKey().equals(obsResourceAbsolutePath))
197203
.map(this::transformObsObjectToStorageEntity)
198-
.collect(Collectors.toList());
204+
.collect(Collectors.toList()));
205+
206+
return storageEntities;
199207
}
200208

201209
@Override
@@ -266,6 +274,21 @@ protected StorageEntity transformObsObjectToStorageEntity(ObsObject object) {
266274
.build();
267275
}
268276

277+
private StorageEntity transformCommonPrefixToStorageEntity(String commonPrefix) {
278+
String absolutePath = transformObsKeyToAbsolutePath(commonPrefix);
279+
ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath);
280+
281+
StorageEntity entity = new StorageEntity();
282+
entity.setFileName(new File(absolutePath).getName());
283+
entity.setFullName(absolutePath);
284+
entity.setDirectory(resourceMetaData.isDirectory());
285+
entity.setType(resourceMetaData.getResourceType());
286+
entity.setSize(0L);
287+
entity.setCreateTime(null);
288+
entity.setUpdateTime(null);
289+
return entity;
290+
}
291+
269292
private String transformAbsolutePathToObsKey(String absolutePath) {
270293
ResourceMetadata resourceMetaData = getResourceMetaData(absolutePath);
271294
if (resourceMetaData.isDirectory()) {

‎dolphinscheduler-storage-plugin/dolphinscheduler-storage-obs/src/test/java/org/apache/dolphinscheduler/plugin/storage/obs/ObsStorageOperatorTest.java‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,172 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.storage.obs;
19+
20+
import static org.assertj.core.api.Assertions.assertThat;
21+
import static org.mockito.ArgumentMatchers.any;
22+
import static org.mockito.Mockito.mock;
23+
import static org.mockito.Mockito.when;
24+
25+
import org.apache.dolphinscheduler.plugin.storage.api.StorageEntity;
26+
import org.apache.dolphinscheduler.spi.enums.ResourceType;
27+
28+
import java.util.Arrays;
29+
import java.util.Collections;
30+
import java.util.Date;
31+
import java.util.List;
32+
33+
import org.junit.jupiter.api.BeforeEach;
34+
import org.junit.jupiter.api.Test;
35+
import org.junit.jupiter.api.extension.ExtendWith;
36+
import org.mockito.MockedConstruction;
37+
import org.mockito.Mockito;
38+
import org.mockito.junit.jupiter.MockitoExtension;
39+
40+
import com.obs.services.ObsClient;
41+
import com.obs.services.model.ListObjectsRequest;
42+
import com.obs.services.model.ObjectListing;
43+
import com.obs.services.model.ObjectMetadata;
44+
import com.obs.services.model.ObsObject;
45+
46+
@ExtendWith(MockitoExtension.class)
47+
public class ObsStorageOperatorTest {
48+
49+
private ObsClient mockObsClient;
50+
private ObsStorageOperator obsStorageOperator;
51+
52+
@BeforeEach
53+
public void setUp() {
54+
try (
55+
MockedConstruction<ObsClient> construction = Mockito.mockConstruction(ObsClient.class,
56+
(mock, context) -> {
57+
when(mock.headBucket("test-bucket")).thenReturn(true);
58+
})) {
59+
ObsStorageProperties properties = ObsStorageProperties.builder()
60+
.accessKeyId("testAccessKey")
61+
.accessKeySecret("testSecretKey")
62+
.bucketName("test-bucket")
63+
.endPoint("https://obs.cn-north-4.myhuaweicloud.com")
64+
.resourceUploadPath("tmp/dolphinscheduler")
65+
.build();
66+
obsStorageOperator = new ObsStorageOperator(properties);
67+
mockObsClient = construction.constructed().get(0);
68+
}
69+
}
70+
71+
private ObjectListing mockObjectListing(List<ObsObject> objects, List<String> commonPrefixes) {
72+
ObjectListing objectListing = mock(ObjectListing.class);
73+
when(objectListing.getObjects()).thenReturn(objects);
74+
when(objectListing.getCommonPrefixes()).thenReturn(commonPrefixes);
75+
return objectListing;
76+
}
77+
78+
private ObsObject createObsObject(String key, long size) {
79+
ObsObject obsObject = new ObsObject();
80+
obsObject.setObjectKey(key);
81+
ObjectMetadata metadata = new ObjectMetadata();
82+
metadata.setContentLength(size);
83+
metadata.setLastModified(new Date());
84+
obsObject.setMetadata(metadata);
85+
return obsObject;
86+
}
87+
88+
@Test
89+
public void testListStorageEntity_withFilesAndDirectories() {
90+
ObsObject fileObject = createObsObject("tmp/dolphinscheduler/default/resources/demo.sql", 1024L);
91+
ObsObject dirSelfObject = createObsObject("tmp/dolphinscheduler/default/resources/", 0L);
92+
93+
ObjectListing objectListing = mockObjectListing(
94+
Arrays.asList(dirSelfObject, fileObject),
95+
Arrays.asList("tmp/dolphinscheduler/default/resources/subDir1/",
96+
"tmp/dolphinscheduler/default/resources/subDir2/"));
97+
98+
when(mockObsClient.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
99+
100+
List<StorageEntity> result =
101+
obsStorageOperator.listStorageEntity("tmp/dolphinscheduler/default/resources");
102+
103+
assertThat(result).hasSize(3);
104+
105+
StorageEntity dir1 = result.get(0);
106+
assertThat(dir1.getFileName()).isEqualTo("subDir1");
107+
assertThat(dir1.isDirectory()).isTrue();
108+
assertThat(dir1.getSize()).isEqualTo(0L);
109+
assertThat(dir1.getCreateTime()).isNull();
110+
111+
StorageEntity dir2 = result.get(1);
112+
assertThat(dir2.getFileName()).isEqualTo("subDir2");
113+
assertThat(dir2.isDirectory()).isTrue();
114+
115+
StorageEntity file = result.get(2);
116+
assertThat(file.getFileName()).isEqualTo("demo.sql");
117+
assertThat(file.isDirectory()).isFalse();
118+
assertThat(file.getSize()).isEqualTo(1024L);
119+
assertThat(file.getType()).isEqualTo(ResourceType.FILE);
120+
}
121+
122+
@Test
123+
public void testListStorageEntity_onlyFiles() {
124+
ObsObject fileObject = createObsObject("tmp/dolphinscheduler/default/resources/test.sql", 512L);
125+
126+
ObjectListing objectListing = mockObjectListing(
127+
Collections.singletonList(fileObject),
128+
Collections.emptyList());
129+
130+
when(mockObsClient.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
131+
132+
List<StorageEntity> result =
133+
obsStorageOperator.listStorageEntity("tmp/dolphinscheduler/default/resources");
134+
135+
assertThat(result).hasSize(1);
136+
assertThat(result.get(0).getFileName()).isEqualTo("test.sql");
137+
assertThat(result.get(0).isDirectory()).isFalse();
138+
}
139+
140+
@Test
141+
public void testListStorageEntity_onlyDirectories() {
142+
ObsObject dirSelf = createObsObject("tmp/dolphinscheduler/default/resources/", 0L);
143+
144+
ObjectListing objectListing = mockObjectListing(
145+
Collections.singletonList(dirSelf),
146+
Collections.singletonList("tmp/dolphinscheduler/default/resources/dir1/"));
147+
148+
when(mockObsClient.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
149+
150+
List<StorageEntity> result =
151+
obsStorageOperator.listStorageEntity("tmp/dolphinscheduler/default/resources");
152+
153+
assertThat(result).hasSize(1);
154+
assertThat(result.get(0).getFileName()).isEqualTo("dir1");
155+
assertThat(result.get(0).isDirectory()).isTrue();
156+
assertThat(result.get(0).getSize()).isEqualTo(0L);
157+
}
158+
159+
@Test
160+
public void testListStorageEntity_empty() {
161+
ObjectListing objectListing = mockObjectListing(
162+
Collections.emptyList(),
163+
Collections.emptyList());
164+
165+
when(mockObsClient.listObjects(any(ListObjectsRequest.class))).thenReturn(objectListing);
166+
167+
List<StorageEntity> result =
168+
obsStorageOperator.listStorageEntity("tmp/dolphinscheduler/default/resources");
169+
170+
assertThat(result).isEmpty();
171+
}
172+
}

0 commit comments

Comments
 (0)