diff --git a/pom.xml b/pom.xml
index a1c7cbd..2250813 100644
--- a/pom.xml
+++ b/pom.xml
@@ -74,7 +74,7 @@
com.aliyun.openservices
aliyun-log
- 0.6.75
+ 0.6.87
diff --git a/src/main/java/com/aliyun/openservices/loghub/client/LogHubClientAdapter.java b/src/main/java/com/aliyun/openservices/loghub/client/LogHubClientAdapter.java
index 3a404a6..c3fe168 100755
--- a/src/main/java/com/aliyun/openservices/loghub/client/LogHubClientAdapter.java
+++ b/src/main/java/com/aliyun/openservices/loghub/client/LogHubClientAdapter.java
@@ -7,9 +7,11 @@
import com.aliyun.openservices.log.common.ConsumerGroupShardCheckPoint;
import com.aliyun.openservices.log.exception.LogException;
import com.aliyun.openservices.log.http.client.ClientConfiguration;
+import com.aliyun.openservices.log.request.PullLogsRequest;
import com.aliyun.openservices.log.response.BatchGetLogResponse;
import com.aliyun.openservices.log.response.ConsumerGroupCheckPointResponse;
import com.aliyun.openservices.log.response.ListConsumerGroupResponse;
+import com.aliyun.openservices.log.response.PullLogsResponse;
import com.aliyun.openservices.loghub.client.config.LogHubConfig;
import com.aliyun.openservices.loghub.client.config.LogHubCursorPosition;
import com.aliyun.openservices.loghub.client.exceptions.LogHubClientWorkerException;
@@ -248,10 +250,17 @@ public String GetCursor(final int shard, final long time) throws LogException {
}
}
- public BatchGetLogResponse BatchGetLogs(final int shard, final int lines, final String cursor) throws LogException {
+ public PullLogsResponse PullLogs(final int shard, final String cursor, LogHubConfig config) throws LogException {
lock.readLock().lock();
try {
- return client.BatchGetLog(project, logstore, shard, lines, cursor);
+ int lines = config.getMaxFetchLogGroupSize();
+ PullLogsRequest pullLogsRequest = new PullLogsRequest(project, logstore, shard, lines, cursor);
+ String query = config.getQuery();
+ if (query != null && !query.isEmpty()) {
+ pullLogsRequest.setQuery(query);
+ pullLogsRequest.setPullMode("scan_on_stream");
+ }
+ return client.pullLogs(pullLogsRequest);
} finally {
lock.readLock().unlock();
}
diff --git a/src/main/java/com/aliyun/openservices/loghub/client/LogHubFetchTask.java b/src/main/java/com/aliyun/openservices/loghub/client/LogHubFetchTask.java
index fdd1b2b..eec4862 100755
--- a/src/main/java/com/aliyun/openservices/loghub/client/LogHubFetchTask.java
+++ b/src/main/java/com/aliyun/openservices/loghub/client/LogHubFetchTask.java
@@ -2,9 +2,8 @@
import com.aliyun.openservices.log.common.LogGroupData;
import com.aliyun.openservices.log.exception.LogException;
-import com.aliyun.openservices.log.response.BatchGetLogResponse;
+import com.aliyun.openservices.log.response.PullLogsResponse;
import com.aliyun.openservices.loghub.client.config.LogHubConfig;
-import com.aliyun.openservices.loghub.client.throttle.ResourceBarrier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -32,17 +31,16 @@ public TaskResult call() {
Exception exception = null;
for (int attempt = 0; ; attempt++) {
try {
- BatchGetLogResponse response = loghubClient.BatchGetLogs(
- shardId, config.getMaxFetchLogGroupSize(), cursor);
- List fetchedData = response.GetLogGroups();
+ PullLogsResponse response = loghubClient.PullLogs(shardId, cursor, config);
+ List fetchedData = response.getLogGroups();
LOG.debug("shard {}, cursor {}, next cursor {}, response size: {}", shardId, cursor,
- response.GetNextCursor(), response.GetCount());
- String nextCursor = response.GetNextCursor();
+ response.getNextCursor(), response.getCount());
+ String nextCursor = response.getNextCursor();
if (nextCursor.isEmpty()) {
LOG.info("Shard {} next cursor is empty, set to current cursor {}", shardId, cursor);
nextCursor = cursor;
}
- return new FetchTaskResult(fetchedData, cursor, nextCursor, response.GetRawSize());
+ return new FetchTaskResult(fetchedData, cursor, nextCursor, response.getRawSize());
} catch (LogException lex) {
if (attempt == 0 && lex.GetErrorCode().toLowerCase().contains("invalidcursor")) {
// If checkpoint is invalid, such as expired cursor, will
diff --git a/src/main/java/com/aliyun/openservices/loghub/client/config/LogHubConfig.java b/src/main/java/com/aliyun/openservices/loghub/client/config/LogHubConfig.java
index 8f00376..c4c2007 100755
--- a/src/main/java/com/aliyun/openservices/loghub/client/config/LogHubConfig.java
+++ b/src/main/java/com/aliyun/openservices/loghub/client/config/LogHubConfig.java
@@ -23,6 +23,7 @@ public enum ConsumePosition {
private String endpoint;
private String project;
private String logstore;
+ private String query;
private String accessId;
private String accessKey;
private LogHubCursorPosition initialPosition;
@@ -74,6 +75,19 @@ public LogHubConfig(String consumerGroup,
this.initialPosition = convertPosition(position);
}
+ public LogHubConfig(String consumerGroup,
+ String consumer,
+ String endpoint,
+ String project,
+ String logstore,
+ String accessId,
+ String accessKey,
+ ConsumePosition position,
+ String query) {
+ this(consumerGroup, consumer, endpoint, project, logstore, accessId, accessKey, position);
+ this.setQuery(query);
+ }
+
private static LogHubCursorPosition convertPosition(ConsumePosition position) {
switch (position) {
case BEGIN_CURSOR:
@@ -329,6 +343,14 @@ public void setMaxInProgressingDataSizeInMB(int maxInProgressingDataSizeInMB) {
this.maxInProgressingDataSizeInMB = maxInProgressingDataSizeInMB;
}
+ public String getQuery() {
+ return query;
+ }
+
+ public void setQuery(String query) {
+ this.query = query;
+ }
+
@Override
public String toString() {
return "LogHubConfig{" +
@@ -337,6 +359,7 @@ public String toString() {
", endpoint='" + endpoint + '\'' +
", project='" + project + '\'' +
", logstore='" + logstore + '\'' +
+ ", query ='" + query + '\'' +
", accessId='" + accessId + '\'' +
", initialPosition=" + initialPosition +
", startTimestamp=" + startTimestamp +