Skip to content

Commit 4d7f927

Browse files
ruanwenjunruanwenjun
authored
[Improvement-17361][TaskPlugin] Use logger marker to exclude the system log in task instance log content. (#17378)
1 parent b9b6d51 commit 4d7f927

14 files changed

Lines changed: 94 additions & 74 deletions

File tree

‎dolphinscheduler-master/src/test/resources/logback.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@
4343
<file>${taskInstanceLogFullPath}</file>
4444
<encoder>
4545
<pattern>
46-
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} - %message%n
46+
%date{yyyy-MM-dd HH:mm:ss.SSS} %-5level - %message%n
4747
</pattern>
4848
<charset>UTF-8</charset>
4949
</encoder>

‎dolphinscheduler-standalone-server/src/main/resources/logback-spring.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@
6464
<file>${taskInstanceLogFullPath}</file>
6565
<encoder>
6666
<pattern>
67-
%date{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%thread] - %message%n
67+
%date{yyyy-MM-dd HH:mm:ss.SSS} %-5level - %message%n
6868
</pattern>
6969
<charset>UTF-8</charset>
7070
</encoder>

‎dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/AbstractTaskExecutor.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.dolphinscheduler.common.utils.JSONUtils;
2222
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
2323
import org.apache.dolphinscheduler.plugin.task.api.log.TaskInstanceLogHeader;
24+
import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogMarkers;
2425
import org.apache.dolphinscheduler.task.executor.eventbus.TaskExecutorEventBus;
2526
import org.apache.dolphinscheduler.task.executor.events.TaskExecutorStartedLifecycleEvent;
2627
import org.apache.dolphinscheduler.task.executor.exceptions.TaskExecutorRuntimeException;
@@ -125,9 +126,9 @@ protected void transitTaskExecutorState(final TaskExecutorState taskExecutorStat
125126
protected abstract void doTriggerTaskPlugin();
126127

127128
protected void initializeTaskContext() {
128-
log.info("Begin to initialize taskContext");
129129
taskExecutionContext.setStartTime(System.currentTimeMillis());
130-
log.info("End initialize taskContext {}", JSONUtils.toPrettyJsonString(taskExecutionContext));
130+
log.info(TaskLogMarkers.excludeInTaskLog(), "Initialized taskContext {}",
131+
JSONUtils.toPrettyJsonString(taskExecutionContext));
131132
}
132133

133134
private void publishTaskRunningEvent() {

‎dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/eventbus/TaskExecutorEventBus.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
import org.apache.dolphinscheduler.common.utils.JSONUtils;
2121
import org.apache.dolphinscheduler.eventbus.AbstractDelayEventBus;
22+
import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogMarkers;
2223
import org.apache.dolphinscheduler.task.executor.events.AbstractTaskExecutorLifecycleEvent;
2324

2425
import lombok.extern.slf4j.Slf4j;
@@ -28,7 +29,8 @@ public class TaskExecutorEventBus extends AbstractDelayEventBus<AbstractTaskExec
2829

2930
public void publish(final AbstractTaskExecutorLifecycleEvent event) {
3031
super.publish(event);
31-
log.info("Publish {}: {}", event.getClass().getSimpleName(), JSONUtils.toPrettyJsonString(event));
32+
log.info(TaskLogMarkers.excludeInTaskLog(), "Publish {}: {}", event.getClass().getSimpleName(),
33+
JSONUtils.toPrettyJsonString(event));
3234
}
3335

3436
}

‎dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/eventbus/TaskExecutorEventBusCoordinator.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
2323
import org.apache.dolphinscheduler.common.utils.JSONUtils;
24+
import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogMarkers;
2425
import org.apache.dolphinscheduler.task.executor.ITaskExecutor;
2526
import org.apache.dolphinscheduler.task.executor.ITaskExecutorRepository;
2627
import org.apache.dolphinscheduler.task.executor.events.AbstractTaskExecutorLifecycleEvent;
@@ -187,9 +188,9 @@ private void doFireTaskExecutorEventBus(final ITaskExecutor taskExecutor) {
187188
"Unsupported TaskExecutorLifecycleEvent: " + taskExecutorLifecycleEvent);
188189
}
189190
}
190-
log.info("Success fire {}: {} ",
191+
log.info(TaskLogMarkers.excludeInTaskLog(), "Success fire {}: {} ",
191192
taskExecutorLifecycleEvent.getClass().getSimpleName(),
192-
JSONUtils.toPrettyJsonString(taskExecutorLifecycleEvent));
193+
JSONUtils.toJsonString(taskExecutorLifecycleEvent));
193194
} catch (Exception e) {
194195
log.error("Fire TaskExecutorLifecycleEvent: {} error", taskExecutorLifecycleEvent, e);
195196
}

‎dolphinscheduler-task-executor/src/main/java/org/apache/dolphinscheduler/task/executor/eventbus/TaskExecutorLifecycleEventRemoteReporter.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.dolphinscheduler.common.exception.BaseException;
2121
import org.apache.dolphinscheduler.common.thread.BaseDaemonThread;
2222
import org.apache.dolphinscheduler.common.utils.JSONUtils;
23+
import org.apache.dolphinscheduler.plugin.task.api.log.TaskLogMarkers;
2324
import org.apache.dolphinscheduler.task.executor.ITaskExecutor;
2425
import org.apache.dolphinscheduler.task.executor.ITaskExecutorRepository;
2526
import org.apache.dolphinscheduler.task.executor.events.IReportableTaskExecutorLifecycleEvent;
@@ -107,7 +108,8 @@ public void run() {
107108
public void reportTaskExecutorLifecycleEvent(final IReportableTaskExecutorLifecycleEvent reportableTaskExecutorLifecycleEvent) {
108109
eventChannelsLock.lock();
109110
try {
110-
log.debug("Report : {}", JSONUtils.toPrettyJsonString(reportableTaskExecutorLifecycleEvent));
111+
log.info(TaskLogMarkers.excludeInTaskLog(), "Report : {}",
112+
JSONUtils.toPrettyJsonString(reportableTaskExecutorLifecycleEvent));
111113
int taskInstanceId = reportableTaskExecutorLifecycleEvent.getTaskInstanceId();
112114
eventChannels.computeIfAbsent(
113115
taskInstanceId,

‎dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/TaskInstanceLogHeader.java‎

Lines changed: 6 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -22,45 +22,17 @@
2222
@Slf4j
2323
public class TaskInstanceLogHeader {
2424

25-
private static final String INITIALIZE_TASK_CONTEXT_HEADER = new StringBuilder()
26-
.append("\n")
27-
.append("************************************************************************************************")
28-
.append("\n")
29-
.append("********************************* Initialize task context ************************************")
30-
.append("\n")
31-
.append("************************************************************************************************")
32-
.toString();
33-
private static final String LOAD_TASK_INSTANCE_PLUGIN_HEADER = new StringBuilder()
34-
.append("\n")
35-
.append("***********************************************************************************************")
36-
.append("\n")
37-
.append("********************************* Load task instance plugin *********************************")
38-
.append("\n")
39-
.append("***********************************************************************************************")
40-
.toString();
25+
public static final String DOLPHIN_EMOJI = "\uD83D\uDC2C";
26+
27+
private static final String INITIALIZE_TASK_CONTEXT_HEADER = DOLPHIN_EMOJI + " Initialize Task Context";
28+
private static final String LOAD_TASK_INSTANCE_PLUGIN_HEADER = DOLPHIN_EMOJI + " Load Task Instance Plugin";
29+
private static final String EXECUTE_TASK_HEADER = DOLPHIN_EMOJI + " Execute Task Instance";
30+
private static final String FINALIZE_TASK_HEADER = DOLPHIN_EMOJI + " Finalize Task Instance";
4131

4232
public static void printInitializeTaskContextHeader() {
4333
log.info(INITIALIZE_TASK_CONTEXT_HEADER);
4434
}
4535

46-
private static final String EXECUTE_TASK_HEADER = new StringBuilder()
47-
.append("\n")
48-
.append("************************************************************************************************")
49-
.append("\n")
50-
.append("********************************* Execute task instance *************************************")
51-
.append("\n")
52-
.append("***********************************************************************************************")
53-
.toString();
54-
55-
private static final String FINALIZE_TASK_HEADER = new StringBuilder()
56-
.append("\n")
57-
.append("************************************************************************************************")
58-
.append("\n")
59-
.append("********************************* Finalize task instance ************************************")
60-
.append("\n")
61-
.append("***********************************************************************************************")
62-
.toString();
63-
6436
public static void printLoadTaskInstancePluginHeader() {
6537
log.info(LOAD_TASK_INSTANCE_PLUGIN_HEADER);
6638
}

‎dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/TaskLogDiscriminator.java‎

Lines changed: 4 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919

2020
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
2121

22+
import lombok.Getter;
23+
import lombok.Setter;
2224
import lombok.extern.slf4j.Slf4j;
2325

2426
import org.slf4j.MDC;
@@ -30,6 +32,8 @@
3032
* Task Log Discriminator
3133
*/
3234
@Slf4j
35+
@Getter
36+
@Setter
3337
public class TaskLogDiscriminator extends AbstractDiscriminator<ILoggingEvent> {
3438

3539
private String key;
@@ -44,26 +48,4 @@ public String getDiscriminatingValue(ILoggingEvent event) {
4448
}
4549
return taskInstanceLogPath;
4650
}
47-
48-
@Override
49-
public void start() {
50-
started = true;
51-
}
52-
53-
@Override
54-
public String getKey() {
55-
return key;
56-
}
57-
58-
public void setKey(String key) {
59-
this.key = key;
60-
}
61-
62-
public String getLogBase() {
63-
return logBase;
64-
}
65-
66-
public void setLogBase(String logBase) {
67-
this.logBase = logBase;
68-
}
6951
}

‎dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/TaskLogFilter.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,12 @@
1919

2020
import org.apache.dolphinscheduler.plugin.task.api.utils.LogUtils;
2121

22+
import org.apache.commons.lang3.StringUtils;
23+
2224
import lombok.extern.slf4j.Slf4j;
2325

2426
import org.slf4j.MDC;
27+
import org.slf4j.Marker;
2528

2629
import ch.qos.logback.classic.spi.ILoggingEvent;
2730
import ch.qos.logback.core.filter.Filter;
@@ -35,6 +38,24 @@ public class TaskLogFilter extends Filter<ILoggingEvent> {
3538

3639
@Override
3740
public FilterReply decide(ILoggingEvent event) {
38-
return MDC.get(LogUtils.TASK_INSTANCE_LOG_FULL_PATH_MDC_KEY) == null ? FilterReply.DENY : FilterReply.ACCEPT;
41+
String taskInstanceLogPath = MDC.get(LogUtils.TASK_INSTANCE_LOG_FULL_PATH_MDC_KEY);
42+
// If the taskInstanceLogPath is empty, it means that the log is not related to a task instance.
43+
if (StringUtils.isEmpty(taskInstanceLogPath)) {
44+
return FilterReply.DENY;
45+
}
46+
47+
// todo: Only when the master is includeInTaskLog, the log should be included in the task instance log, we can
48+
// remove `excludeInTaskLog` after change the log in plugin
49+
final Marker marker = event.getMarker();
50+
if (marker == null) {
51+
return FilterReply.ACCEPT;
52+
}
53+
if (marker.contains(TaskLogMarkers.includeInTaskLog())) {
54+
return FilterReply.ACCEPT;
55+
}
56+
if (marker.contains(TaskLogMarkers.excludeInTaskLog())) {
57+
return FilterReply.DENY;
58+
}
59+
return FilterReply.ACCEPT;
3960
}
4061
}

‎dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/log/TaskLogMarkers.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.dolphinscheduler.plugin.task.api.log;
19+
20+
import org.slf4j.Marker;
21+
import org.slf4j.MarkerFactory;
22+
23+
public class TaskLogMarkers {
24+
25+
private static final Marker TASK_LOGGER_EXCLUDE_MARKER = MarkerFactory.getMarker("TASK_LOGGER_EXCLUDE");
26+
27+
private static final Marker TASK_LOGGER_INCLUDE_MARKER = MarkerFactory.getMarker("TASK_LOGGER_INCLUDE");
28+
29+
/**
30+
* The marker used to exclude logs from the task instance log file.
31+
*/
32+
public static Marker excludeInTaskLog() {
33+
return TASK_LOGGER_EXCLUDE_MARKER;
34+
}
35+
36+
public static Marker includeInTaskLog() {
37+
return TASK_LOGGER_INCLUDE_MARKER;
38+
39+
}
40+
}

0 commit comments

Comments
 (0)