Source code

001/**
002 * Copyright (C) 2006-2025 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.runtime.input;
017
018import static java.lang.Thread.sleep;
019import static java.util.concurrent.TimeUnit.MILLISECONDS;
020import static org.talend.sdk.component.runtime.input.Streaming.RetryStrategy;
021
022import java.io.IOException;
023import java.io.InvalidObjectException;
024import java.io.ObjectStreamException;
025import java.io.Serializable;
026import java.lang.annotation.Annotation;
027import java.lang.reflect.Method;
028import java.lang.reflect.Parameter;
029import java.util.concurrent.ExecutorService;
030import java.util.concurrent.Executors;
031import java.util.concurrent.Future;
032import java.util.concurrent.Semaphore;
033import java.util.concurrent.TimeoutException;
034import java.util.concurrent.atomic.AtomicBoolean;
035
036import javax.annotation.PostConstruct;
037
038import org.talend.sdk.component.api.configuration.Option;
039import org.talend.sdk.component.runtime.input.Streaming.RetryConfiguration;
040import org.talend.sdk.component.runtime.input.Streaming.StopStrategy;
041
042import lombok.extern.slf4j.Slf4j;
043
044@Slf4j
045public class StreamingInputImpl extends InputImpl {
046
047    private RetryConfiguration retryConfiguration;
048
049    private transient Thread shutdownHook;
050
051    private final AtomicBoolean running = new AtomicBoolean();
052
053    private transient Semaphore semaphore;
054
055    private StopStrategy stopStrategy;
056
057    private transient long readRecords = 0L;
058
059    public StreamingInputImpl(final String rootName, final String name, final String plugin,
060            final Serializable instance, final RetryConfiguration retryConfiguration, final StopStrategy stopStrategy) {
061        super(rootName, name, plugin, instance);
062        shutdownHook = new Thread(() -> running.compareAndSet(true, false),
063                getClass().getName() + "_" + rootName() + "-" + name() + "_" + hashCode());
064        this.retryConfiguration = retryConfiguration;
065        this.stopStrategy = stopStrategy;
066        log.debug("[StreamingInputImpl] Created with retryStrategy: {}, stopStrategy: {}.", this.retryConfiguration,
067                this.stopStrategy);
068    }
069
070    protected StreamingInputImpl() {
071        // no-op
072    }
073
074    @Override
075    protected Object readNext() {
076        if (!running.get()) {
077            return null;
078        }
079        if (stopStrategy.isActive() && stopStrategy.shouldStop(readRecords)) {
080            log.debug("[readNext] stopStrategy condition validated.");
081            return null;
082        }
083        try {
084            semaphore.acquire();
085        } catch (final InterruptedException e) {
086            Thread.currentThread().interrupt();
087            return null;
088        }
089        try {
090            final RetryStrategy strategy = retryConfiguration.getStrategy();
091            int retries = retryConfiguration.getMaxRetries();
092            while (running.get() && retries > 0) {
093                Object next = null;
094                if (stopStrategy.isActive() && stopStrategy.getMaxActiveTime() > -1) {
095                    // Some connectors do not block input and return null (rabbitmq for instance). Thus, the future
096                    // timeout is never reached and retryStrategy is run then. So, need to check timeout in the loop.
097                    if (stopStrategy.shouldStop(readRecords)) {
098                        log.debug("[readNext] shouldStop now! Duration {}ms",
099                                System.currentTimeMillis() - stopStrategy.getStartedAtTime());
100                        return null;
101                    }
102                    final ExecutorService executor = Executors.newSingleThreadExecutor();
103                    final Future<Object> reader = executor.submit(super::readNext);
104                    // manage job latency...
105                    // timeout + hardcoded grace period
106                    final long maxActiveTimeWithGracePeriod =
107                            stopStrategy.getMaxActiveTime() + Streaming.MAX_DURATION_TIME_MS_GRACE_PERIOD;
108                    final long estimatedTimeout = maxActiveTimeWithGracePeriod
109                            - (System.currentTimeMillis() - stopStrategy.getStartedAtTime());
110                    final long timeout = estimatedTimeout < -1 ? 10 : estimatedTimeout;
111                    log.debug(
112                            "[readNext] Applying duration strategy for reading record: will interrupt in {}ms (estimated:{} ms Duration:{}ms).",
113                            timeout, estimatedTimeout, maxActiveTimeWithGracePeriod);
114                    try {
115                        next = reader.get(timeout, MILLISECONDS);
116                    } catch (TimeoutException e) {
117                        log.debug("[readNext] Read record: timeout received.");
118                        reader.cancel(true);
119                        return next;
120                    } catch (Exception e) {
121                        // nop
122                    } finally {
123                        executor.shutdownNow();
124                    }
125                } else {
126                    next = super.readNext();
127                }
128                if (next != null) {
129                    strategy.reset();
130                    readRecords++;
131                    return next;
132                }
133
134                retries--;
135                try {
136                    final long millis = strategy.nextPauseDuration();
137                    if (millis < 0) { // assume it means "give up"
138                        prepareStop();
139                    } else if (millis > 0) { // we can wait 1s but not minutes to quit
140                        if (millis < 1000) {
141                            sleep(millis);
142                        } else {
143                            long remaining = millis;
144                            while (running.get() && remaining > 0) {
145                                final long current = Math.min(remaining, 250);
146                                remaining -= current;
147                                sleep(current);
148                            }
149                        }
150                    } // else if millis == 0 no need to call any method
151                } catch (final InterruptedException e) {
152                    prepareStop(); // stop the stream
153                }
154            }
155            return null;
156        } finally {
157            semaphore.release();
158        }
159    }
160
161    @Override
162    protected void init() {
163        super.init();
164        semaphore = new Semaphore(1);
165    }
166
167    @Override
168    protected Object[] evaluateParameters(final Class<? extends Annotation> marker, final Method method) {
169        if (marker != PostConstruct.class) {
170            return super.evaluateParameters(marker, method);
171        }
172
173        final Object[] args = new Object[method.getParameters().length];
174        for (int i = 0; i < method.getParameters().length; i++) {
175            final Parameter parameter = method.getParameters()[i];
176
177            final Option annotation = parameter.getAnnotation(Option.class);
178            if (annotation == null) {
179                args[i] = null;
180            } else if (Option.MAX_DURATION_PARAMETER.equals(annotation.value())) {
181                if (parameter.getType() == Integer.class || parameter.getType() == int.class) {
182                    args[i] = (int) stopStrategy.getMaxActiveTime();
183                } else if (parameter.getType() == Long.class || parameter.getType() == long.class) {
184                    args[i] = stopStrategy.getMaxActiveTime();
185                } else {
186                    log.warn("The parameter {} is marked as timeout but type is not long.", parameter.getName());
187                    args[i] = null;
188                }
189            } else if (Option.MAX_RECORDS_PARAMETER.equals(annotation.value())) {
190                if (parameter.getType() == Integer.class || parameter.getType() == int.class) {
191                    args[i] = (int) stopStrategy.getMaxReadRecords();
192                } else if (parameter.getType() == Long.class || parameter.getType() == long.class) {
193                    args[i] = stopStrategy.getMaxReadRecords();
194                } else {
195                    log.warn("The parameter {} is marked as max records limitation but type is not long.",
196                            parameter.getName());
197                    args[i] = null;
198                }
199
200            }
201        }
202        return args;
203    }
204
205    @Override
206    public void start() {
207        super.start();
208        running.compareAndSet(false, true);
209        Runtime.getRuntime().addShutdownHook(shutdownHook);
210    }
211
212    @Override
213    public void stop() {
214        prepareStop();
215        super.stop();
216    }
217
218    private void prepareStop() {
219        running.compareAndSet(true, false);
220        if (shutdownHook != null) {
221            try {
222                Runtime.getRuntime().removeShutdownHook(shutdownHook);
223            } catch (final IllegalStateException itse) {
224                // ok to ignore
225            }
226        }
227        try {
228            semaphore.acquire();
229        } catch (final InterruptedException e) {
230            Thread.currentThread().interrupt();
231        }
232    }
233
234    @Override
235    protected Object writeReplace() throws ObjectStreamException {
236        return new StreamSerializationReplacer(plugin(), rootName(), name(), serializeDelegate(), retryConfiguration,
237                stopStrategy);
238    }
239
240    private static class StreamSerializationReplacer extends SerializationReplacer {
241
242        private final RetryConfiguration retryConfiguration;
243
244        private final StopStrategy stopStrategy;
245
246        StreamSerializationReplacer(final String plugin, final String component, final String name, final byte[] value,
247                final RetryConfiguration retryConfiguration, final StopStrategy stopStrategy) {
248            super(plugin, component, name, value);
249            this.retryConfiguration = retryConfiguration;
250            this.stopStrategy = stopStrategy;
251        }
252
253        protected Object readResolve() throws ObjectStreamException {
254            try {
255                return new StreamingInputImpl(component, name, plugin, loadDelegate(), retryConfiguration,
256                        stopStrategy);
257            } catch (final IOException | ClassNotFoundException e) {
258                final InvalidObjectException invalidObjectException = new InvalidObjectException(e.getMessage());
259                invalidObjectException.initCause(e);
260                throw invalidObjectException;
261            }
262        }
263    }
264}