001/**
002 * Copyright (C) 2006-2025 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.runtime.input;
017
018import static java.util.Collections.emptySet;
019import static java.util.Optional.ofNullable;
020
021import java.io.Serializable;
022import java.util.Set;
023import java.util.concurrent.TimeUnit;
024import java.util.function.Supplier;
025
026import org.talend.sdk.component.api.service.configuration.LocalConfiguration;
027import org.talend.sdk.component.runtime.serialization.ContainerFinder;
028
029import lombok.AllArgsConstructor;
030import lombok.Data;
031import lombok.NoArgsConstructor;
032import lombok.extern.slf4j.Slf4j;
033
034@Slf4j
035public class Streaming {
036
037    public static final int MAX_DURATION_TIME_MS_GRACE_PERIOD = 3_000;
038
039    private static Supplier<LocalConfiguration> defaultLocalConfiguration = () -> new LocalConfiguration() {
040
041        @Override
042        public String get(final String key) {
043            return null;
044        }
045
046        @Override
047        public Set<String> keys() {
048            return emptySet();
049        }
050    };
051
052    public static RetryConfiguration loadRetryConfiguration(final String plugin) {
053        // note: this configuratoin could be read on the mapper too and distributed
054        final LocalConfiguration configuration = ofNullable(ContainerFinder.Instance.get().find(plugin))
055                .map(it -> it.findService(LocalConfiguration.class))
056                .orElseGet(defaultLocalConfiguration);
057        final int maxRetries = ofNullable(configuration.get("talend.input.streaming.retry.maxRetries"))
058                .map(Integer::parseInt)
059                .orElse(Integer.MAX_VALUE);
060        return new RetryConfiguration(maxRetries, getStrategy(configuration));
061    }
062
063    public static RetryStrategy getStrategy(final LocalConfiguration configuration) {
064        switch (ofNullable(configuration.get("talend.input.streaming.retry.strategy")).orElse("constant")) {
065            case "exponential":
066                return new RetryConfiguration.ExponentialBackoff(
067                        ofNullable(configuration.get("talend.input.streaming.retry.exponential.exponent"))
068                                .map(Double::parseDouble)
069                                .orElse(1.5),
070                        ofNullable(configuration.get("talend.input.streaming.retry.exponential.randomizationFactor"))
071                                .map(Double::parseDouble)
072                                .orElse(.5),
073                        ofNullable(configuration.get("talend.input.streaming.retry.exponential.maxDuration"))
074                                .map(Long::parseLong)
075                                .orElse(TimeUnit.MINUTES.toMillis(5)),
076                        ofNullable(configuration.get("talend.input.streaming.retry.exponential.initialBackOff"))
077                                .map(Long::parseLong)
078                                .orElse(TimeUnit.SECONDS.toMillis(1)),
079                        0);
080            case "constant":
081            default:
082                return new RetryConfiguration.Constant(
083                        ofNullable(configuration.get("talend.input.streaming.retry.constant.timeout"))
084                                .map(Long::parseLong)
085                                .orElse(500L));
086        }
087    }
088
089    public static StopStrategy loadStopStrategy(final String plugin,
090            final java.util.Map<String, String> internalConfiguration) {
091        final LocalConfiguration configuration = ofNullable(ContainerFinder.Instance.get().find(plugin))
092                .map(it -> it.findService(LocalConfiguration.class))
093                .orElseGet(defaultLocalConfiguration);
094        final Long maxReadRecords = ofNullable(internalConfiguration.entrySet()
095                .stream()
096                .filter(e -> e.getKey().startsWith("$maxRecords") || e.getKey().contains(".$maxRecords"))
097                .findFirst()
098                .map(e -> e.getValue())
099                .map(Long::parseLong)).get()
100                .orElseGet(() -> ofNullable(System.getProperty(
101                        String.format("%s.talend.input.streaming.maxRecords", plugin)))
102                        .map(Long::parseLong)
103                        .orElseGet(() -> ofNullable(
104                                configuration.get("talend.input.streaming.maxRecords"))
105                                .map(Long::parseLong)
106                                .orElseGet(() -> null)));
107        Long maxActiveTime = ofNullable(internalConfiguration.entrySet()
108                .stream()
109                .filter(e -> e.getKey().startsWith("$maxDurationMs") || e.getKey().contains(".$maxDurationMs"))
110                .findFirst()
111                .map(e -> e.getValue())
112                .map(Long::parseLong)).get()
113                .orElseGet(() -> ofNullable(System.getProperty(
114                        String.format("%s.talend.input.streaming.maxDurationMs", plugin)))
115                        .map(Long::parseLong)
116                        .orElseGet(() -> ofNullable(
117                                configuration.get("talend.input.streaming.maxDurationMs"))
118                                .map(Long::parseLong)
119                                .orElseGet(() -> null)));
120        log.debug("[loadStopStrategy] Records: {}; Duration: {}.", maxReadRecords, maxActiveTime);
121        return new StopConfiguration(maxReadRecords, maxActiveTime, null);
122    }
123
124    public interface RetryStrategy {
125
126        long nextPauseDuration();
127
128        void reset();
129    }
130
131    @Data
132    @NoArgsConstructor
133    @AllArgsConstructor
134    public static class RetryConfiguration implements Serializable {
135
136        private int maxRetries;
137
138        private RetryStrategy strategy;
139
140        @Data
141        @NoArgsConstructor
142        @AllArgsConstructor
143        public static class Constant implements Serializable, RetryStrategy {
144
145            private long timeout;
146
147            @Override
148            public long nextPauseDuration() {
149                return timeout;
150            }
151
152            @Override
153            public void reset() {
154                // no-op
155            }
156        }
157
158        @Data
159        @NoArgsConstructor
160        @AllArgsConstructor
161        public static class ExponentialBackoff implements Serializable, RetryStrategy {
162
163            private double exponent;
164
165            private double randomizationFactor;
166
167            private long max;
168
169            private long initialBackOff;
170
171            // state
172            private int iteration;
173
174            @Override
175            public long nextPauseDuration() {
176                final double currentIntervalMillis = Math.min(initialBackOff * Math.pow(exponent, iteration), max);
177                final double randomOffset = (Math.random() * 2 - 1) * randomizationFactor * currentIntervalMillis;
178                final long nextBackoffMillis = Math.min(max, Math.round(currentIntervalMillis + randomOffset));
179                iteration += 1;
180                return nextBackoffMillis;
181            }
182
183            @Override
184            public void reset() {
185                iteration = 0;
186            }
187        }
188    }
189
190    public interface StopStrategy {
191
192        /**
193         * Check if the stop strategy is active according specified conditions.
194         * 
195         * @return true if strategy is active, false otherwise.
196         */
197        boolean isActive();
198
199        /**
200         * Check the stop strategy conditions.
201         * 
202         * @param read - already read records.
203         * @return true if the lifecycle should be stopped, false otherwise.
204         */
205        boolean shouldStop(long read);
206
207        /**
208         * Maximum records to read.
209         * 
210         * @return max number of records to read.
211         */
212        long getMaxReadRecords();
213
214        /**
215         * Maximum duration the lifecycle should run in ms.
216         * 
217         * @return max activity duration.
218         */
219        long getMaxActiveTime();
220
221        /**
222         * The system time in millis when the lifecycle started.
223         * 
224         * @return started time.
225         */
226        long getStartedAtTime();
227
228    }
229
230    @Data
231    public static class StopConfiguration implements StopStrategy, Serializable {
232
233        private long maxReadRecords;
234
235        private long maxActiveTime;
236
237        private long startedAtTime;
238
239        public StopConfiguration() {
240            maxReadRecords = -1L;
241            maxActiveTime = -1L;
242            startedAtTime = System.currentTimeMillis();
243        }
244
245        public StopConfiguration(final Long maxRecords, final Long maxTime, final Long start) {
246            maxReadRecords = maxRecords == null ? -1L : maxRecords;
247            maxActiveTime = maxTime == null ? -1L : maxTime;
248            startedAtTime = start == null ? System.currentTimeMillis() : start;
249        }
250
251        @Override
252        public boolean isActive() {
253            return (maxReadRecords > -1) || (maxActiveTime > -1);
254        }
255
256        private boolean hasEnoughRecords(final long read) {
257            return maxReadRecords != -1 && read >= maxReadRecords;
258        }
259
260        private boolean isTimePassed() {
261            return maxActiveTime != -1 && System.currentTimeMillis() - startedAtTime >= maxActiveTime;
262        }
263
264        @Override
265        public boolean shouldStop(final long readRecords) {
266            return hasEnoughRecords(readRecords) || isTimePassed();
267        }
268    }
269
270}