001/** 002 * Copyright (C) 2006-2025 Talend Inc. - www.talend.com 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016package org.talend.sdk.component.runtime.input; 017 018import static java.util.Collections.emptySet; 019import static java.util.Optional.ofNullable; 020 021import java.io.Serializable; 022import java.util.Set; 023import java.util.concurrent.TimeUnit; 024import java.util.function.Supplier; 025 026import org.talend.sdk.component.api.service.configuration.LocalConfiguration; 027import org.talend.sdk.component.runtime.serialization.ContainerFinder; 028 029import lombok.AllArgsConstructor; 030import lombok.Data; 031import lombok.NoArgsConstructor; 032import lombok.extern.slf4j.Slf4j; 033 034@Slf4j 035public class Streaming { 036 037 public static final int MAX_DURATION_TIME_MS_GRACE_PERIOD = 3_000; 038 039 private static Supplier<LocalConfiguration> defaultLocalConfiguration = () -> new LocalConfiguration() { 040 041 @Override 042 public String get(final String key) { 043 return null; 044 } 045 046 @Override 047 public Set<String> keys() { 048 return emptySet(); 049 } 050 }; 051 052 public static RetryConfiguration loadRetryConfiguration(final String plugin) { 053 // note: this configuratoin could be read on the mapper too and distributed 054 final LocalConfiguration configuration = ofNullable(ContainerFinder.Instance.get().find(plugin)) 055 .map(it -> it.findService(LocalConfiguration.class)) 056 .orElseGet(defaultLocalConfiguration); 057 final int maxRetries = ofNullable(configuration.get("talend.input.streaming.retry.maxRetries")) 058 .map(Integer::parseInt) 059 .orElse(Integer.MAX_VALUE); 060 return new RetryConfiguration(maxRetries, getStrategy(configuration)); 061 } 062 063 public static RetryStrategy getStrategy(final LocalConfiguration configuration) { 064 switch (ofNullable(configuration.get("talend.input.streaming.retry.strategy")).orElse("constant")) { 065 case "exponential": 066 return new RetryConfiguration.ExponentialBackoff( 067 ofNullable(configuration.get("talend.input.streaming.retry.exponential.exponent")) 068 .map(Double::parseDouble) 069 .orElse(1.5), 070 ofNullable(configuration.get("talend.input.streaming.retry.exponential.randomizationFactor")) 071 .map(Double::parseDouble) 072 .orElse(.5), 073 ofNullable(configuration.get("talend.input.streaming.retry.exponential.maxDuration")) 074 .map(Long::parseLong) 075 .orElse(TimeUnit.MINUTES.toMillis(5)), 076 ofNullable(configuration.get("talend.input.streaming.retry.exponential.initialBackOff")) 077 .map(Long::parseLong) 078 .orElse(TimeUnit.SECONDS.toMillis(1)), 079 0); 080 case "constant": 081 default: 082 return new RetryConfiguration.Constant( 083 ofNullable(configuration.get("talend.input.streaming.retry.constant.timeout")) 084 .map(Long::parseLong) 085 .orElse(500L)); 086 } 087 } 088 089 public static StopStrategy loadStopStrategy(final String plugin, 090 final java.util.Map<String, String> internalConfiguration) { 091 final LocalConfiguration configuration = ofNullable(ContainerFinder.Instance.get().find(plugin)) 092 .map(it -> it.findService(LocalConfiguration.class)) 093 .orElseGet(defaultLocalConfiguration); 094 final Long maxReadRecords = ofNullable(internalConfiguration.entrySet() 095 .stream() 096 .filter(e -> e.getKey().startsWith("$maxRecords") || e.getKey().contains(".$maxRecords")) 097 .findFirst() 098 .map(e -> e.getValue()) 099 .map(Long::parseLong)).get() 100 .orElseGet(() -> ofNullable(System.getProperty( 101 String.format("%s.talend.input.streaming.maxRecords", plugin))) 102 .map(Long::parseLong) 103 .orElseGet(() -> ofNullable( 104 configuration.get("talend.input.streaming.maxRecords")) 105 .map(Long::parseLong) 106 .orElseGet(() -> null))); 107 Long maxActiveTime = ofNullable(internalConfiguration.entrySet() 108 .stream() 109 .filter(e -> e.getKey().startsWith("$maxDurationMs") || e.getKey().contains(".$maxDurationMs")) 110 .findFirst() 111 .map(e -> e.getValue()) 112 .map(Long::parseLong)).get() 113 .orElseGet(() -> ofNullable(System.getProperty( 114 String.format("%s.talend.input.streaming.maxDurationMs", plugin))) 115 .map(Long::parseLong) 116 .orElseGet(() -> ofNullable( 117 configuration.get("talend.input.streaming.maxDurationMs")) 118 .map(Long::parseLong) 119 .orElseGet(() -> null))); 120 log.debug("[loadStopStrategy] Records: {}; Duration: {}.", maxReadRecords, maxActiveTime); 121 return new StopConfiguration(maxReadRecords, maxActiveTime, null); 122 } 123 124 public interface RetryStrategy { 125 126 long nextPauseDuration(); 127 128 void reset(); 129 } 130 131 @Data 132 @NoArgsConstructor 133 @AllArgsConstructor 134 public static class RetryConfiguration implements Serializable { 135 136 private int maxRetries; 137 138 private RetryStrategy strategy; 139 140 @Data 141 @NoArgsConstructor 142 @AllArgsConstructor 143 public static class Constant implements Serializable, RetryStrategy { 144 145 private long timeout; 146 147 @Override 148 public long nextPauseDuration() { 149 return timeout; 150 } 151 152 @Override 153 public void reset() { 154 // no-op 155 } 156 } 157 158 @Data 159 @NoArgsConstructor 160 @AllArgsConstructor 161 public static class ExponentialBackoff implements Serializable, RetryStrategy { 162 163 private double exponent; 164 165 private double randomizationFactor; 166 167 private long max; 168 169 private long initialBackOff; 170 171 // state 172 private int iteration; 173 174 @Override 175 public long nextPauseDuration() { 176 final double currentIntervalMillis = Math.min(initialBackOff * Math.pow(exponent, iteration), max); 177 final double randomOffset = (Math.random() * 2 - 1) * randomizationFactor * currentIntervalMillis; 178 final long nextBackoffMillis = Math.min(max, Math.round(currentIntervalMillis + randomOffset)); 179 iteration += 1; 180 return nextBackoffMillis; 181 } 182 183 @Override 184 public void reset() { 185 iteration = 0; 186 } 187 } 188 } 189 190 public interface StopStrategy { 191 192 /** 193 * Check if the stop strategy is active according specified conditions. 194 * 195 * @return true if strategy is active, false otherwise. 196 */ 197 boolean isActive(); 198 199 /** 200 * Check the stop strategy conditions. 201 * 202 * @param read - already read records. 203 * @return true if the lifecycle should be stopped, false otherwise. 204 */ 205 boolean shouldStop(long read); 206 207 /** 208 * Maximum records to read. 209 * 210 * @return max number of records to read. 211 */ 212 long getMaxReadRecords(); 213 214 /** 215 * Maximum duration the lifecycle should run in ms. 216 * 217 * @return max activity duration. 218 */ 219 long getMaxActiveTime(); 220 221 /** 222 * The system time in millis when the lifecycle started. 223 * 224 * @return started time. 225 */ 226 long getStartedAtTime(); 227 228 } 229 230 @Data 231 public static class StopConfiguration implements StopStrategy, Serializable { 232 233 private long maxReadRecords; 234 235 private long maxActiveTime; 236 237 private long startedAtTime; 238 239 public StopConfiguration() { 240 maxReadRecords = -1L; 241 maxActiveTime = -1L; 242 startedAtTime = System.currentTimeMillis(); 243 } 244 245 public StopConfiguration(final Long maxRecords, final Long maxTime, final Long start) { 246 maxReadRecords = maxRecords == null ? -1L : maxRecords; 247 maxActiveTime = maxTime == null ? -1L : maxTime; 248 startedAtTime = start == null ? System.currentTimeMillis() : start; 249 } 250 251 @Override 252 public boolean isActive() { 253 return (maxReadRecords > -1) || (maxActiveTime > -1); 254 } 255 256 private boolean hasEnoughRecords(final long read) { 257 return maxReadRecords != -1 && read >= maxReadRecords; 258 } 259 260 private boolean isTimePassed() { 261 return maxActiveTime != -1 && System.currentTimeMillis() - startedAtTime >= maxActiveTime; 262 } 263 264 @Override 265 public boolean shouldStop(final long readRecords) { 266 return hasEnoughRecords(readRecords) || isTimePassed(); 267 } 268 } 269 270}