Source code

001/**
002 * Copyright (C) 2006-2020 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.server.service;
017
018import static java.util.Collections.emptyMap;
019import static java.util.Collections.emptySet;
020import static java.util.Collections.unmodifiableSet;
021import static java.util.Locale.ROOT;
022import static java.util.Optional.ofNullable;
023import static java.util.function.Function.identity;
024import static java.util.stream.Collectors.joining;
025import static java.util.stream.Collectors.toList;
026import static java.util.stream.Collectors.toMap;
027import static lombok.AccessLevel.PACKAGE;
028
029import java.io.BufferedInputStream;
030import java.io.BufferedOutputStream;
031import java.io.BufferedReader;
032import java.io.ByteArrayInputStream;
033import java.io.ByteArrayOutputStream;
034import java.io.IOException;
035import java.io.InputStream;
036import java.io.Reader;
037import java.io.StringReader;
038import java.nio.file.Files;
039import java.nio.file.Path;
040import java.nio.file.StandardCopyOption;
041import java.nio.file.StandardOpenOption;
042import java.util.Collection;
043import java.util.HashMap;
044import java.util.Map;
045import java.util.Properties;
046import java.util.Set;
047import java.util.concurrent.ConcurrentHashMap;
048import java.util.function.Supplier;
049import java.util.jar.Attributes;
050import java.util.jar.JarEntry;
051import java.util.jar.JarOutputStream;
052import java.util.jar.Manifest;
053import java.util.stream.Stream;
054
055import javax.annotation.PostConstruct;
056import javax.enterprise.context.ApplicationScoped;
057import javax.enterprise.inject.spi.CDI;
058import javax.inject.Inject;
059
060import org.talend.sdk.component.api.service.configuration.LocalConfiguration;
061import org.talend.sdk.component.classloader.ConfigurableClassLoader;
062import org.talend.sdk.component.container.Container;
063import org.talend.sdk.component.dependencies.maven.Artifact;
064import org.talend.sdk.component.path.PathFactory;
065import org.talend.sdk.component.runtime.manager.ComponentManager;
066import org.talend.sdk.component.server.configuration.ComponentServerConfiguration;
067
068import lombok.Getter;
069import lombok.RequiredArgsConstructor;
070import lombok.extern.slf4j.Slf4j;
071
072@Slf4j
073@ApplicationScoped
074public class VirtualDependenciesService {
075
076    @Getter(PACKAGE)
077    private final String virtualGroupId = "virtual.talend.component.server.generated.";
078
079    @Getter(PACKAGE)
080    private final String configurationArtifactIdPrefix = "user-local-configuration-";
081
082    private final Enrichment noCustomization = new Enrichment(false, null, null, null);
083
084    @Inject
085    private ComponentServerConfiguration configuration;
086
087    private final Map<String, Enrichment> enrichmentsPerContainer = new HashMap<>();
088
089    @Getter
090    private final Map<Artifact, Path> artifactMapping = new ConcurrentHashMap<>();
091
092    private final Map<Artifact, Supplier<InputStream>> configurationArtifactMapping = new ConcurrentHashMap<>();
093
094    private Path provisioningM2Base;
095
096    @PostConstruct
097    private void init() {
098        final String m2 = configuration.getUserExtensionsAutoM2Provisioning();
099        switch (m2) {
100        case "skip":
101            provisioningM2Base = null;
102            break;
103        case "auto":
104            provisioningM2Base = findStudioM2();
105            break;
106        default:
107            provisioningM2Base = PathFactory.get(m2);
108        }
109        log.debug("m2 provisioning base: {}", provisioningM2Base);
110    }
111
112    public void onDeploy(final String pluginId) {
113        if (!configuration.getUserExtensions().isPresent()) {
114            enrichmentsPerContainer.put(pluginId, noCustomization);
115            return;
116        }
117        final Path extensions = PathFactory
118                .get(configuration.getUserExtensions().orElseThrow(IllegalArgumentException::new))
119                .resolve(pluginId);
120        if (!Files.exists(extensions)) {
121            log.debug("'{}' does not exist so no extension will be added to family '{}'", extensions, pluginId);
122            enrichmentsPerContainer.put(pluginId, noCustomization);
123            return;
124        }
125
126        final Path userConfig = extensions.resolve("user-configuration.properties");
127        final Map<Artifact, Path> userJars = findJars(extensions, pluginId);
128        final Properties userConfiguration = loadUserConfiguration(pluginId, userConfig, userJars);
129        if (userConfiguration.isEmpty() && userJars.isEmpty()) {
130            log.debug("No customization for container '{}'", pluginId);
131            enrichmentsPerContainer.put(pluginId, noCustomization);
132            return;
133        }
134
135        final Map<String, String> customConfigAsMap = userConfiguration
136                .stringPropertyNames()
137                .stream()
138                .collect(toMap(identity(), userConfiguration::getProperty));
139        log
140                .debug("Set up customization for container '{}' (has-configuration={}, jars={})", pluginId,
141                        !userConfiguration.isEmpty(), userJars);
142
143        if (userConfiguration.isEmpty()) {
144            enrichmentsPerContainer.put(pluginId, new Enrichment(true, customConfigAsMap, null, userJars.keySet()));
145        } else {
146            final byte[] localConfigurationJar = generateConfigurationJar(pluginId, userConfiguration);
147            final Artifact configurationArtifact;
148            try {
149                configurationArtifact = new Artifact(groupIdFor(pluginId), configurationArtifactIdPrefix + pluginId,
150                        "jar", "", Long.toString(Files.getLastModifiedTime(userConfig).toMillis()), "compile");
151            } catch (final IOException e) {
152                throw new IllegalStateException(e);
153            }
154            doProvision(configurationArtifact, () -> new ByteArrayInputStream(localConfigurationJar));
155            enrichmentsPerContainer
156                    .put(pluginId, new Enrichment(true, customConfigAsMap, configurationArtifact, userJars.keySet()));
157            configurationArtifactMapping
158                    .put(configurationArtifact, () -> new ByteArrayInputStream(localConfigurationJar));
159        }
160        userJars.forEach((artifact, file) -> doProvision(artifact, () -> {
161            try {
162                return Files.newInputStream(file, StandardOpenOption.READ);
163            } catch (final IOException e) {
164                throw new IllegalStateException(e);
165            }
166        }));
167
168        artifactMapping.putAll(userJars);
169    }
170
171    public void onUnDeploy(final Container plugin) {
172        final Enrichment enrichment = enrichmentsPerContainer.remove(plugin.getId());
173        if (enrichment == null || enrichment == noCustomization) {
174            return;
175        }
176
177        if (enrichment.userArtifacts != null) {
178            enrichment.userArtifacts.forEach(artifactMapping::remove);
179            enrichment.userArtifacts.clear();
180        }
181        if (enrichment.configurationArtifact != null) {
182            configurationArtifactMapping.remove(enrichment.configurationArtifact);
183        }
184    }
185
186    public boolean isVirtual(final String gav) {
187        return gav.startsWith(virtualGroupId);
188    }
189
190    public Enrichment getEnrichmentFor(final String pluginId) {
191        return enrichmentsPerContainer.get(pluginId);
192    }
193
194    public Stream<Artifact> userArtifactsFor(final String pluginId) {
195        final Enrichment enrichment = enrichmentsPerContainer.get(pluginId);
196        if (enrichment == null || !enrichment.customized) {
197            return Stream.empty();
198        }
199        final Stream<Artifact> userJars = enrichment.userArtifacts.stream();
200        if (enrichment.configurationArtifact != null) {
201            // config is added but will be ignored cause not physically here
202            // however it ensures our rest service returns right data
203            return Stream.concat(userJars, Stream.of(enrichment.configurationArtifact));
204        }
205        return userJars;
206    }
207
208    public Supplier<InputStream> retrieveArtifact(final Artifact artifact) {
209        return ofNullable(artifactMapping.get(artifact)).map(it -> (Supplier<InputStream>) () -> {
210            try {
211                return Files.newInputStream(it);
212            } catch (final IOException e) {
213                throw new IllegalStateException(e);
214            }
215        }).orElseGet(() -> configurationArtifactMapping.get(artifact));
216    }
217
218    public String groupIdFor(final String family) {
219        return virtualGroupId + sanitizedForGav(family);
220    }
221
222    private byte[] generateConfigurationJar(final String family, final Properties userConfiguration) {
223        final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
224        final Manifest manifest = new Manifest();
225        final Attributes mainAttributes = manifest.getMainAttributes();
226        mainAttributes.put(Attributes.Name.MANIFEST_VERSION, "1.0");
227        mainAttributes.putValue("Created-By", "Talend Component Kit Server");
228        mainAttributes.putValue("Talend-Time", Long.toString(System.currentTimeMillis()));
229        mainAttributes.putValue("Talend-Family-Name", family);
230        try (final JarOutputStream jar = new JarOutputStream(new BufferedOutputStream(outputStream), manifest)) {
231            jar.putNextEntry(new JarEntry("TALEND-INF/local-configuration.properties"));
232            userConfiguration.store(jar, "Configuration of the family " + family);
233            jar.closeEntry();
234        } catch (final IOException e) {
235            throw new IllegalStateException(e);
236        }
237        return outputStream.toByteArray();
238    }
239
240    private Properties loadUserConfiguration(final String plugin, final Path userConfig,
241            final Map<Artifact, Path> userJars) {
242        final Properties properties = new Properties();
243        if (!Files.exists(userConfig)) {
244            return properties;
245        }
246        final String content;
247        try (final BufferedReader stream = Files.newBufferedReader(userConfig)) {
248            content = stream.lines().collect(joining("\n"));
249        } catch (final IOException e) {
250            throw new IllegalStateException(e);
251        }
252        try (final Reader reader = new StringReader(replaceByGav(plugin, content, userJars))) {
253            properties.load(reader);
254        } catch (final IOException e) {
255            throw new IllegalStateException(e);
256        }
257        return properties;
258    }
259
260    // handle "userJar(name)" function in the config, normally not needed since jars are in the context already
261    // note: if we make it more complex, switch to a real parser or StrSubstitutor
262    String replaceByGav(final String plugin, final String content, final Map<Artifact, Path> userJars) {
263        final StringBuilder output = new StringBuilder();
264        final String prefixFn = "userJar(";
265        int fnIdx = content.indexOf(prefixFn);
266        int previousEnd = 0;
267        if (fnIdx < 0) {
268            output.append(content);
269        } else {
270            while (fnIdx >= 0) {
271                final int end = content.indexOf(')', fnIdx);
272                output.append(content, previousEnd, fnIdx);
273                output.append(toGav(plugin, content.substring(fnIdx + prefixFn.length(), end), userJars));
274                fnIdx = content.indexOf(prefixFn, end);
275                if (fnIdx < 0) {
276                    if (end < content.length() - 1) {
277                        output.append(content, end + 1, content.length());
278                    }
279                } else {
280                    previousEnd = end + 1;
281                }
282            }
283        }
284        return output.toString();
285    }
286
287    private String toGav(final String plugin, final String jarNameWithoutExtension,
288            final Map<Artifact, Path> userJars) {
289        return groupIdFor(plugin) + ':' + jarNameWithoutExtension + ":jar:"
290                + userJars
291                        .keySet()
292                        .stream()
293                        .filter(it -> it.getArtifact().equals(jarNameWithoutExtension))
294                        .findFirst()
295                        .map(Artifact::getVersion)
296                        .orElse("unknown");
297    }
298
299    private Map<Artifact, Path> findJars(final Path familyFolder, final String family) {
300        if (!Files.isDirectory(familyFolder)) {
301            return emptyMap();
302        }
303        try {
304            return Files
305                    .list(familyFolder)
306                    .filter(file -> file.getFileName().toString().endsWith(".jar"))
307                    .collect(toMap(it -> {
308                        try {
309                            return new Artifact(groupIdFor(family), toArtifact(it), "jar", "",
310                                    Long.toString(Files.getLastModifiedTime(it).toMillis()), "compile");
311                        } catch (final IOException e) {
312                            throw new IllegalStateException(e);
313                        }
314                    }, identity()));
315        } catch (final IOException e) {
316            throw new IllegalStateException(e);
317        }
318    }
319
320    private String toArtifact(final Path file) {
321        final String name = file.getFileName().toString();
322        return name.substring(0, name.length() - ".jar".length());
323    }
324
325    private String sanitizedForGav(final String name) {
326        return name.replace(' ', '_').toLowerCase(ROOT);
327    }
328
329    // note: we don't want to provision based on our real m2, only studio one for now
330    private Path findStudioM2() {
331        if (System.getProperty("talend.studio.version") != null && System.getProperty("osgi.bundles") != null) {
332            final Path localM2 = PathFactory.get(System.getProperty("talend.component.server.maven.repository", ""));
333            if (Files.isDirectory(localM2)) {
334                return localM2;
335            }
336        }
337        return null;
338    }
339
340    private void doProvision(final Artifact artifact, final Supplier<InputStream> newInputStream) {
341        if (provisioningM2Base == null) {
342            log.debug("No m2 to provision, skipping {}", artifact);
343            return;
344        }
345        final Path target = provisioningM2Base.resolve(artifact.toPath());
346        if (target.toFile().exists()) {
347            log.debug("{} already exists, skipping", target);
348            return;
349        }
350        final Path parentFile = target.getParent();
351        if (!Files.exists(parentFile)) {
352            try {
353                Files.createDirectories(parentFile);
354            } catch (final IOException e) {
355                throw new IllegalArgumentException("Can't create " + parentFile, e);
356            }
357        }
358        try (final InputStream stream = new BufferedInputStream(newInputStream.get())) {
359            Files.copy(stream, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES);
360        } catch (final IOException e) {
361            throw new IllegalStateException(e);
362        }
363    }
364
365    @RequiredArgsConstructor
366    private static class Enrichment {
367
368        private final boolean customized;
369
370        private final Map<String, String> customConfiguration;
371
372        private final Artifact configurationArtifact;
373
374        private final Collection<Artifact> userArtifacts;
375    }
376
377    public static class LocalConfigurationImpl implements LocalConfiguration {
378
379        private final VirtualDependenciesService delegate;
380
381        public LocalConfigurationImpl() {
382            delegate = CDI.current().select(VirtualDependenciesService.class).get();
383        }
384
385        @Override
386        public String get(final String key) {
387            if (key == null || !key.contains(".")) {
388                return null;
389            }
390            final String plugin = key.substring(0, key.indexOf('.'));
391            final Enrichment enrichment = delegate.getEnrichmentFor(plugin);
392            if (enrichment == null || enrichment.customConfiguration == null) {
393                return null;
394            }
395            return enrichment.customConfiguration.get(key.substring(plugin.length() + 1));
396        }
397
398        @Override
399        public Set<String> keys() {
400            final ClassLoader loader = Thread.currentThread().getContextClassLoader();
401            if (!ConfigurableClassLoader.class.isInstance(loader)) {
402                return emptySet();
403            }
404            final String id = ConfigurableClassLoader.class.cast(loader).getId();
405            final Enrichment enrichment = delegate.getEnrichmentFor(id);
406            if (enrichment == null || enrichment.customConfiguration == null) {
407                return emptySet();
408            }
409            return unmodifiableSet(enrichment.customConfiguration.keySet());
410        }
411    }
412
413    public static class UserContainerClasspathContributor implements ComponentManager.ContainerClasspathContributor {
414
415        private final VirtualDependenciesService delegate;
416
417        public UserContainerClasspathContributor() {
418            delegate = CDI.current().select(VirtualDependenciesService.class).get();
419        }
420
421        @Override
422        public Collection<Artifact> findContributions(final String pluginId) {
423            delegate.onDeploy(pluginId);
424            return delegate.userArtifactsFor(pluginId).collect(toList());
425        }
426
427        @Override
428        public boolean canResolve(final String path) {
429            return delegate.isVirtual(path.replace('/', '.'));
430        }
431
432        @Override
433        public Path resolve(final String path) {
434            if (path.contains('/' + delegate.getConfigurationArtifactIdPrefix())) {
435                return null; // not needed, will be enriched on the fly, see LocalConfigurationImpl
436            }
437            final String[] segments = path.split("/");
438            if (segments.length < 9) {
439                return null;
440            }
441            // ex: virtual.talend.component.server.generated.<plugin id>:<artifact>:jar:dynamic
442            final String group = delegate
443                    .groupIdFor(Stream.of(segments).skip(5).limit(segments.length - 5 - 3).collect(joining(".")));
444            final String artifact = segments[segments.length - 3];
445            return delegate
446                    .getArtifactMapping()
447                    .entrySet()
448                    .stream()
449                    .filter(it -> it.getKey().getGroup().equals(group) && it.getKey().getArtifact().equals(artifact))
450                    .findFirst()
451                    .map(Map.Entry::getValue)
452                    .orElse(null);
453        }
454    }
455}