Source code

001/**
002 * Copyright (C) 2006-2020 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.server.front;
017
018import static java.util.Collections.emptyList;
019import static java.util.Collections.emptyMap;
020import static java.util.Collections.singletonList;
021import static java.util.Collections.singletonMap;
022import static java.util.Optional.ofNullable;
023import static java.util.stream.Collectors.toList;
024import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE;
025import static org.talend.sdk.component.server.front.model.ErrorDictionary.COMPONENT_MISSING;
026import static org.talend.sdk.component.server.front.model.ErrorDictionary.DESIGN_MODEL_MISSING;
027import static org.talend.sdk.component.server.front.model.ErrorDictionary.PLUGIN_MISSING;
028
029import java.io.BufferedInputStream;
030import java.io.IOException;
031import java.io.InputStream;
032import java.nio.file.Files;
033import java.nio.file.Path;
034import java.util.Collections;
035import java.util.HashMap;
036import java.util.Iterator;
037import java.util.List;
038import java.util.Locale;
039import java.util.Map;
040import java.util.Objects;
041import java.util.Optional;
042import java.util.concurrent.ConcurrentHashMap;
043import java.util.concurrent.ConcurrentMap;
044import java.util.function.Function;
045import java.util.function.Predicate;
046import java.util.function.Supplier;
047import java.util.stream.Stream;
048
049import javax.annotation.PostConstruct;
050import javax.enterprise.context.ApplicationScoped;
051import javax.enterprise.event.Observes;
052import javax.inject.Inject;
053import javax.ws.rs.WebApplicationException;
054import javax.ws.rs.core.MediaType;
055import javax.ws.rs.core.Response;
056import javax.ws.rs.core.StreamingOutput;
057
058import org.talend.sdk.component.container.Container;
059import org.talend.sdk.component.dependencies.maven.Artifact;
060import org.talend.sdk.component.design.extension.DesignModel;
061import org.talend.sdk.component.runtime.manager.ComponentFamilyMeta;
062import org.talend.sdk.component.runtime.manager.ComponentManager;
063import org.talend.sdk.component.runtime.manager.ContainerComponentRegistry;
064import org.talend.sdk.component.runtime.manager.extension.ComponentContexts;
065import org.talend.sdk.component.server.api.ComponentResource;
066import org.talend.sdk.component.server.configuration.ComponentServerConfiguration;
067import org.talend.sdk.component.server.dao.ComponentDao;
068import org.talend.sdk.component.server.dao.ComponentFamilyDao;
069import org.talend.sdk.component.server.front.base.internal.RequestKey;
070import org.talend.sdk.component.server.front.model.ComponentDetail;
071import org.talend.sdk.component.server.front.model.ComponentDetailList;
072import org.talend.sdk.component.server.front.model.ComponentId;
073import org.talend.sdk.component.server.front.model.ComponentIndex;
074import org.talend.sdk.component.server.front.model.ComponentIndices;
075import org.talend.sdk.component.server.front.model.Dependencies;
076import org.talend.sdk.component.server.front.model.DependencyDefinition;
077import org.talend.sdk.component.server.front.model.ErrorDictionary;
078import org.talend.sdk.component.server.front.model.Icon;
079import org.talend.sdk.component.server.front.model.Link;
080import org.talend.sdk.component.server.front.model.SimplePropertyDefinition;
081import org.talend.sdk.component.server.front.model.error.ErrorPayload;
082import org.talend.sdk.component.server.lang.MapCache;
083import org.talend.sdk.component.server.service.ActionsService;
084import org.talend.sdk.component.server.service.ComponentManagerService;
085import org.talend.sdk.component.server.service.ExtensionComponentMetadataManager;
086import org.talend.sdk.component.server.service.IconResolver;
087import org.talend.sdk.component.server.service.LocaleMapper;
088import org.talend.sdk.component.server.service.PropertiesService;
089import org.talend.sdk.component.server.service.SimpleQueryLanguageCompiler;
090import org.talend.sdk.component.server.service.VirtualDependenciesService;
091import org.talend.sdk.component.server.service.event.DeployedComponent;
092import org.talend.sdk.component.spi.component.ComponentExtension;
093
094import lombok.extern.slf4j.Slf4j;
095
096@Slf4j
097@ApplicationScoped
098public class ComponentResourceImpl implements ComponentResource {
099
100    private final ConcurrentMap<RequestKey, ComponentIndices> indicesPerRequest = new ConcurrentHashMap<>();
101
102    @Inject
103    private ComponentManager manager;
104
105    @Inject
106    private ComponentManagerService componentManagerService;
107
108    @Inject
109    private ComponentDao componentDao;
110
111    @Inject
112    private ComponentFamilyDao componentFamilyDao;
113
114    @Inject
115    private LocaleMapper localeMapper;
116
117    @Inject
118    private ActionsService actionsService;
119
120    @Inject
121    private PropertiesService propertiesService;
122
123    @Inject
124    private IconResolver iconResolver;
125
126    @Inject
127    private ComponentServerConfiguration configuration;
128
129    @Inject
130    private VirtualDependenciesService virtualDependenciesService;
131
132    @Inject
133    private ExtensionComponentMetadataManager virtualComponents;
134
135    @Inject
136    private MapCache caches;
137
138    @Inject
139    private SimpleQueryLanguageCompiler queryLanguageCompiler;
140
141    private Map<String, Function<ComponentIndex, Object>> componentEvaluators = new HashMap<>();
142
143    @PostConstruct
144    private void setupRuntime() {
145        log.info("Initializing " + getClass());
146
147        // preload some highly used data
148        getIndex("en", false, null);
149
150        componentEvaluators.put("plugin", c -> c.getId().getPlugin());
151        componentEvaluators.put("id", c -> c.getId().getId());
152        componentEvaluators.put("familyId", c -> c.getId().getFamilyId());
153        componentEvaluators.put("name", c -> c.getId().getName());
154        componentEvaluators.put("metadata", component -> {
155            final Iterator<SimplePropertyDefinition> iterator =
156                    getDetail("en", new String[] { component.getId().getId() })
157                            .getDetails()
158                            .iterator()
159                            .next()
160                            .getProperties()
161                            .iterator();
162            if (iterator.hasNext()) {
163                return iterator.next().getMetadata();
164            }
165            return Collections.emptyMap();
166        });
167    }
168
169    public void clearCache(@Observes final DeployedComponent deployedComponent) {
170        indicesPerRequest.clear();
171    }
172
173    @Override
174    public Dependencies getDependencies(final String[] ids) {
175        if (ids.length == 0) {
176            return new Dependencies(emptyMap());
177        }
178        final Map<String, DependencyDefinition> dependencies = new HashMap<>();
179        for (final String id : ids) {
180            if (virtualComponents.isExtensionEntity(id)) {
181                final DependencyDefinition deps = ofNullable(virtualComponents.getDependenciesFor(id))
182                        .orElseGet(() -> new DependencyDefinition(emptyList()));
183                dependencies.put(id, deps);
184            } else {
185                final ComponentFamilyMeta.BaseMeta<Object> meta = componentDao.findById(id);
186                dependencies.put(meta.getId(), getDependenciesFor(meta));
187            }
188        }
189        return new Dependencies(dependencies);
190    }
191
192    @Override
193    public StreamingOutput getDependency(final String id) {
194        final ComponentFamilyMeta.BaseMeta<?> component = componentDao.findById(id);
195        final Supplier<InputStream> streamProvider;
196        if (component != null) { // local dep
197            final Path file = componentManagerService
198                    .manager()
199                    .findPlugin(component.getParent().getPlugin())
200                    .orElseThrow(() -> new WebApplicationException(Response
201                            .status(Response.Status.NOT_FOUND)
202                            .type(APPLICATION_JSON_TYPE)
203                            .entity(new ErrorPayload(PLUGIN_MISSING, "No plugin matching the id: " + id))
204                            .build()))
205                    .getContainerFile()
206                    .orElseThrow(() -> new WebApplicationException(Response
207                            .status(Response.Status.NOT_FOUND)
208                            .type(APPLICATION_JSON_TYPE)
209                            .entity(new ErrorPayload(PLUGIN_MISSING, "No dependency matching the id: " + id))
210                            .build()));
211            if (!Files.exists(file)) {
212                return onMissingJar(id);
213            }
214            streamProvider = () -> {
215                try {
216                    return Files.newInputStream(file);
217                } catch (final IOException e) {
218                    throw new IllegalStateException(e);
219                }
220            };
221        } else { // just try to resolve it locally, note we would need to ensure some security here
222            final Artifact artifact = Artifact.from(id);
223            if (virtualDependenciesService.isVirtual(id)) {
224                streamProvider = virtualDependenciesService.retrieveArtifact(artifact);
225                if (streamProvider == null) {
226                    return onMissingJar(id);
227                }
228            } else {
229                final Path file = componentManagerService.manager().getContainer().resolve(artifact.toPath());
230                if (!Files.exists(file)) {
231                    return onMissingJar(id);
232                }
233                streamProvider = () -> {
234                    try {
235                        return Files.newInputStream(file);
236                    } catch (final IOException e) {
237                        throw new IllegalStateException(e);
238                    }
239                };
240            }
241        }
242        return output -> {
243            final byte[] buffer = new byte[40960]; // 5k
244            try (final InputStream stream = new BufferedInputStream(streamProvider.get(), buffer.length)) {
245                int count;
246                while ((count = stream.read(buffer)) >= 0) {
247                    if (count == 0) {
248                        continue;
249                    }
250                    output.write(buffer, 0, count);
251                }
252            }
253        };
254    }
255
256    @Override
257    public ComponentIndices getIndex(final String language, final boolean includeIconContent, final String query) {
258        final Locale locale = localeMapper.mapLocale(language);
259        caches.evictIfNeeded(indicesPerRequest, configuration.getMaxCacheSize() - 1);
260        return indicesPerRequest.computeIfAbsent(new RequestKey(locale, includeIconContent, query), k -> {
261            final Predicate<ComponentIndex> filter = queryLanguageCompiler.compile(query, componentEvaluators);
262            return new ComponentIndices(Stream
263                    .concat(findDeployedComponents(includeIconContent, locale),
264                            virtualComponents
265                                    .getDetails()
266                                    .stream()
267                                    .map(detail -> new ComponentIndex(detail.getId(), detail.getDisplayName(),
268                                            detail.getId().getFamily(), new Icon(detail.getIcon(), null, null),
269                                            new Icon(virtualComponents.getFamilyIconFor(detail.getId().getFamilyId()),
270                                                    null, null),
271                                            detail.getVersion(), singletonList(detail.getId().getFamily()),
272                                            detail.getLinks())))
273                    .filter(filter)
274                    .collect(toList()));
275        });
276    }
277
278    @Override
279    public Response familyIcon(final String id) {
280        if (virtualComponents.isExtensionEntity(id)) { // todo or just use front bundle?
281            return Response
282                    .status(Response.Status.NOT_FOUND)
283                    .entity(new ErrorPayload(ErrorDictionary.ICON_MISSING, "No icon for family: " + id))
284                    .type(APPLICATION_JSON_TYPE)
285                    .build();
286        }
287
288        // todo: add caching if SvgIconResolver becomes used a lot - not the case ATM
289        final ComponentFamilyMeta meta = componentFamilyDao.findById(id);
290        if (meta == null) {
291            return Response
292                    .status(Response.Status.NOT_FOUND)
293                    .entity(new ErrorPayload(ErrorDictionary.FAMILY_MISSING, "No family for identifier: " + id))
294                    .type(APPLICATION_JSON_TYPE)
295                    .build();
296        }
297        final Optional<Container> plugin = manager.findPlugin(meta.getPlugin());
298        if (!plugin.isPresent()) {
299            return Response
300                    .status(Response.Status.NOT_FOUND)
301                    .entity(new ErrorPayload(ErrorDictionary.PLUGIN_MISSING,
302                            "No plugin '" + meta.getPlugin() + "' for identifier: " + id))
303                    .type(APPLICATION_JSON_TYPE)
304                    .build();
305        }
306
307        final IconResolver.Icon iconContent = iconResolver.resolve(plugin.get(), meta.getIcon());
308        if (iconContent == null) {
309            return Response
310                    .status(Response.Status.NOT_FOUND)
311                    .entity(new ErrorPayload(ErrorDictionary.ICON_MISSING, "No icon for family identifier: " + id))
312                    .type(APPLICATION_JSON_TYPE)
313                    .build();
314        }
315
316        return Response.ok(iconContent.getBytes()).type(iconContent.getType()).build();
317    }
318
319    @Override
320    public Response icon(final String id) {
321        if (virtualComponents.isExtensionEntity(id)) { // todo if the front bundle is not sufficient
322            return Response
323                    .status(Response.Status.NOT_FOUND)
324                    .entity(new ErrorPayload(ErrorDictionary.ICON_MISSING, "No icon for family: " + id))
325                    .type(APPLICATION_JSON_TYPE)
326                    .build();
327        }
328
329        // todo: add caching if SvgIconResolver becomes used a lot - not the case ATM
330        final ComponentFamilyMeta.BaseMeta<Object> meta = componentDao.findById(id);
331        if (meta == null) {
332            return Response
333                    .status(Response.Status.NOT_FOUND)
334                    .entity(new ErrorPayload(ErrorDictionary.COMPONENT_MISSING, "No component for identifier: " + id))
335                    .type(APPLICATION_JSON_TYPE)
336                    .build();
337        }
338
339        final Optional<Container> plugin = manager.findPlugin(meta.getParent().getPlugin());
340        if (!plugin.isPresent()) {
341            return Response
342                    .status(Response.Status.NOT_FOUND)
343                    .entity(new ErrorPayload(ErrorDictionary.PLUGIN_MISSING,
344                            "No plugin '" + meta.getParent().getPlugin() + "' for identifier: " + id))
345                    .type(APPLICATION_JSON_TYPE)
346                    .build();
347        }
348
349        final IconResolver.Icon iconContent = iconResolver.resolve(plugin.get(), meta.getIcon());
350        if (iconContent == null) {
351            return Response
352                    .status(Response.Status.NOT_FOUND)
353                    .entity(new ErrorPayload(ErrorDictionary.ICON_MISSING, "No icon for identifier: " + id))
354                    .type(APPLICATION_JSON_TYPE)
355                    .build();
356        }
357
358        return Response.ok(iconContent.getBytes()).type(iconContent.getType()).build();
359    }
360
361    @Override
362    public Map<String, String> migrate(final String id, final int version, final Map<String, String> config) {
363        if (virtualComponents.isExtensionEntity(id)) {
364            return config;
365        }
366        return ofNullable(componentDao.findById(id))
367                .orElseThrow(() -> new WebApplicationException(Response
368                        .status(Response.Status.NOT_FOUND)
369                        .entity(new ErrorPayload(ErrorDictionary.COMPONENT_MISSING, "Didn't find component " + id))
370                        .build()))
371                .getMigrationHandler()
372                .get()
373                .migrate(version, config);
374    }
375
376    @Override // TODO: max ids.length
377    public ComponentDetailList getDetail(final String language, final String[] ids) {
378        if (ids == null || ids.length == 0) {
379            return new ComponentDetailList(emptyList());
380        }
381
382        final Map<String, ErrorPayload> errors = new HashMap<>();
383        final List<ComponentDetail> details = Stream.of(ids).map(id -> {
384            if (virtualComponents.isExtensionEntity(id)) {
385                return virtualComponents.findComponentById(id).orElseGet(() -> {
386                    errors.put(id, new ErrorPayload(COMPONENT_MISSING, "No virtual component '" + id + "'"));
387                    return null;
388                });
389            }
390            return ofNullable(componentDao.findById(id)).map(meta -> {
391                final Optional<Container> plugin = manager.findPlugin(meta.getParent().getPlugin());
392                if (!plugin.isPresent()) {
393                    errors
394                            .put(meta.getId(), new ErrorPayload(PLUGIN_MISSING,
395                                    "No plugin '" + meta.getParent().getPlugin() + "'"));
396                    return null;
397                }
398
399                final Container container = plugin.get();
400                final Optional<DesignModel> model = ofNullable(meta.get(DesignModel.class));
401                if (!model.isPresent()) {
402                    errors
403                            .put(meta.getId(),
404                                    new ErrorPayload(DESIGN_MODEL_MISSING, "No design model '" + meta.getId() + "'"));
405                    return null;
406                }
407
408                final Locale locale = localeMapper.mapLocale(language);
409                final boolean isProcessor = ComponentFamilyMeta.ProcessorMeta.class.isInstance(meta);
410
411                final ComponentDetail componentDetail = new ComponentDetail();
412                componentDetail.setLinks(emptyList() /* todo ? */);
413                componentDetail.setId(createMetaId(container, meta));
414                componentDetail.setVersion(meta.getVersion());
415                componentDetail.setIcon(meta.getIcon());
416                componentDetail.setInputFlows(model.get().getInputFlows());
417                componentDetail.setOutputFlows(model.get().getOutputFlows());
418                componentDetail.setType(isProcessor ? "processor" : "input");
419                componentDetail
420                        .setDisplayName(
421                                meta.findBundle(container.getLoader(), locale).displayName().orElse(meta.getName()));
422                componentDetail
423                        .setProperties(propertiesService
424                                .buildProperties(meta.getParameterMetas().get(), container.getLoader(), locale, null)
425                                .collect(toList()));
426                componentDetail
427                        .setActions(actionsService
428                                .findActions(meta.getParent().getName(), container, locale, meta,
429                                        meta.getParent().findBundle(container.getLoader(), locale)));
430                if (isProcessor) {
431                    componentDetail.setMetadata(emptyMap());
432                } else {
433                    componentDetail
434                            .setMetadata(singletonMap("mapper::infinite", Boolean
435                                    .toString(ComponentFamilyMeta.PartitionMapperMeta.class.cast(meta).isInfinite())));
436                }
437
438                return componentDetail;
439            }).orElseGet(() -> {
440                errors.put(id, new ErrorPayload(COMPONENT_MISSING, "No component '" + id + "'"));
441                return null;
442            });
443        }).filter(Objects::nonNull).collect(toList());
444
445        if (!errors.isEmpty()) {
446            throw new WebApplicationException(Response.status(Response.Status.BAD_REQUEST).entity(errors).build());
447        }
448
449        return new ComponentDetailList(details);
450    }
451
452    private Stream<ComponentIndex> findDeployedComponents(final boolean includeIconContent, final Locale locale) {
453        return manager
454                .find(c -> c
455                        .execute(() -> c.get(ContainerComponentRegistry.class).getComponents().values().stream())
456                        .flatMap(component -> Stream
457                                .concat(component
458                                        .getPartitionMappers()
459                                        .values()
460                                        .stream()
461                                        .map(mapper -> toComponentIndex(c, locale, c.getId(), mapper,
462                                                c.get(ComponentManager.OriginalId.class), includeIconContent)),
463                                        component
464                                                .getProcessors()
465                                                .values()
466                                                .stream()
467                                                .map(proc -> toComponentIndex(c, locale, c.getId(), proc,
468                                                        c.get(ComponentManager.OriginalId.class),
469                                                        includeIconContent)))));
470    }
471
472    private DependencyDefinition getDependenciesFor(final ComponentFamilyMeta.BaseMeta<?> meta) {
473        final ComponentFamilyMeta familyMeta = meta.getParent();
474        final Optional<Container> container = componentManagerService.manager().findPlugin(familyMeta.getPlugin());
475        return new DependencyDefinition(container.map(c -> {
476            final ComponentExtension.ComponentContext context =
477                    c.get(ComponentContexts.class).getContexts().get(meta.getType());
478            final ComponentExtension extension = context.owningExtension();
479            final Stream<Artifact> deps = c.findDependencies();
480            final Stream<Artifact> artifacts;
481            if (configuration.getAddExtensionDependencies() && extension != null) {
482                final List<Artifact> dependencies = deps.collect(toList());
483                final Stream<Artifact> addDeps = getExtensionDependencies(extension, dependencies);
484                artifacts = Stream.concat(dependencies.stream(), addDeps);
485            } else {
486                artifacts = deps;
487            }
488            return artifacts.map(Artifact::toCoordinate).collect(toList());
489        }).orElseThrow(() -> new IllegalArgumentException("Can't find container '" + meta.getId() + "'")));
490    }
491
492    private Stream<Artifact> getExtensionDependencies(final ComponentExtension extension,
493            final List<Artifact> filtered) {
494        return extension
495                .getAdditionalDependencies()
496                .stream()
497                .map(Artifact::from)
498                // filter required artifacts if they are already present in the list.
499                .filter(extArtifact -> filtered
500                        .stream()
501                        .map(d -> d.getGroup() + ":" + d.getArtifact())
502                        .noneMatch(ga -> ga.equals(extArtifact.getGroup() + ":" + extArtifact.getArtifact())));
503    }
504
505    private ComponentId createMetaId(final Container container, final ComponentFamilyMeta.BaseMeta<Object> meta) {
506        return new ComponentId(meta.getId(), meta.getParent().getId(), meta.getParent().getPlugin(),
507                ofNullable(container.get(ComponentManager.OriginalId.class))
508                        .map(ComponentManager.OriginalId::getValue)
509                        .orElse(container.getId()),
510                meta.getParent().getName(), meta.getName());
511    }
512
513    private ComponentIndex toComponentIndex(final Container container, final Locale locale, final String plugin,
514            final ComponentFamilyMeta.BaseMeta meta, final ComponentManager.OriginalId originalId,
515            final boolean includeIcon) {
516        final ClassLoader loader = container.getLoader();
517        final String icon = meta.getIcon();
518        final String familyIcon = meta.getParent().getIcon();
519        final IconResolver.Icon iconContent = iconResolver.resolve(container, icon);
520        final IconResolver.Icon iconFamilyContent = iconResolver.resolve(container, familyIcon);
521        final String familyDisplayName =
522                meta.getParent().findBundle(loader, locale).displayName().orElse(meta.getParent().getName());
523        final List<String> categories = ofNullable(meta.getParent().getCategories())
524                .map(vals -> vals
525                        .stream()
526                        .map(this::normalizeCategory)
527                        .map(category -> category.replace("${family}", meta.getParent().getName())) // not
528                                                                                                    // i18n-ed
529                                                                                                    // yet
530                        .map(category -> meta
531                                .getParent()
532                                .findBundle(loader, locale)
533                                .category(category)
534                                .orElseGet(() -> category
535                                        .replace("/" + meta.getParent().getName() + "/",
536                                                "/" + familyDisplayName + "/")))
537                        .collect(toList()))
538                .orElseGet(Collections::emptyList);
539        return new ComponentIndex(
540                new ComponentId(meta.getId(), meta.getParent().getId(), plugin,
541                        ofNullable(originalId).map(ComponentManager.OriginalId::getValue).orElse(plugin),
542                        meta.getParent().getName(), meta.getName()),
543                meta.findBundle(loader, locale).displayName().orElse(meta.getName()), familyDisplayName,
544                new Icon(icon, iconContent == null ? null : iconContent.getType(),
545                        !includeIcon ? null : (iconContent == null ? null : iconContent.getBytes())),
546                new Icon(familyIcon, iconFamilyContent == null ? null : iconFamilyContent.getType(),
547                        !includeIcon ? null : (iconFamilyContent == null ? null : iconFamilyContent.getBytes())),
548                meta.getVersion(), categories, singletonList(new Link("Detail",
549                        "/component/details?identifiers=" + meta.getId(), MediaType.APPLICATION_JSON)));
550    }
551
552    private String normalizeCategory(final String category) {
553        // we prevent root categories and always append the family in this case
554        if (!category.contains("${family}")) {
555            return category + "/${family}";
556        }
557        return category;
558    }
559
560    private StreamingOutput onMissingJar(final String id) {
561        throw new WebApplicationException(Response
562                .status(Response.Status.NOT_FOUND)
563                .type(APPLICATION_JSON_TYPE)
564                .entity(new ErrorPayload(PLUGIN_MISSING, "No file found for: " + id))
565                .build());
566    }
567}