001/**
002 * Copyright (C) 2006-2025 Talend Inc. - www.talend.com
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016package org.talend.sdk.component.runtime.record;
017
018import static java.util.stream.Collectors.toList;
019import static java.util.stream.Collectors.toSet;
020
021import java.io.ObjectStreamException;
022import java.io.Serializable;
023import java.lang.reflect.Constructor;
024import java.lang.reflect.InvocationTargetException;
025import java.lang.reflect.Method;
026import java.math.BigDecimal;
027import java.time.Instant;
028import java.time.ZonedDateTime;
029import java.time.format.DateTimeFormatter;
030import java.util.Base64;
031import java.util.Collection;
032import java.util.Collections;
033import java.util.Date;
034import java.util.List;
035import java.util.Map;
036import java.util.Set;
037import java.util.concurrent.ConcurrentHashMap;
038import java.util.function.Function;
039import java.util.function.Supplier;
040import java.util.stream.Collector;
041import java.util.stream.Stream;
042
043import javax.json.JsonArray;
044import javax.json.JsonArrayBuilder;
045import javax.json.JsonBuilderFactory;
046import javax.json.JsonNumber;
047import javax.json.JsonObject;
048import javax.json.JsonObjectBuilder;
049import javax.json.JsonString;
050import javax.json.JsonValue;
051import javax.json.bind.Jsonb;
052import javax.json.spi.JsonProvider;
053
054import org.apache.johnzon.core.JsonLongImpl;
055import org.apache.johnzon.jsonb.extension.JsonValueReader;
056import org.talend.sdk.component.api.record.Record;
057import org.talend.sdk.component.api.record.Schema;
058import org.talend.sdk.component.api.service.record.RecordBuilderFactory;
059import org.talend.sdk.component.runtime.record.json.OutputRecordHolder;
060import org.talend.sdk.component.runtime.record.json.PojoJsonbProvider;
061
062import lombok.Data;
063
064public class RecordConverters implements Serializable {
065
066    public <T> Record toRecord(final MappingMetaRegistry registry, final T data, final Supplier<Jsonb> jsonbProvider,
067            final Supplier<RecordBuilderFactory> recordBuilderProvider) {
068        if (data == null) {
069            return null;
070        }
071        if (Record.class.isInstance(data)) {
072            return Record.class.cast(data);
073        }
074        if (JsonObject.class.isInstance(data)) {
075            return json2Record(recordBuilderProvider.get(), JsonObject.class.cast(data));
076        }
077
078        final MappingMeta meta = registry.find(data.getClass());
079        if (meta.isLinearMapping()) {
080            return meta.newRecord(data, recordBuilderProvider.get());
081        }
082
083        final Jsonb jsonb = jsonbProvider.get();
084        if (!String.class.isInstance(data) && !data.getClass().isPrimitive()
085                && PojoJsonbProvider.class.isInstance(jsonb)) {
086            final Jsonb pojoMapper = PojoJsonbProvider.class.cast(jsonb).get();
087            final OutputRecordHolder holder = new OutputRecordHolder(data);
088            try (final OutputRecordHolder stream = holder) {
089                pojoMapper.toJson(data, stream);
090            }
091            return holder.getRecord();
092        }
093        return json2Record(recordBuilderProvider.get(), jsonb.fromJson(jsonb.toJson(data), JsonObject.class));
094    }
095
096    private Record json2Record(final RecordBuilderFactory factory, final JsonObject object) {
097        final Record.Builder builder = factory.newRecordBuilder();
098        object.forEach((key, value) -> {
099            switch (value.getValueType()) {
100                case ARRAY: {
101                    final List<Object> items =
102                            value.asJsonArray().stream().map(it -> mapJson(factory, it)).collect(toList());
103                    builder
104                            .withArray(factory
105                                    .newEntryBuilder()
106                                    .withName(key)
107                                    .withType(Schema.Type.ARRAY)
108                                    .withElementSchema(getArrayElementSchema(factory, items))
109                                    .build(), items);
110                    break;
111                }
112                case OBJECT: {
113                    final Record record = json2Record(factory, value.asJsonObject());
114                    builder
115                            .withRecord(factory
116                                    .newEntryBuilder()
117                                    .withName(key)
118                                    .withType(Schema.Type.RECORD)
119                                    .withElementSchema(record.getSchema())
120                                    .build(), record);
121                    break;
122                }
123                case TRUE:
124                case FALSE:
125                    builder.withBoolean(key, JsonValue.TRUE.equals(value));
126                    break;
127                case STRING:
128                    builder.withString(key, JsonString.class.cast(value).getString());
129                    break;
130                case NUMBER:
131                    final JsonNumber number = JsonNumber.class.cast(value);
132                    builder.withDouble(key, number.doubleValue());
133                    break;
134                case NULL:
135                    break;
136                default:
137                    throw new IllegalArgumentException("Unsupported value type: " + value);
138            }
139        });
140        return builder.build();
141    }
142
143    private Schema getArrayElementSchema(final RecordBuilderFactory factory, final List<Object> items) {
144        if (items.isEmpty()) {
145            return factory.newSchemaBuilder(Schema.Type.STRING).build();
146        }
147        final Schema firstSchema = toSchema(factory, items.iterator().next());
148        switch (firstSchema.getType()) {
149            case RECORD:
150                return items.stream().map(it -> toSchema(factory, it)).reduce(null, (s1, s2) -> {
151                    if (s1 == null) {
152                        return s2;
153                    }
154                    if (s2 == null) { // unlikely
155                        return s1;
156                    }
157                    final Set<String> names1 = s1.getAllEntries().map(Schema.Entry::getName).collect(toSet());
158                    final Set<String> names2 = s2.getAllEntries().map(Schema.Entry::getName).collect(toSet());
159                    if (!names1.equals(names2)) {
160                        // here we are not good since values will not be right anymore,
161                        // forbidden for current version anyway but potentially supported later
162                        final Schema.Builder builder = factory.newSchemaBuilder(Schema.Type.RECORD);
163                        s1.getAllEntries().forEach(builder::withEntry);
164                        s2.getAllEntries().filter(it -> !(names1.contains(it.getName()))).forEach(builder::withEntry);
165                        return builder.build();
166                    }
167                    return s1;
168                });
169            default:
170                return firstSchema;
171        }
172    }
173
174    private Object mapJson(final RecordBuilderFactory factory, final JsonValue it) {
175        if (JsonObject.class.isInstance(it)) {
176            return json2Record(factory, JsonObject.class.cast(it));
177        }
178        if (JsonArray.class.isInstance(it)) {
179            return JsonArray.class.cast(it).stream().map(i -> mapJson(factory, i)).collect(toList());
180        }
181        if (JsonString.class.isInstance(it)) {
182            return JsonString.class.cast(it).getString();
183        }
184        if (JsonNumber.class.isInstance(it)) {
185            return JsonNumber.class.cast(it).numberValue();
186        }
187        if (JsonValue.FALSE.equals(it)) {
188            return false;
189        }
190        if (JsonValue.TRUE.equals(it)) {
191            return true;
192        }
193        if (JsonValue.NULL.equals(it)) {
194            return null;
195        }
196        return it;
197    }
198
199    public static Schema toSchema(final RecordBuilderFactory factory, final Object next) {
200        if (String.class.isInstance(next) || JsonString.class.isInstance(next)) {
201            return factory.newSchemaBuilder(Schema.Type.STRING).build();
202        }
203        if (Integer.class.isInstance(next)) {
204            return factory.newSchemaBuilder(Schema.Type.INT).build();
205        }
206        if (Long.class.isInstance(next) || JsonLongImpl.class.isInstance(next)) {
207            return factory.newSchemaBuilder(Schema.Type.LONG).build();
208        }
209        if (Float.class.isInstance(next)) {
210            return factory.newSchemaBuilder(Schema.Type.FLOAT).build();
211        }
212        if (JsonNumber.class.isInstance(next)) {
213            return factory.newSchemaBuilder(Schema.Type.DOUBLE).build();
214        }
215        if (Double.class.isInstance(next) || JsonNumber.class.isInstance(next)) {
216            return factory.newSchemaBuilder(Schema.Type.DOUBLE).build();
217        }
218        if (Boolean.class.isInstance(next) || JsonValue.TRUE.equals(next) || JsonValue.FALSE.equals(next)) {
219            return factory.newSchemaBuilder(Schema.Type.BOOLEAN).build();
220        }
221        if (Date.class.isInstance(next) || ZonedDateTime.class.isInstance(next) || Instant.class.isInstance(next)) {
222            return factory.newSchemaBuilder(Schema.Type.DATETIME).build();
223        }
224        if (BigDecimal.class.isInstance(next)) {
225            return factory.newSchemaBuilder(Schema.Type.DECIMAL).build();
226        }
227        if (byte[].class.isInstance(next)) {
228            return factory.newSchemaBuilder(Schema.Type.BYTES).build();
229        }
230        if (Collection.class.isInstance(next) || JsonArray.class.isInstance(next)) {
231            final Collection collection = Collection.class.cast(next);
232            if (collection.isEmpty()) {
233                return factory.newSchemaBuilder(Schema.Type.STRING).build();
234            }
235            return factory
236                    .newSchemaBuilder(Schema.Type.ARRAY)
237                    .withElementSchema(toSchema(factory, collection.iterator().next()))
238                    .build();
239        }
240        if (Record.class.isInstance(next)) {
241            return Record.class.cast(next).getSchema();
242        }
243        throw new IllegalArgumentException("unsupported type for " + next);
244    }
245
246    public Object toType(final MappingMetaRegistry registry, final Object data, final Class<?> parameterType,
247            final Supplier<JsonBuilderFactory> factorySupplier, final Supplier<JsonProvider> providerSupplier,
248            final Supplier<Jsonb> jsonbProvider, final Supplier<RecordBuilderFactory> recordBuilderProvider) {
249        return toType(registry, data, parameterType, factorySupplier, providerSupplier, jsonbProvider,
250                recordBuilderProvider, Collections.emptyMap());
251    }
252
253    public Object toType(final MappingMetaRegistry registry, final Object data, final Class<?> parameterType,
254            final Supplier<JsonBuilderFactory> factorySupplier, final Supplier<JsonProvider> providerSupplier,
255            final Supplier<Jsonb> jsonbProvider, final Supplier<RecordBuilderFactory> recordBuilderProvider,
256            final java.util.Map<String, String> metadata) {
257        if (parameterType.isInstance(data)) {
258            return data;
259        }
260
261        final JsonObject inputAsJson;
262        if (JsonObject.class.isInstance(data)) {
263            if (JsonObject.class == parameterType) {
264                return data;
265            }
266            inputAsJson = JsonObject.class.cast(data);
267        } else if (Record.class.isInstance(data)) {
268            final Record record = Record.class.cast(data);
269            if (!JsonObject.class.isAssignableFrom(parameterType)) {
270                final MappingMeta mappingMeta = registry.find(parameterType);
271                if (mappingMeta.isLinearMapping()) {
272                    return mappingMeta.newInstance(record, metadata);
273                }
274            }
275            final JsonObject asJson = toJson(factorySupplier, providerSupplier, record);
276            if (JsonObject.class == parameterType) {
277                return asJson;
278            }
279            inputAsJson = asJson;
280        } else {
281            if (parameterType == Record.class) {
282                return toRecord(registry, data, jsonbProvider, recordBuilderProvider);
283            }
284            final Jsonb jsonb = jsonbProvider.get();
285            inputAsJson = jsonb.fromJson(jsonb.toJson(data), JsonObject.class);
286        }
287        return jsonbProvider.get().fromJson(new JsonValueReader<>(inputAsJson), parameterType);
288    }
289
290    private JsonObject toJson(final Supplier<JsonBuilderFactory> factorySupplier,
291            final Supplier<JsonProvider> providerSupplier, final Record record) {
292        return buildRecord(factorySupplier.get(), providerSupplier, record).build();
293    }
294
295    private JsonObjectBuilder buildRecord(final JsonBuilderFactory factory,
296            final Supplier<JsonProvider> providerSupplier, final Record record) {
297        final Schema schema = record.getSchema();
298        final JsonObjectBuilder builder = factory.createObjectBuilder();
299        schema.getEntries().forEach(entry -> {
300            final String name = entry.getName();
301            switch (entry.getType()) {
302                case STRING: {
303                    final String value = record.get(String.class, name);
304                    if (value != null) {
305                        builder.add(name, value);
306                    }
307                    break;
308                }
309                case INT: {
310                    final Integer value = record.get(Integer.class, name);
311                    if (value != null) {
312                        builder.add(name, value);
313                    }
314                    break;
315                }
316                case LONG: {
317                    final Long value = record.get(Long.class, name);
318                    if (value != null) {
319                        builder.add(name, value);
320                    }
321                    break;
322                }
323                case FLOAT: {
324                    final Float value = record.get(Float.class, name);
325                    if (value != null) {
326                        builder.add(name, value);
327                    }
328                    break;
329                }
330                case DOUBLE: {
331                    final Double value = record.get(Double.class, name);
332                    if (value != null) {
333                        builder.add(name, value);
334                    }
335                    break;
336                }
337                case BOOLEAN: {
338                    final Boolean value = record.get(Boolean.class, name);
339                    if (value != null) {
340                        builder.add(name, value);
341                    }
342                    break;
343                }
344                case BYTES: {
345                    final byte[] value = record.get(byte[].class, name);
346                    if (value != null) {
347                        builder.add(name, Base64.getEncoder().encodeToString(value));
348                    }
349                    break;
350                }
351                case DATETIME: {
352                    final ZonedDateTime value = record.get(ZonedDateTime.class, name);
353                    if (value != null) {
354                        builder.add(name, value.format(DateTimeFormatter.ISO_ZONED_DATE_TIME));
355                    }
356                    break;
357                }
358                case DECIMAL: {
359                    final BigDecimal value = record.get(BigDecimal.class, name);
360                    if (value != null) {
361                        builder.add(name, value.toString());
362                    }
363                    break;
364                }
365                case RECORD: {
366                    final Record value = record.get(Record.class, name);
367                    if (value != null) {
368                        builder.add(name, buildRecord(factory, providerSupplier, value));
369                    }
370                    break;
371                }
372                case ARRAY:
373                    final Collection<?> collection = record.get(Collection.class, name);
374                    if (collection == null) {
375                        break;
376                    }
377                    if (collection.isEmpty()) {
378                        builder.add(name, factory.createArrayBuilder().build());
379                    } else { // only homogeneous collections
380                        final Object item = collection.iterator().next();
381                        if (String.class.isInstance(item)) {
382                            final JsonProvider jsonProvider = providerSupplier.get();
383                            builder.add(name,
384                                    toArray(factory, v -> jsonProvider.createValue(String.class.cast(v)), collection));
385                        } else if (Double.class.isInstance(item)) {
386                            final JsonProvider jsonProvider = providerSupplier.get();
387                            builder.add(name,
388                                    toArray(factory, v -> jsonProvider.createValue(Double.class.cast(v)), collection));
389                        } else if (Float.class.isInstance(item)) {
390                            final JsonProvider jsonProvider = providerSupplier.get();
391                            builder.add(name,
392                                    toArray(factory, v -> jsonProvider.createValue(Float.class.cast(v)), collection));
393                        } else if (Integer.class.isInstance(item)) {
394                            final JsonProvider jsonProvider = providerSupplier.get();
395                            builder.add(name,
396                                    toArray(factory, v -> jsonProvider.createValue(Integer.class.cast(v)), collection));
397                        } else if (Long.class.isInstance(item)) {
398                            final JsonProvider jsonProvider = providerSupplier.get();
399                            builder.add(name,
400                                    toArray(factory, v -> jsonProvider.createValue(Long.class.cast(v)), collection));
401                        } else if (Boolean.class.isInstance(item)) {
402                            builder.add(name, toArray(factory,
403                                    v -> Boolean.class.cast(v) ? JsonValue.TRUE : JsonValue.FALSE, collection));
404                        } else if (ZonedDateTime.class.isInstance(item)) {
405                            final JsonProvider jsonProvider = providerSupplier.get();
406                            builder.add(name,
407                                    toArray(factory,
408                                            v -> jsonProvider.createValue(
409                                                    ZonedDateTime.class.cast(v).toInstant().toEpochMilli()),
410                                            collection));
411                        } else if (Date.class.isInstance(item)) {
412                            final JsonProvider jsonProvider = providerSupplier.get();
413                            builder.add(name,
414                                    toArray(factory,
415                                            v -> jsonProvider.createValue(Date.class.cast(v).getTime()),
416                                            collection));
417                        } else if (Record.class.isInstance(item)) {
418                            builder.add(name,
419                                    toArray(factory,
420                                            v -> buildRecord(factory, providerSupplier, Record.class.cast(v)).build(),
421                                            collection));
422                        } else if (JsonValue.class.isInstance(item)) {
423                            builder.add(name, toArray(factory, JsonValue.class::cast, collection));
424                        } // else throw?
425                    }
426                    break;
427                default:
428                    throw new IllegalArgumentException("Unsupported type: " + entry.getType() + " for '" + name + "'");
429            }
430        });
431        return builder;
432    }
433
434    private JsonArray toArray(final JsonBuilderFactory factory, final Function<Object, JsonValue> valueFactory,
435            final Collection<?> collection) {
436        final Collector<JsonValue, JsonArrayBuilder, JsonArray> collector = Collector
437                .of(factory::createArrayBuilder, JsonArrayBuilder::add, JsonArrayBuilder::addAll,
438                        JsonArrayBuilder::build);
439        return collection.stream().map(valueFactory).collect(collector);
440    }
441
442    public <T> T coerce(final Class<T> expectedType, final Object value, final String name) {
443        if (value == null) {
444            return null;
445        }
446
447        // here mean get(Object.class, name) return origin store type, like DATETIME return long, is expected?
448        if (!expectedType.isInstance(value)) {
449            return expectedType.cast(MappingUtils.coerce(expectedType, value, name));
450        }
451
452        return expectedType.cast(value);
453    }
454
455    @Data
456    public static class MappingMeta {
457
458        private final boolean linearMapping;
459
460        private final Class<?> rowStruct;
461
462        private Object recordVisitor;
463
464        private Method visitRecord;
465
466        private Object rowStructVisitor;
467
468        private Method visitRowStruct;
469
470        public MappingMeta(final Class<?> type, final MappingMetaRegistry registry) {
471            linearMapping = Stream.of(type.getInterfaces()).anyMatch(it -> it.getName().startsWith("routines.system."));
472            rowStruct = type;
473        }
474
475        public Object newInstance(final Record record) {
476            return newInstance(record, Collections.emptyMap());
477        }
478
479        public Object newInstance(final Record record, final java.util.Map<String, String> metadata) {
480            if (recordVisitor == null) {
481                try {
482                    final String className = "org.talend.sdk.component.runtime.di.record.DiRecordVisitor";
483                    final Class<?> visitorClass = getClass().getClassLoader().loadClass(className);
484                    final Constructor<?> constructor = visitorClass.getDeclaredConstructors()[0];
485                    constructor.setAccessible(true);
486                    recordVisitor = constructor.newInstance(rowStruct, metadata);
487                    visitRecord = visitorClass.getDeclaredMethod("visit", Record.class);
488                } catch (final NoClassDefFoundError | ClassNotFoundException | InstantiationException
489                        | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
490                    if (e.getMessage().matches(".*routines.system.Dynamic.*")) {
491                        throw new IllegalStateException("TOS does not support dynamic type", e);
492                    }
493                    throw new IllegalStateException(e);
494                }
495            }
496            try {
497                return visitRecord.invoke(recordVisitor, record);
498            } catch (final IllegalAccessException | InvocationTargetException e) {
499                throw new IllegalStateException(e);
500            }
501        }
502
503        public <T> Record newRecord(final T data, final RecordBuilderFactory factory) {
504            if (rowStructVisitor == null) {
505                try {
506                    final String className = "org.talend.sdk.component.runtime.di.record.DiRowStructVisitor";
507                    final Class<?> visitorClass = getClass().getClassLoader().loadClass(className);
508                    final Constructor<?> constructor = visitorClass.getConstructors()[0];
509                    constructor.setAccessible(true);
510                    rowStructVisitor = constructor.newInstance();
511                    visitRowStruct = visitorClass.getMethod("get", Object.class, RecordBuilderFactory.class);
512                } catch (final NoClassDefFoundError | ClassNotFoundException | InstantiationException
513                        | IllegalAccessException | InvocationTargetException | NoSuchMethodException e) {
514                    if (e.getMessage().matches(".*routines.system.Dynamic.*")) {
515                        throw new IllegalStateException("TOS does not support dynamic type", e);
516                    }
517                    throw new IllegalStateException(e);
518                }
519            }
520            try {
521                return Record.class.cast(visitRowStruct.invoke(rowStructVisitor, data, factory));
522            } catch (final IllegalAccessException | InvocationTargetException e) {
523                throw new IllegalStateException(e);
524            }
525        }
526
527    }
528
529    @Data
530    public static class MappingMetaRegistry implements Serializable {
531
532        protected final Map<Class<?>, MappingMeta> registry = new ConcurrentHashMap<>();
533
534        private Object writeReplace() throws ObjectStreamException {
535            return new Factory(); // don't serialize the mapping, recalculate it lazily
536        }
537
538        public MappingMeta find(final Class<?> parameterType) {
539            final MappingMeta meta = registry.get(parameterType);
540            if (meta != null) {
541                return meta;
542            }
543            final MappingMeta mappingMeta = new MappingMeta(parameterType, this);
544            final MappingMeta existing = registry.putIfAbsent(parameterType, mappingMeta);
545            if (existing != null) {
546                return existing;
547            }
548            return mappingMeta;
549        }
550
551        public static class Factory implements Serializable {
552
553            private Object readResolve() throws ObjectStreamException {
554                return new MappingMetaRegistry();
555            }
556        }
557    }
558}