1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 package eu.fbk.knowledgestore.elastic;
17
18 import com.google.common.base.Charsets;
19 import com.google.common.base.Preconditions;
20 import com.google.common.collect.BoundType;
21 import com.google.common.collect.Range;
22 import eu.fbk.knowledgestore.data.Data;
23 import eu.fbk.knowledgestore.data.Record;
24 import org.elasticsearch.action.get.GetResponse;
25 import org.elasticsearch.common.xcontent.XContentBuilder;
26 import org.elasticsearch.index.query.FilterBuilder;
27 import org.elasticsearch.index.query.FilterBuilders;
28 import org.elasticsearch.index.query.QueryBuilders;
29 import org.elasticsearch.index.query.RangeFilterBuilder;
30 import org.elasticsearch.search.SearchHit;
31 import org.openrdf.model.Literal;
32 import org.openrdf.model.URI;
33 import org.openrdf.model.Value;
34 import org.openrdf.model.vocabulary.XMLSchema;
35 import org.slf4j.Logger;
36 import org.slf4j.LoggerFactory;
37
38 import java.io.IOException;
39 import java.util.*;
40
41 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
42
43
44
45
46
47
48 public final class Utility {
49
50 private static final Logger LOGGER = LoggerFactory.getLogger(Utility.class);
51 private static final String ID_PROPERTY_NAME = "ID";
52 private static final char URI_ESCAPE = (char) 1;
53
54 private static String encodeString(URI uri, URIHandler handler) throws IOException {
55 if (uri == null) {
56 return null;
57 }
58 return handler.encode(uri);
59 }
60
61 private static Object decodeString(String str, URIHandler handler) {
62 Object res = str;
63 if (handler.isEncodedUri(str)) {
64 res = handler.decode(str);
65 }
66 return res;
67 }
68
69
70
71
72
73
74
75
76 public static boolean areEquals(Record exp, Record res) {
77 if (!exp.getID().equals(res.getID())) {
78 LOGGER.info("differend ids expected: " + exp.getID() + " ; found: " + res.getID());
79 }
80 List<URI> properties = exp.getProperties();
81 List<URI> resProperties = res.getProperties();
82
83 if (properties.size() != resProperties.size()) {
84 LOGGER.info("different number of properties; expected: " + properties.size() + " found: " + resProperties
85 .size());
86 return false;
87 }
88
89 properties.sort(Data.getTotalComparator());
90 resProperties.sort(Data.getTotalComparator());
91
92 for (int i = 0; i < properties.size(); i++) {
93 URI prop = properties.get(i);
94 URI resProp = resProperties.get(i);
95 if (!prop.equals(resProp)) {
96 LOGGER.debug("different property name; expected: " + prop + " found: " + resProp);
97 return false;
98 }
99
100 List<Object> expValues = exp.get(prop);
101 List<Object> resValues = res.get(prop);
102
103 if (resValues == null && expValues != null) {
104 LOGGER.debug("found null values expected not null");
105 return false;
106 }
107
108 if (resValues != null && expValues == null) {
109 LOGGER.debug("found not null values expected null");
110 return false;
111 }
112
113 if (resValues != null && expValues != null) {
114 if (resValues.size() != expValues.size()) {
115 LOGGER.debug(
116 "different number of values under " + prop + " expected: " + expValues.size() + " found: "
117 + resValues.size());
118 return false;
119 }
120
121 Iterator<Object> expIt = expValues.iterator();
122 Iterator<Object> resIt = resValues.iterator();
123 while (expIt.hasNext()) {
124 Object expValue = expIt.next();
125 Object resValue = resIt.next();
126 if (expValue instanceof Record) {
127 if (!areEquals((Record) expValue, (Record) resValue)) {
128 LOGGER.debug("different Records under " + prop);
129 return false;
130 }
131 } else if (expValue instanceof Value) {
132
133 if (expValue instanceof URI) {
134 if (!((URI) expValue).equals((URI) resValue)) {
135 LOGGER.debug("differet URIs; expected: " + ((URI) expValue).toString() + " found: "
136 + ((URI) resValue).toString());
137 return false;
138 }
139 } else {
140 if (expValue instanceof Literal) {
141 if (!((Literal) expValue).equals((Literal) resValue)) {
142 LOGGER.debug("differet Literals; expected: " + ((Literal) expValue).toString()
143 + " found: " + ((Literal) resValue).toString());
144 return false;
145 }
146 } else {
147 throw new IllegalArgumentException(
148 "unknow type in Record property, Value subclass that is neither URI nor Literal: "
149 + expValue.getClass());
150 }
151 }
152
153 } else {
154 throw new IllegalArgumentException("unknow type in Record property: " + expValue.getClass());
155 }
156 }
157 }
158 }
159 return true;
160 }
161
162
163
164
165
166
167
168
169 public static boolean areEquals(List<Record> expResult, List<Record> result) {
170 LOGGER.debug(
171 "\n\nexpected(" + expResult.size() + "): " + expResult + "\n\nfound(" + result.size() + "): " + result);
172 if (expResult.size() != result.size()) {
173 LOGGER.debug(
174 "the 2 List<Record> have a different number of entries expected: " + expResult.size() + " , found: "
175 + result.size());
176 return false;
177 }
178
179 for (int i = 0; i < result.size(); i++) {
180 if (!areEquals(expResult.get(i), result.get(i))) {
181 return false;
182 }
183 }
184 return true;
185 }
186
187
188
189
190
191
192
193
194
195
196 private static XContentBuilder literalSerializer(Literal literal, XContentBuilder builder) throws IOException {
197 final URI datatype = literal.getDatatype();
198 if (datatype == null || datatype.equals(XMLSchema.STRING)) {
199 builder.value(Data.convert(literal, String.class));
200 } else if (datatype.equals(XMLSchema.BOOLEAN)) {
201 builder.value(Data.convert(literal, Boolean.class));
202 } else if (datatype.equals(XMLSchema.DATE) || datatype.equals(XMLSchema.DATETIME)) {
203 builder.value(Data.convert(literal, Date.class));
204 } else if (datatype.equals(XMLSchema.INT)) {
205 builder.value(Data.convert(literal, Integer.class));
206 } else if (datatype.equals(XMLSchema.LONG)) {
207 builder.value(Data.convert(literal, Long.class));
208 } else if (datatype.equals(XMLSchema.DOUBLE)) {
209 builder.value(Data.convert(literal, Double.class));
210 } else if (datatype.equals(XMLSchema.FLOAT)) {
211 builder.value(Data.convert(literal, Float.class));
212 } else if (datatype.equals(XMLSchema.SHORT)) {
213 builder.value(Data.convert(literal, Short.class));
214 } else if (datatype.equals(XMLSchema.BYTE)) {
215 builder.value(Data.convert(literal, Byte.class));
216 } else if (datatype.equals(XMLSchema.DECIMAL)) {
217 String tmp = literalToString(literal);
218 builder.value(tmp.getBytes(Charsets.UTF_8));
219 } else if (datatype.equals(XMLSchema.INTEGER)) {
220 String tmp = literalToString(literal);
221 builder.value(tmp.getBytes(Charsets.UTF_8));
222 } else if (datatype.equals(XMLSchema.NON_NEGATIVE_INTEGER)
223 || datatype.equals(XMLSchema.NON_POSITIVE_INTEGER)
224 || datatype.equals(XMLSchema.NEGATIVE_INTEGER)
225 || datatype.equals(XMLSchema.POSITIVE_INTEGER)) {
226 String tmp = literalToString(literal);
227 builder.value(tmp.getBytes(Charsets.UTF_8));
228 } else if (datatype.equals(XMLSchema.NORMALIZEDSTRING) || datatype.equals(XMLSchema.TOKEN)
229 || datatype.equals(XMLSchema.NMTOKEN) || datatype.equals(XMLSchema.LANGUAGE)
230 || datatype.equals(XMLSchema.NAME) || datatype.equals(XMLSchema.NCNAME)) {
231 String tmp = literalToString(literal);
232 builder.value(tmp.getBytes(Charsets.UTF_8));
233 } else {
234 throw new IllegalArgumentException("can not serialize this literal: " + literal);
235 }
236 return builder;
237 }
238
239
240
241
242
243
244
245 public static FilterBuilder buildTermFilter(String elKey, Object item, MappingHandler mapper, URIHandler handler)
246 throws IOException {
247 LOGGER.debug("creating term filter: " + elKey + " = " + item);
248
249
250
251 if (item instanceof URI) {
252 String encodedURI = encodeString((URI) item, handler);
253 LOGGER.debug("item is a URI: " + encodedURI);
254 return FilterBuilders.queryFilter(QueryBuilders.matchPhraseQuery(elKey, encodedURI));
255
256 }
257 if (item instanceof String) {
258 LOGGER.debug("item is a string: " + item);
259
260
261 return FilterBuilders.queryFilter(QueryBuilders.matchPhraseQuery(elKey, item.toString()));
262 }
263 if (item instanceof Date) {
264 LOGGER.debug("item is a Date: " + item);
265 return FilterBuilders.termFilter(elKey, (Date) item);
266 }
267 if (item instanceof Boolean) {
268 LOGGER.debug("item is a Boolean: " + item);
269 return FilterBuilders.termFilter(elKey, (boolean) item);
270 }
271
272 if (item instanceof Number) {
273 if (item instanceof Short) {
274 LOGGER.debug("item is a Short: " + item);
275 return FilterBuilders.termFilter(elKey, (short) item);
276 }
277 if (item instanceof Byte) {
278 LOGGER.debug("item is a Byte: " + item);
279 return FilterBuilders.termFilter(elKey, (byte) item);
280 }
281 if (item instanceof Integer) {
282 LOGGER.debug("item is a Integer: " + item);
283 return FilterBuilders.termFilter(elKey, (int) item);
284 }
285 if (item instanceof Double) {
286 LOGGER.debug("item is a Double: " + item.getClass());
287 return FilterBuilders.termFilter(elKey, (double) item);
288 }
289 if (item instanceof Float) {
290 LOGGER.debug("item is a Float: " + item);
291 return FilterBuilders.termFilter(elKey, (float) item);
292 }
293 if (item instanceof Long) {
294 LOGGER.debug("item is a Long: " + item);
295 return FilterBuilders.termFilter(elKey, (long) item);
296 } else {
297 LOGGER.debug("found unhandled Number type in the xPath conditions: {} ; can not create the filter.",
298 item.getClass());
299 return null;
300 }
301 }
302 LOGGER.debug("found unhandled type in the xPath conditions: {} ; can not create the filter.", item.getClass());
303 return null;
304 }
305
306
307
308
309
310
311
312
313 private static RangeFilterBuilder setFromToRange(RangeFilterBuilder res, Range item, Class<?> itemClass,
314 URIHandler handler) throws IOException {
315 Object lower = null;
316 Object upper = null;
317 if (item.hasLowerBound()) {
318 lower = item.lowerEndpoint();
319 if (lower instanceof URI) {
320 lower = encodeString((URI) lower, handler);
321 }
322 lower = Data.convert(lower, itemClass);
323 LOGGER.debug("FROM: " + lower.toString() + " class: " + lower.getClass());
324 }
325 if (item.hasUpperBound()) {
326 upper = item.upperEndpoint();
327 if (upper instanceof URI) {
328 upper = encodeString((URI) upper, handler);
329 }
330 upper = Data.convert(upper, itemClass);
331 LOGGER.debug("TO: " + upper.toString() + " class: " + upper.getClass());
332 }
333
334 if (lower == null && upper == null) {
335 LOGGER.debug("upper bound and lower bound are both null");
336 return null;
337 }
338
339
340 if (lower instanceof Number || upper instanceof Number) {
341
342 if (lower instanceof Short || lower instanceof Byte || lower instanceof Integer || lower instanceof Long
343 || upper instanceof Short || upper instanceof Byte || upper instanceof Integer
344 || upper instanceof Long) {
345 LOGGER.debug("integer");
346 if (lower != null)
347 {
348 res = res.from(((Number) lower).intValue());
349 }
350 if (upper != null)
351 {
352 res = res.to(((Number) upper).intValue());
353 }
354
355 } else if (lower instanceof Float || lower instanceof Double || upper instanceof Float
356 || upper instanceof Double) {
357 LOGGER.debug("double");
358 if (lower != null) {
359 res = res.from(((Number) lower).doubleValue());
360 }
361 if (upper != null) {
362 res = res.to(((Number) upper).doubleValue());
363 }
364 } else {
365 LOGGER.warn("can not build a Range filter with this Number class {}", item.getClass());
366 return null;
367 }
368
369 } else {
370 if (lower instanceof String
371 || upper instanceof String) {
372 LOGGER.debug("string");
373 if (lower != null) {
374 res = res.from((String) lower);
375 }
376 if (upper != null) {
377 res = res.to((String) upper);
378 }
379
380 } else if (lower instanceof Date || upper instanceof Date) {
381 LOGGER.debug("date");
382 if (lower != null) {
383 LOGGER.debug("lower: " + lower);
384 res = res.from(((Date) lower));
385 }
386 if (upper != null) {
387 LOGGER.debug("upper: " + upper);
388 res = res.to(((Date) upper));
389 }
390
391 } else {
392 if (lower instanceof Boolean || upper instanceof Boolean) {
393 LOGGER.debug("boolean");
394 if (lower != null) {
395 LOGGER.debug("lower: " + lower);
396 res = res.from(((boolean) lower));
397 }
398 if (upper != null) {
399 LOGGER.debug("upper: " + upper);
400 res = res.to(((boolean) upper));
401 }
402 } else {
403 LOGGER.warn("can not build a Range filter with this class {}", item.getClass());
404 return null;
405 }
406 }
407 }
408
409
410 if (lower != null) {
411 LOGGER.debug("include lower: " + item.lowerBoundType().equals(BoundType.CLOSED));
412 res = res.includeLower(item.lowerBoundType().equals(BoundType.CLOSED));
413 }
414 if (upper != null) {
415 LOGGER.debug("include upper: " + item.upperBoundType().equals(BoundType.CLOSED));
416 res = res.includeUpper(item.upperBoundType().equals(BoundType.CLOSED));
417 }
418 return res;
419 }
420
421
422
423
424
425
426
427
428
429
430 public static FilterBuilder buildRangeFilter(String elKey, Range item, MappingHandler mapper, URIHandler handler)
431 throws IOException {
432 Preconditions.checkNotNull(item);
433 LOGGER.debug("range filter on " + elKey);
434 RangeFilterBuilder res = FilterBuilders.rangeFilter(elKey);
435 LOGGER.debug("set from-to");
436 return setFromToRange(res, item, mapper.getValueClass(elKey), handler);
437 }
438
439
440
441
442
443
444
445
446 private static XContentBuilder singleObjectSerializer(Object value, XContentBuilder builder, URIHandler handler)
447 throws IOException {
448 if (value instanceof Value) {
449 if (value instanceof URI) {
450 builder.value(encodeString((URI) value, handler));
451 } else if (value instanceof Literal)
452 {
453 builder = literalSerializer((Literal) value, builder);
454 }
455
456 return builder;
457 }
458 if (value instanceof Record) {
459 builder.startObject();
460 String id = encodeString(((Record) value).getID(), handler);
461 if (id != null) {
462 builder.field(ID_PROPERTY_NAME, id);
463 }
464 builder = serialize((Record) value, builder, handler).endObject();
465 return builder;
466 }
467 throw new UnsupportedOperationException(
468 "can not serialize this class of Objects: " + value.getClass() + " ; value: " + value.toString());
469 }
470
471
472
473
474
475
476
477
478 private static XContentBuilder collectionSerializer(Collection<?> values, XContentBuilder builder,
479 URIHandler handler) throws IOException {
480 for (Object obj : values) {
481 builder = singleObjectSerializer(obj, builder,
482 handler);
483 }
484 return builder;
485 }
486
487
488
489
490
491
492
493
494 private static XContentBuilder serialize(Record record, XContentBuilder builder, URIHandler handler)
495 throws IOException {
496 String propString;
497 for (URI property : record.getProperties()) {
498 Object value = record.get(property);
499 propString = encodeString(property, handler);
500
501 if (((Collection) value).size() == 1) {
502 value = ((Collection) value).iterator().next();
503 }
504 if (value instanceof Collection) {
505 builder.startArray(propString);
506 builder = collectionSerializer((Collection) value, builder, handler);
507 builder.endArray();
508
509 } else {
510 builder.field(propString);
511 builder = singleObjectSerializer(value, builder, handler);
512 }
513 }
514 return builder;
515 }
516
517
518
519
520
521
522
523 public static XContentBuilder serialize(Record record, URIHandler handler) throws IOException {
524 if (record == null) {
525 return null;
526 }
527 XContentBuilder builder = jsonBuilder().startObject();
528 builder = serialize(record, builder, handler);
529 builder.endObject();
530 return builder;
531 }
532
533
534
535
536
537
538
539 private static Object deserializeSingleObject(Object value, String propName, MappingHandler mapper,
540 URIHandler handler) throws IOException {
541 Class<?> valueClass = mapper.getValueClass(propName);
542 if (valueClass.equals(Record.class)) {
543 return deserialize((Map) value, Record.create(), mapper, handler);
544 }
545 if (valueClass.equals(byte[].class)) {
546 String res;
547
548 byte[] valueByte = Base64.getDecoder().decode(((String) value));
549 res = new String(valueByte, Charsets.UTF_8);
550
551
552
553
554
555
556
557
558 return Utility.stringToLiteral(res);
559 }
560 if (valueClass.equals(String.class)) {
561 return Utility.decodeString((String) value, handler);
562 }
563
564 return Data.convert(value, valueClass);
565 }
566
567
568
569
570
571
572
573 private static Collection<?> deserializeCollection(Collection<?> values, String propName, MappingHandler mapper,
574 URIHandler handler) throws IOException {
575 ArrayList<Object> res = new ArrayList<>();
576 for (Object obj : values) {
577 res.add(deserializeSingleObject(obj, propName, mapper, handler));
578 }
579 return res;
580 }
581
582
583
584
585
586
587
588
589 private static Record deserialize(Map<String, Object> source, Record res, MappingHandler mapper, URIHandler handler)
590 throws IOException {
591 if (source == null) {
592 return res;
593 }
594
595 for (String propStr : source.keySet()) {
596 if (propStr.equals(ID_PROPERTY_NAME)) {
597 res.setID((URI) decodeString((String) source.get(ID_PROPERTY_NAME), handler));
598 } else {
599 URI propName = handler.decode(propStr);
600 Object value = source.get(propStr);
601 if (value instanceof Collection) {
602 res.add(propName, deserializeCollection((Collection) value, propStr, mapper, handler));
603 } else {
604 res.add(propName, deserializeSingleObject(value, propStr, mapper, handler));
605 }
606 }
607 }
608 return res;
609 }
610
611
612
613
614
615
616
617
618 public static Record deserialize(GetResponse response, MappingHandler mapper, URIHandler handler)
619 throws IOException {
620 if (response == null) {
621 LOGGER.debug("null GetReponse");
622 return null;
623 }
624 if (!response.isExists()) {
625 LOGGER.trace("response not exists");
626 return null;
627 }
628
629 Record res = Record.create();
630 res.setID(handler.decode(response.getId()));
631
632 if (response.isSourceEmpty()) {
633 return res;
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652 }
653
654 return deserialize(response.getSource(), res, mapper, handler);
655 }
656
657
658
659
660
661
662
663
664 public static Record deserialize(SearchHit response, MappingHandler mapper, URIHandler handler) throws IOException {
665 if (response == null) {
666 return null;
667 }
668 Record res = Record.create();
669 res.setID(handler.decode(response.getId()));
670 if (response.isSourceEmpty()) {
671 throw new UnsupportedOperationException("deserialization of projected object not supported");
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688 }
689
690 return deserialize(response.getSource(), res, mapper, handler);
691 }
692
693
694
695
696
697
698
699 protected static String literalToString(Literal obj) {
700
701 return Data.toString(obj, null);
702 }
703
704
705
706
707
708
709
710 protected static Literal stringToLiteral(String s) throws IOException {
711 return (Literal) Data.parseValue(s, null);
712 }
713 }