forked from D-Net/dnet-hadoop
Compare commits
1721 Commits
deduptesti
...
beta
Author | SHA1 | Date |
---|---|---|
Andreas Czerniak | 1622b40842 | |
Andreas Czerniak | 854da6c0e7 | |
Andreas Czerniak | 2d0a4d5b85 | |
Andreas Czerniak | c777b50995 | |
Claudio Atzori | 97ad94d7d9 | |
Sandro La Bruzzo | 7a3819144d | |
Claudio Atzori | f0ea2410e5 | |
Claudio Atzori | b37bc277c4 | |
Miriam Baglioni | 42e8f76778 | |
Miriam Baglioni | a7c4d0d16d | |
Miriam Baglioni | 7bf12ad24a | |
Miriam Baglioni | a75fb8c47a | |
Miriam Baglioni | 4d517ed9ec | |
Miriam Baglioni | e7d5a39c03 | |
Claudio Atzori | dbd6fa1d65 | |
Miriam Baglioni | 4993666d73 | |
Sandro La Bruzzo | 1b9e8378b3 | |
Sandro La Bruzzo | 57e2c4b749 | |
Sandro La Bruzzo | b78d2b71f0 | |
Claudio Atzori | 0f2144b5e0 | |
Claudio Atzori | dcd282977c | |
Claudio Atzori | 4f212652ca | |
Sandro La Bruzzo | 0163dadb7f | |
Miriam Baglioni | 904e1c2667 | |
Miriam Baglioni | 064f9bbd87 | |
Miriam Baglioni | 93f26fb742 | |
Miriam Baglioni | b7e450070b | |
Miriam Baglioni | af8a33638d | |
Miriam Baglioni | 639190370a | |
Miriam Baglioni | adccc2346a | |
Claudio Atzori | 908294d86e | |
Claudio Atzori | 58f8998e3d | |
Claudio Atzori | 174c3037e1 | |
Claudio Atzori | 045d767013 | |
Claudio Atzori | cb30770a0b | |
Claudio Atzori | bd59b58efb | |
Claudio Atzori | a6977197b3 | |
Miriam Baglioni | 4c60ee1718 | |
Miriam Baglioni | 92fd69e25d | |
Claudio Atzori | fe7e5f4748 | |
Claudio Atzori | bcea4e3a9b | |
Miriam Baglioni | a706ba0c08 | |
Antonis Lempesis | 81ee654271 | |
Antonis Lempesis | 7551e52e95 | |
Miriam Baglioni | 7a1b440413 | |
Claudio Atzori | 278cf08421 | |
Claudio Atzori | cccb16900c | |
Miriam Baglioni | 2a67ee13ec | |
Miriam Baglioni | 5c4fee3533 | |
Miriam Baglioni | 69e9ea9eeb | |
Miriam Baglioni | 31b26d48ac | |
Miriam Baglioni | bf3a9505e0 | |
Miriam Baglioni | 10579c0dd0 | |
Miriam Baglioni | 6116fc5d40 | |
Miriam Baglioni | b81efb6a9d | |
Miriam Baglioni | 73175ba086 | |
Miriam Baglioni | de6c4c8968 | |
Miriam Baglioni | b352fbe453 | |
Miriam Baglioni | 34ac56565d | |
Miriam Baglioni | 20ef1d657f | |
Miriam Baglioni | 813f856d3f | |
Miriam Baglioni | 2c126ed014 | |
Miriam Baglioni | bf52a1847b | |
Miriam Baglioni | 0807fdb65a | |
Miriam Baglioni | b5e11a3a0a | |
Miriam Baglioni | c5739c4266 | |
Miriam Baglioni | da5f6260aa | |
Miriam Baglioni | 4849270c55 | |
Claudio Atzori | 8d18500069 | |
Miriam Baglioni | 9d19b057b8 | |
Miriam Baglioni | be0acccf42 | |
Miriam Baglioni | 89ea9fa0e1 | |
Antonis Lempesis | 16539d7360 | |
Antonis Lempesis | 3edd661608 | |
Antonis Lempesis | a4c0cbb98c | |
Miriam Baglioni | e24a7f3496 | |
Miriam Baglioni | d1ae219cb4 | |
Miriam Baglioni | 460e6b95d6 | |
Sandro La Bruzzo | 3920d68992 | |
Antonis Lempesis | 58996972d9 | |
dimitrispie | c1cdec09a9 | |
Miriam Baglioni | 3cc1b7b153 | |
Miriam Baglioni | 5e5dfd619c | |
Miriam Baglioni | 63b648b0dd | |
Sandro La Bruzzo | b881ee5ef8 | |
Sandro La Bruzzo | 63952018c0 | |
Sandro La Bruzzo | e5bff64f2e | |
Claudio Atzori | e30e5ac8a8 | |
Claudio Atzori | 1790fa2d44 | |
Miriam Baglioni | 56409d1281 | |
Miriam Baglioni | a3592b463a | |
Miriam Baglioni | 22d4b5619b | |
Miriam Baglioni | 6fb6236cd4 | |
Claudio Atzori | aff3ddc8d2 | |
Miriam Baglioni | 573bd17cbb | |
Miriam Baglioni | 4eb8276493 | |
Miriam Baglioni | 936578aaf1 | |
Miriam Baglioni | 8d755cca80 | |
Claudio Atzori | 98eb292c59 | |
Claudio Atzori | 5e17247bb6 | |
Claudio Atzori | b70ecccea0 | |
Claudio Atzori | 25dc7929a9 | |
Claudio Atzori | eb43eda42a | |
Claudio Atzori | 41c70c607d | |
Alessia Bardi | 8f1e018ceb | |
Alessia Bardi | cba63e9f82 | |
Alessia Bardi | e53228401b | |
Claudio Atzori | adf17452b0 | |
Claudio Atzori | e6e177dda0 | |
Alessia Bardi | 6b5d7688a4 | |
Miriam Baglioni | b113586207 | |
Sandro La Bruzzo | 5d51b3dd4a | |
Miriam Baglioni | d9836f0cf3 | |
Miriam Baglioni | d1df01ff1e | |
Sandro La Bruzzo | ed0c352799 | |
Miriam Baglioni | 96a7d46278 | |
Sandro La Bruzzo | e9f285ec4d | |
Sandro La Bruzzo | bf880e2508 | |
Sandro La Bruzzo | 81bf604059 | |
Sandro La Bruzzo | 7af0bbd0b1 | |
Claudio Atzori | 9132727793 | |
Claudio Atzori | 08795cbd30 | |
Miriam Baglioni | f430688ff7 | |
Miriam Baglioni | 4bb1d43afc | |
Sandro La Bruzzo | 0fa0ce33d6 | |
Sandro La Bruzzo | f7011b90d8 | |
Claudio Atzori | 372633880f | |
Claudio Atzori | dd0b2e5244 | |
Claudio Atzori | c4c705aa46 | |
Claudio Atzori | 863a2f9db3 | |
Claudio Atzori | 9cac283bec | |
Miriam Baglioni | d9f80488cc | |
Miriam Baglioni | 58bc3f223a | |
Miriam Baglioni | 8905a39bf3 | |
Miriam Baglioni | 87eedad898 | |
Claudio Atzori | 3b19821f3c | |
Claudio Atzori | cfa4560769 | |
Claudio Atzori | d85af6fc25 | |
Claudio Atzori | 4fe7888817 | |
Claudio Atzori | 01e5e0142a | |
Claudio Atzori | 0df9574a6f | |
Claudio Atzori | 014e872ae1 | |
Claudio Atzori | 5c6d328537 | |
dimitrispie | 09fc2afdca | |
dimitrispie | 8750a71502 | |
dimitrispie | 25fc8abf77 | |
Antonis Lempesis | 0b4163ee0b | |
dimitrispie | 29f69f2f89 | |
Sandro La Bruzzo | bb7f556eff | |
Sandro La Bruzzo | 1e1f5e4fe0 | |
Miriam Baglioni | ac07ed8251 | |
Miriam Baglioni | 5fd0e610bf | |
Sandro La Bruzzo | feea154e89 | |
Sandro La Bruzzo | 028a8acad8 | |
Sandro La Bruzzo | 2164a2a889 | |
Sandro La Bruzzo | a7cf277d98 | |
Sandro La Bruzzo | 483d3039d1 | |
Sandro La Bruzzo | 93fe8ce8b2 | |
Sandro La Bruzzo | 35e20b0647 | |
Miriam Baglioni | fdb75b180e | |
Miriam Baglioni | 0506fa2654 | |
Sandro La Bruzzo | 6110a2b984 | |
Sandro La Bruzzo | 65ebe1019b | |
Sandro La Bruzzo | 155d8bf83f | |
Sandro La Bruzzo | 3426451d3f | |
Sandro La Bruzzo | 75298ec442 | |
Sandro La Bruzzo | 4542a2338b | |
Claudio Atzori | 90c2a4987e | |
Claudio Atzori | e5a2c596b2 | |
Claudio Atzori | f4538f3c4c | |
Claudio Atzori | 2b46b87f56 | |
Miriam Baglioni | 9fae872181 | |
Sandro La Bruzzo | fc03c99805 | |
Sandro La Bruzzo | 8a7c7d36db | |
Sandro La Bruzzo | 0c0d561bc4 | |
Claudio Atzori | 62fa61f3cf | |
Claudio Atzori | bd9a43cefd | |
Claudio Atzori | 3974fa7dc1 | |
Claudio Atzori | a24b9f8268 | |
Claudio Atzori | c0750fb17c | |
Claudio Atzori | bb5dca7979 | |
Miriam Baglioni | 793b5a8e5f | |
Miriam Baglioni | 5dc5792722 | |
Miriam Baglioni | 0136a8c266 | |
Miriam Baglioni | 1b79c0ee79 | |
Claudio Atzori | 10a32f287f | |
Antonis Lempesis | cb3adb90f4 | |
Antonis Lempesis | c283406829 | |
Claudio Atzori | e0395719d7 | |
Claudio Atzori | 82a4e4efae | |
Miriam Baglioni | 6d4a1c57ee | |
Sandro La Bruzzo | 9c82d670b8 | |
Sandro La Bruzzo | 1f5ee116ed | |
Sandro La Bruzzo | 2fd9ceac13 | |
Sandro La Bruzzo | 60ae874dcb | |
Sandro La Bruzzo | 2506d7a679 | |
Sandro La Bruzzo | cded363b55 | |
Miriam Baglioni | 4094f2bb9a | |
Miriam Baglioni | ec8b0219ff | |
Miriam Baglioni | 2bbece2ca5 | |
Sandro La Bruzzo | 2d67020c59 | |
Claudio Atzori | 49f897ef29 | |
Miriam Baglioni | 28ea532ece | |
Sandro La Bruzzo | 18c1d70ef4 | |
Sandro La Bruzzo | a1cafaf2e3 | |
Miriam Baglioni | 7c96e3fd46 | |
Miriam Baglioni | c7c0c3187b | |
Miriam Baglioni | c6a9f0a1a8 | |
Miriam Baglioni | 99d86134f5 | |
Claudio Atzori | 0a727d325d | |
Claudio Atzori | bafa2990f3 | |
Claudio Atzori | 668ac25224 | |
Claudio Atzori | 7d0a03f607 | |
Claudio Atzori | 941a50a2fc | |
Claudio Atzori | 7c804acda8 | |
Sandro La Bruzzo | efa09057db | |
Sandro La Bruzzo | 48923e46a1 | |
Claudio Atzori | d2c787d416 | |
Claudio Atzori | 975b10b711 | |
Claudio Atzori | 1ecceea788 | |
Miriam Baglioni | 4ec88c718c | |
Miriam Baglioni | 6f1a434e90 | |
Miriam Baglioni | 157d33ebf9 | |
Claudio Atzori | 7b81607035 | |
Miriam Baglioni | 6595135a1a | |
Miriam Baglioni | 43cae4ad88 | |
Miriam Baglioni | b3f9370125 | |
Miriam Baglioni | 92d0e18b55 | |
Miriam Baglioni | 881113743f | |
Miriam Baglioni | 47ccb53c4f | |
Miriam Baglioni | ffb0ce1d59 | |
Miriam Baglioni | 716021546e | |
Claudio Atzori | 1f2a3d1af0 | |
Sandro La Bruzzo | 3469cc2b1d | |
Sandro La Bruzzo | a7763d2492 | |
Miriam Baglioni | b8bdabfae9 | |
Miriam Baglioni | e5498052e8 | |
Miriam Baglioni | 935062edec | |
Antonis Lempesis | 26f086dd64 | |
Claudio Atzori | 8bdca3413f | |
Claudio Atzori | 148289150f | |
Sandro La Bruzzo | 2ca0a436ad | |
Sandro La Bruzzo | 9cb195314f | |
Miriam Baglioni | 6d3c4c4abe | |
Miriam Baglioni | 8cc50ecee0 | |
Miriam Baglioni | 88b73f4f49 | |
Miriam Baglioni | c371b23077 | |
Miriam Baglioni | 9e214ce0eb | |
Sandro La Bruzzo | 6477a40670 | |
Miriam Baglioni | 6f7ca539c6 | |
Miriam Baglioni | a7d50c499b | |
Antonis Lempesis | 91354c6068 | |
Miriam Baglioni | 94918a673c | |
Miriam Baglioni | 4c70201412 | |
Miriam Baglioni | 8442efd8d1 | |
Claudio Atzori | 5681e89544 | |
Miriam Baglioni | a22c29fba1 | |
Miriam Baglioni | c10ff6928c | |
Miriam Baglioni | 0857849a86 | |
Miriam Baglioni | df7ee77c7a | |
Miriam Baglioni | de63d29b6f | |
Miriam Baglioni | d50057b2d9 | |
Miriam Baglioni | edf55395e9 | |
Miriam Baglioni | d97ea82a29 | |
Miriam Baglioni | 96769b4481 | |
Miriam Baglioni | 683fe093cf | |
Miriam Baglioni | b2bb8d9d79 | |
Miriam Baglioni | 779318961c | |
Miriam Baglioni | 2480e590d1 | |
Miriam Baglioni | b9d124bb7c | |
Sandro La Bruzzo | 7bd224f051 | |
Claudio Atzori | 7fa49f6956 | |
Antonis Lempesis | f78afb5ef9 | |
Miriam Baglioni | 2aca6bfa0a | |
Miriam Baglioni | 09f36cffb8 | |
Claudio Atzori | 1225ba0b92 | |
Sandro La Bruzzo | d9cbca83f7 | |
Sandro La Bruzzo | 1be9aa0a5f | |
Sandro La Bruzzo | 4acfa8fa2e | |
Miriam Baglioni | d0ef7d91c5 | |
Sandro La Bruzzo | aafdffa6b3 | |
Sandro La Bruzzo | 034304b33a | |
Claudio Atzori | 6b34ba737e | |
Claudio Atzori | d147295c2f | |
Claudio Atzori | 3702fe478d | |
Sandro La Bruzzo | ac36aa7d1c | |
Sandro La Bruzzo | aeeebd573b | |
Sandro La Bruzzo | ab3a99d3e9 | |
Sandro La Bruzzo | ae4e99a471 | |
Claudio Atzori | 4f8970f8ed | |
Claudio Atzori | 00b78b9c58 | |
Claudio Atzori | c01dd0c925 | |
Miriam Baglioni | 652114c641 | |
Claudio Atzori | d0cf2963f0 | |
Claudio Atzori | 59f76b50d4 | |
Claudio Atzori | bc3372093e | |
Antonis Lempesis | 241dcf6df1 | |
Claudio Atzori | 515e068a78 | |
Claudio Atzori | 512e7b0170 | |
Claudio Atzori | d517c71458 | |
Claudio Atzori | e9157c67aa | |
Claudio Atzori | 98f37c8d81 | |
Claudio Atzori | c8850456e9 | |
Sandro La Bruzzo | c9870c5122 | |
Sandro La Bruzzo | f8329bc110 | |
Claudio Atzori | 7a73010acd | |
Miriam Baglioni | c7f6cd2591 | |
Sandro La Bruzzo | a894d7adf3 | |
miconis | 5f780a6ba1 | |
Miriam Baglioni | 1315952702 | |
Miriam Baglioni | 1cc09adfaa | |
Miriam Baglioni | 76d41602be | |
Miriam Baglioni | 46f82c7c8f | |
Sandro La Bruzzo | 7b15b88d4c | |
Antonis Lempesis | 41ecb1eb61 | |
Antonis Lempesis | 4b7c8dff2d | |
Sandro La Bruzzo | 51a03c0a50 | |
Claudio Atzori | dd568ec88b | |
Claudio Atzori | 14fbf92ad6 | |
Claudio Atzori | b292e4a700 | |
miconis | 995c1eddaf | |
Miriam Baglioni | 5d9cc2452d | |
miconis | 326bf63775 | |
Miriam Baglioni | 16b28494a9 | |
Miriam Baglioni | 63933808d4 | |
Sandro La Bruzzo | f2c8356ccf | |
Sandro La Bruzzo | 7387416e90 | |
Claudio Atzori | 914b3e92cb | |
Sandro La Bruzzo | 511da98d0c | |
Miriam Baglioni | fec40bdd95 | |
Miriam Baglioni | 83f51f1812 | |
Sandro La Bruzzo | 5606014b17 | |
Claudio Atzori | 2f61054cd1 | |
Claudio Atzori | 83c90c7180 | |
Serafeim Chatzopoulos | 201ce71cc1 | |
Serafeim Chatzopoulos | e468a7b96b | |
Serafeim Chatzopoulos | de81007302 | |
Sandro La Bruzzo | 8f99d2af86 | |
Alessia Bardi | c48c43fa9e | |
Alessia Bardi | 8d3b60f446 | |
miconis | 611ca511db | |
miconis | 9646b9fd98 | |
Sandro La Bruzzo | 2557bb41f5 | |
Sandro La Bruzzo | b84e0cabeb | |
Sandro La Bruzzo | f258bbb927 | |
Sandro La Bruzzo | 991b06bd0b | |
Claudio Atzori | cb7efe12ac | |
Miriam Baglioni | e653756e3d | |
dimitrispie | 3f25d2efb2 | |
dimitrispie | 13687fd887 | |
Miriam Baglioni | 9814c3e700 | |
Miriam Baglioni | c4ccd7b32c | |
Miriam Baglioni | c8321ad31a | |
Claudio Atzori | 60a6a9a583 | |
Sandro La Bruzzo | 66702b1973 | |
Sandro La Bruzzo | 477cb10715 | |
Sandro La Bruzzo | be79d74e3d | |
Claudio Atzori | 35619b93ee | |
Claudio Atzori | 474117c2e8 | |
Miriam Baglioni | 476a4708d6 | |
Miriam Baglioni | 5ec69889db | |
Claudio Atzori | a53acfbc06 | |
Alessia Bardi | b924276e18 | |
Antonis Lempesis | a1e1cf32d7 | |
Antonis Lempesis | f358cabb2b | |
Miriam Baglioni | eedf7c3310 | |
Miriam Baglioni | f2118d771a | |
Claudio Atzori | df15a4dc9f | |
Claudio Atzori | 7fa60e166e | |
Antonis Lempesis | 421d55265d | |
Enrico Ottonello | 92a63f78fe | |
Enrico Ottonello | 0c74f5667e | |
miconis | 853333bdde | |
Antonis Lempesis | 8b681dcf1b | |
Claudio Atzori | 71cfa386bc | |
Antonis Lempesis | 2943287d10 | |
Antonis Lempesis | dd2329849f | |
Claudio Atzori | 09c2eb7f62 | |
Claudio Atzori | 954a16c213 | |
Miriam Baglioni | e9ccdf853f | |
Claudio Atzori | 12766bf5f2 | |
Claudio Atzori | 663b1556d7 | |
Claudio Atzori | ebf53a1616 | |
Enrico Ottonello | 8b804e7fe1 | |
Enrico Ottonello | aefa36c54b | |
Antonis Lempesis | de9bf3a161 | |
Antonis Lempesis | 9b1936701c | |
Antonis Lempesis | 8fc89ae822 | |
Antonis Lempesis | 461bf90ca6 | |
Antonis Lempesis | 43852bac0e | |
Antonis Lempesis | f13cca7e83 | |
Antonis Lempesis | c6ada217a1 | |
Antonis Lempesis | 1250ae197f | |
Antonis Lempesis | ccee451dde | |
Sandro La Bruzzo | aed29156c7 | |
Sandro La Bruzzo | 370dddb2fa | |
Sandro La Bruzzo | 3c6fc2096c | |
Sandro La Bruzzo | d4dadf6d77 | |
Sandro La Bruzzo | 9f8a80deb7 | |
Alessia Bardi | 3762b17f7b | |
Sandro La Bruzzo | e8b3cb9147 | |
Alessia Bardi | ccf4103a25 | |
Sandro La Bruzzo | 45898c71ac | |
Alessia Bardi | 00a28c0080 | |
Alessia Bardi | f19b04d41b | |
Alessia Bardi | 412d2cb16a | |
Alessia Bardi | 3bcac7e88c | |
Alessia Bardi | 931f430129 | |
Alessia Bardi | 4c1474e693 | |
Miriam Baglioni | 5f8ccbc365 | |
Miriam Baglioni | 882abb40e4 | |
Miriam Baglioni | 45c62609af | |
Miriam Baglioni | 35880c0e7b | |
Miriam Baglioni | f3b6c392c1 | |
Miriam Baglioni | 65822400ce | |
Alessia Bardi | a053e1513c | |
Alessia Bardi | 812bd54c57 | |
Miriam Baglioni | a65d3caaea | |
Miriam Baglioni | e5cf11d088 | |
Claudio Atzori | 7c0c67bdd6 | |
Claudio Atzori | 82086f3422 | |
Claudio Atzori | bc7068106c | |
Claudio Atzori | 2c0a05f11a | |
Claudio Atzori | d43667d857 | |
Miriam Baglioni | 5856ca8a7b | |
Miriam Baglioni | 6fec71e8d2 | |
Miriam Baglioni | ed7e28490a | |
Claudio Atzori | 7743d0f919 | |
Miriam Baglioni | 6eb7508995 | |
Claudio Atzori | f74adc4752 | |
Claudio Atzori | 5f0903d50d | |
Claudio Atzori | 17cefe6a97 | |
Claudio Atzori | 7ee2757fcd | |
Claudio Atzori | c3ad4ab701 | |
Claudio Atzori | baed5e3337 | |
Claudio Atzori | 3359f73fcf | |
Claudio Atzori | 4e6575a428 | |
Miriam Baglioni | f4ec81c92c | |
Miriam Baglioni | dc8b05b39e | |
Miriam Baglioni | 32fd75691f | |
Miriam Baglioni | dfd1e53c69 | |
Miriam Baglioni | 01db1f8bc4 | |
Miriam Baglioni | 964a46ca21 | |
Miriam Baglioni | eaf077fc34 | |
Miriam Baglioni | 5f674efb0c | |
Miriam Baglioni | 5cd5714530 | |
Miriam Baglioni | 58f241f4a2 | |
Miriam Baglioni | f3d575f749 | |
Miriam Baglioni | a5f6edfa6c | |
Miriam Baglioni | ed183d878e | |
Miriam Baglioni | 8769dd8eef | |
Miriam Baglioni | 6b9e1bf2e3 | |
Miriam Baglioni | d57b2bb927 | |
Miriam Baglioni | 9da74b544a | |
Miriam Baglioni | ab8abd61bb | |
Miriam Baglioni | 335a824e34 | |
Miriam Baglioni | f0845e9865 | |
Miriam Baglioni | 7a789423aa | |
Miriam Baglioni | e9fc3ef3bc | |
Miriam Baglioni | 4317211a2b | |
Miriam Baglioni | b62cd656a7 | |
Miriam Baglioni | d36e925277 | |
Miriam Baglioni | 7402daf51a | |
Miriam Baglioni | 733bcaecf6 | |
Miriam Baglioni | bfe8f5335c | |
Miriam Baglioni | 6e84b3951f | |
Claudio Atzori | e91ffcd2f3 | |
Claudio Atzori | 9587d4aee8 | |
Claudio Atzori | 86d940044c | |
Claudio Atzori | 8cdce59e0e | |
Miriam Baglioni | 08dd2b2102 | |
Miriam Baglioni | ac417ca798 | |
Miriam Baglioni | e33daaeee8 | |
Miriam Baglioni | 9650eea497 | |
Miriam Baglioni | 785db1d5b2 | |
Miriam Baglioni | 95e5482bbb | |
Miriam Baglioni | cc3d72df0e | |
Miriam Baglioni | b966329833 | |
Miriam Baglioni | 8ad7c71417 | |
Miriam Baglioni | 0e1a6bec20 | |
Miriam Baglioni | c6a2a780a9 | |
Miriam Baglioni | b6b58bba28 | |
Miriam Baglioni | 804589eb30 | |
Miriam Baglioni | d688749ad9 | |
Miriam Baglioni | 524c06e028 | |
Miriam Baglioni | 7aa3260729 | |
Miriam Baglioni | 55fc500d8d | |
Miriam Baglioni | f9b6b45d85 | |
Miriam Baglioni | 8229632839 | |
Miriam Baglioni | b1c6140ebf | |
Miriam Baglioni | 52c18c2697 | |
Miriam Baglioni | 8da3a25cf6 | |
Claudio Atzori | 9f4db73f30 | |
Claudio Atzori | 61d811ba53 | |
Claudio Atzori | 2ee21da43b | |
Miriam Baglioni | b954fe9ba8 | |
Miriam Baglioni | b688567db5 | |
Miriam Baglioni | 9731a6144a | |
Miriam Baglioni | a90bac3bc9 | |
Miriam Baglioni | bd0d7bfba7 | |
Miriam Baglioni | 8daaa32e90 | |
Miriam Baglioni | bc9e3a06ba | |
Claudio Atzori | d64a942a76 | |
Miriam Baglioni | 2efa5abda5 | |
Claudio Atzori | 577f3b1ac8 | |
Miriam Baglioni | da20fceaf7 | |
Claudio Atzori | 964f97ed4d | |
Miriam Baglioni | 54a6cbb244 | |
Miriam Baglioni | b7079804cb | |
Miriam Baglioni | a5f82f442b | |
Miriam Baglioni | b6dcf89d22 | |
Miriam Baglioni | eff499af9f | |
Claudio Atzori | a45b95ccc1 | |
Miriam Baglioni | 5d70f842eb | |
Miriam Baglioni | c3931557e3 | |
Claudio Atzori | 66f398fe6f | |
Miriam Baglioni | 6bd1eca7e0 | |
Miriam Baglioni | 73dc082927 | |
Miriam Baglioni | ee13da9258 | |
Miriam Baglioni | bd096f5170 | |
Miriam Baglioni | 5faeefbda8 | |
Miriam Baglioni | 1965e4eece | |
Claudio Atzori | 83c04e5d28 | |
Miriam Baglioni | b4eb026c8b | |
Miriam Baglioni | c7b71647c6 | |
Miriam Baglioni | eb8c3f8594 | |
Miriam Baglioni | e94ae0b1de | |
Miriam Baglioni | 67ba4c40e0 | |
Miriam Baglioni | eccf3851b0 | |
Sandro La Bruzzo | 74afe43c3a | |
Miriam Baglioni | 1e952cccf6 | |
Miriam Baglioni | 8ba8c77f92 | |
Miriam Baglioni | 8f7623e77a | |
Sandro La Bruzzo | 3fc820203b | |
Miriam Baglioni | a7bf314fd2 | |
Miriam Baglioni | 9831725073 | |
Miriam Baglioni | 100e54e6c8 | |
Miriam Baglioni | 461b8a29a0 | |
Miriam Baglioni | 327cddde33 | |
Miriam Baglioni | 17292c6641 | |
Miriam Baglioni | ee7ccb98dc | |
Miriam Baglioni | 90e91486e2 | |
Miriam Baglioni | 1e859706a3 | |
Miriam Baglioni | 72df8f9232 | |
Miriam Baglioni | ff1ce75e33 | |
Claudio Atzori | e826aae848 | |
Claudio Atzori | fd55c77d97 | |
Antonis Lempesis | 117c3d5c67 | |
Miriam Baglioni | 1695d45bd4 | |
Miriam Baglioni | 7c6ea2f4c7 | |
Miriam Baglioni | d8b9b0553b | |
Miriam Baglioni | 613bd3bde0 | |
Miriam Baglioni | d1807781c0 | |
Miriam Baglioni | 1d6ac3715b | |
Claudio Atzori | e244f73165 | |
Claudio Atzori | 11e26c020a | |
Claudio Atzori | 19620eed46 | |
Claudio Atzori | 5219d56be5 | |
Claudio Atzori | 4f78565c04 | |
Claudio Atzori | a6a38cca9e | |
Miriam Baglioni | 9bc4fd3b69 | |
Miriam Baglioni | 2fc89fc9b5 | |
Claudio Atzori | 081fe92a21 | |
Claudio Atzori | 576693d782 | |
Claudio Atzori | 55e6470f44 | |
Sandro La Bruzzo | 6358f92c3a | |
Antonis Lempesis | 26af0320d0 | |
Claudio Atzori | 7b172e7cd9 | |
Claudio Atzori | c53d106e80 | |
Claudio Atzori | 6e3554a45e | |
Sandro La Bruzzo | b1b0cc3f15 | |
Miriam Baglioni | baad01cadc | |
Claudio Atzori | e725c88ebb | |
Claudio Atzori | 5d08ad86ae | |
Claudio Atzori | e87e1805c4 | |
Claudio Atzori | f83dd70e1c | |
Claudio Atzori | 5f7330d407 | |
Claudio Atzori | 1923c1ce21 | |
Claudio Atzori | dc55ed4acd | |
Claudio Atzori | 908f57a475 | |
Sandro La Bruzzo | 3721df7aa6 | |
Michele Artini | 6aef3e8f46 | |
Antonis Lempesis | 4afa5215a9 | |
Antonis Lempesis | 3d1580fa9b | |
Claudio Atzori | 4c5a71ba2f | |
Claudio Atzori | a9961a1835 | |
Claudio Atzori | e1797c0a42 | |
Claudio Atzori | 6dddad86ee | |
Sandro La Bruzzo | 3d8f0f629b | |
Antonis Lempesis | 9b181ffa73 | |
Alessia Bardi | df8715a1ec | |
Michele Artini | 3e2a2d6e71 | |
Alessia Bardi | c806387d4b | |
Alessia Bardi | 9594343725 | |
Claudio Atzori | 2fff24df55 | |
Michele Artini | 9f1c7b8e17 | |
Claudio Atzori | b346feed36 | |
Antonis Lempesis | 4a9741825d | |
Miriam Baglioni | 3d2bba3d5d | |
Miriam Baglioni | cc0d3d8a7b | |
Michele Artini | e6f1773d63 | |
Miriam Baglioni | 80d5b3b4de | |
Miriam Baglioni | 5fe016dcbc | |
Miriam Baglioni | 73ed7374a9 | |
Miriam Baglioni | 43e62fcae9 | |
Michele Artini | c72c960ffb | |
Michele Artini | 1fb572a33a | |
Miriam Baglioni | 708d0ade34 | |
Sandro La Bruzzo | 16c91203bd | |
Miriam Baglioni | 6c936943aa | |
Miriam Baglioni | 0424f47494 | |
Michele Artini | 52e2315ba2 | |
Claudio Atzori | d267dce520 | |
Sandro La Bruzzo | 825d9f0289 | |
Claudio Atzori | 5aa7d16d1b | |
Claudio Atzori | 998b66855a | |
Antonis Lempesis | 1a28a69cac | |
Miriam Baglioni | 74f801b689 | |
Miriam Baglioni | 35e395eae8 | |
Miriam Baglioni | eb07f7f40f | |
Antonis Lempesis | ed185fd7ed | |
Antonis Lempesis | f3b9570354 | |
Sandro La Bruzzo | 848aabbb6c | |
Sandro La Bruzzo | 8fac10c91e | |
Sandro La Bruzzo | 3920c69bc8 | |
Antonis Lempesis | f9fbb0f261 | |
Claudio Atzori | a0393607a7 | |
Claudio Atzori | 5b6844b969 | |
Sandro La Bruzzo | d9e3b89937 | |
Sandro La Bruzzo | cfde63a7c3 | |
Sandro La Bruzzo | 4a439c3863 | |
Claudio Atzori | bc835d2024 | |
Claudio Atzori | ffdb2a3ea3 | |
Sandro La Bruzzo | ca74e8dd02 | |
Sandro La Bruzzo | 43e9380cd3 | |
Sandro La Bruzzo | 058b636d4d | |
Sandro La Bruzzo | 62ae36a3d2 | |
Miriam Baglioni | 63553a76b3 | |
Miriam Baglioni | 1a5b114906 | |
Sandro La Bruzzo | d94565862a | |
Sandro La Bruzzo | 31d2d6d41e | |
Miriam Baglioni | b226ba4439 | |
Alessia Bardi | 9069958479 | |
Claudio Atzori | 10d7b4f0b4 | |
Claudio Atzori | 77e8c6c7f7 | |
Miriam Baglioni | 83fe31c92e | |
Miriam Baglioni | dd81c36b60 | |
Miriam Baglioni | 54acc5373b | |
Miriam Baglioni | b420b11ed3 | |
Claudio Atzori | 65934888a1 | |
Claudio Atzori | 5947cddafc | |
Claudio Atzori | 0977baf41d | |
Miriam Baglioni | 13cf444f85 | |
Claudio Atzori | 5e5f65a3c3 | |
Miriam Baglioni | 662c396354 | |
Miriam Baglioni | 59530a14fb | |
Miriam Baglioni | 199123b74b | |
Miriam Baglioni | c4b18e6ccb | |
Miriam Baglioni | acd6056330 | |
Miriam Baglioni | 3bc9a05bc9 | |
Miriam Baglioni | 34506df1b6 | |
Claudio Atzori | bf9e0d2d4f | |
Claudio Atzori | 9913b6073c | |
Sandro La Bruzzo | 7e2caafe84 | |
Enrico Ottonello | 2dc50c0999 | |
Enrico Ottonello | 66604bb2b4 | |
Enrico Ottonello | 7840cc6526 | |
Miriam Baglioni | 4da46bb62f | |
Enrico Ottonello | a65667d217 | |
Sandro La Bruzzo | 10068c00ea | |
Miriam Baglioni | 09ad7b2a9e | |
Miriam Baglioni | f4f7c6f9d3 | |
Miriam Baglioni | 6222adf176 | |
Miriam Baglioni | 981b1018f6 | |
Sandro La Bruzzo | 3d8e2aa146 | |
Miriam Baglioni | 441701c85c | |
Sandro La Bruzzo | c35c117601 | |
Miriam Baglioni | 1cdd09cd8e | |
Sandro La Bruzzo | 4cb65bc64a | |
Miriam Baglioni | 774cdb190e | |
Miriam Baglioni | 886617afd0 | |
Miriam Baglioni | 320cf02d96 | |
Miriam Baglioni | 52ce35d57b | |
Miriam Baglioni | 970b387b8d | |
Miriam Baglioni | eae10c5894 | |
Miriam Baglioni | c028feef4f | |
Miriam Baglioni | d70f8c96fd | |
Miriam Baglioni | 5e38c7f42d | |
Claudio Atzori | 734de62474 | |
Miriam Baglioni | d418c309f5 | |
Miriam Baglioni | 618d2de2da | |
Miriam Baglioni | 59615da65e | |
Miriam Baglioni | 084b4ef999 | |
Claudio Atzori | fa720c1da4 | |
Miriam Baglioni | 8f322a73cb | |
Miriam Baglioni | 72397ea1ba | |
Miriam Baglioni | 5295d10691 | |
Claudio Atzori | 9629569e22 | |
Claudio Atzori | f13e11e3f7 | |
Miriam Baglioni | e9a17ec899 | |
Miriam Baglioni | 8429aed6c6 | |
Miriam Baglioni | 39b1a6edf6 | |
Miriam Baglioni | 9a58f1b93d | |
Miriam Baglioni | 13c66e16be | |
Miriam Baglioni | 6410ab71d8 | |
Miriam Baglioni | 65a242646d | |
Miriam Baglioni | 4b432fbee8 | |
Miriam Baglioni | 87a6e2b967 | |
Miriam Baglioni | 69fd40fd30 | |
Miriam Baglioni | 86e50f7311 | |
Miriam Baglioni | da88c850c6 | |
Miriam Baglioni | 2f66fedfec | |
Miriam Baglioni | f5486ffb14 | |
Claudio Atzori | e0061232e9 | |
Claudio Atzori | bc4b86c27c | |
Claudio Atzori | 28a66af425 | |
Claudio Atzori | 783988af06 | |
Claudio Atzori | 9038fdc771 | |
Sandro La Bruzzo | bbe8193930 | |
Claudio Atzori | ae2b47b29d | |
Sandro La Bruzzo | 57c74c73c6 | |
Sandro La Bruzzo | 61ccb54fde | |
Sandro La Bruzzo | 9f5a0f3ab6 | |
Sandro La Bruzzo | 09fccf8000 | |
Sandro La Bruzzo | 0ea576745f | |
Sandro La Bruzzo | cd17e19044 | |
Miriam Baglioni | c30f3ce647 | |
Sandro La Bruzzo | 8a034e46e1 | |
Claudio Atzori | b7b8e0986e | |
Sandro La Bruzzo | 0799ac9fb6 | |
Sandro La Bruzzo | 4d53402712 | |
Sandro La Bruzzo | a4a54a3786 | |
Sandro La Bruzzo | a01dbe0ab0 | |
Claudio Atzori | fdcff42e46 | |
Claudio Atzori | 777536ce91 | |
Claudio Atzori | bc014023c8 | |
Claudio Atzori | 32bdfdccbc | |
Andreas Czerniak | ebf3f47a02 | |
Claudio Atzori | f580cb77e1 | |
Sandro La Bruzzo | ed684874f2 | |
Sandro La Bruzzo | 8535506c22 | |
Sandro La Bruzzo | 4c54bd8742 | |
Andreas Czerniak | 3531802710 | |
Sandro La Bruzzo | 7d8db2eb8a | |
Sandro La Bruzzo | c952c8d236 | |
Claudio Atzori | 70ded407bb | |
Miriam Baglioni | 7177c25261 | |
Miriam Baglioni | 0892cad4e8 | |
Claudio Atzori | 350a0823bd | |
Antonis Lempesis | 89e6f46682 | |
Sandro La Bruzzo | e4b84ef5d6 | |
Sandro La Bruzzo | 8fa0841898 | |
Sandro La Bruzzo | c6fa8598e1 | |
Antonis Lempesis | 829caee4fd | |
Sandro La Bruzzo | 84b834c893 | |
Sandro La Bruzzo | 1a6b398968 | |
Miriam Baglioni | bc34347643 | |
Miriam Baglioni | 86f47afcc7 | |
Miriam Baglioni | 03767ea8e6 | |
Miriam Baglioni | f8eec0ca9a | |
Miriam Baglioni | 149f85ddf5 | |
Miriam Baglioni | e487b5544c | |
Miriam Baglioni | 1503ccbbb5 | |
Miriam Baglioni | 1299bfb357 | |
Sandro La Bruzzo | 623a0c4edb | |
Miriam Baglioni | cf758f4f91 | |
Miriam Baglioni | 801763a0fa | |
Miriam Baglioni | a74de1cda2 | |
Miriam Baglioni | 06074ea7d3 | |
Miriam Baglioni | 8b8ffe82dc | |
Miriam Baglioni | 50cc21d92e | |
Claudio Atzori | 6d3f960238 | |
Antonis Lempesis | ae18171212 | |
Antonis Lempesis | 87f14a3899 | |
Sandro La Bruzzo | db933ebd21 | |
Sandro La Bruzzo | 7e08655e5f | |
Sandro La Bruzzo | 075055eaca | |
Sandro La Bruzzo | f36f92287d | |
Claudio Atzori | 986a8011ec | |
Antonis Lempesis | 018c4eb52c | |
Sandro La Bruzzo | 511ec14c63 | |
Claudio Atzori | af42377d0e | |
Sandro La Bruzzo | ad50415167 | |
Sandro La Bruzzo | 80e15cc455 | |
Claudio Atzori | 67afd06cd1 | |
Claudio Atzori | 2e8fd2c531 | |
Claudio Atzori | 4dc9ebf217 | |
Claudio Atzori | 50fc5a64a0 | |
Claudio Atzori | 5edcc6832a | |
Sandro La Bruzzo | 080a280bea | |
Sandro La Bruzzo | 1dc0c59e20 | |
Sandro La Bruzzo | dc66cf615b | |
Sandro La Bruzzo | 507e42102a | |
Sandro La Bruzzo | a167543637 | |
Sandro La Bruzzo | 4fe7b75644 | |
Sandro La Bruzzo | 3990165d05 | |
Claudio Atzori | 2dd5449c13 | |
Claudio Atzori | fd54ecf7bd | |
Miriam Baglioni | 180d671127 | |
Miriam Baglioni | 13c96622c9 | |
Miriam Baglioni | b486ae498f | |
Miriam Baglioni | 464c2ddde3 | |
Miriam Baglioni | 6aca0d8ebb | |
Miriam Baglioni | 3585e53da3 | |
Claudio Atzori | 41b551562e | |
Sandro La Bruzzo | 3100166d29 | |
Claudio Atzori | 74833d04f1 | |
Claudio Atzori | 7243a40c88 | |
Sandro La Bruzzo | dfcf78cf24 | |
Sandro La Bruzzo | cc0f2b11fb | |
Miriam Baglioni | 95885bcf12 | |
Miriam Baglioni | 2550a73981 | |
Miriam Baglioni | 1c47c0d786 | |
Miriam Baglioni | 7deac55138 | |
Antonis Lempesis | f7c0b80e35 | |
Miriam Baglioni | 66e7ef892f | |
Miriam Baglioni | 4f47ad0891 | |
Miriam Baglioni | 9f9dd00b94 | |
Miriam Baglioni | 63d74ee379 | |
Miriam Baglioni | 6ebc236657 | |
Miriam Baglioni | f7379255b6 | |
Miriam Baglioni | d6e21bb6ea | |
Miriam Baglioni | 4da141bd7c | |
Miriam Baglioni | ce0cfd79e0 | |
Miriam Baglioni | 93efe4de82 | |
Michele Artini | ada063ce70 | |
Michele Artini | 83132ee99a | |
Miriam Baglioni | cf360d7c97 | |
Miriam Baglioni | 8873e6b6d1 | |
Miriam Baglioni | 0f1acdf6b6 | |
Sandro La Bruzzo | aeb8132627 | |
Sandro La Bruzzo | efbea1e01a | |
Miriam Baglioni | 75780fc636 | |
Claudio Atzori | 2039bb9f5f | |
Claudio Atzori | dd19c4ac5a | |
Claudio Atzori | e9e86a237d | |
Claudio Atzori | 10bd6ca194 | |
Claudio Atzori | a900bfb874 | |
Sandro La Bruzzo | dd997c49e0 | |
Antonis Lempesis | d413b24611 | |
Claudio Atzori | 741077dbca | |
Miriam Baglioni | 32b0c27217 | |
Sandro La Bruzzo | 0d1f37302f | |
Miriam Baglioni | dc07f1079b | |
Miriam Baglioni | 8d2e086e48 | |
Miriam Baglioni | f33521d338 | |
Miriam Baglioni | bc12e9819e | |
Sandro La Bruzzo | 0cdb7ccdaa | |
Sandro La Bruzzo | 5b724d9972 | |
Sandro La Bruzzo | e57294ac99 | |
Michele Artini | ede2749822 | |
Michele Artini | f0fbfdcfae | |
Michele Artini | e950750262 | |
Michele Artini | 03a510859a | |
Michele Artini | e9f2b6037c | |
Sandro La Bruzzo | 02ef46535f | |
Sandro La Bruzzo | aeadc5a366 | |
Claudio Atzori | 96238152cb | |
Michele Artini | ad56a44fda | |
Claudio Atzori | 83722ebc47 | |
Claudio Atzori | eb6acfbabc | |
Claudio Atzori | 6e3a4e9237 | |
Claudio Atzori | ac3d090e9e | |
Michele Artini | 4fa5671d16 | |
Claudio Atzori | c3d92247d3 | |
Claudio Atzori | d512062b58 | |
Claudio Atzori | 5e4b91d9ef | |
Sandro La Bruzzo | bced804151 | |
Claudio Atzori | 4f58418184 | |
Miriam Baglioni | abd88f663d | |
Miriam Baglioni | c844877de2 | |
Miriam Baglioni | 073d76864d | |
Miriam Baglioni | 4c8b4a774c | |
Enrico Ottonello | abdd0ade1f | |
Miriam Baglioni | 53b9d87fec | |
Miriam Baglioni | 1ee8f13580 | |
Miriam Baglioni | e07c3ba089 | |
Miriam Baglioni | 54f6e2f693 | |
Miriam Baglioni | 7180505519 | |
Miriam Baglioni | 2eb1a8b344 | |
Enrico Ottonello | d0945c3c78 | |
Enrico Ottonello | 1265dadc90 | |
Enrico Ottonello | 0821d8e97d | |
Enrico Ottonello | ae7bd24d79 | |
Enrico Ottonello | 4d6c473bf1 | |
Claudio Atzori | 9d725efdc1 | |
Miriam Baglioni | 9610224671 | |
Claudio Atzori | 863b56b6ce | |
Claudio Atzori | ae5c28e54f | |
Miriam Baglioni | aa45b4df9b | |
Miriam Baglioni | 052c837843 | |
Claudio Atzori | b695932ae4 | |
Claudio Atzori | ea9b00ce56 | |
Claudio Atzori | 2e70aa43f0 | |
Claudio Atzori | b572f56763 | |
Claudio Atzori | 2578b7fbb3 | |
Miriam Baglioni | dc0ad8d2e0 | |
Claudio Atzori | 232dce83db | |
Claudio Atzori | aef2977ad0 | |
Miriam Baglioni | 02b80cf24f | |
Claudio Atzori | c4a23c2f4d | |
Claudio Atzori | ba03f549d7 | |
Claudio Atzori | 239d0f0a9a | |
Antonis Lempesis | 168edcbde3 | |
Michele Artini | e56ccec536 | |
Michele Artini | c1e20de7cf | |
Claudio Atzori | a9f512103b | |
Claudio Atzori | eeb8bcf075 | |
Claudio Atzori | 2cbf15f4fb | |
Enrico Ottonello | e13926cdd0 | |
Claudio Atzori | f19feceaf0 | |
Claudio Atzori | 1bd70fa2c6 | |
Claudio Atzori | ca3f3a7687 | |
Claudio Atzori | 0358ae16ce | |
Claudio Atzori | 23b8883ab1 | |
Claudio Atzori | 609eb711b3 | |
Claudio Atzori | 1517bf7c92 | |
Sandro La Bruzzo | d9a0bbda7b | |
Sandro La Bruzzo | 6424cd9062 | |
Sandro La Bruzzo | 073dcea2aa | |
Claudio Atzori | d4c3476152 | |
Claudio Atzori | da9d6f3887 | |
Sandro La Bruzzo | 54217d73ff | |
Claudio Atzori | d1cbee8413 | |
Claudio Atzori | 3797543600 | |
Claudio Atzori | 3925eb6a79 | |
Claudio Atzori | 25254885b9 | |
Claudio Atzori | 8a0de2fc18 | |
Sandro La Bruzzo | 7dc824fc23 | |
Michele Artini | d82071ba6c | |
Claudio Atzori | d4a30fabe3 | |
Claudio Atzori | dccaf173cf | |
Claudio Atzori | 8c96a82a03 | |
Claudio Atzori | 50fc128ff7 | |
Claudio Atzori | 2e1eb96f9a | |
Claudio Atzori | b1785ba77c | |
Sandro La Bruzzo | 1adfc41d23 | |
Claudio Atzori | fb930b84d3 | |
Claudio Atzori | 923d19ea8e | |
Sandro La Bruzzo | 714b71bd21 | |
Claudio Atzori | ba86835951 | |
Claudio Atzori | c00be646f3 | |
Michele Artini | f4bd2b5619 | |
Michele Artini | 49910aedca | |
Claudio Atzori | 5cc3e6d61c | |
Michele Artini | b4877da363 | |
Alessia Bardi | 9a20057615 | |
Michele Artini | 6692128234 | |
Alessia Bardi | a801999e75 | |
Michele Artini | a278d67175 | |
Claudio Atzori | f6ccd54d87 | |
Claudio Atzori | 91e7220f20 | |
Michele Artini | f77ba34126 | |
Michele Artini | 7c5cd86927 | |
Michele Artini | b5cf505cc6 | |
Enrico Ottonello | c537986b7c | |
Sandro La Bruzzo | 2129e9caa7 | |
Claudio Atzori | 5afa7d3e0c | |
Alessia Bardi | e6075bb917 | |
Claudio Atzori | ac77a245a3 | |
Claudio Atzori | f783e60ff7 | |
Sandro La Bruzzo | 63c0303137 | |
Sandro La Bruzzo | 74484d2823 | |
Claudio Atzori | dd2e0a81f4 | |
Claudio Atzori | 233d849f90 | |
Claudio Atzori | fcd13f5350 | |
Claudio Atzori | 4028176559 | |
Sandro La Bruzzo | c74b03d59c | |
Sandro La Bruzzo | 7f8848ecdd | |
Claudio Atzori | 27ab8a704d | |
Claudio Atzori | a7cf449b36 | |
Claudio Atzori | 82de6fb634 | |
Claudio Atzori | fa42026590 | |
Claudio Atzori | ef4bfd82e2 | |
Claudio Atzori | faa8f6f4e2 | |
miconis | 6d5c14e030 | |
Claudio Atzori | c2bb03c8b5 | |
Claudio Atzori | 7ed107be53 | |
Claudio Atzori | c25238480c | |
Claudio Atzori | 99cfb027fa | |
Miriam Baglioni | 72e5aa3b42 | |
Miriam Baglioni | 4ae6fba01d | |
Miriam Baglioni | 7d1b8b7f64 | |
miconis | d0e3366c34 | |
miconis | 3c12eeadce | |
Claudio Atzori | e5abbec2ba | |
Claudio Atzori | 55964cbd81 | |
Claudio Atzori | 8f309b72ff | |
Claudio Atzori | 52244f813a | |
Sandro La Bruzzo | fd29307b84 | |
Claudio Atzori | 815b9f4d56 | |
Claudio Atzori | d0d477cca3 | |
miconis | 0393cdce42 | |
miconis | cadd0a5de8 | |
Sandro La Bruzzo | e06c7f32f6 | |
Sandro La Bruzzo | dbe0d0378e | |
Antonis Lempesis | 625d993cd9 | |
Antonis Lempesis | 25d0512fbd | |
Sandro La Bruzzo | 524e5f3092 | |
Sandro La Bruzzo | cdfe01bbae | |
Sandro La Bruzzo | 3ae67b7a1d | |
Sandro La Bruzzo | a16e5299f9 | |
Claudio Atzori | 45057440c1 | |
Enrico Ottonello | 34ca792a55 | |
Enrico Ottonello | 27068aacd1 | |
miconis | 7ad573d023 | |
Sandro La Bruzzo | 67085da305 | |
Sandro La Bruzzo | 644aa8f40c | |
Sandro La Bruzzo | 7d6a80e2f2 | |
Claudio Atzori | 8704d32780 | |
Claudio Atzori | ba4b4c74d8 | |
Claudio Atzori | 906d50563c | |
Claudio Atzori | 3d58f95522 | |
Antonis Lempesis | 03d36fadea | |
miconis | f64e57c112 | |
miconis | 176a5e493d | |
miconis | 3525a8f504 | |
Claudio Atzori | 745fa92db8 | |
Claudio Atzori | 083c2959dc | |
Sandro La Bruzzo | 3f77bfceb0 | |
Claudio Atzori | 3125cef545 | |
Sandro La Bruzzo | 44a0064df6 | |
Sandro La Bruzzo | 479abd10cb | |
Claudio Atzori | 710cd1e8f2 | |
Claudio Atzori | d1ca025b0b | |
miconis | 1542196a33 | |
miconis | 369ed1cd8a | |
Andreas Czerniak | 52fbece3b3 | |
Andreas Czerniak | d7614c1f85 | |
Andreas Czerniak | 3b694074ff | |
Claudio Atzori | 511c0521e5 | |
Claudio Atzori | 72dcadd8e6 | |
Claudio Atzori | 902d05f548 | |
miconis | d442e25cbc | |
miconis | dcff9cecdf | |
Andreas Czerniak | 34df35926c | |
miconis | 11b22b2d23 | |
miconis | 0857100fb8 | |
miconis | bf685d849f | |
Miriam Baglioni | 70e391d427 | |
miconis | eaaefb8b4c | |
miconis | c39c82dfe9 | |
Claudio Atzori | 37b65cc3ad | |
Claudio Atzori | 1e7e5180fa | |
Claudio Atzori | e686b8de8d | |
Claudio Atzori | ee34cc51c3 | |
Claudio Atzori | 70e49ed53c | |
Claudio Atzori | 7941d7be29 | |
Claudio Atzori | 879e8cc7ef | |
Claudio Atzori | 72ce741ea6 | |
Enrico Ottonello | 59ec5137e1 | |
Sandro La Bruzzo | 616d2ecce2 | |
Miriam Baglioni | 4b6e514f02 | |
Claudio Atzori | 27681b876c | |
Claudio Atzori | 9237d55d7f | |
Claudio Atzori | 7f4e9479ec | |
Claudio Atzori | 940556f6d3 | |
miconis | 2709d08fc2 | |
miconis | f446580e9f | |
Claudio Atzori | 3becaa5539 | |
Claudio Atzori | a0837ac357 | |
Claudio Atzori | 48f2b6127e | |
miconis | 2355cc4e9b | |
Sandro La Bruzzo | 1dfda3624e | |
Claudio Atzori | b5b7dc2104 | |
Enrico Ottonello | 91d8660982 | |
Enrico Ottonello | ebd67b8c8f | |
Claudio Atzori | 827e7e37db | |
miconis | 28c1cdd132 | |
miconis | 5dfb66b0fa | |
miconis | 348b0ef921 | |
Claudio Atzori | 751125fdf9 | |
Claudio Atzori | 1e423fdc07 | |
Claudio Atzori | e5ebb500cf | |
Claudio Atzori | b75ad76f79 | |
Claudio Atzori | 8db248aa13 | |
Sandro La Bruzzo | 625e4c29c4 | |
Claudio Atzori | b4febed138 | |
Claudio Atzori | 431cbe9955 | |
Sandro La Bruzzo | c392936b97 | |
Sandro La Bruzzo | c73072079d | |
Sandro La Bruzzo | 098914dcff | |
miconis | 0fe40b08e4 | |
miconis | 98854b0124 | |
Claudio Atzori | 5a043e95ea | |
Claudio Atzori | a4e82a65aa | |
Claudio Atzori | 3256b9c836 | |
Claudio Atzori | 75144dacb3 | |
Claudio Atzori | 9588bfba81 | |
Claudio Atzori | 972d5a3d98 | |
Sandro La Bruzzo | 25d5663d97 | |
Sandro La Bruzzo | 5f98ea74a9 | |
Sandro La Bruzzo | b4805b989d | |
Claudio Atzori | 734232d3b9 | |
Sandro La Bruzzo | 76b10090fc | |
Claudio Atzori | a3dac32f16 | |
Sandro La Bruzzo | 2be0428047 | |
Claudio Atzori | 8257f9a2bc | |
Sandro La Bruzzo | 7c97a4d900 | |
Sandro La Bruzzo | cc5bbafa5d | |
Claudio Atzori | 3b2da86f0a | |
Claudio Atzori | 640b885706 | |
Claudio Atzori | 61a2551e74 | |
Claudio Atzori | 9cac6da9bd | |
Antonis Lempesis | 0ba0a6b9da | |
Antonis Lempesis | 60ebdf2dbe | |
Antonis Lempesis | 236435b470 | |
Antonis Lempesis | 3c75a05044 | |
Claudio Atzori | 19f3580b3d | |
Claudio Atzori | d3cb923f24 | |
Sandro La Bruzzo | 4bb3bcafa5 | |
Sandro La Bruzzo | a8e5d0ea0d | |
Sandro La Bruzzo | f5e7c57654 | |
Claudio Atzori | f74e464942 | |
Antonis Lempesis | fa1ec5b5e9 | |
Claudio Atzori | c801ab6c1d | |
Claudio Atzori | 9917d7e01c | |
Claudio Atzori | 01630f638d | |
Claudio Atzori | b3f3b895e5 | |
Claudio Atzori | 765f9bdee7 | |
Claudio Atzori | 59532b0919 | |
Claudio Atzori | d525785497 | |
Sandro La Bruzzo | bbe1a7c69a | |
Sandro La Bruzzo | a2169ccf07 | |
Claudio Atzori | f468c7f0d7 | |
Claudio Atzori | 76441f4edd | |
Claudio Atzori | 8d2bb24512 | |
Claudio Atzori | acbe3119a4 | |
Antonis Lempesis | f40c150a0d | |
Claudio Atzori | fa7930d2e2 | |
Antonis Lempesis | 6147ee4950 | |
Antonis Lempesis | c5fbad8093 | |
Claudio Atzori | 55f6ff5f55 | |
Claudio Atzori | e8789b0cdb | |
Claudio Atzori | ec80b7ade3 | |
Claudio Atzori | 36f750cd1d | |
Claudio Atzori | b73dce3e3a | |
Antonis Lempesis | 27796343ca | |
Enrico Ottonello | 20c0438f11 | |
Enrico Ottonello | 70cb100647 | |
Enrico Ottonello | bd3b16402b | |
Claudio Atzori | e76c4f62c1 | |
miconis | 1a85020572 | |
Enrico Ottonello | ca1800510a | |
Enrico Ottonello | 53d7023460 | |
Claudio Atzori | 7df2461ccc | |
Enrico Ottonello | d43ea88caf | |
Claudio Atzori | b830e33392 | |
Claudio Atzori | dc98c39500 | |
Claudio Atzori | 271e88537b | |
Claudio Atzori | 9c899f4433 | |
Claudio Atzori | fc3fa5e343 | |
Enrico Ottonello | 975823b968 | |
Miriam Baglioni | 896919e735 | |
Antonis Lempesis | d90767c733 | |
Antonis Lempesis | 3681afbe04 | |
Antonis Lempesis | c5502eba8f | |
Antonis Lempesis | 33c85d4e66 | |
Antonis Lempesis | b8e96c8ae7 | |
Antonis Lempesis | bcbfc052b1 | |
Antonis Lempesis | 10a29a4b9a | |
Antonis Lempesis | 8ef66452d5 | |
Antonis Lempesis | a8836e2f5f | |
Claudio Atzori | e7eba9f7e7 | |
Claudio Atzori | 58467aaf1e | |
Claudio Atzori | cc88701f29 | |
Antonis Lempesis | a445c1ac3d | |
Antonis Lempesis | 00d516360f | |
Claudio Atzori | 545f8f3e48 | |
Claudio Atzori | b592d78bb4 | |
Antonis Lempesis | cd1b794409 | |
Claudio Atzori | cf27905a71 | |
Alessia Bardi | bf2830b981 | |
Claudio Atzori | 6f9864c564 | |
Alessia Bardi | 32e81c2d89 | |
Claudio Atzori | 58288a95b8 | |
Claudio Atzori | 1abe6d1ad7 | |
Claudio Atzori | 523a6bfa97 | |
Antonis Lempesis | 1c029b9fc0 | |
Antonis Lempesis | 2c4dcc90ba | |
Sandro La Bruzzo | 7edcc87ed4 | |
Sandro La Bruzzo | 6a37c7f175 | |
Sandro La Bruzzo | b3f5c2351d | |
Sandro La Bruzzo | f216277219 | |
Andreas Czerniak | 5a9017cf18 | |
Claudio Atzori | aa55dedb8a | |
Claudio Atzori | 29c6f7e255 | |
Sandro La Bruzzo | 17e6f1934e | |
Sandro La Bruzzo | ebcc3ec14f | |
Michele Artini | 83d815d0bc | |
Michele Artini | 8c836bf930 | |
Michele Artini | 8c1600398a | |
Claudio Atzori | 3f8f78cbfb | |
Claudio Atzori | b34b5a39ca | |
Michele Artini | 7249cceb53 | |
Claudio Atzori | 73393d3c4d | |
Alessia Bardi | 986dd969d3 | |
miconis | 4b2124a18e | |
Alessia Bardi | c4d1feca74 | |
Alessia Bardi | 09fc7e2f78 | |
Enrico Ottonello | ee4ba7298b | |
Claudio Atzori | bc458d1b54 | |
Claudio Atzori | 82e6c50f3f | |
Claudio Atzori | 62bd3c53ee | |
Claudio Atzori | bae029f828 | |
Claudio Atzori | bebc54d5bf | |
Claudio Atzori | 50add4c61b | |
Miriam Baglioni | 2f5e6647c6 | |
Claudio Atzori | 40df0f987d | |
Claudio Atzori | a8a758925e | |
Michele Artini | 2ee0c3e47e | |
Claudio Atzori | 730973679a | |
Claudio Atzori | deb85706db | |
Sandro La Bruzzo | 4dae5e605d | |
Claudio Atzori | 72c57b28fa | |
Claudio Atzori | 40764cf626 | |
Enrico Ottonello | c238561001 | |
Enrico Ottonello | 465ce39f75 | |
Sandro La Bruzzo | 69c253710b | |
Michele Artini | 3ea8c328ac | |
Michele Artini | 26d2eb946f | |
Claudio Atzori | 4758b58aa2 | |
Claudio Atzori | e04045089f | |
Alessia Bardi | c67329d3ad | |
Michele Artini | 1b9731632b | |
Michele Artini | 820d729e99 | |
Michele Artini | 33f4696d6e | |
Michele Artini | c286d28ad2 | |
Claudio Atzori | 0e8a4f9f1a | |
Alessia Bardi | fd705404a1 | |
Miriam Baglioni | 6190465851 | |
Claudio Atzori | 53884d12c2 | |
Claudio Atzori | ac46c247d2 | |
Claudio Atzori | bde14b149a | |
Claudio Atzori | ca4391aa1c | |
Claudio Atzori | bb89b99b24 | |
Claudio Atzori | 75807ea5ae | |
Sandro La Bruzzo | 4ed1e306b6 | |
Sandro La Bruzzo | 0634674add | |
Claudio Atzori | d62ea1490d | |
Claudio Atzori | 73d772a4b4 | |
Claudio Atzori | 8eaa1fd4b4 | |
Sandro La Bruzzo | bead34d11a | |
Sandro La Bruzzo | 6ff234d81b | |
Sandro La Bruzzo | b6b835ef49 | |
Sandro La Bruzzo | e423634cb6 | |
Sandro La Bruzzo | 8ee82576c6 | |
Sandro La Bruzzo | 0276180039 | |
Michele Artini | d942d0c77d | |
Sandro La Bruzzo | 0f8e2ecce6 | |
Sandro La Bruzzo | 99cf3a8ea4 | |
Sandro La Bruzzo | 2da8bf7429 | |
Sandro La Bruzzo | 686e7b507c | |
Sandro La Bruzzo | 98b9498b57 | |
Michele Artini | 38f2508c87 | |
Sandro La Bruzzo | 184e7b3856 | |
Sandro La Bruzzo | 150a617bd1 | |
Claudio Atzori | f1a852f278 | |
Claudio Atzori | 9c32119dc2 | |
Claudio Atzori | 885e0dd926 | |
Claudio Atzori | 2890511613 | |
Claudio Atzori | 4eb9ed35b1 | |
Claudio Atzori | cd379eb5e3 | |
Alessia Bardi | 505477f36f | |
Alessia Bardi | ded6ed8d7d | |
Claudio Atzori | 3465c8ccee | |
Sandro La Bruzzo | a54848a59c | |
Sandro La Bruzzo | ffb092b8d3 | |
Sandro La Bruzzo | cda210a2ca | |
Claudio Atzori | 07a0ccfc96 | |
miconis | c7e2d5a59a | |
Claudio Atzori | 646dab7f68 | |
Claudio Atzori | 34d653de41 | |
Miriam Baglioni | fe36895c53 | |
miconis | 8fea29177c | |
Dimitris | 3e8d2a6b2d | |
Michele Artini | f667e94a31 | |
Michele Artini | cfbcdc95bc | |
Michele Artini | 69ba3203c0 | |
Michele Artini | fafb5b2e08 | |
Michele Artini | b230d44411 | |
Michele Artini | b9d90e95b8 | |
Michele Artini | 64b0b0bfb3 | |
Michele Artini | e3e0ab1de1 | |
Michele Artini | 26a941315a | |
Michele Artini | 6f4d1a37f0 | |
Michele Artini | 1391341d06 | |
Michele Artini | 3c9cbd19f3 | |
Michele Artini | 467aa77279 | |
Michele Artini | 10f3f7eca7 | |
Michele Artini | ff41a7b3a4 | |
Michele Artini | 223fa660cb | |
Michele Artini | ac91e495fc | |
Claudio Atzori | 80cf55ef2e | |
Claudio Atzori | 41500669e2 | |
Claudio Atzori | 2a7a10809e | |
Claudio Atzori | 5bd999efe7 | |
Claudio Atzori | d6686dd7cf | |
Claudio Atzori | 34229970e6 | |
Claudio Atzori | 1361c9eb0c | |
Claudio Atzori | ab2fe9266a | |
Claudio Atzori | 7c722f3fdc | |
Claudio Atzori | 8879704ba0 | |
Claudio Atzori | 26e9d55c13 | |
Sandro La Bruzzo | 7834a35768 | |
Sandro La Bruzzo | e79445a8b4 | |
Sandro La Bruzzo | 8765020b85 | |
Sandro La Bruzzo | b0dc92786f | |
Claudio Atzori | 7185158942 | |
Claudio Atzori | 28460c2cd1 | |
Claudio Atzori | 60649ac7d2 | |
Claudio Atzori | 723b01f9e9 | |
Claudio Atzori | 6848d0c3d7 | |
Claudio Atzori | d8b5f43a7e | |
Claudio Atzori | 7bfc35df5e | |
Antonis Lempesis | be5969a8c2 | |
miconis | 794e22b09c | |
miconis | 1e1aab83e3 | |
Claudio Atzori | 6cb0dc3f43 | |
Claudio Atzori | 573a8a3272 | |
Antonis Lempesis | 2a074c3b2b | |
Claudio Atzori | 47270d9af5 | |
Claudio Atzori | 2e503ee101 | |
Claudio Atzori | 5a3e2199b2 | |
Claudio Atzori | 03319d3bd9 | |
Claudio Atzori | add7e1693b | |
Alessia Bardi | f9a8fd8bbd | |
Claudio Atzori | 4766495f5b | |
Claudio Atzori | de00094ebc | |
Michele Artini | f9dc1e45fd | |
Sandro La Bruzzo | f92bd56f56 | |
Sandro La Bruzzo | 1f6c8a9e83 | |
Enrico Ottonello | b2de598c1a | |
Claudio Atzori | 9f1181290e | |
Claudio Atzori | 6299f75807 | |
Michele Artini | 0a0f62bd01 | |
Michele Artini | 12fa5d122a | |
Michele Artini | 991e675dc6 | |
Michele Artini | 3e19cf7b4a | |
Claudio Atzori | b6f08ce226 | |
Claudio Atzori | e8ef8c63d4 | |
Claudio Atzori | 7d325e2c57 | |
Claudio Atzori | 152916890f | |
Michele Artini | a203aee32a | |
Claudio Atzori | 1506f49052 | |
Michele Artini | d03756c962 | |
Michele Artini | 399548f221 | |
Michele Artini | 38da1c282a | |
Dimitris | dc9c2f3272 | |
Enrico Ottonello | efe4c2a9c5 | |
Enrico Ottonello | 858efbfad1 | |
Claudio Atzori | 61cd129ded | |
Claudio Atzori | ce7a319e01 | |
Claudio Atzori | 7fe2433137 | |
Claudio Atzori | d9532446eb | |
Claudio Atzori | 1eaad89a3c | |
Michele Artini | 933b4c1ada | |
Michele Artini | 2e7df07328 | |
Michele Artini | 94bfed1c84 | |
Claudio Atzori | 3c10941376 | |
Claudio Atzori | 12e2f930c8 | |
Miriam Baglioni | b7adbc7c3e | |
Alessia Bardi | 112da6d76a | |
Alessia Bardi | bece04b330 | |
Alessia Bardi | 426b76ee8e | |
Claudio Atzori | ff72fcd91a | |
Claudio Atzori | 4705144918 | |
Claudio Atzori | 211aa04726 | |
Claudio Atzori | db4e400a0b | |
Claudio Atzori | ada21ad920 | |
Miriam Baglioni | 6fbc67a959 | |
Claudio Atzori | 3c5ce1dada | |
Miriam Baglioni | 212b52614f | |
Michele Artini | 1bc9adc10d | |
Claudio Atzori | fcd7689b50 | |
Michele Artini | 5f21a356fd | |
Michele Artini | 370a5e650b | |
Antonis Lempesis | aead9efd24 | |
Antonis Lempesis | 77a3a6d82e | |
Antonis Lempesis | 91226117b3 | |
Antonis Lempesis | b7f29db126 | |
Antonis Lempesis | ded2392275 | |
Antonis Lempesis | 1a87a1effd | |
Michele Artini | 75bf708351 | |
Michele Artini | 620e1307a3 | |
Enrico Ottonello | 2233750a37 | |
Claudio Atzori | 491ad24750 | |
Claudio Atzori | 27e96767e0 | |
Claudio Atzori | fba11eef2a | |
Claudio Atzori | 2fcc24b36e | |
Claudio Atzori | 197f286fa4 | |
Sandro La Bruzzo | 7f8b93de72 | |
Sandro La Bruzzo | 302baab67b | |
Enrico Ottonello | 5c65e602d3 | |
Michele Artini | d6934f370e | |
Michele Artini | 5de8a7276f | |
Claudio Atzori | 5e8509bef7 | |
Claudio Atzori | 026ad40633 | |
Claudio Atzori | 21ddcf3a73 | |
Enrico Ottonello | fa1855a4b8 | |
Enrico Ottonello | b1b589ada1 | |
Sandro La Bruzzo | 620e585b63 | |
Sandro La Bruzzo | b31dd126fb | |
Enrico Ottonello | 8812ab65e1 | |
Claudio Atzori | a104a632df | |
Claudio Atzori | 5b4e1142a8 | |
Antonis Lempesis | b1ed1afdcc | |
Antonis Lempesis | 7cb113e088 | |
Antonis Lempesis | d23ccae0d5 | |
Miriam Baglioni | 5fb65ffc4a | |
Miriam Baglioni | ea88dc3401 | |
Miriam Baglioni | 4c58bd1c93 | |
Miriam Baglioni | 05c452f58d | |
Enrico Ottonello | 53b22c1937 | |
Enrico Ottonello | 1b1e9ea67c | |
Antonis Lempesis | 413afcfed5 | |
Antonis Lempesis | 0948536614 | |
Sandro La Bruzzo | 7da679542f | |
Sandro La Bruzzo | 6ba8037cc7 | |
Claudio Atzori | cfb55effd9 | |
Claudio Atzori | 74242e450e | |
Miriam Baglioni | d5efa6963a | |
Claudio Atzori | 873c358d1d | |
Miriam Baglioni | cd285e98bc | |
Miriam Baglioni | 51c582c08c | |
Miriam Baglioni | 4b0d1530a2 | |
Claudio Atzori | faa977df7e | |
Claudio Atzori | 57f448b7a4 | |
Alessia Bardi | 2d15667b4a | |
Alessia Bardi | a417624670 | |
Claudio Atzori | 943b961cf6 | |
Claudio Atzori | 893ac4a77b | |
Miriam Baglioni | f8468c9c22 | |
Miriam Baglioni | 888175baf7 | |
Miriam Baglioni | 3d62d99d5d | |
Miriam Baglioni | 17680296b9 | |
Miriam Baglioni | 5b3ed70808 | |
Miriam Baglioni | 62ff4999e3 | |
Miriam Baglioni | 45d06c45c7 | |
Miriam Baglioni | 0051ebede5 | |
Miriam Baglioni | 719da15f04 | |
Miriam Baglioni | e819155eb2 | |
Miriam Baglioni | db36e11912 | |
Claudio Atzori | 349e7246aa | |
Enrico Ottonello | f2df3ead74 | |
Enrico Ottonello | 40c4559e92 | |
Claudio Atzori | 2c407e775e | |
Antonis Lempesis | 815d6b25d9 | |
Claudio Atzori | 758d27745d | |
Claudio Atzori | 596a2a459d | |
Claudio Atzori | e731a7658d | |
Claudio Atzori | fa66e5b6b8 | |
Claudio Atzori | 5151850a19 | |
Claudio Atzori | a104d2b6ad | |
Claudio Atzori | d0d5525d40 | |
Claudio Atzori | 13eae4b31e | |
Claudio Atzori | 76363a8512 | |
Claudio Atzori | c1b9a4045a | |
Miriam Baglioni | 124591a7f3 | |
Miriam Baglioni | 1a89f8211c | |
Miriam Baglioni | 5fbe54ef54 | |
Miriam Baglioni | ed01e5a5e1 | |
Miriam Baglioni | d4ddde2ef2 | |
Miriam Baglioni | f5e5e92a10 | |
Miriam Baglioni | 1df94b85b4 | |
Miriam Baglioni | 66c0e3e574 | |
Claudio Atzori | db0181b8af | |
Sandro La Bruzzo | ec3e238de6 | |
Claudio Atzori | 1372a4d1bf | |
Claudio Atzori | e208b03755 | |
Claudio Atzori | dfd6205b95 | |
Miriam Baglioni | 90d4369fd2 | |
Miriam Baglioni | 6750e33d69 | |
Miriam Baglioni | b2c455f883 | |
Miriam Baglioni | 1f130cdf92 | |
Miriam Baglioni | e758d5d9b4 | |
Miriam Baglioni | 87a9f616ae | |
Miriam Baglioni | e7e418e444 | |
Miriam Baglioni | 305e3d0c9c | |
Miriam Baglioni | 21ce175d17 | |
Miriam Baglioni | bde6d337dd | |
Miriam Baglioni | b37b9352d7 | |
Sandro La Bruzzo | 264723ffd8 | |
Claudio Atzori | 36173c13a5 | |
Claudio Atzori | eeebd5a920 | |
Claudio Atzori | e1a1bb3ee4 | |
Enrico Ottonello | 99a086f0c6 | |
Miriam Baglioni | 72bb0fe360 | |
Miriam Baglioni | 00874a8ce6 | |
Miriam Baglioni | 39f4a20873 | |
Miriam Baglioni | 7e14452a87 | |
Miriam Baglioni | c167a18057 | |
Miriam Baglioni | 54a309bb6b | |
Miriam Baglioni | 35ecea8842 | |
Miriam Baglioni | b9b6bdb2e6 | |
Miriam Baglioni | 7e940f1991 | |
Miriam Baglioni | 62928ef7a5 | |
Claudio Atzori | 33bae02451 | |
Claudio Atzori | e43ab07af6 | |
Miriam Baglioni | 3319440c53 | |
Miriam Baglioni | 00c377dac2 | |
Miriam Baglioni | 44db258dc4 | |
Miriam Baglioni | 1832708c42 | |
Miriam Baglioni | 73dbb79602 | |
Claudio Atzori | c016cc050a | |
Enrico Ottonello | 5c17e768b2 | |
Enrico Ottonello | 5c9a727895 | |
Enrico Ottonello | 97c8111847 | |
Miriam Baglioni | 259c67ce36 | |
Miriam Baglioni | 0a9db67eec | |
Miriam Baglioni | d362f2637d | |
Miriam Baglioni | cf3f47563f | |
Miriam Baglioni | 24c56fa7a3 | |
Claudio Atzori | d48f388fb2 | |
Claudio Atzori | 46bde9c13f | |
Claudio Atzori | 7c9feaf9e7 | |
Claudio Atzori | fcbb05eb21 | |
Claudio Atzori | 3f34757c63 | |
Michele Artini | 293da47ad9 | |
Michele Artini | ab08d12c46 | |
Claudio Atzori | e503271abe | |
Claudio Atzori | 0374d34c3e | |
Miriam Baglioni | fafb688887 | |
Miriam Baglioni | 906db690d2 | |
Claudio Atzori | ede7fae6c8 | |
Miriam Baglioni | 5402062ff5 | |
Miriam Baglioni | a172a37ad1 | |
Miriam Baglioni | 46ba3793f6 | |
Claudio Atzori | 5218718e8b | |
Claudio Atzori | d9e07a242b | |
Claudio Atzori | 29dcff0f34 | |
Miriam Baglioni | 57cac36898 | |
Claudio Atzori | 12acf25519 | |
Claudio Atzori | 8177ce7939 | |
Alessia Bardi | 10e673660f | |
Alessia Bardi | be7b310cef | |
Michele Artini | 33da2e3d6c | |
Antonis Lempesis | 01a6e03989 | |
Alessia Bardi | 8f87020a50 | |
Alessia Bardi | 7e0a76a8ac | |
Enrico Ottonello | 2b0c9bbb7e | |
Enrico Ottonello | c0c2e05eae | |
Claudio Atzori | cfc01f136e | |
Claudio Atzori | 628ca54dd3 | |
Dimitris | bbcf6b7c8b | |
Enrico Ottonello | c796adae24 | |
Claudio Atzori | 6ab1ce53c9 | |
Claudio Atzori | 4de8c8b237 | |
Dimitris | 3e24c9b176 | |
Claudio Atzori | 331d621800 | |
Claudio Atzori | 5d4e34e26a | |
Claudio Atzori | 768bc5304c | |
Claudio Atzori | 93f7b7974f | |
Claudio Atzori | 2facfefc19 | |
Claudio Atzori | 528231a287 | |
Enrico Ottonello | 005f849674 | |
Enrico Ottonello | 9a2fa9dc2f | |
Claudio Atzori | 2bed29eb09 | |
Claudio Atzori | 13e36a4da0 | |
Enrico Ottonello | 13f28fa225 | |
Enrico Ottonello | 2af21150c5 | |
Claudio Atzori | 9b0fb9e958 | |
Claudio Atzori | 75324ae58a | |
Claudio Atzori | 822971f54f | |
Enrico Ottonello | 1f861f2b0d | |
Claudio Atzori | 9841488482 | |
Antonis Lempesis | 99ebaee347 | |
Claudio Atzori | e3d3481fb9 | |
Antonis Lempesis | f14e65f6a3 | |
Antonis Lempesis | c02c7741c9 | |
Antonis Lempesis | e603fa5847 | |
Enrico Ottonello | fea2451658 | |
Claudio Atzori | 18d9aad70c | |
Enrico Ottonello | 1513174d7e | |
Michele Artini | 40160d171f | |
Sandro La Bruzzo | 8e1d43aab2 | |
Sandro La Bruzzo | 027ef2326c | |
Sandro La Bruzzo | cd27df91a1 | |
Enrico Ottonello | 6bc7dbeca7 | |
Claudio Atzori | d10447e747 | |
Claudio Atzori | 2d76497488 | |
Claudio Atzori | 144216fb88 | |
Miriam Baglioni | f8e9bda24c | |
Miriam Baglioni | afa0b1489b | |
Miriam Baglioni | 7ebdfacee9 | |
Miriam Baglioni | be5ed8f554 | |
Claudio Atzori | 2148a51fae | |
Claudio Atzori | 4625b7486e | |
Claudio Atzori | f5f346dd2b | |
Miriam Baglioni | e9ac471ae9 | |
Miriam Baglioni | f45c23316f | |
Miriam Baglioni | e9d948786d | |
Miriam Baglioni | b90a945c49 | |
Miriam Baglioni | bac307155a | |
Miriam Baglioni | 9c9d50f486 | |
Miriam Baglioni | 5669890934 | |
Miriam Baglioni | 6a89f59be9 | |
Miriam Baglioni | 56150d7e5e | |
Miriam Baglioni | 16c54a96f8 | |
Miriam Baglioni | d9d8de63cc | |
Miriam Baglioni | 0cac5436ff | |
Alessia Bardi | 51808b5afd | |
Alessia Bardi | e6becf8659 | |
Alessia Bardi | 0abe0eee33 | |
Alessia Bardi | f6ab238f5d | |
Sandro La Bruzzo | 3581244daf | |
Sandro La Bruzzo | 66efb39634 | |
Miriam Baglioni | c010a8442f | |
Miriam Baglioni | 8ec7a61188 | |
Miriam Baglioni | 8b4f7bf492 | |
Miriam Baglioni | c209284ca7 | |
Miriam Baglioni | 08806deddf | |
Miriam Baglioni | 7d2eda43ca | |
Miriam Baglioni | cbbb1bdc54 | |
Miriam Baglioni | 7d95a5e2b4 | |
Miriam Baglioni | d4382b54df | |
Miriam Baglioni | 1124ac29fc | |
Dimitris | 32bf943979 | |
Miriam Baglioni | dabb33e018 | |
Miriam Baglioni | 0fba08eae4 | |
Miriam Baglioni | b828587252 | |
Miriam Baglioni | f747e303ac | |
Miriam Baglioni | 16baf5b69e | |
Miriam Baglioni | a9eef9c852 | |
Miriam Baglioni | 5f4de9a962 | |
Miriam Baglioni | 10d8bbada8 | |
Miriam Baglioni | 14bf2e7238 | |
Dimitris | b8a3392b59 | |
Miriam Baglioni | 78fdb11c3f | |
Miriam Baglioni | d6e8dc0313 | |
Miriam Baglioni | 4cf4454341 | |
Miriam Baglioni | c8f32dd109 | |
Miriam Baglioni | 3582eba565 | |
Miriam Baglioni | d2374e3b9e | |
Miriam Baglioni | 5d3012eeb4 | |
Miriam Baglioni | 1bd638d291 | |
Miriam Baglioni | 3241ec1777 | |
Miriam Baglioni | cc68855a1e | |
Miriam Baglioni | 1cb60aede4 | |
Enrico Ottonello | 9818e74a70 | |
Enrico Ottonello | 210a50e4f4 | |
Enrico Ottonello | b0290dbcb7 | |
Enrico Ottonello | a38ab57062 | |
Enrico Ottonello | 1139d6568d | |
Enrico Ottonello | c58db1c8ea | |
Enrico Ottonello | 846ba30873 | |
Enrico Ottonello | c3114ba0ae | |
Enrico Ottonello | c295c71ca0 | |
Enrico Ottonello | ab083f9946 | |
Miriam Baglioni | 959f30811e | |
Miriam Baglioni | 11b7eaae09 | |
Miriam Baglioni | 32bffb0134 | |
Miriam Baglioni | 25cbcf6114 | |
Miriam Baglioni | 01117a46e1 | |
Miriam Baglioni | cfb5766c6b | |
Miriam Baglioni | fcaedac980 | |
Miriam Baglioni | 983a12ed15 | |
Miriam Baglioni | 8b36d19182 | |
Miriam Baglioni | ed5239f9ec | |
Miriam Baglioni | 3a8c524fce | |
Miriam Baglioni | ccd48dd78a | |
Miriam Baglioni | 3e5497b336 | |
Miriam Baglioni | 2ac2b537b6 | |
Miriam Baglioni | 54800fb9b0 | |
Miriam Baglioni | de6c4d46d8 | |
Enrico Ottonello | a97ad20c7b | |
Enrico Ottonello | fefbcfb106 | |
Enrico Ottonello | 7cffd14fb0 | |
Enrico Ottonello | 9e8e7fe6ef | |
Miriam Baglioni | c2b5c780ff | |
Miriam Baglioni | e2ceefe9be | |
Miriam Baglioni | 1f893e63dc | |
Enrico Ottonello | 538f299767 | |
Enrico Ottonello | eb8c9b2348 | |
Miriam Baglioni | b72a7dad46 | |
Miriam Baglioni | 8694bb9b31 | |
Miriam Baglioni | 428f1022fd | |
Miriam Baglioni | c90a0d39dd | |
Miriam Baglioni | bd5a72929b | |
Miriam Baglioni | 8a069a4fea | |
Miriam Baglioni | 34fa96f3b1 | |
Miriam Baglioni | 5fb2949cb8 | |
Miriam Baglioni | 2a540b6c01 | |
Miriam Baglioni | da103c399a | |
Miriam Baglioni | 630a6a1fe7 | |
Miriam Baglioni | 40c8d2de7b | |
Miriam Baglioni | bef79d3bdf | |
Miriam Baglioni | 85203c16e3 | |
Miriam Baglioni | 2c783793ba | |
Miriam Baglioni | c325acef3f | |
Miriam Baglioni | 6b8c5034fc | |
Miriam Baglioni | f26382fa51 | |
Miriam Baglioni | 4584cf6334 | |
Miriam Baglioni | f5bae426f7 | |
Miriam Baglioni | 11b80899d7 | |
Miriam Baglioni | f6bf888016 | |
Miriam Baglioni | 66d0e0d3f2 | |
Miriam Baglioni | 1c593a9cfe | |
Miriam Baglioni | e42b2f5ae2 | |
Miriam Baglioni | f81ee22418 | |
Miriam Baglioni | 387be43fd4 | |
Miriam Baglioni | c5858afb88 | |
Miriam Baglioni | d407852ac2 | |
Miriam Baglioni | 47c21a8961 | |
Miriam Baglioni | 5570678c65 | |
Miriam Baglioni | dc5096a327 | |
Miriam Baglioni | 1791cf2e78 | |
Miriam Baglioni | c7f944a533 | |
Enrico Ottonello | 0377b40fba | |
Enrico Ottonello | 196f36c6ed | |
Enrico Ottonello | c82b15b5f4 | |
Enrico Ottonello | a6acb37689 | |
Michele Artini | 3adedd0a68 | |
Enrico Ottonello | ca37d3427b | |
Enrico Ottonello | 1729cc5cf3 | |
Enrico Ottonello | 5525f57ec8 | |
Enrico Ottonello | b7b6be12a5 | |
Enrico Ottonello | b2213b6435 | |
Enrico Ottonello | c5e149c46e | |
Enrico Ottonello | d6498278ed | |
Enrico Ottonello | fcbb4c1489 |
|
@ -3,10 +3,10 @@
|
||||||
*.iws
|
*.iws
|
||||||
*.ipr
|
*.ipr
|
||||||
*.iml
|
*.iml
|
||||||
*.ipr
|
|
||||||
*.iws
|
|
||||||
*~
|
*~
|
||||||
.vscode
|
.vscode
|
||||||
|
.metals
|
||||||
|
.bloop
|
||||||
.classpath
|
.classpath
|
||||||
/*/.classpath
|
/*/.classpath
|
||||||
/*/*/.classpath
|
/*/*/.classpath
|
||||||
|
@ -24,4 +24,5 @@
|
||||||
spark-warehouse
|
spark-warehouse
|
||||||
/**/job-override.properties
|
/**/job-override.properties
|
||||||
/**/*.log
|
/**/*.log
|
||||||
|
/**/.factorypath
|
||||||
|
/**/.scalafmt.conf
|
||||||
|
|
|
@ -1,2 +1,2 @@
|
||||||
# dnet-hadoop
|
# dnet-hadoop
|
||||||
Dnet-hadoop is a tool for
|
Dnet-hadoop is the project that defined all the OOZIE workflows for the OpenAIRE Graph construction, processing, provisioning.
|
|
@ -8,8 +8,6 @@ import java.util.List;
|
||||||
import org.apache.commons.lang.ArrayUtils;
|
import org.apache.commons.lang.ArrayUtils;
|
||||||
import org.apache.commons.lang.StringUtils;
|
import org.apache.commons.lang.StringUtils;
|
||||||
import org.apache.maven.plugin.AbstractMojo;
|
import org.apache.maven.plugin.AbstractMojo;
|
||||||
import org.apache.maven.plugin.MojoExecutionException;
|
|
||||||
import org.apache.maven.plugin.MojoFailureException;
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Generates oozie properties which were not provided from commandline.
|
* Generates oozie properties which were not provided from commandline.
|
||||||
|
@ -27,7 +25,7 @@ public class GenerateOoziePropertiesMojo extends AbstractMojo {
|
||||||
};
|
};
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void execute() throws MojoExecutionException, MojoFailureException {
|
public void execute() {
|
||||||
if (System.getProperties().containsKey(PROPERTY_NAME_WF_SOURCE_DIR)
|
if (System.getProperties().containsKey(PROPERTY_NAME_WF_SOURCE_DIR)
|
||||||
&& !System.getProperties().containsKey(PROPERTY_NAME_SANDBOX_NAME)) {
|
&& !System.getProperties().containsKey(PROPERTY_NAME_SANDBOX_NAME)) {
|
||||||
String generatedSandboxName = generateSandboxName(
|
String generatedSandboxName = generateSandboxName(
|
||||||
|
@ -46,24 +44,24 @@ public class GenerateOoziePropertiesMojo extends AbstractMojo {
|
||||||
/**
|
/**
|
||||||
* Generates sandbox name from workflow source directory.
|
* Generates sandbox name from workflow source directory.
|
||||||
*
|
*
|
||||||
* @param wfSourceDir
|
* @param wfSourceDir workflow source directory
|
||||||
* @return generated sandbox name
|
* @return generated sandbox name
|
||||||
*/
|
*/
|
||||||
private String generateSandboxName(String wfSourceDir) {
|
private String generateSandboxName(String wfSourceDir) {
|
||||||
// utilize all dir names until finding one of the limiters
|
// utilize all dir names until finding one of the limiters
|
||||||
List<String> sandboxNameParts = new ArrayList<String>();
|
List<String> sandboxNameParts = new ArrayList<>();
|
||||||
String[] tokens = StringUtils.split(wfSourceDir, File.separatorChar);
|
String[] tokens = StringUtils.split(wfSourceDir, File.separatorChar);
|
||||||
ArrayUtils.reverse(tokens);
|
ArrayUtils.reverse(tokens);
|
||||||
if (tokens.length > 0) {
|
if (tokens.length > 0) {
|
||||||
for (String token : tokens) {
|
for (String token : tokens) {
|
||||||
for (String limiter : limiters) {
|
for (String limiter : limiters) {
|
||||||
if (limiter.equals(token)) {
|
if (limiter.equals(token)) {
|
||||||
return sandboxNameParts.size() > 0
|
return !sandboxNameParts.isEmpty()
|
||||||
? StringUtils.join(sandboxNameParts.toArray())
|
? StringUtils.join(sandboxNameParts.toArray())
|
||||||
: null;
|
: null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (sandboxNameParts.size() > 0) {
|
if (!sandboxNameParts.isEmpty()) {
|
||||||
sandboxNameParts.add(0, File.separator);
|
sandboxNameParts.add(0, File.separator);
|
||||||
}
|
}
|
||||||
sandboxNameParts.add(0, token);
|
sandboxNameParts.add(0, token);
|
||||||
|
|
|
@ -16,6 +16,7 @@ import java.io.File;
|
||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
import java.util.Arrays;
|
import java.util.Arrays;
|
||||||
import java.util.Collections;
|
import java.util.Collections;
|
||||||
|
@ -289,7 +290,7 @@ public class WritePredefinedProjectProperties extends AbstractMojo {
|
||||||
*/
|
*/
|
||||||
protected List<String> getEscapeChars(String escapeChars) {
|
protected List<String> getEscapeChars(String escapeChars) {
|
||||||
List<String> tokens = getListFromCSV(escapeChars);
|
List<String> tokens = getListFromCSV(escapeChars);
|
||||||
List<String> realTokens = new ArrayList<String>();
|
List<String> realTokens = new ArrayList<>();
|
||||||
for (String token : tokens) {
|
for (String token : tokens) {
|
||||||
String realToken = getRealToken(token);
|
String realToken = getRealToken(token);
|
||||||
realTokens.add(realToken);
|
realTokens.add(realToken);
|
||||||
|
@ -324,7 +325,7 @@ public class WritePredefinedProjectProperties extends AbstractMojo {
|
||||||
* @return content
|
* @return content
|
||||||
*/
|
*/
|
||||||
protected String getContent(String comment, Properties properties, List<String> escapeTokens) {
|
protected String getContent(String comment, Properties properties, List<String> escapeTokens) {
|
||||||
List<String> names = new ArrayList<String>(properties.stringPropertyNames());
|
List<String> names = new ArrayList<>(properties.stringPropertyNames());
|
||||||
Collections.sort(names);
|
Collections.sort(names);
|
||||||
StringBuilder sb = new StringBuilder();
|
StringBuilder sb = new StringBuilder();
|
||||||
if (!StringUtils.isBlank(comment)) {
|
if (!StringUtils.isBlank(comment)) {
|
||||||
|
@ -352,7 +353,7 @@ public class WritePredefinedProjectProperties extends AbstractMojo {
|
||||||
throws MojoExecutionException {
|
throws MojoExecutionException {
|
||||||
try {
|
try {
|
||||||
String content = getContent(comment, properties, escapeTokens);
|
String content = getContent(comment, properties, escapeTokens);
|
||||||
FileUtils.writeStringToFile(file, content, ENCODING_UTF8);
|
FileUtils.writeStringToFile(file, content, StandardCharsets.UTF_8);
|
||||||
} catch (IOException e) {
|
} catch (IOException e) {
|
||||||
throw new MojoExecutionException("Error creating properties file", e);
|
throw new MojoExecutionException("Error creating properties file", e);
|
||||||
}
|
}
|
||||||
|
@ -399,9 +400,9 @@ public class WritePredefinedProjectProperties extends AbstractMojo {
|
||||||
*/
|
*/
|
||||||
protected static final List<String> getListFromCSV(String csv) {
|
protected static final List<String> getListFromCSV(String csv) {
|
||||||
if (StringUtils.isBlank(csv)) {
|
if (StringUtils.isBlank(csv)) {
|
||||||
return new ArrayList<String>();
|
return new ArrayList<>();
|
||||||
}
|
}
|
||||||
List<String> list = new ArrayList<String>();
|
List<String> list = new ArrayList<>();
|
||||||
String[] tokens = StringUtils.split(csv, ",");
|
String[] tokens = StringUtils.split(csv, ",");
|
||||||
for (String token : tokens) {
|
for (String token : tokens) {
|
||||||
list.add(token.trim());
|
list.add(token.trim());
|
||||||
|
|
|
@ -9,18 +9,18 @@ import org.junit.jupiter.api.BeforeEach;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
/** @author mhorst, claudio.atzori */
|
/** @author mhorst, claudio.atzori */
|
||||||
public class GenerateOoziePropertiesMojoTest {
|
class GenerateOoziePropertiesMojoTest {
|
||||||
|
|
||||||
private final GenerateOoziePropertiesMojo mojo = new GenerateOoziePropertiesMojo();
|
private final GenerateOoziePropertiesMojo mojo = new GenerateOoziePropertiesMojo();
|
||||||
|
|
||||||
@BeforeEach
|
@BeforeEach
|
||||||
public void clearSystemProperties() {
|
void clearSystemProperties() {
|
||||||
System.clearProperty(PROPERTY_NAME_SANDBOX_NAME);
|
System.clearProperty(PROPERTY_NAME_SANDBOX_NAME);
|
||||||
System.clearProperty(PROPERTY_NAME_WF_SOURCE_DIR);
|
System.clearProperty(PROPERTY_NAME_WF_SOURCE_DIR);
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteEmpty() throws Exception {
|
void testExecuteEmpty() throws Exception {
|
||||||
// execute
|
// execute
|
||||||
mojo.execute();
|
mojo.execute();
|
||||||
|
|
||||||
|
@ -29,7 +29,7 @@ public class GenerateOoziePropertiesMojoTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteSandboxNameAlreadySet() throws Exception {
|
void testExecuteSandboxNameAlreadySet() throws Exception {
|
||||||
// given
|
// given
|
||||||
String workflowSourceDir = "eu/dnetlib/dhp/wf/transformers";
|
String workflowSourceDir = "eu/dnetlib/dhp/wf/transformers";
|
||||||
String sandboxName = "originalSandboxName";
|
String sandboxName = "originalSandboxName";
|
||||||
|
@ -44,7 +44,7 @@ public class GenerateOoziePropertiesMojoTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteEmptyWorkflowSourceDir() throws Exception {
|
void testExecuteEmptyWorkflowSourceDir() throws Exception {
|
||||||
// given
|
// given
|
||||||
String workflowSourceDir = "";
|
String workflowSourceDir = "";
|
||||||
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||||
|
@ -57,7 +57,7 @@ public class GenerateOoziePropertiesMojoTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteNullSandboxNameGenerated() throws Exception {
|
void testExecuteNullSandboxNameGenerated() throws Exception {
|
||||||
// given
|
// given
|
||||||
String workflowSourceDir = "eu/dnetlib/dhp/";
|
String workflowSourceDir = "eu/dnetlib/dhp/";
|
||||||
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||||
|
@ -70,7 +70,7 @@ public class GenerateOoziePropertiesMojoTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecute() throws Exception {
|
void testExecute() throws Exception {
|
||||||
// given
|
// given
|
||||||
String workflowSourceDir = "eu/dnetlib/dhp/wf/transformers";
|
String workflowSourceDir = "eu/dnetlib/dhp/wf/transformers";
|
||||||
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||||
|
@ -83,7 +83,7 @@ public class GenerateOoziePropertiesMojoTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteWithoutRoot() throws Exception {
|
void testExecuteWithoutRoot() throws Exception {
|
||||||
// given
|
// given
|
||||||
String workflowSourceDir = "wf/transformers";
|
String workflowSourceDir = "wf/transformers";
|
||||||
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
System.setProperty(PROPERTY_NAME_WF_SOURCE_DIR, workflowSourceDir);
|
||||||
|
|
|
@ -20,7 +20,7 @@ import org.mockito.junit.jupiter.MockitoExtension;
|
||||||
|
|
||||||
/** @author mhorst, claudio.atzori */
|
/** @author mhorst, claudio.atzori */
|
||||||
@ExtendWith(MockitoExtension.class)
|
@ExtendWith(MockitoExtension.class)
|
||||||
public class WritePredefinedProjectPropertiesTest {
|
class WritePredefinedProjectPropertiesTest {
|
||||||
|
|
||||||
@Mock
|
@Mock
|
||||||
private MavenProject mavenProject;
|
private MavenProject mavenProject;
|
||||||
|
@ -39,7 +39,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
// ----------------------------------- TESTS ---------------------------------------------
|
// ----------------------------------- TESTS ---------------------------------------------
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteEmpty() throws Exception {
|
void testExecuteEmpty() throws Exception {
|
||||||
// execute
|
// execute
|
||||||
mojo.execute();
|
mojo.execute();
|
||||||
|
|
||||||
|
@ -50,7 +50,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteWithProjectProperties() throws Exception {
|
void testExecuteWithProjectProperties() throws Exception {
|
||||||
// given
|
// given
|
||||||
String key = "projectPropertyKey";
|
String key = "projectPropertyKey";
|
||||||
String value = "projectPropertyValue";
|
String value = "projectPropertyValue";
|
||||||
|
@ -70,7 +70,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test()
|
@Test()
|
||||||
public void testExecuteWithProjectPropertiesAndInvalidOutputFile(@TempDir File testFolder) {
|
void testExecuteWithProjectPropertiesAndInvalidOutputFile(@TempDir File testFolder) {
|
||||||
// given
|
// given
|
||||||
String key = "projectPropertyKey";
|
String key = "projectPropertyKey";
|
||||||
String value = "projectPropertyValue";
|
String value = "projectPropertyValue";
|
||||||
|
@ -84,7 +84,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteWithProjectPropertiesExclusion(@TempDir File testFolder) throws Exception {
|
void testExecuteWithProjectPropertiesExclusion(@TempDir File testFolder) throws Exception {
|
||||||
// given
|
// given
|
||||||
String key = "projectPropertyKey";
|
String key = "projectPropertyKey";
|
||||||
String value = "projectPropertyValue";
|
String value = "projectPropertyValue";
|
||||||
|
@ -108,7 +108,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteWithProjectPropertiesInclusion(@TempDir File testFolder) throws Exception {
|
void testExecuteWithProjectPropertiesInclusion(@TempDir File testFolder) throws Exception {
|
||||||
// given
|
// given
|
||||||
String key = "projectPropertyKey";
|
String key = "projectPropertyKey";
|
||||||
String value = "projectPropertyValue";
|
String value = "projectPropertyValue";
|
||||||
|
@ -132,7 +132,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteIncludingPropertyKeysFromFile(@TempDir File testFolder) throws Exception {
|
void testExecuteIncludingPropertyKeysFromFile(@TempDir File testFolder) throws Exception {
|
||||||
// given
|
// given
|
||||||
String key = "projectPropertyKey";
|
String key = "projectPropertyKey";
|
||||||
String value = "projectPropertyValue";
|
String value = "projectPropertyValue";
|
||||||
|
@ -164,7 +164,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteIncludingPropertyKeysFromClasspathResource(@TempDir File testFolder)
|
void testExecuteIncludingPropertyKeysFromClasspathResource(@TempDir File testFolder)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
// given
|
// given
|
||||||
String key = "projectPropertyKey";
|
String key = "projectPropertyKey";
|
||||||
|
@ -194,7 +194,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteIncludingPropertyKeysFromBlankLocation() {
|
void testExecuteIncludingPropertyKeysFromBlankLocation() {
|
||||||
// given
|
// given
|
||||||
String key = "projectPropertyKey";
|
String key = "projectPropertyKey";
|
||||||
String value = "projectPropertyValue";
|
String value = "projectPropertyValue";
|
||||||
|
@ -214,7 +214,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteIncludingPropertyKeysFromXmlFile(@TempDir File testFolder)
|
void testExecuteIncludingPropertyKeysFromXmlFile(@TempDir File testFolder)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
// given
|
// given
|
||||||
String key = "projectPropertyKey";
|
String key = "projectPropertyKey";
|
||||||
|
@ -247,7 +247,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteIncludingPropertyKeysFromInvalidXmlFile(@TempDir File testFolder)
|
void testExecuteIncludingPropertyKeysFromInvalidXmlFile(@TempDir File testFolder)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
// given
|
// given
|
||||||
String key = "projectPropertyKey";
|
String key = "projectPropertyKey";
|
||||||
|
@ -273,7 +273,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteWithQuietModeOn(@TempDir File testFolder) throws Exception {
|
void testExecuteWithQuietModeOn(@TempDir File testFolder) throws Exception {
|
||||||
// given
|
// given
|
||||||
mojo.setQuiet(true);
|
mojo.setQuiet(true);
|
||||||
mojo.setIncludePropertyKeysFromFiles(new String[] {
|
mojo.setIncludePropertyKeysFromFiles(new String[] {
|
||||||
|
@ -290,7 +290,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteIncludingPropertyKeysFromInvalidFile() {
|
void testExecuteIncludingPropertyKeysFromInvalidFile() {
|
||||||
// given
|
// given
|
||||||
mojo.setIncludePropertyKeysFromFiles(new String[] {
|
mojo.setIncludePropertyKeysFromFiles(new String[] {
|
||||||
"invalid location"
|
"invalid location"
|
||||||
|
@ -301,7 +301,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteWithEnvironmentProperties(@TempDir File testFolder) throws Exception {
|
void testExecuteWithEnvironmentProperties(@TempDir File testFolder) throws Exception {
|
||||||
// given
|
// given
|
||||||
mojo.setIncludeEnvironmentVariables(true);
|
mojo.setIncludeEnvironmentVariables(true);
|
||||||
|
|
||||||
|
@ -318,7 +318,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteWithSystemProperties(@TempDir File testFolder) throws Exception {
|
void testExecuteWithSystemProperties(@TempDir File testFolder) throws Exception {
|
||||||
// given
|
// given
|
||||||
String key = "systemPropertyKey";
|
String key = "systemPropertyKey";
|
||||||
String value = "systemPropertyValue";
|
String value = "systemPropertyValue";
|
||||||
|
@ -337,7 +337,7 @@ public class WritePredefinedProjectPropertiesTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testExecuteWithSystemPropertiesAndEscapeChars(@TempDir File testFolder)
|
void testExecuteWithSystemPropertiesAndEscapeChars(@TempDir File testFolder)
|
||||||
throws Exception {
|
throws Exception {
|
||||||
// given
|
// given
|
||||||
String key = "systemPropertyKey ";
|
String key = "systemPropertyKey ";
|
||||||
|
|
|
@ -15,16 +15,27 @@
|
||||||
<snapshotRepository>
|
<snapshotRepository>
|
||||||
<id>dnet45-snapshots</id>
|
<id>dnet45-snapshots</id>
|
||||||
<name>DNet45 Snapshots</name>
|
<name>DNet45 Snapshots</name>
|
||||||
<url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-snapshots</url>
|
<url>https://maven.d4science.org/nexus/content/repositories/dnet45-snapshots</url>
|
||||||
<layout>default</layout>
|
<layout>default</layout>
|
||||||
</snapshotRepository>
|
</snapshotRepository>
|
||||||
<repository>
|
<repository>
|
||||||
<id>dnet45-releases</id>
|
<id>dnet45-releases</id>
|
||||||
<url>http://maven.research-infrastructures.eu/nexus/content/repositories/dnet45-releases</url>
|
<url>https://maven.d4science.org/nexus/content/repositories/dnet45-releases</url>
|
||||||
</repository>
|
</repository>
|
||||||
|
<site>
|
||||||
|
<id>DHPSite</id>
|
||||||
|
<url>${dhp.site.stage.path}/dhp-build/dhp-code-style</url>
|
||||||
|
</site>
|
||||||
</distributionManagement>
|
</distributionManagement>
|
||||||
|
|
||||||
<build>
|
<build>
|
||||||
|
<extensions>
|
||||||
|
<extension>
|
||||||
|
<groupId>org.apache.maven.wagon</groupId>
|
||||||
|
<artifactId>wagon-ssh</artifactId>
|
||||||
|
<version>2.10</version>
|
||||||
|
</extension>
|
||||||
|
</extensions>
|
||||||
<pluginManagement>
|
<pluginManagement>
|
||||||
<plugins>
|
<plugins>
|
||||||
<plugin>
|
<plugin>
|
||||||
|
@ -35,7 +46,7 @@
|
||||||
<plugin>
|
<plugin>
|
||||||
<groupId>org.apache.maven.plugins</groupId>
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
<artifactId>maven-site-plugin</artifactId>
|
<artifactId>maven-site-plugin</artifactId>
|
||||||
<version>3.7.1</version>
|
<version>3.9.1</version>
|
||||||
</plugin>
|
</plugin>
|
||||||
</plugins>
|
</plugins>
|
||||||
</pluginManagement>
|
</pluginManagement>
|
||||||
|
@ -43,6 +54,7 @@
|
||||||
|
|
||||||
<properties>
|
<properties>
|
||||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
<dhp.site.stage.path>sftp://dnet-hadoop@static-web.d4science.org/dnet-hadoop</dhp.site.stage.path>
|
||||||
</properties>
|
</properties>
|
||||||
|
|
||||||
</project>
|
</project>
|
|
@ -0,0 +1,21 @@
|
||||||
|
style = defaultWithAlign
|
||||||
|
|
||||||
|
align.openParenCallSite = false
|
||||||
|
align.openParenDefnSite = false
|
||||||
|
align.tokens = [{code = "->"}, {code = "<-"}, {code = "=>", owner = "Case"}]
|
||||||
|
continuationIndent.callSite = 2
|
||||||
|
continuationIndent.defnSite = 2
|
||||||
|
danglingParentheses = true
|
||||||
|
indentOperator = spray
|
||||||
|
maxColumn = 120
|
||||||
|
newlines.alwaysBeforeTopLevelStatements = true
|
||||||
|
project.excludeFilters = [".*\\.sbt"]
|
||||||
|
rewrite.rules = [AvoidInfix]
|
||||||
|
rewrite.rules = [ExpandImportSelectors]
|
||||||
|
rewrite.rules = [RedundantBraces]
|
||||||
|
rewrite.rules = [RedundantParens]
|
||||||
|
rewrite.rules = [SortImports]
|
||||||
|
rewrite.rules = [SortModifiers]
|
||||||
|
rewrite.rules = [PreferCurlyFors]
|
||||||
|
spaces.inImportCurlyBraces = false
|
||||||
|
unindentTopLevelOperators = true
|
|
@ -0,0 +1,21 @@
|
||||||
|
<?xml version="1.0" encoding="ISO-8859-1"?>
|
||||||
|
<project xmlns="http://maven.apache.org/DECORATION/1.8.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/DECORATION/1.8.0 https://maven.apache.org/xsd/decoration-1.8.0.xsd"
|
||||||
|
name="DHP-Aggregation">
|
||||||
|
<skin>
|
||||||
|
<groupId>org.apache.maven.skins</groupId>
|
||||||
|
<artifactId>maven-fluido-skin</artifactId>
|
||||||
|
<version>1.8</version>
|
||||||
|
</skin>
|
||||||
|
<poweredBy>
|
||||||
|
<logo name="OpenAIRE Research Graph" href="https://graph.openaire.eu/"
|
||||||
|
img="https://graph.openaire.eu/assets/common-assets/logo-large-graph.png"/>
|
||||||
|
</poweredBy>
|
||||||
|
<body>
|
||||||
|
<links>
|
||||||
|
<item name="Code" href="https://code-repo.d4science.org/" />
|
||||||
|
</links>
|
||||||
|
<menu ref="modules" />
|
||||||
|
<menu ref="reports"/>
|
||||||
|
</body>
|
||||||
|
</project>
|
|
@ -10,6 +10,9 @@
|
||||||
<packaging>pom</packaging>
|
<packaging>pom</packaging>
|
||||||
|
|
||||||
<description>This module is a container for the build tools used in dnet-hadoop</description>
|
<description>This module is a container for the build tools used in dnet-hadoop</description>
|
||||||
|
<properties>
|
||||||
|
<maven.javadoc.skip>true</maven.javadoc.skip>
|
||||||
|
</properties>
|
||||||
|
|
||||||
<modules>
|
<modules>
|
||||||
<module>dhp-code-style</module>
|
<module>dhp-code-style</module>
|
||||||
|
@ -17,4 +20,12 @@
|
||||||
<module>dhp-build-properties-maven-plugin</module>
|
<module>dhp-build-properties-maven-plugin</module>
|
||||||
</modules>
|
</modules>
|
||||||
|
|
||||||
|
|
||||||
|
<distributionManagement>
|
||||||
|
<site>
|
||||||
|
<id>DHPSite</id>
|
||||||
|
<url>${dhp.site.stage.path}/dhp-build/</url>
|
||||||
|
</site>
|
||||||
|
</distributionManagement>
|
||||||
|
|
||||||
</project>
|
</project>
|
||||||
|
|
|
@ -0,0 +1,22 @@
|
||||||
|
<?xml version="1.0" encoding="ISO-8859-1"?>
|
||||||
|
<project xmlns="http://maven.apache.org/DECORATION/1.8.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/DECORATION/1.8.0 https://maven.apache.org/xsd/decoration-1.8.0.xsd"
|
||||||
|
name="DHP-Aggregation">
|
||||||
|
<skin>
|
||||||
|
<groupId>org.apache.maven.skins</groupId>
|
||||||
|
<artifactId>maven-fluido-skin</artifactId>
|
||||||
|
<version>1.8</version>
|
||||||
|
</skin>
|
||||||
|
<poweredBy>
|
||||||
|
<logo name="OpenAIRE Research Graph" href="https://graph.openaire.eu/"
|
||||||
|
img="https://graph.openaire.eu/assets/common-assets/logo-large-graph.png"/>
|
||||||
|
</poweredBy>
|
||||||
|
<body>
|
||||||
|
<links>
|
||||||
|
<item name="Code" href="https://code-repo.d4science.org/" />
|
||||||
|
</links>
|
||||||
|
|
||||||
|
<menu ref="modules" />
|
||||||
|
<menu ref="reports"/>
|
||||||
|
</body>
|
||||||
|
</project>
|
|
@ -6,13 +6,58 @@
|
||||||
<groupId>eu.dnetlib.dhp</groupId>
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
<artifactId>dhp</artifactId>
|
<artifactId>dhp</artifactId>
|
||||||
<version>1.2.4-SNAPSHOT</version>
|
<version>1.2.4-SNAPSHOT</version>
|
||||||
<relativePath>../</relativePath>
|
<relativePath>../pom.xml</relativePath>
|
||||||
|
|
||||||
</parent>
|
</parent>
|
||||||
|
|
||||||
<artifactId>dhp-common</artifactId>
|
<artifactId>dhp-common</artifactId>
|
||||||
<packaging>jar</packaging>
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<distributionManagement>
|
||||||
|
<site>
|
||||||
|
<id>DHPSite</id>
|
||||||
|
<url>${dhp.site.stage.path}/dhp-common</url>
|
||||||
|
</site>
|
||||||
|
</distributionManagement>
|
||||||
|
|
||||||
<description>This module contains common utilities meant to be used across the dnet-hadoop submodules</description>
|
<description>This module contains common utilities meant to be used across the dnet-hadoop submodules</description>
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>net.alchim31.maven</groupId>
|
||||||
|
<artifactId>scala-maven-plugin</artifactId>
|
||||||
|
<version>${net.alchim31.maven.version}</version>
|
||||||
|
<executions>
|
||||||
|
<execution>
|
||||||
|
<id>scala-compile-first</id>
|
||||||
|
<phase>initialize</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>add-source</goal>
|
||||||
|
<goal>compile</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<id>scala-test-compile</id>
|
||||||
|
<phase>process-test-resources</phase>
|
||||||
|
<goals>
|
||||||
|
<goal>testCompile</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
<execution>
|
||||||
|
<id>scala-doc</id>
|
||||||
|
<phase>process-resources</phase> <!-- or wherever -->
|
||||||
|
<goals>
|
||||||
|
<goal>doc</goal>
|
||||||
|
</goals>
|
||||||
|
</execution>
|
||||||
|
</executions>
|
||||||
|
<configuration>
|
||||||
|
<scalaVersion>${scala.version}</scalaVersion>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
|
||||||
|
</build>
|
||||||
|
|
||||||
<dependencies>
|
<dependencies>
|
||||||
|
|
||||||
|
@ -20,6 +65,15 @@
|
||||||
<groupId>org.apache.hadoop</groupId>
|
<groupId>org.apache.hadoop</groupId>
|
||||||
<artifactId>hadoop-common</artifactId>
|
<artifactId>hadoop-common</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.github.sisyphsu</groupId>
|
||||||
|
<artifactId>dateparser</artifactId>
|
||||||
|
</dependency>
|
||||||
|
<dependency>
|
||||||
|
<groupId>me.xuender</groupId>
|
||||||
|
<artifactId>unidecode</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>org.apache.spark</groupId>
|
<groupId>org.apache.spark</groupId>
|
||||||
<artifactId>spark-core_2.11</artifactId>
|
<artifactId>spark-core_2.11</artifactId>
|
||||||
|
@ -29,12 +83,6 @@
|
||||||
<artifactId>spark-sql_2.11</artifactId>
|
<artifactId>spark-sql_2.11</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>eu.dnetlib.dhp</groupId>
|
|
||||||
<artifactId>dhp-schemas</artifactId>
|
|
||||||
<version>${project.version}</version>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>commons-cli</groupId>
|
<groupId>commons-cli</groupId>
|
||||||
<artifactId>commons-cli</artifactId>
|
<artifactId>commons-cli</artifactId>
|
||||||
|
@ -59,11 +107,6 @@
|
||||||
<groupId>com.fasterxml.jackson.core</groupId>
|
<groupId>com.fasterxml.jackson.core</groupId>
|
||||||
<artifactId>jackson-databind</artifactId>
|
<artifactId>jackson-databind</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
<!-- https://mvnrepository.com/artifact/com.rabbitmq/amqp-client -->
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.rabbitmq</groupId>
|
|
||||||
<artifactId>amqp-client</artifactId>
|
|
||||||
</dependency>
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>net.sf.saxon</groupId>
|
<groupId>net.sf.saxon</groupId>
|
||||||
<artifactId>Saxon-HE</artifactId>
|
<artifactId>Saxon-HE</artifactId>
|
||||||
|
@ -104,10 +147,24 @@
|
||||||
<artifactId>dnet-pace-core</artifactId>
|
<artifactId>dnet-pace-core</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.apache.httpcomponents</groupId>
|
||||||
|
<artifactId>httpclient</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>org.mongodb</groupId>
|
||||||
|
<artifactId>mongo-java-driver</artifactId>
|
||||||
|
</dependency>
|
||||||
|
|
||||||
<dependency>
|
<dependency>
|
||||||
<groupId>eu.dnetlib.dhp</groupId>
|
<groupId>eu.dnetlib.dhp</groupId>
|
||||||
<artifactId>dhp-schemas</artifactId>
|
<artifactId>dhp-schemas</artifactId>
|
||||||
<version>${project.version}</version>
|
</dependency>
|
||||||
|
|
||||||
|
<dependency>
|
||||||
|
<groupId>com.opencsv</groupId>
|
||||||
|
<artifactId>opencsv</artifactId>
|
||||||
</dependency>
|
</dependency>
|
||||||
</dependencies>
|
</dependencies>
|
||||||
|
|
||||||
|
|
|
@ -1,48 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.collector.worker.model;
|
|
||||||
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
public class ApiDescriptor {
|
|
||||||
|
|
||||||
private String id;
|
|
||||||
|
|
||||||
private String baseUrl;
|
|
||||||
|
|
||||||
private String protocol;
|
|
||||||
|
|
||||||
private Map<String, String> params = new HashMap<>();
|
|
||||||
|
|
||||||
public String getBaseUrl() {
|
|
||||||
return baseUrl;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setBaseUrl(final String baseUrl) {
|
|
||||||
this.baseUrl = baseUrl;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getId() {
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setId(final String id) {
|
|
||||||
this.id = id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Map<String, String> getParams() {
|
|
||||||
return params;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setParams(final HashMap<String, String> params) {
|
|
||||||
this.params = params;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getProtocol() {
|
|
||||||
return protocol;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setProtocol(final String protocol) {
|
|
||||||
this.protocol = protocol;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,119 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.data.mdstore.manager.common.model;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
import java.util.UUID;
|
|
||||||
|
|
||||||
import javax.persistence.Column;
|
|
||||||
import javax.persistence.Entity;
|
|
||||||
import javax.persistence.Id;
|
|
||||||
import javax.persistence.Table;
|
|
||||||
|
|
||||||
@Entity
|
|
||||||
@Table(name = "mdstores")
|
|
||||||
public class MDStore implements Serializable {
|
|
||||||
|
|
||||||
/** */
|
|
||||||
private static final long serialVersionUID = 3160530489149700055L;
|
|
||||||
|
|
||||||
@Id
|
|
||||||
@Column(name = "id")
|
|
||||||
private String id;
|
|
||||||
|
|
||||||
@Column(name = "format")
|
|
||||||
private String format;
|
|
||||||
|
|
||||||
@Column(name = "layout")
|
|
||||||
private String layout;
|
|
||||||
|
|
||||||
@Column(name = "interpretation")
|
|
||||||
private String interpretation;
|
|
||||||
|
|
||||||
@Column(name = "datasource_name")
|
|
||||||
private String datasourceName;
|
|
||||||
|
|
||||||
@Column(name = "datasource_id")
|
|
||||||
private String datasourceId;
|
|
||||||
|
|
||||||
@Column(name = "api_id")
|
|
||||||
private String apiId;
|
|
||||||
|
|
||||||
public String getId() {
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setId(final String id) {
|
|
||||||
this.id = id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getFormat() {
|
|
||||||
return format;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setFormat(final String format) {
|
|
||||||
this.format = format;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getLayout() {
|
|
||||||
return layout;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setLayout(final String layout) {
|
|
||||||
this.layout = layout;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getInterpretation() {
|
|
||||||
return interpretation;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setInterpretation(final String interpretation) {
|
|
||||||
this.interpretation = interpretation;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getDatasourceName() {
|
|
||||||
return datasourceName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDatasourceName(final String datasourceName) {
|
|
||||||
this.datasourceName = datasourceName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getDatasourceId() {
|
|
||||||
return datasourceId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDatasourceId(final String datasourceId) {
|
|
||||||
this.datasourceId = datasourceId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getApiId() {
|
|
||||||
return apiId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setApiId(final String apiId) {
|
|
||||||
this.apiId = apiId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static MDStore newInstance(
|
|
||||||
final String format, final String layout, final String interpretation) {
|
|
||||||
return newInstance(format, layout, interpretation, null, null, null);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static MDStore newInstance(
|
|
||||||
final String format,
|
|
||||||
final String layout,
|
|
||||||
final String interpretation,
|
|
||||||
final String dsName,
|
|
||||||
final String dsId,
|
|
||||||
final String apiId) {
|
|
||||||
final MDStore md = new MDStore();
|
|
||||||
md.setId("md-" + UUID.randomUUID());
|
|
||||||
md.setFormat(format);
|
|
||||||
md.setLayout(layout);
|
|
||||||
md.setInterpretation(interpretation);
|
|
||||||
md.setDatasourceName(dsName);
|
|
||||||
md.setDatasourceId(dsId);
|
|
||||||
md.setApiId(apiId);
|
|
||||||
return md;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,51 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.data.mdstore.manager.common.model;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
|
|
||||||
import javax.persistence.Column;
|
|
||||||
import javax.persistence.Entity;
|
|
||||||
import javax.persistence.Id;
|
|
||||||
import javax.persistence.Table;
|
|
||||||
|
|
||||||
@Entity
|
|
||||||
@Table(name = "mdstore_current_versions")
|
|
||||||
public class MDStoreCurrentVersion implements Serializable {
|
|
||||||
|
|
||||||
/** */
|
|
||||||
private static final long serialVersionUID = -4757725888593745773L;
|
|
||||||
|
|
||||||
@Id
|
|
||||||
@Column(name = "mdstore")
|
|
||||||
private String mdstore;
|
|
||||||
|
|
||||||
@Column(name = "current_version")
|
|
||||||
private String currentVersion;
|
|
||||||
|
|
||||||
public String getMdstore() {
|
|
||||||
return mdstore;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setMdstore(final String mdstore) {
|
|
||||||
this.mdstore = mdstore;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getCurrentVersion() {
|
|
||||||
return currentVersion;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setCurrentVersion(final String currentVersion) {
|
|
||||||
this.currentVersion = currentVersion;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static MDStoreCurrentVersion newInstance(final String mdId, final String versionId) {
|
|
||||||
final MDStoreCurrentVersion cv = new MDStoreCurrentVersion();
|
|
||||||
cv.setMdstore(mdId);
|
|
||||||
cv.setCurrentVersion(versionId);
|
|
||||||
return cv;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static MDStoreCurrentVersion newInstance(final MDStoreVersion v) {
|
|
||||||
return newInstance(v.getMdstore(), v.getId());
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,99 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.data.mdstore.manager.common.model;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
import java.util.Date;
|
|
||||||
|
|
||||||
import javax.persistence.Column;
|
|
||||||
import javax.persistence.Entity;
|
|
||||||
import javax.persistence.Id;
|
|
||||||
import javax.persistence.Table;
|
|
||||||
import javax.persistence.Temporal;
|
|
||||||
import javax.persistence.TemporalType;
|
|
||||||
|
|
||||||
@Entity
|
|
||||||
@Table(name = "mdstore_versions")
|
|
||||||
public class MDStoreVersion implements Serializable {
|
|
||||||
|
|
||||||
/** */
|
|
||||||
private static final long serialVersionUID = -4763494442274298339L;
|
|
||||||
|
|
||||||
@Id
|
|
||||||
@Column(name = "id")
|
|
||||||
private String id;
|
|
||||||
|
|
||||||
@Column(name = "mdstore")
|
|
||||||
private String mdstore;
|
|
||||||
|
|
||||||
@Column(name = "writing")
|
|
||||||
private boolean writing;
|
|
||||||
|
|
||||||
@Column(name = "readcount")
|
|
||||||
private int readCount = 0;
|
|
||||||
|
|
||||||
@Column(name = "lastupdate")
|
|
||||||
@Temporal(TemporalType.TIMESTAMP)
|
|
||||||
private Date lastUpdate;
|
|
||||||
|
|
||||||
@Column(name = "size")
|
|
||||||
private long size = 0;
|
|
||||||
|
|
||||||
public static MDStoreVersion newInstance(final String mdId, final boolean writing) {
|
|
||||||
final MDStoreVersion t = new MDStoreVersion();
|
|
||||||
t.setId(mdId + "-" + new Date().getTime());
|
|
||||||
t.setMdstore(mdId);
|
|
||||||
t.setLastUpdate(null);
|
|
||||||
t.setWriting(writing);
|
|
||||||
t.setReadCount(0);
|
|
||||||
t.setSize(0);
|
|
||||||
return t;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getId() {
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setId(final String id) {
|
|
||||||
this.id = id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getMdstore() {
|
|
||||||
return mdstore;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setMdstore(final String mdstore) {
|
|
||||||
this.mdstore = mdstore;
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean isWriting() {
|
|
||||||
return writing;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setWriting(final boolean writing) {
|
|
||||||
this.writing = writing;
|
|
||||||
}
|
|
||||||
|
|
||||||
public int getReadCount() {
|
|
||||||
return readCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setReadCount(final int readCount) {
|
|
||||||
this.readCount = readCount;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Date getLastUpdate() {
|
|
||||||
return lastUpdate;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setLastUpdate(final Date lastUpdate) {
|
|
||||||
this.lastUpdate = lastUpdate;
|
|
||||||
}
|
|
||||||
|
|
||||||
public long getSize() {
|
|
||||||
return size;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setSize(final long size) {
|
|
||||||
this.size = size;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,143 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.data.mdstore.manager.common.model;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
import java.util.Date;
|
|
||||||
|
|
||||||
import javax.persistence.Column;
|
|
||||||
import javax.persistence.Entity;
|
|
||||||
import javax.persistence.Id;
|
|
||||||
import javax.persistence.Table;
|
|
||||||
import javax.persistence.Temporal;
|
|
||||||
import javax.persistence.TemporalType;
|
|
||||||
|
|
||||||
@Entity
|
|
||||||
@Table(name = "mdstores_with_info")
|
|
||||||
public class MDStoreWithInfo implements Serializable {
|
|
||||||
|
|
||||||
/** */
|
|
||||||
private static final long serialVersionUID = -8445784770687571492L;
|
|
||||||
|
|
||||||
@Id
|
|
||||||
@Column(name = "id")
|
|
||||||
private String id;
|
|
||||||
|
|
||||||
@Column(name = "format")
|
|
||||||
private String format;
|
|
||||||
|
|
||||||
@Column(name = "layout")
|
|
||||||
private String layout;
|
|
||||||
|
|
||||||
@Column(name = "interpretation")
|
|
||||||
private String interpretation;
|
|
||||||
|
|
||||||
@Column(name = "datasource_name")
|
|
||||||
private String datasourceName;
|
|
||||||
|
|
||||||
@Column(name = "datasource_id")
|
|
||||||
private String datasourceId;
|
|
||||||
|
|
||||||
@Column(name = "api_id")
|
|
||||||
private String apiId;
|
|
||||||
|
|
||||||
@Column(name = "current_version")
|
|
||||||
private String currentVersion;
|
|
||||||
|
|
||||||
@Column(name = "lastupdate")
|
|
||||||
@Temporal(TemporalType.TIMESTAMP)
|
|
||||||
private Date lastUpdate;
|
|
||||||
|
|
||||||
@Column(name = "size")
|
|
||||||
private long size = 0;
|
|
||||||
|
|
||||||
@Column(name = "n_versions")
|
|
||||||
private long numberOfVersions = 0;
|
|
||||||
|
|
||||||
public String getId() {
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setId(final String id) {
|
|
||||||
this.id = id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getFormat() {
|
|
||||||
return format;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setFormat(final String format) {
|
|
||||||
this.format = format;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getLayout() {
|
|
||||||
return layout;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setLayout(final String layout) {
|
|
||||||
this.layout = layout;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getInterpretation() {
|
|
||||||
return interpretation;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setInterpretation(final String interpretation) {
|
|
||||||
this.interpretation = interpretation;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getDatasourceName() {
|
|
||||||
return datasourceName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDatasourceName(final String datasourceName) {
|
|
||||||
this.datasourceName = datasourceName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getDatasourceId() {
|
|
||||||
return datasourceId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDatasourceId(final String datasourceId) {
|
|
||||||
this.datasourceId = datasourceId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getApiId() {
|
|
||||||
return apiId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setApiId(final String apiId) {
|
|
||||||
this.apiId = apiId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getCurrentVersion() {
|
|
||||||
return currentVersion;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setCurrentVersion(final String currentVersion) {
|
|
||||||
this.currentVersion = currentVersion;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Date getLastUpdate() {
|
|
||||||
return lastUpdate;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setLastUpdate(final Date lastUpdate) {
|
|
||||||
this.lastUpdate = lastUpdate;
|
|
||||||
}
|
|
||||||
|
|
||||||
public long getSize() {
|
|
||||||
return size;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setSize(final long size) {
|
|
||||||
this.size = size;
|
|
||||||
}
|
|
||||||
|
|
||||||
public long getNumberOfVersions() {
|
|
||||||
return numberOfVersions;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setNumberOfVersions(final long numberOfVersions) {
|
|
||||||
this.numberOfVersions = numberOfVersions;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,10 +1,7 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.application;
|
package eu.dnetlib.dhp.application;
|
||||||
|
|
||||||
import java.io.ByteArrayInputStream;
|
import java.io.*;
|
||||||
import java.io.ByteArrayOutputStream;
|
|
||||||
import java.io.Serializable;
|
|
||||||
import java.io.StringWriter;
|
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.zip.GZIPInputStream;
|
import java.util.zip.GZIPInputStream;
|
||||||
import java.util.zip.GZIPOutputStream;
|
import java.util.zip.GZIPOutputStream;
|
||||||
|
@ -12,17 +9,21 @@ import java.util.zip.GZIPOutputStream;
|
||||||
import org.apache.commons.cli.*;
|
import org.apache.commons.cli.*;
|
||||||
import org.apache.commons.codec.binary.Base64;
|
import org.apache.commons.codec.binary.Base64;
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
public class ArgumentApplicationParser implements Serializable {
|
public class ArgumentApplicationParser implements Serializable {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(ArgumentApplicationParser.class);
|
||||||
|
|
||||||
private final Options options = new Options();
|
private final Options options = new Options();
|
||||||
private final Map<String, String> objectMap = new HashMap<>();
|
private final Map<String, String> objectMap = new HashMap<>();
|
||||||
|
|
||||||
private final List<String> compressedValues = new ArrayList<>();
|
private final List<String> compressedValues = new ArrayList<>();
|
||||||
|
|
||||||
public ArgumentApplicationParser(final String json_configuration) throws Exception {
|
public ArgumentApplicationParser(final String json_configuration) throws IOException {
|
||||||
final ObjectMapper mapper = new ObjectMapper();
|
final ObjectMapper mapper = new ObjectMapper();
|
||||||
final OptionsParameter[] configuration = mapper.readValue(json_configuration, OptionsParameter[].class);
|
final OptionsParameter[] configuration = mapper.readValue(json_configuration, OptionsParameter[].class);
|
||||||
createOptionMap(configuration);
|
createOptionMap(configuration);
|
||||||
|
@ -33,7 +34,6 @@ public class ArgumentApplicationParser implements Serializable {
|
||||||
}
|
}
|
||||||
|
|
||||||
private void createOptionMap(final OptionsParameter[] configuration) {
|
private void createOptionMap(final OptionsParameter[] configuration) {
|
||||||
|
|
||||||
Arrays
|
Arrays
|
||||||
.stream(configuration)
|
.stream(configuration)
|
||||||
.map(
|
.map(
|
||||||
|
@ -47,10 +47,6 @@ public class ArgumentApplicationParser implements Serializable {
|
||||||
return o;
|
return o;
|
||||||
})
|
})
|
||||||
.forEach(options::addOption);
|
.forEach(options::addOption);
|
||||||
|
|
||||||
// HelpFormatter formatter = new HelpFormatter();
|
|
||||||
// formatter.printHelp("myapp", null, options, null, true);
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String decompressValue(final String abstractCompressed) {
|
public static String decompressValue(final String abstractCompressed) {
|
||||||
|
@ -60,13 +56,13 @@ public class ArgumentApplicationParser implements Serializable {
|
||||||
final StringWriter stringWriter = new StringWriter();
|
final StringWriter stringWriter = new StringWriter();
|
||||||
IOUtils.copy(gis, stringWriter);
|
IOUtils.copy(gis, stringWriter);
|
||||||
return stringWriter.toString();
|
return stringWriter.toString();
|
||||||
} catch (Throwable e) {
|
} catch (IOException e) {
|
||||||
System.out.println("Wrong value to decompress:" + abstractCompressed);
|
log.error("Wrong value to decompress: {}", abstractCompressed);
|
||||||
throw new RuntimeException(e);
|
throw new IllegalArgumentException(e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String compressArgument(final String value) throws Exception {
|
public static String compressArgument(final String value) throws IOException {
|
||||||
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
ByteArrayOutputStream out = new ByteArrayOutputStream();
|
||||||
GZIPOutputStream gzip = new GZIPOutputStream(out);
|
GZIPOutputStream gzip = new GZIPOutputStream(out);
|
||||||
gzip.write(value.getBytes());
|
gzip.write(value.getBytes());
|
||||||
|
@ -74,7 +70,7 @@ public class ArgumentApplicationParser implements Serializable {
|
||||||
return java.util.Base64.getEncoder().encodeToString(out.toByteArray());
|
return java.util.Base64.getEncoder().encodeToString(out.toByteArray());
|
||||||
}
|
}
|
||||||
|
|
||||||
public void parseArgument(final String[] args) throws Exception {
|
public void parseArgument(final String[] args) throws ParseException {
|
||||||
CommandLineParser parser = new BasicParser();
|
CommandLineParser parser = new BasicParser();
|
||||||
CommandLine cmd = parser.parse(options, args);
|
CommandLine cmd = parser.parse(options, args);
|
||||||
Arrays
|
Arrays
|
||||||
|
|
|
@ -9,9 +9,6 @@ public class OptionsParameter {
|
||||||
private boolean paramRequired;
|
private boolean paramRequired;
|
||||||
private boolean compressed;
|
private boolean compressed;
|
||||||
|
|
||||||
public OptionsParameter() {
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getParamName() {
|
public String getParamName() {
|
||||||
return paramName;
|
return paramName;
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,48 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.collection;
|
||||||
|
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
public class ApiDescriptor {
|
||||||
|
|
||||||
|
private String id;
|
||||||
|
|
||||||
|
private String baseUrl;
|
||||||
|
|
||||||
|
private String protocol;
|
||||||
|
|
||||||
|
private Map<String, String> params = new HashMap<>();
|
||||||
|
|
||||||
|
public String getBaseUrl() {
|
||||||
|
return baseUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setBaseUrl(final String baseUrl) {
|
||||||
|
this.baseUrl = baseUrl;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getId() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setId(final String id) {
|
||||||
|
this.id = id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, String> getParams() {
|
||||||
|
return params;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setParams(final Map<String, String> params) {
|
||||||
|
this.params = params;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getProtocol() {
|
||||||
|
return protocol;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setProtocol(final String protocol) {
|
||||||
|
this.protocol = protocol;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,61 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.common;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import com.google.common.collect.Maps;
|
||||||
|
|
||||||
|
public class Constants {
|
||||||
|
|
||||||
|
public static final Map<String, String> accessRightsCoarMap = Maps.newHashMap();
|
||||||
|
public static final Map<String, String> coarCodeLabelMap = Maps.newHashMap();
|
||||||
|
|
||||||
|
public static String COAR_ACCESS_RIGHT_SCHEMA = "http://vocabularies.coar-repositories.org/documentation/access_rights/";
|
||||||
|
|
||||||
|
private Constants() {
|
||||||
|
}
|
||||||
|
|
||||||
|
static {
|
||||||
|
accessRightsCoarMap.put("OPEN", "c_abf2");
|
||||||
|
accessRightsCoarMap.put("RESTRICTED", "c_16ec");
|
||||||
|
accessRightsCoarMap.put("OPEN SOURCE", "c_abf2");
|
||||||
|
accessRightsCoarMap.put("CLOSED", "c_14cb");
|
||||||
|
accessRightsCoarMap.put("EMBARGO", "c_f1cf");
|
||||||
|
}
|
||||||
|
|
||||||
|
static {
|
||||||
|
coarCodeLabelMap.put("c_abf2", "OPEN");
|
||||||
|
coarCodeLabelMap.put("c_16ec", "RESTRICTED");
|
||||||
|
coarCodeLabelMap.put("c_14cb", "CLOSED");
|
||||||
|
coarCodeLabelMap.put("c_f1cf", "EMBARGO");
|
||||||
|
}
|
||||||
|
|
||||||
|
public static final String SEQUENCE_FILE_NAME = "/sequence_file";
|
||||||
|
public static final String REPORT_FILE_NAME = "/report";
|
||||||
|
public static final String MDSTORE_DATA_PATH = "/store";
|
||||||
|
public static final String MDSTORE_SIZE_PATH = "/size";
|
||||||
|
|
||||||
|
public static final String COLLECTION_MODE = "collectionMode";
|
||||||
|
public static final String METADATA_ENCODING = "metadataEncoding";
|
||||||
|
public static final String OOZIE_WF_PATH = "oozieWfPath";
|
||||||
|
public static final String DNET_MESSAGE_MGR_URL = "dnetMessageManagerURL";
|
||||||
|
|
||||||
|
public static final String MAX_NUMBER_OF_RETRY = "maxNumberOfRetry";
|
||||||
|
public static final String REQUEST_DELAY = "requestDelay";
|
||||||
|
public static final String RETRY_DELAY = "retryDelay";
|
||||||
|
public static final String CONNECT_TIMEOUT = "connectTimeOut";
|
||||||
|
public static final String READ_TIMEOUT = "readTimeOut";
|
||||||
|
public static final String FROM_DATE_OVERRIDE = "fromDateOverride";
|
||||||
|
public static final String UNTIL_DATE_OVERRIDE = "untilDateOverride";
|
||||||
|
|
||||||
|
public static final String CONTENT_TOTALITEMS = "TotalItems";
|
||||||
|
public static final String CONTENT_INVALIDRECORDS = "InvalidRecords";
|
||||||
|
public static final String CONTENT_TRANSFORMEDRECORDS = "transformedItems";
|
||||||
|
|
||||||
|
// IETF Draft and used by Repositories like ZENODO , not included in APACHE HTTP java packages
|
||||||
|
// see https://ietf-wg-httpapi.github.io/ratelimit-headers/draft-ietf-httpapi-ratelimit-headers.html
|
||||||
|
public static final String HTTPHEADER_IETF_DRAFT_RATELIMIT_LIMIT = "X-RateLimit-Limit";
|
||||||
|
public static final String HTTPHEADER_IETF_DRAFT_RATELIMIT_REMAINING = "X-RateLimit-Remaining";
|
||||||
|
public static final String HTTPHEADER_IETF_DRAFT_RATELIMIT_RESET = "X-RateLimit-Reset";
|
||||||
|
|
||||||
|
}
|
|
@ -14,7 +14,7 @@ public class DbClient implements Closeable {
|
||||||
|
|
||||||
private static final Log log = LogFactory.getLog(DbClient.class);
|
private static final Log log = LogFactory.getLog(DbClient.class);
|
||||||
|
|
||||||
private Connection connection;
|
private final Connection connection;
|
||||||
|
|
||||||
public DbClient(final String address, final String login, final String password) {
|
public DbClient(final String address, final String login, final String password) {
|
||||||
|
|
||||||
|
|
|
@ -28,7 +28,7 @@ public class HdfsSupport {
|
||||||
* @param configuration Configuration of hadoop env
|
* @param configuration Configuration of hadoop env
|
||||||
*/
|
*/
|
||||||
public static boolean exists(String path, Configuration configuration) {
|
public static boolean exists(String path, Configuration configuration) {
|
||||||
logger.info("Removing path: {}", path);
|
logger.info("Checking existence for path: {}", path);
|
||||||
return rethrowAsRuntimeException(
|
return rethrowAsRuntimeException(
|
||||||
() -> {
|
() -> {
|
||||||
Path f = new Path(path);
|
Path f = new Path(path);
|
||||||
|
|
|
@ -0,0 +1,116 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.common;
|
||||||
|
|
||||||
|
import java.io.BufferedInputStream;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
|
||||||
|
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
|
||||||
|
import org.apache.hadoop.fs.*;
|
||||||
|
|
||||||
|
public class MakeTarArchive implements Serializable {
|
||||||
|
|
||||||
|
private static TarArchiveOutputStream getTar(FileSystem fileSystem, String outputPath) throws IOException {
|
||||||
|
Path hdfsWritePath = new Path(outputPath);
|
||||||
|
if (fileSystem.exists(hdfsWritePath)) {
|
||||||
|
fileSystem.delete(hdfsWritePath, true);
|
||||||
|
|
||||||
|
}
|
||||||
|
return new TarArchiveOutputStream(fileSystem.create(hdfsWritePath).getWrappedStream());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void write(FileSystem fileSystem, String inputPath, String outputPath, String dir_name)
|
||||||
|
throws IOException {
|
||||||
|
|
||||||
|
Path hdfsWritePath = new Path(outputPath);
|
||||||
|
if (fileSystem.exists(hdfsWritePath)) {
|
||||||
|
fileSystem.delete(hdfsWritePath, true);
|
||||||
|
|
||||||
|
}
|
||||||
|
try (TarArchiveOutputStream ar = new TarArchiveOutputStream(
|
||||||
|
fileSystem.create(hdfsWritePath).getWrappedStream())) {
|
||||||
|
|
||||||
|
RemoteIterator<LocatedFileStatus> iterator = fileSystem
|
||||||
|
.listFiles(
|
||||||
|
new Path(inputPath), true);
|
||||||
|
|
||||||
|
while (iterator.hasNext()) {
|
||||||
|
writeCurrentFile(fileSystem, dir_name, iterator, ar, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void tarMaxSize(FileSystem fileSystem, String inputPath, String outputPath, String dir_name,
|
||||||
|
int gBperSplit) throws IOException {
|
||||||
|
final long bytesPerSplit = 1024L * 1024L * 1024L * gBperSplit;
|
||||||
|
|
||||||
|
long sourceSize = fileSystem.getContentSummary(new Path(inputPath)).getSpaceConsumed();
|
||||||
|
|
||||||
|
if (sourceSize < bytesPerSplit) {
|
||||||
|
write(fileSystem, inputPath, outputPath + ".tar", dir_name);
|
||||||
|
} else {
|
||||||
|
int partNum = 0;
|
||||||
|
|
||||||
|
RemoteIterator<LocatedFileStatus> fileStatusListIterator = fileSystem
|
||||||
|
.listFiles(
|
||||||
|
new Path(inputPath), true);
|
||||||
|
boolean next = fileStatusListIterator.hasNext();
|
||||||
|
while (next) {
|
||||||
|
TarArchiveOutputStream ar = getTar(fileSystem, outputPath + "_" + (partNum + 1) + ".tar");
|
||||||
|
|
||||||
|
long current_size = 0;
|
||||||
|
while (next && current_size < bytesPerSplit) {
|
||||||
|
current_size = writeCurrentFile(fileSystem, dir_name, fileStatusListIterator, ar, current_size);
|
||||||
|
next = fileStatusListIterator.hasNext();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
partNum += 1;
|
||||||
|
ar.close();
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
private static long writeCurrentFile(FileSystem fileSystem, String dir_name,
|
||||||
|
RemoteIterator<LocatedFileStatus> fileStatusListIterator,
|
||||||
|
TarArchiveOutputStream ar, long current_size) throws IOException {
|
||||||
|
LocatedFileStatus fileStatus = fileStatusListIterator.next();
|
||||||
|
|
||||||
|
Path p = fileStatus.getPath();
|
||||||
|
String p_string = p.toString();
|
||||||
|
if (!p_string.endsWith("_SUCCESS")) {
|
||||||
|
String name = p_string.substring(p_string.lastIndexOf("/") + 1);
|
||||||
|
if (name.startsWith("part-") & name.length() > 10) {
|
||||||
|
String tmp = name.substring(0, 10);
|
||||||
|
if (name.contains(".")) {
|
||||||
|
tmp += name.substring(name.indexOf("."));
|
||||||
|
}
|
||||||
|
name = tmp;
|
||||||
|
}
|
||||||
|
TarArchiveEntry entry = new TarArchiveEntry(dir_name + "/" + name);
|
||||||
|
entry.setSize(fileStatus.getLen());
|
||||||
|
current_size += fileStatus.getLen();
|
||||||
|
ar.putArchiveEntry(entry);
|
||||||
|
|
||||||
|
InputStream is = fileSystem.open(fileStatus.getPath());
|
||||||
|
|
||||||
|
BufferedInputStream bis = new BufferedInputStream(is);
|
||||||
|
|
||||||
|
int count;
|
||||||
|
byte[] data = new byte[1024];
|
||||||
|
while ((count = bis.read(data, 0, data.length)) != -1) {
|
||||||
|
ar.write(data, 0, count);
|
||||||
|
}
|
||||||
|
bis.close();
|
||||||
|
ar.closeArchiveEntry();
|
||||||
|
|
||||||
|
}
|
||||||
|
return current_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,122 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.common;
|
||||||
|
|
||||||
|
import java.io.Closeable;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Optional;
|
||||||
|
import java.util.stream.StreamSupport;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.bson.Document;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.google.common.collect.Iterables;
|
||||||
|
import com.mongodb.BasicDBObject;
|
||||||
|
import com.mongodb.MongoClient;
|
||||||
|
import com.mongodb.MongoClientURI;
|
||||||
|
import com.mongodb.QueryBuilder;
|
||||||
|
import com.mongodb.client.FindIterable;
|
||||||
|
import com.mongodb.client.MongoCollection;
|
||||||
|
import com.mongodb.client.MongoDatabase;
|
||||||
|
|
||||||
|
public class MdstoreClient implements Closeable {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(MdstoreClient.class);
|
||||||
|
|
||||||
|
private final MongoClient client;
|
||||||
|
private final MongoDatabase db;
|
||||||
|
|
||||||
|
private static final String COLL_METADATA = "metadata";
|
||||||
|
private static final String COLL_METADATA_MANAGER = "metadataManager";
|
||||||
|
|
||||||
|
public MdstoreClient(final String baseUrl, final String dbName) {
|
||||||
|
this.client = new MongoClient(new MongoClientURI(baseUrl));
|
||||||
|
this.db = getDb(client, dbName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public MongoCollection<Document> mdStore(final String mdId) {
|
||||||
|
BasicDBObject query = (BasicDBObject) QueryBuilder.start("mdId").is(mdId).get();
|
||||||
|
|
||||||
|
log.info("querying current mdId: {}", query.toJson());
|
||||||
|
|
||||||
|
final String currentId = Optional
|
||||||
|
.ofNullable(getColl(db, COLL_METADATA_MANAGER, true).find(query))
|
||||||
|
.map(FindIterable::first)
|
||||||
|
.map(d -> d.getString("currentId"))
|
||||||
|
.orElseThrow(() -> new IllegalArgumentException("cannot find current mdstore id for: " + mdId));
|
||||||
|
|
||||||
|
log.info("currentId: {}", currentId);
|
||||||
|
|
||||||
|
return getColl(db, currentId, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, String> validCollections(
|
||||||
|
final String mdFormat, final String mdLayout, final String mdInterpretation) {
|
||||||
|
|
||||||
|
final Map<String, String> transactions = new HashMap<>();
|
||||||
|
for (final Document entry : getColl(db, COLL_METADATA_MANAGER, true).find()) {
|
||||||
|
final String mdId = entry.getString("mdId");
|
||||||
|
final String currentId = entry.getString("currentId");
|
||||||
|
if (StringUtils.isNoneBlank(mdId, currentId)) {
|
||||||
|
transactions.put(mdId, currentId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
final Map<String, String> res = new HashMap<>();
|
||||||
|
for (final Document entry : getColl(db, COLL_METADATA, true).find()) {
|
||||||
|
if (entry.getString("format").equals(mdFormat)
|
||||||
|
&& entry.getString("layout").equals(mdLayout)
|
||||||
|
&& entry.getString("interpretation").equals(mdInterpretation)
|
||||||
|
&& transactions.containsKey(entry.getString("mdId"))) {
|
||||||
|
res.put(entry.getString("mdId"), transactions.get(entry.getString("mdId")));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
private MongoDatabase getDb(final MongoClient client, final String dbName) {
|
||||||
|
if (!Iterables.contains(client.listDatabaseNames(), dbName)) {
|
||||||
|
final String err = String.format("Database '%s' not found in %s", dbName, client.getAddress());
|
||||||
|
log.warn(err);
|
||||||
|
throw new IllegalArgumentException(err);
|
||||||
|
}
|
||||||
|
return client.getDatabase(dbName);
|
||||||
|
}
|
||||||
|
|
||||||
|
private MongoCollection<Document> getColl(
|
||||||
|
final MongoDatabase db, final String collName, final boolean abortIfMissing) {
|
||||||
|
if (!Iterables.contains(db.listCollectionNames(), collName)) {
|
||||||
|
final String err = String
|
||||||
|
.format(
|
||||||
|
String.format("Missing collection '%s' in database '%s'", collName, db.getName()));
|
||||||
|
log.warn(err);
|
||||||
|
if (abortIfMissing) {
|
||||||
|
throw new IllegalArgumentException(err);
|
||||||
|
} else {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return db.getCollection(collName);
|
||||||
|
}
|
||||||
|
|
||||||
|
public Iterable<String> listRecords(final String collName) {
|
||||||
|
final MongoCollection<Document> coll = getColl(db, collName, false);
|
||||||
|
return coll == null
|
||||||
|
? new ArrayList<>()
|
||||||
|
: () -> StreamSupport
|
||||||
|
.stream(coll.find().spliterator(), false)
|
||||||
|
.filter(e -> e.containsKey("body"))
|
||||||
|
.map(e -> e.getString("body"))
|
||||||
|
.iterator();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
client.close();
|
||||||
|
}
|
||||||
|
}
|
|
@ -24,7 +24,6 @@ import com.google.common.hash.Hashing;
|
||||||
*/
|
*/
|
||||||
public class PacePerson {
|
public class PacePerson {
|
||||||
|
|
||||||
private static final String UTF8 = "UTF-8";
|
|
||||||
private List<String> name = Lists.newArrayList();
|
private List<String> name = Lists.newArrayList();
|
||||||
private List<String> surname = Lists.newArrayList();
|
private List<String> surname = Lists.newArrayList();
|
||||||
private List<String> fullname = Lists.newArrayList();
|
private List<String> fullname = Lists.newArrayList();
|
||||||
|
|
|
@ -0,0 +1,45 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.common.aggregation;
|
||||||
|
|
||||||
|
import java.io.Closeable;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Objects;
|
||||||
|
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.message.MessageSender;
|
||||||
|
import eu.dnetlib.dhp.utils.DHPUtils;
|
||||||
|
|
||||||
|
public class AggregatorReport extends LinkedHashMap<String, String> implements Closeable {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(AggregatorReport.class);
|
||||||
|
|
||||||
|
private transient MessageSender messageSender;
|
||||||
|
|
||||||
|
public AggregatorReport() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public AggregatorReport(MessageSender messageSender) {
|
||||||
|
this.messageSender = messageSender;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void ongoing(Long current, Long total) {
|
||||||
|
messageSender.sendMessage(current, total);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void close() throws IOException {
|
||||||
|
if (Objects.nonNull(messageSender)) {
|
||||||
|
log.info("closing report: ");
|
||||||
|
this.forEach((k, v) -> log.info("{} - {}", k, v));
|
||||||
|
|
||||||
|
Map<String, String> m = new HashMap<>();
|
||||||
|
m.put(getClass().getSimpleName().toLowerCase(), DHPUtils.MAPPER.writeValueAsString(values()));
|
||||||
|
messageSender.sendReport(m);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -13,9 +13,9 @@ import okio.Source;
|
||||||
|
|
||||||
public class InputStreamRequestBody extends RequestBody {
|
public class InputStreamRequestBody extends RequestBody {
|
||||||
|
|
||||||
private InputStream inputStream;
|
private final InputStream inputStream;
|
||||||
private MediaType mediaType;
|
private final MediaType mediaType;
|
||||||
private long lenght;
|
private final long lenght;
|
||||||
|
|
||||||
public static RequestBody create(final MediaType mediaType, final InputStream inputStream, final long len) {
|
public static RequestBody create(final MediaType mediaType, final InputStream inputStream, final long len) {
|
||||||
|
|
||||||
|
|
|
@ -3,6 +3,10 @@ package eu.dnetlib.dhp.common.api;
|
||||||
|
|
||||||
import java.io.*;
|
import java.io.*;
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
|
import java.util.concurrent.TimeUnit;
|
||||||
|
|
||||||
|
import org.apache.http.HttpHeaders;
|
||||||
|
import org.apache.http.entity.ContentType;
|
||||||
|
|
||||||
import com.google.gson.Gson;
|
import com.google.gson.Gson;
|
||||||
|
|
||||||
|
@ -42,7 +46,7 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
this.deposition_id = deposition_id;
|
this.deposition_id = deposition_id;
|
||||||
}
|
}
|
||||||
|
|
||||||
public ZenodoAPIClient(String urlString, String access_token) throws IOException {
|
public ZenodoAPIClient(String urlString, String access_token) {
|
||||||
|
|
||||||
this.urlString = urlString;
|
this.urlString = urlString;
|
||||||
this.access_token = access_token;
|
this.access_token = access_token;
|
||||||
|
@ -50,19 +54,20 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Brand new deposition in Zenodo. It sets the deposition_id and the bucket where to store the files to upload
|
* Brand new deposition in Zenodo. It sets the deposition_id and the bucket where to store the files to upload
|
||||||
|
*
|
||||||
* @return response code
|
* @return response code
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public int newDeposition() throws IOException {
|
public int newDeposition() throws IOException {
|
||||||
String json = "{}";
|
String json = "{}";
|
||||||
OkHttpClient httpClient = new OkHttpClient();
|
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
|
||||||
|
|
||||||
RequestBody body = RequestBody.create(MEDIA_TYPE_JSON, json);
|
RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(urlString)
|
.url(urlString)
|
||||||
.addHeader("Content-Type", "application/json") // add request headers
|
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
|
||||||
.addHeader("Authorization", "Bearer " + access_token)
|
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||||
.post(body)
|
.post(body)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
@ -86,18 +91,23 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Upload files in Zenodo.
|
* Upload files in Zenodo.
|
||||||
|
*
|
||||||
* @param is the inputStream for the file to upload
|
* @param is the inputStream for the file to upload
|
||||||
* @param file_name the name of the file as it will appear on Zenodo
|
* @param file_name the name of the file as it will appear on Zenodo
|
||||||
* @param len the size of the file
|
* @param len the size of the file
|
||||||
* @return the response code
|
* @return the response code
|
||||||
*/
|
*/
|
||||||
public int uploadIS(InputStream is, String file_name, long len) throws IOException {
|
public int uploadIS(InputStream is, String file_name, long len) throws IOException {
|
||||||
OkHttpClient httpClient = new OkHttpClient();
|
OkHttpClient httpClient = new OkHttpClient.Builder()
|
||||||
|
.writeTimeout(600, TimeUnit.SECONDS)
|
||||||
|
.readTimeout(600, TimeUnit.SECONDS)
|
||||||
|
.connectTimeout(600, TimeUnit.SECONDS)
|
||||||
|
.build();
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(bucket + "/" + file_name)
|
.url(bucket + "/" + file_name)
|
||||||
.addHeader("Content-Type", "application/zip") // add request headers
|
.addHeader(HttpHeaders.CONTENT_TYPE, "application/zip") // add request headers
|
||||||
.addHeader("Authorization", "Bearer " + access_token)
|
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||||
.put(InputStreamRequestBody.create(MEDIA_TYPE_ZIP, is, len))
|
.put(InputStreamRequestBody.create(MEDIA_TYPE_ZIP, is, len))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
@ -110,20 +120,21 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Associates metadata information to the current deposition
|
* Associates metadata information to the current deposition
|
||||||
|
*
|
||||||
* @param metadata the metadata
|
* @param metadata the metadata
|
||||||
* @return response code
|
* @return response code
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
public int sendMretadata(String metadata) throws IOException {
|
public int sendMretadata(String metadata) throws IOException {
|
||||||
|
|
||||||
OkHttpClient httpClient = new OkHttpClient();
|
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
|
||||||
|
|
||||||
RequestBody body = RequestBody.create(MEDIA_TYPE_JSON, metadata);
|
RequestBody body = RequestBody.create(metadata, MEDIA_TYPE_JSON);
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(urlString + "/" + deposition_id)
|
.url(urlString + "/" + deposition_id)
|
||||||
.addHeader("Content-Type", "application/json") // add request headers
|
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
|
||||||
.addHeader("Authorization", "Bearer " + access_token)
|
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||||
.put(body)
|
.put(body)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
@ -140,6 +151,7 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* To publish the current deposition. It works for both new deposition or new version of an old deposition
|
* To publish the current deposition. It works for both new deposition or new version of an old deposition
|
||||||
|
*
|
||||||
* @return response code
|
* @return response code
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
*/
|
*/
|
||||||
|
@ -147,12 +159,14 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
|
|
||||||
String json = "{}";
|
String json = "{}";
|
||||||
|
|
||||||
OkHttpClient httpClient = new OkHttpClient();
|
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
|
||||||
|
|
||||||
|
RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(urlString + "/" + deposition_id + "/actions/publish")
|
.url(urlString + "/" + deposition_id + "/actions/publish")
|
||||||
.addHeader("Authorization", "Bearer " + access_token)
|
.addHeader("Authorization", "Bearer " + access_token)
|
||||||
.post(RequestBody.create(MEDIA_TYPE_JSON, json))
|
.post(body)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
try (Response response = httpClient.newCall(request).execute()) {
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
|
@ -166,11 +180,12 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* To create a new version of an already published deposition.
|
* To create a new version of an already published deposition. It sets the deposition_id and the bucket to be used
|
||||||
* It sets the deposition_id and the bucket to be used for the new version.
|
* for the new version.
|
||||||
* @param concept_rec_id the concept record id of the deposition for which to create a new version. It is
|
*
|
||||||
* the last part of the url for the DOI Zenodo suggests to use to cite all versions:
|
* @param concept_rec_id the concept record id of the deposition for which to create a new version. It is the last
|
||||||
* DOI: 10.xxx/zenodo.656930 concept_rec_id = 656930
|
* part of the url for the DOI Zenodo suggests to use to cite all versions: DOI: 10.xxx/zenodo.656930
|
||||||
|
* concept_rec_id = 656930
|
||||||
* @return response code
|
* @return response code
|
||||||
* @throws IOException
|
* @throws IOException
|
||||||
* @throws MissingConceptDoiException
|
* @throws MissingConceptDoiException
|
||||||
|
@ -179,12 +194,14 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
setDepositionId(concept_rec_id);
|
setDepositionId(concept_rec_id);
|
||||||
String json = "{}";
|
String json = "{}";
|
||||||
|
|
||||||
OkHttpClient httpClient = new OkHttpClient();
|
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
|
||||||
|
|
||||||
|
RequestBody body = RequestBody.create(json, MEDIA_TYPE_JSON);
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(urlString + "/" + deposition_id + "/actions/newversion")
|
.url(urlString + "/" + deposition_id + "/actions/newversion")
|
||||||
.addHeader("Authorization", "Bearer " + access_token)
|
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||||
.post(RequestBody.create(MEDIA_TYPE_JSON, json))
|
.post(body)
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
try (Response response = httpClient.newCall(request).execute()) {
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
|
@ -201,6 +218,41 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* To finish uploading a version or new deposition not published
|
||||||
|
* It sets the deposition_id and the bucket to be used
|
||||||
|
*
|
||||||
|
*
|
||||||
|
* @param deposition_id the deposition id of the not yet published upload
|
||||||
|
* concept_rec_id = 656930
|
||||||
|
* @return response code
|
||||||
|
* @throws IOException
|
||||||
|
* @throws MissingConceptDoiException
|
||||||
|
*/
|
||||||
|
public int uploadOpenDeposition(String deposition_id) throws IOException, MissingConceptDoiException {
|
||||||
|
|
||||||
|
this.deposition_id = deposition_id;
|
||||||
|
|
||||||
|
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
|
||||||
|
|
||||||
|
Request request = new Request.Builder()
|
||||||
|
.url(urlString + "/" + deposition_id)
|
||||||
|
.addHeader("Authorization", "Bearer " + access_token)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
try (Response response = httpClient.newCall(request).execute()) {
|
||||||
|
|
||||||
|
if (!response.isSuccessful())
|
||||||
|
throw new IOException("Unexpected code " + response + response.body().string());
|
||||||
|
|
||||||
|
ZenodoModel zenodoModel = new Gson().fromJson(response.body().string(), ZenodoModel.class);
|
||||||
|
bucket = zenodoModel.getLinks().getBucket();
|
||||||
|
return response.code();
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
private void setDepositionId(String concept_rec_id) throws IOException, MissingConceptDoiException {
|
private void setDepositionId(String concept_rec_id) throws IOException, MissingConceptDoiException {
|
||||||
|
|
||||||
ZenodoModelList zenodoModelList = new Gson().fromJson(getPrevDepositions(), ZenodoModelList.class);
|
ZenodoModelList zenodoModelList = new Gson().fromJson(getPrevDepositions(), ZenodoModelList.class);
|
||||||
|
@ -217,12 +269,12 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getPrevDepositions() throws IOException {
|
private String getPrevDepositions() throws IOException {
|
||||||
OkHttpClient httpClient = new OkHttpClient();
|
OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(600, TimeUnit.SECONDS).build();
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(urlString)
|
.url(urlString)
|
||||||
.addHeader("Content-Type", "application/json") // add request headers
|
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
|
||||||
.addHeader("Authorization", "Bearer " + access_token)
|
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||||
.get()
|
.get()
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
@ -238,12 +290,14 @@ public class ZenodoAPIClient implements Serializable {
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getBucket(String url) throws IOException {
|
private String getBucket(String url) throws IOException {
|
||||||
OkHttpClient httpClient = new OkHttpClient();
|
OkHttpClient httpClient = new OkHttpClient.Builder()
|
||||||
|
.connectTimeout(600, TimeUnit.SECONDS)
|
||||||
|
.build();
|
||||||
|
|
||||||
Request request = new Request.Builder()
|
Request request = new Request.Builder()
|
||||||
.url(url)
|
.url(url)
|
||||||
.addHeader("Content-Type", "application/json") // add request headers
|
.addHeader(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON.toString()) // add request headers
|
||||||
.addHeader("Authorization", "Bearer " + access_token)
|
.addHeader(HttpHeaders.AUTHORIZATION, "Bearer " + access_token)
|
||||||
.get()
|
.get()
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
|
|
|
@ -32,13 +32,13 @@ public class Creator {
|
||||||
|
|
||||||
public static Creator newInstance(String name, String affiliation, String orcid) {
|
public static Creator newInstance(String name, String affiliation, String orcid) {
|
||||||
Creator c = new Creator();
|
Creator c = new Creator();
|
||||||
if (!(name == null)) {
|
if (name != null) {
|
||||||
c.name = name;
|
c.name = name;
|
||||||
}
|
}
|
||||||
if (!(affiliation == null)) {
|
if (affiliation != null) {
|
||||||
c.affiliation = affiliation;
|
c.affiliation = affiliation;
|
||||||
}
|
}
|
||||||
if (!(orcid == null)) {
|
if (orcid != null) {
|
||||||
c.orcid = orcid;
|
c.orcid = orcid;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,17 +3,12 @@ package eu.dnetlib.dhp.common.api.zenodo;
|
||||||
|
|
||||||
import java.io.Serializable;
|
import java.io.Serializable;
|
||||||
|
|
||||||
import net.minidev.json.annotate.JsonIgnore;
|
|
||||||
|
|
||||||
public class File implements Serializable {
|
public class File implements Serializable {
|
||||||
private String checksum;
|
private String checksum;
|
||||||
private String filename;
|
private String filename;
|
||||||
private long filesize;
|
private long filesize;
|
||||||
private String id;
|
private String id;
|
||||||
|
|
||||||
@JsonIgnore
|
|
||||||
// private Links links;
|
|
||||||
|
|
||||||
public String getChecksum() {
|
public String getChecksum() {
|
||||||
return checksum;
|
return checksum;
|
||||||
}
|
}
|
||||||
|
@ -46,13 +41,4 @@ public class File implements Serializable {
|
||||||
this.id = id;
|
this.id = id;
|
||||||
}
|
}
|
||||||
|
|
||||||
// @JsonIgnore
|
|
||||||
// public Links getLinks() {
|
|
||||||
// return links;
|
|
||||||
// }
|
|
||||||
//
|
|
||||||
// @JsonIgnore
|
|
||||||
// public void setLinks(Links links) {
|
|
||||||
// this.links = links;
|
|
||||||
// }
|
|
||||||
}
|
}
|
||||||
|
|
|
@ -0,0 +1,32 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.common.collection;
|
||||||
|
|
||||||
|
public class CollectorException extends Exception {
|
||||||
|
|
||||||
|
/** */
|
||||||
|
private static final long serialVersionUID = -290723075076039757L;
|
||||||
|
|
||||||
|
public CollectorException() {
|
||||||
|
super();
|
||||||
|
}
|
||||||
|
|
||||||
|
public CollectorException(
|
||||||
|
final String message,
|
||||||
|
final Throwable cause,
|
||||||
|
final boolean enableSuppression,
|
||||||
|
final boolean writableStackTrace) {
|
||||||
|
super(message, cause, enableSuppression, writableStackTrace);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CollectorException(final String message, final Throwable cause) {
|
||||||
|
super(message, cause);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CollectorException(final String message) {
|
||||||
|
super(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
public CollectorException(final Throwable cause) {
|
||||||
|
super(cause);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,56 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.common.collection;
|
||||||
|
|
||||||
|
import java.io.*;
|
||||||
|
import java.nio.charset.StandardCharsets;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
|
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||||
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.opencsv.bean.CsvToBeanBuilder;
|
||||||
|
|
||||||
|
public class GetCSV {
|
||||||
|
|
||||||
|
public static final char DEFAULT_DELIMITER = ',';
|
||||||
|
|
||||||
|
private GetCSV() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void getCsv(FileSystem fileSystem, BufferedReader reader, String hdfsPath,
|
||||||
|
String modelClass) throws IOException, ClassNotFoundException {
|
||||||
|
getCsv(fileSystem, reader, hdfsPath, modelClass, DEFAULT_DELIMITER);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void getCsv(FileSystem fileSystem, Reader reader, String hdfsPath,
|
||||||
|
String modelClass, char delimiter) throws IOException, ClassNotFoundException {
|
||||||
|
|
||||||
|
Path hdfsWritePath = new Path(hdfsPath);
|
||||||
|
FSDataOutputStream fsDataOutputStream = null;
|
||||||
|
if (fileSystem.exists(hdfsWritePath)) {
|
||||||
|
fileSystem.delete(hdfsWritePath, false);
|
||||||
|
}
|
||||||
|
fsDataOutputStream = fileSystem.create(hdfsWritePath);
|
||||||
|
|
||||||
|
try (BufferedWriter writer = new BufferedWriter(
|
||||||
|
new OutputStreamWriter(fsDataOutputStream, StandardCharsets.UTF_8))) {
|
||||||
|
|
||||||
|
final ObjectMapper mapper = new ObjectMapper();
|
||||||
|
|
||||||
|
@SuppressWarnings("unchecked")
|
||||||
|
final List lines = new CsvToBeanBuilder(reader)
|
||||||
|
.withType(Class.forName(modelClass))
|
||||||
|
.withSeparator(delimiter)
|
||||||
|
.build()
|
||||||
|
.parse();
|
||||||
|
|
||||||
|
for (Object line : lines) {
|
||||||
|
writer.write(mapper.writeValueAsString(line));
|
||||||
|
writer.newLine();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,94 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.common.collection;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bundles the http connection parameters driving the client behaviour.
|
||||||
|
*/
|
||||||
|
public class HttpClientParams {
|
||||||
|
|
||||||
|
// Defaults
|
||||||
|
public static int _maxNumberOfRetry = 3;
|
||||||
|
public static int _requestDelay = 0; // milliseconds
|
||||||
|
public static int _retryDelay = 10; // seconds
|
||||||
|
public static int _connectTimeOut = 10; // seconds
|
||||||
|
public static int _readTimeOut = 30; // seconds
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Maximum number of allowed retires before failing
|
||||||
|
*/
|
||||||
|
private int maxNumberOfRetry;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Delay between request (Milliseconds)
|
||||||
|
*/
|
||||||
|
private int requestDelay;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Time to wait after a failure before retrying (Seconds)
|
||||||
|
*/
|
||||||
|
private int retryDelay;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Connect timeout (Seconds)
|
||||||
|
*/
|
||||||
|
private int connectTimeOut;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Read timeout (Seconds)
|
||||||
|
*/
|
||||||
|
private int readTimeOut;
|
||||||
|
|
||||||
|
public HttpClientParams() {
|
||||||
|
this(_maxNumberOfRetry, _requestDelay, _retryDelay, _connectTimeOut, _readTimeOut);
|
||||||
|
}
|
||||||
|
|
||||||
|
public HttpClientParams(int maxNumberOfRetry, int requestDelay, int retryDelay, int connectTimeOut,
|
||||||
|
int readTimeOut) {
|
||||||
|
this.maxNumberOfRetry = maxNumberOfRetry;
|
||||||
|
this.requestDelay = requestDelay;
|
||||||
|
this.retryDelay = retryDelay;
|
||||||
|
this.connectTimeOut = connectTimeOut;
|
||||||
|
this.readTimeOut = readTimeOut;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getMaxNumberOfRetry() {
|
||||||
|
return maxNumberOfRetry;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setMaxNumberOfRetry(int maxNumberOfRetry) {
|
||||||
|
this.maxNumberOfRetry = maxNumberOfRetry;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getRequestDelay() {
|
||||||
|
return requestDelay;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRequestDelay(int requestDelay) {
|
||||||
|
this.requestDelay = requestDelay;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getRetryDelay() {
|
||||||
|
return retryDelay;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRetryDelay(int retryDelay) {
|
||||||
|
this.retryDelay = retryDelay;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setConnectTimeOut(int connectTimeOut) {
|
||||||
|
this.connectTimeOut = connectTimeOut;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getConnectTimeOut() {
|
||||||
|
return connectTimeOut;
|
||||||
|
}
|
||||||
|
|
||||||
|
public int getReadTimeOut() {
|
||||||
|
return readTimeOut;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setReadTimeOut(int readTimeOut) {
|
||||||
|
this.readTimeOut = readTimeOut;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,263 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.common.collection;
|
||||||
|
|
||||||
|
import static eu.dnetlib.dhp.utils.DHPUtils.*;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.io.InputStream;
|
||||||
|
import java.net.*;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.commons.lang3.math.NumberUtils;
|
||||||
|
import org.apache.http.HttpHeaders;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.common.Constants;
|
||||||
|
import eu.dnetlib.dhp.common.aggregation.AggregatorReport;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Migrated from https://svn.driver.research-infrastructures.eu/driver/dnet45/modules/dnet-modular-collector-service/trunk/src/main/java/eu/dnetlib/data/collector/plugins/HttpConnector.java
|
||||||
|
*
|
||||||
|
* @author jochen, michele, andrea, alessia, claudio, andreas
|
||||||
|
*/
|
||||||
|
public class HttpConnector2 {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(HttpConnector2.class);
|
||||||
|
|
||||||
|
private static final String REPORT_PREFIX = "http:";
|
||||||
|
|
||||||
|
private HttpClientParams clientParams;
|
||||||
|
|
||||||
|
private String responseType = null;
|
||||||
|
|
||||||
|
private static final String userAgent = "Mozilla/5.0 (compatible; OAI; +http://www.openaire.eu)";
|
||||||
|
|
||||||
|
public HttpConnector2() {
|
||||||
|
this(new HttpClientParams());
|
||||||
|
}
|
||||||
|
|
||||||
|
public HttpConnector2(HttpClientParams clientParams) {
|
||||||
|
this.clientParams = clientParams;
|
||||||
|
CookieHandler.setDefault(new CookieManager(null, CookiePolicy.ACCEPT_ALL));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @see HttpConnector2#getInputSource(java.lang.String, AggregatorReport)
|
||||||
|
*/
|
||||||
|
public InputStream getInputSourceAsStream(final String requestUrl) throws CollectorException {
|
||||||
|
return IOUtils.toInputStream(getInputSource(requestUrl));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @see HttpConnector2#getInputSource(java.lang.String, AggregatorReport)
|
||||||
|
*/
|
||||||
|
public String getInputSource(final String requestUrl) throws CollectorException {
|
||||||
|
return attemptDownloadAsString(requestUrl, 1, new AggregatorReport());
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Given the URL returns the content via HTTP GET
|
||||||
|
*
|
||||||
|
* @param requestUrl the URL
|
||||||
|
* @param report the list of errors
|
||||||
|
* @return the content of the downloaded resource
|
||||||
|
* @throws CollectorException when retrying more than maxNumberOfRetry times
|
||||||
|
*/
|
||||||
|
public String getInputSource(final String requestUrl, AggregatorReport report)
|
||||||
|
throws CollectorException {
|
||||||
|
return attemptDownloadAsString(requestUrl, 1, report);
|
||||||
|
}
|
||||||
|
|
||||||
|
private String attemptDownloadAsString(final String requestUrl, final int retryNumber,
|
||||||
|
final AggregatorReport report) throws CollectorException {
|
||||||
|
|
||||||
|
try (InputStream s = attemptDownload(requestUrl, retryNumber, report)) {
|
||||||
|
return IOUtils.toString(s);
|
||||||
|
} catch (IOException e) {
|
||||||
|
log.error(e.getMessage(), e);
|
||||||
|
throw new CollectorException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private InputStream attemptDownload(final String requestUrl, final int retryNumber,
|
||||||
|
final AggregatorReport report) throws CollectorException, IOException {
|
||||||
|
|
||||||
|
if (retryNumber > getClientParams().getMaxNumberOfRetry()) {
|
||||||
|
final String msg = String
|
||||||
|
.format(
|
||||||
|
"Max number of retries (%s/%s) exceeded, failing.",
|
||||||
|
retryNumber, getClientParams().getMaxNumberOfRetry());
|
||||||
|
log.error(msg);
|
||||||
|
throw new CollectorException(msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info("Request attempt {} [{}]", retryNumber, requestUrl);
|
||||||
|
|
||||||
|
InputStream input = null;
|
||||||
|
|
||||||
|
try {
|
||||||
|
if (getClientParams().getRequestDelay() > 0) {
|
||||||
|
backoffAndSleep(getClientParams().getRequestDelay());
|
||||||
|
}
|
||||||
|
final HttpURLConnection urlConn = (HttpURLConnection) new URL(requestUrl).openConnection();
|
||||||
|
urlConn.setInstanceFollowRedirects(false);
|
||||||
|
urlConn.setReadTimeout(getClientParams().getReadTimeOut() * 1000);
|
||||||
|
urlConn.setConnectTimeout(getClientParams().getConnectTimeOut() * 1000);
|
||||||
|
urlConn.addRequestProperty(HttpHeaders.USER_AGENT, userAgent);
|
||||||
|
|
||||||
|
if (log.isDebugEnabled()) {
|
||||||
|
logHeaderFields(urlConn);
|
||||||
|
}
|
||||||
|
|
||||||
|
int retryAfter = obtainRetryAfter(urlConn.getHeaderFields());
|
||||||
|
String rateLimit = urlConn.getHeaderField(Constants.HTTPHEADER_IETF_DRAFT_RATELIMIT_LIMIT);
|
||||||
|
String rateRemaining = urlConn.getHeaderField(Constants.HTTPHEADER_IETF_DRAFT_RATELIMIT_REMAINING);
|
||||||
|
|
||||||
|
if ((rateLimit != null) && (rateRemaining != null) && (Integer.parseInt(rateRemaining) < 9)) {
|
||||||
|
if (retryAfter > 0) {
|
||||||
|
backoffAndSleep(retryAfter);
|
||||||
|
} else {
|
||||||
|
backoffAndSleep(1000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (is2xx(urlConn.getResponseCode())) {
|
||||||
|
input = urlConn.getInputStream();
|
||||||
|
responseType = urlConn.getContentType();
|
||||||
|
return input;
|
||||||
|
}
|
||||||
|
if (is3xx(urlConn.getResponseCode())) {
|
||||||
|
// REDIRECTS
|
||||||
|
final String newUrl = obtainNewLocation(urlConn.getHeaderFields());
|
||||||
|
log.info("The requested url has been moved to {}", newUrl);
|
||||||
|
report
|
||||||
|
.put(
|
||||||
|
REPORT_PREFIX + urlConn.getResponseCode(),
|
||||||
|
String.format("Moved to: %s", newUrl));
|
||||||
|
urlConn.disconnect();
|
||||||
|
if (retryAfter > 0) {
|
||||||
|
backoffAndSleep(retryAfter);
|
||||||
|
}
|
||||||
|
return attemptDownload(newUrl, retryNumber + 1, report);
|
||||||
|
}
|
||||||
|
if (is4xx(urlConn.getResponseCode()) || is5xx(urlConn.getResponseCode())) {
|
||||||
|
switch (urlConn.getResponseCode()) {
|
||||||
|
case HttpURLConnection.HTTP_NOT_FOUND:
|
||||||
|
case HttpURLConnection.HTTP_BAD_GATEWAY:
|
||||||
|
case HttpURLConnection.HTTP_UNAVAILABLE:
|
||||||
|
case HttpURLConnection.HTTP_GATEWAY_TIMEOUT:
|
||||||
|
if (retryAfter > 0) {
|
||||||
|
log
|
||||||
|
.warn(
|
||||||
|
"{} - waiting and repeating request after suggested retry-after {} sec.",
|
||||||
|
requestUrl, retryAfter);
|
||||||
|
backoffAndSleep(retryAfter * 1000);
|
||||||
|
} else {
|
||||||
|
log
|
||||||
|
.warn(
|
||||||
|
"{} - waiting and repeating request after default delay of {} sec.",
|
||||||
|
requestUrl, getClientParams().getRetryDelay());
|
||||||
|
backoffAndSleep(retryNumber * getClientParams().getRetryDelay() * 1000);
|
||||||
|
}
|
||||||
|
report.put(REPORT_PREFIX + urlConn.getResponseCode(), requestUrl);
|
||||||
|
urlConn.disconnect();
|
||||||
|
return attemptDownload(requestUrl, retryNumber + 1, report);
|
||||||
|
default:
|
||||||
|
report
|
||||||
|
.put(
|
||||||
|
REPORT_PREFIX + urlConn.getResponseCode(),
|
||||||
|
String
|
||||||
|
.format(
|
||||||
|
"%s Error: %s", requestUrl, urlConn.getResponseMessage()));
|
||||||
|
throw new CollectorException(urlConn.getResponseCode() + " error " + report);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new CollectorException(
|
||||||
|
String
|
||||||
|
.format(
|
||||||
|
"Unexpected status code: %s errors: %s", urlConn.getResponseCode(),
|
||||||
|
MAPPER.writeValueAsString(report)));
|
||||||
|
} catch (MalformedURLException | UnknownHostException e) {
|
||||||
|
log.error(e.getMessage(), e);
|
||||||
|
report.put(e.getClass().getName(), e.getMessage());
|
||||||
|
throw new CollectorException(e.getMessage(), e);
|
||||||
|
} catch (SocketTimeoutException | SocketException e) {
|
||||||
|
log.error(e.getMessage(), e);
|
||||||
|
report.put(e.getClass().getName(), e.getMessage());
|
||||||
|
backoffAndSleep(getClientParams().getRetryDelay() * retryNumber * 1000);
|
||||||
|
return attemptDownload(requestUrl, retryNumber + 1, report);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void logHeaderFields(final HttpURLConnection urlConn) throws IOException {
|
||||||
|
log.debug("StatusCode: {}", urlConn.getResponseMessage());
|
||||||
|
|
||||||
|
for (Map.Entry<String, List<String>> e : urlConn.getHeaderFields().entrySet()) {
|
||||||
|
if (e.getKey() != null) {
|
||||||
|
for (String v : e.getValue()) {
|
||||||
|
log.debug(" key: {} - value: {}", e.getKey(), v);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private void backoffAndSleep(int sleepTimeMs) throws CollectorException {
|
||||||
|
log.info("I'm going to sleep for {}ms", sleepTimeMs);
|
||||||
|
try {
|
||||||
|
Thread.sleep(sleepTimeMs);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
log.error(e.getMessage(), e);
|
||||||
|
throw new CollectorException(e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private int obtainRetryAfter(final Map<String, List<String>> headerMap) {
|
||||||
|
for (String key : headerMap.keySet()) {
|
||||||
|
if ((key != null) && key.equalsIgnoreCase(HttpHeaders.RETRY_AFTER) && (!headerMap.get(key).isEmpty())
|
||||||
|
&& NumberUtils.isCreatable(headerMap.get(key).get(0))) {
|
||||||
|
return Integer.parseInt(headerMap.get(key).get(0)) + 10;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return -1;
|
||||||
|
}
|
||||||
|
|
||||||
|
private String obtainNewLocation(final Map<String, List<String>> headerMap) throws CollectorException {
|
||||||
|
for (String key : headerMap.keySet()) {
|
||||||
|
if ((key != null) && key.equalsIgnoreCase(HttpHeaders.LOCATION) && (headerMap.get(key).size() > 0)) {
|
||||||
|
return headerMap.get(key).get(0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw new CollectorException("The requested url has been MOVED, but 'location' param is MISSING");
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean is2xx(final int statusCode) {
|
||||||
|
return statusCode >= 200 && statusCode <= 299;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean is4xx(final int statusCode) {
|
||||||
|
return statusCode >= 400 && statusCode <= 499;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean is3xx(final int statusCode) {
|
||||||
|
return statusCode >= 300 && statusCode <= 399;
|
||||||
|
}
|
||||||
|
|
||||||
|
private boolean is5xx(final int statusCode) {
|
||||||
|
return statusCode >= 500 && statusCode <= 599;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getResponseType() {
|
||||||
|
return responseType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public HttpClientParams getClientParams() {
|
||||||
|
return clientParams;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setClientParams(HttpClientParams clientParams) {
|
||||||
|
this.clientParams = clientParams;
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,75 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.common.rest;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.Arrays;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.http.client.methods.HttpGet;
|
||||||
|
import org.apache.http.client.methods.HttpPost;
|
||||||
|
import org.apache.http.client.methods.HttpUriRequest;
|
||||||
|
import org.apache.http.entity.StringEntity;
|
||||||
|
import org.apache.http.impl.client.CloseableHttpClient;
|
||||||
|
import org.apache.http.impl.client.HttpClients;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
public class DNetRestClient {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(DNetRestClient.class);
|
||||||
|
|
||||||
|
private static final ObjectMapper mapper = new ObjectMapper();
|
||||||
|
|
||||||
|
private DNetRestClient() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> T doGET(final String url, Class<T> clazz) throws Exception {
|
||||||
|
final HttpGet httpGet = new HttpGet(url);
|
||||||
|
return doHTTPRequest(httpGet, clazz);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String doGET(final String url) throws IOException {
|
||||||
|
final HttpGet httpGet = new HttpGet(url);
|
||||||
|
return doHTTPRequest(httpGet);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <V> String doPOST(final String url, V objParam) throws IOException {
|
||||||
|
final HttpPost httpPost = new HttpPost(url);
|
||||||
|
|
||||||
|
if (objParam != null) {
|
||||||
|
final StringEntity entity = new StringEntity(mapper.writeValueAsString(objParam));
|
||||||
|
httpPost.setEntity(entity);
|
||||||
|
httpPost.setHeader("Accept", "application/json");
|
||||||
|
httpPost.setHeader("Content-type", "application/json");
|
||||||
|
}
|
||||||
|
return doHTTPRequest(httpPost);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T, V> T doPOST(final String url, V objParam, Class<T> clazz) throws IOException {
|
||||||
|
return mapper.readValue(doPOST(url, objParam), clazz);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static String doHTTPRequest(final HttpUriRequest r) throws IOException {
|
||||||
|
try (CloseableHttpClient client = HttpClients.createDefault()) {
|
||||||
|
|
||||||
|
log.info("performing HTTP request, method {} on URI {}", r.getMethod(), r.getURI().toString());
|
||||||
|
log
|
||||||
|
.info(
|
||||||
|
"request headers: {}",
|
||||||
|
Arrays
|
||||||
|
.asList(r.getAllHeaders())
|
||||||
|
.stream()
|
||||||
|
.map(h -> h.getName() + ":" + h.getValue())
|
||||||
|
.collect(Collectors.joining(",")));
|
||||||
|
|
||||||
|
return IOUtils.toString(client.execute(r).getEntity().getContent());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static <T> T doHTTPRequest(final HttpUriRequest r, Class<T> clazz) throws Exception {
|
||||||
|
return mapper.readValue(doHTTPRequest(r), clazz);
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,86 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.common.vocabulary;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.HashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
|
import com.google.common.collect.Maps;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Qualifier;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
|
||||||
|
|
||||||
|
public class Vocabulary implements Serializable {
|
||||||
|
|
||||||
|
private final String id;
|
||||||
|
private final String name;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Code to Term mappings for this Vocabulary.
|
||||||
|
*/
|
||||||
|
private final Map<String, VocabularyTerm> terms = new HashMap<>();
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Synonym to Code mappings for this Vocabulary.
|
||||||
|
*/
|
||||||
|
private final Map<String, String> synonyms = Maps.newHashMap();
|
||||||
|
|
||||||
|
public Vocabulary(final String id, final String name) {
|
||||||
|
this.id = id;
|
||||||
|
this.name = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getId() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected Map<String, VocabularyTerm> getTerms() {
|
||||||
|
return terms;
|
||||||
|
}
|
||||||
|
|
||||||
|
public VocabularyTerm getTerm(final String id) {
|
||||||
|
return Optional.ofNullable(id).map(String::toLowerCase).map(terms::get).orElse(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void addTerm(final String id, final String name) {
|
||||||
|
terms.put(id.toLowerCase(), new VocabularyTerm(id, name));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected boolean termExists(final String id) {
|
||||||
|
return terms.containsKey(id.toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
protected void addSynonym(final String syn, final String termCode) {
|
||||||
|
synonyms.put(syn, termCode.toLowerCase());
|
||||||
|
}
|
||||||
|
|
||||||
|
public VocabularyTerm getTermBySynonym(final String syn) {
|
||||||
|
return getTerm(synonyms.get(syn.toLowerCase()));
|
||||||
|
}
|
||||||
|
|
||||||
|
public Qualifier getTermAsQualifier(final String termId) {
|
||||||
|
if (StringUtils.isBlank(termId)) {
|
||||||
|
return OafMapperUtils.unknown(getId(), getName());
|
||||||
|
} else if (termExists(termId)) {
|
||||||
|
final VocabularyTerm t = getTerm(termId);
|
||||||
|
return OafMapperUtils.qualifier(t.getId(), t.getName(), getId(), getName());
|
||||||
|
} else {
|
||||||
|
return OafMapperUtils.qualifier(termId, termId, getId(), getName());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Qualifier getSynonymAsQualifier(final String syn) {
|
||||||
|
return Optional
|
||||||
|
.ofNullable(getTermBySynonym(syn))
|
||||||
|
.map(term -> getTermAsQualifier(term.getId()))
|
||||||
|
.orElse(null);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,182 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.common.vocabulary;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Qualifier;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.utils.OafMapperUtils;
|
||||||
|
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpException;
|
||||||
|
import eu.dnetlib.enabling.is.lookup.rmi.ISLookUpService;
|
||||||
|
|
||||||
|
public class VocabularyGroup implements Serializable {
|
||||||
|
|
||||||
|
public static final String VOCABULARIES_XQUERY = "for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType') \n"
|
||||||
|
+
|
||||||
|
"let $vocid := $x//VOCABULARY_NAME/@code\n" +
|
||||||
|
"let $vocname := $x//VOCABULARY_NAME/text()\n" +
|
||||||
|
"for $term in ($x//TERM)\n" +
|
||||||
|
"return concat($vocid,' @=@ ',$vocname,' @=@ ',$term/@code,' @=@ ',$term/@english_name)";
|
||||||
|
|
||||||
|
public static final String VOCABULARY_SYNONYMS_XQUERY = "for $x in collection('/db/DRIVER/VocabularyDSResources/VocabularyDSResourceType')\n"
|
||||||
|
+
|
||||||
|
"let $vocid := $x//VOCABULARY_NAME/@code\n" +
|
||||||
|
"let $vocname := $x//VOCABULARY_NAME/text()\n" +
|
||||||
|
"for $term in ($x//TERM)\n" +
|
||||||
|
"for $syn in ($term//SYNONYM/@term)\n" +
|
||||||
|
"return concat($vocid,' @=@ ',$term/@code,' @=@ ', $syn)\n";
|
||||||
|
|
||||||
|
public static VocabularyGroup loadVocsFromIS(ISLookUpService isLookUpService) throws ISLookUpException {
|
||||||
|
|
||||||
|
final VocabularyGroup vocs = new VocabularyGroup();
|
||||||
|
|
||||||
|
for (final String s : isLookUpService.quickSearchProfile(VOCABULARIES_XQUERY)) {
|
||||||
|
final String[] arr = s.split("@=@");
|
||||||
|
if (arr.length == 4) {
|
||||||
|
final String vocId = arr[0].trim();
|
||||||
|
final String vocName = arr[1].trim();
|
||||||
|
final String termId = arr[2].trim();
|
||||||
|
final String termName = arr[3].trim();
|
||||||
|
|
||||||
|
if (!vocs.vocabularyExists(vocId)) {
|
||||||
|
vocs.addVocabulary(vocId, vocName);
|
||||||
|
}
|
||||||
|
|
||||||
|
vocs.addTerm(vocId, termId, termName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (final String s : isLookUpService.quickSearchProfile(VOCABULARY_SYNONYMS_XQUERY)) {
|
||||||
|
final String[] arr = s.split("@=@");
|
||||||
|
if (arr.length == 3) {
|
||||||
|
final String vocId = arr[0].trim();
|
||||||
|
final String termId = arr[1].trim();
|
||||||
|
final String syn = arr[2].trim();
|
||||||
|
|
||||||
|
vocs.addSynonyms(vocId, termId, syn);
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// add the term names as synonyms
|
||||||
|
vocs.vocs.values().forEach(voc -> {
|
||||||
|
voc.getTerms().values().forEach(term -> {
|
||||||
|
voc.addSynonym(term.getName().toLowerCase(), term.getId());
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
return vocs;
|
||||||
|
}
|
||||||
|
|
||||||
|
private final Map<String, Vocabulary> vocs = new HashMap<>();
|
||||||
|
|
||||||
|
public Set<String> vocabularyNames() {
|
||||||
|
return vocs.keySet();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void addVocabulary(final String id, final String name) {
|
||||||
|
vocs.put(id.toLowerCase(), new Vocabulary(id, name));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void addTerm(final String vocId, final String id, final String name) {
|
||||||
|
if (vocabularyExists(vocId)) {
|
||||||
|
vocs.get(vocId.toLowerCase()).addTerm(id, name);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public VocabularyTerm getTerm(final String vocId, final String id) {
|
||||||
|
if (termExists(vocId, id)) {
|
||||||
|
return vocs.get(vocId.toLowerCase()).getTerm(id);
|
||||||
|
} else {
|
||||||
|
return new VocabularyTerm(id, id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public Set<String> getTerms(String vocId) {
|
||||||
|
if (!vocabularyExists(vocId)) {
|
||||||
|
return new HashSet<>();
|
||||||
|
}
|
||||||
|
return vocs
|
||||||
|
.get(vocId.toLowerCase())
|
||||||
|
.getTerms()
|
||||||
|
.values()
|
||||||
|
.stream()
|
||||||
|
.map(VocabularyTerm::getId)
|
||||||
|
.collect(Collectors.toCollection(HashSet::new));
|
||||||
|
}
|
||||||
|
|
||||||
|
public Qualifier lookup(String vocId, String id) {
|
||||||
|
return Optional
|
||||||
|
.ofNullable(getSynonymAsQualifier(vocId, id))
|
||||||
|
.orElse(getTermAsQualifier(vocId, id));
|
||||||
|
}
|
||||||
|
|
||||||
|
public Qualifier getTermAsQualifier(final String vocId, final String id) {
|
||||||
|
if (vocabularyExists(vocId)) {
|
||||||
|
return vocs.get(vocId.toLowerCase()).getTermAsQualifier(id);
|
||||||
|
}
|
||||||
|
return OafMapperUtils.qualifier(id, id, "", "");
|
||||||
|
}
|
||||||
|
|
||||||
|
public Qualifier getSynonymAsQualifier(final String vocId, final String syn) {
|
||||||
|
if (StringUtils.isBlank(vocId)) {
|
||||||
|
return OafMapperUtils.unknown("", "");
|
||||||
|
}
|
||||||
|
return vocs.get(vocId.toLowerCase()).getSynonymAsQualifier(syn);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* getSynonymAsQualifierCaseSensitive
|
||||||
|
*
|
||||||
|
* refelects the situation to check caseSensitive vocabulary
|
||||||
|
*/
|
||||||
|
public Qualifier getSynonymAsQualifierCaseSensitive(final String vocId, final String syn) {
|
||||||
|
if (StringUtils.isBlank(vocId)) {
|
||||||
|
return OafMapperUtils.unknown("", "");
|
||||||
|
}
|
||||||
|
return vocs.get(vocId).getSynonymAsQualifier(syn);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* termExists
|
||||||
|
*
|
||||||
|
* two methods: without and with caseSensitive check
|
||||||
|
*/
|
||||||
|
public boolean termExists(final String vocId, final String id) {
|
||||||
|
return termExists(vocId, id, Boolean.FALSE);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean termExists(final String vocId, final String id, final Boolean caseSensitive) {
|
||||||
|
if (Boolean.TRUE.equals(caseSensitive)) {
|
||||||
|
return vocabularyExists(vocId) && vocs.get(vocId).termExists(id);
|
||||||
|
}
|
||||||
|
return vocabularyExists(vocId) && vocs.get(vocId.toLowerCase()).termExists(id);
|
||||||
|
}
|
||||||
|
|
||||||
|
public boolean vocabularyExists(final String vocId) {
|
||||||
|
return Optional
|
||||||
|
.ofNullable(vocId)
|
||||||
|
.map(String::toLowerCase)
|
||||||
|
.map(vocs::containsKey)
|
||||||
|
.orElse(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
private void addSynonyms(final String vocId, final String termId, final String syn) {
|
||||||
|
String id = Optional
|
||||||
|
.ofNullable(vocId)
|
||||||
|
.map(String::toLowerCase)
|
||||||
|
.orElseThrow(
|
||||||
|
() -> new IllegalArgumentException(
|
||||||
|
String
|
||||||
|
.format(
|
||||||
|
"empty vocabulary id for [term:%s, synonym:%s]", termId, syn)));
|
||||||
|
Optional
|
||||||
|
.ofNullable(vocs.get(id))
|
||||||
|
.orElseThrow(() -> new IllegalArgumentException("missing vocabulary id: " + vocId))
|
||||||
|
.addSynonym(syn.toLowerCase(), termId);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,24 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.common.vocabulary;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
|
||||||
|
public class VocabularyTerm implements Serializable {
|
||||||
|
|
||||||
|
private final String id;
|
||||||
|
private final String name;
|
||||||
|
|
||||||
|
public VocabularyTerm(final String id, final String name) {
|
||||||
|
this.id = id;
|
||||||
|
this.name = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getId() {
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,63 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.message;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.LinkedHashMap;
|
||||||
|
import java.util.Map;
|
||||||
|
|
||||||
|
public class Message implements Serializable {
|
||||||
|
|
||||||
|
private static final long serialVersionUID = 401753881204524893L;
|
||||||
|
|
||||||
|
public static final String CURRENT_PARAM = "current";
|
||||||
|
public static final String TOTAL_PARAM = "total";
|
||||||
|
|
||||||
|
private MessageType messageType;
|
||||||
|
|
||||||
|
private String workflowId;
|
||||||
|
|
||||||
|
private Map<String, String> body;
|
||||||
|
|
||||||
|
public Message() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public Message(final MessageType messageType, final String workflowId) {
|
||||||
|
this(messageType, workflowId, new LinkedHashMap<>());
|
||||||
|
}
|
||||||
|
|
||||||
|
public Message(final MessageType messageType, final String workflowId, final Map<String, String> body) {
|
||||||
|
this.messageType = messageType;
|
||||||
|
this.workflowId = workflowId;
|
||||||
|
this.body = body;
|
||||||
|
}
|
||||||
|
|
||||||
|
public MessageType getMessageType() {
|
||||||
|
return messageType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setMessageType(MessageType messageType) {
|
||||||
|
this.messageType = messageType;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getWorkflowId() {
|
||||||
|
return workflowId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setWorkflowId(final String workflowId) {
|
||||||
|
this.workflowId = workflowId;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Map<String, String> getBody() {
|
||||||
|
return body;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setBody(final Map<String, String> body) {
|
||||||
|
this.body = body;
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public String toString() {
|
||||||
|
return String.format("Message [type=%s, workflowId=%s, body=%s]", messageType, workflowId, body);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,94 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.message;
|
||||||
|
|
||||||
|
import java.util.Map;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
|
||||||
|
import org.apache.http.client.config.RequestConfig;
|
||||||
|
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||||
|
import org.apache.http.client.methods.HttpPut;
|
||||||
|
import org.apache.http.entity.ContentType;
|
||||||
|
import org.apache.http.entity.StringEntity;
|
||||||
|
import org.apache.http.impl.client.CloseableHttpClient;
|
||||||
|
import org.apache.http.impl.client.HttpClients;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
public class MessageSender {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(MessageSender.class);
|
||||||
|
|
||||||
|
private static final int SOCKET_TIMEOUT_MS = 2000;
|
||||||
|
|
||||||
|
private static final int CONNECTION_REQUEST_TIMEOUT_MS = 2000;
|
||||||
|
|
||||||
|
private static final int CONNTECTION_TIMEOUT_MS = 2000;
|
||||||
|
|
||||||
|
private final ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
|
private final String dnetMessageEndpoint;
|
||||||
|
|
||||||
|
private final String workflowId;
|
||||||
|
|
||||||
|
private final ExecutorService executorService = Executors.newCachedThreadPool();
|
||||||
|
|
||||||
|
public MessageSender(final String dnetMessageEndpoint, final String workflowId) {
|
||||||
|
this.workflowId = workflowId;
|
||||||
|
this.dnetMessageEndpoint = dnetMessageEndpoint;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void sendMessage(final Message message) {
|
||||||
|
executorService.submit(() -> _sendMessage(message));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void sendMessage(final Long current, final Long total) {
|
||||||
|
sendMessage(createOngoingMessage(current, total));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void sendReport(final Map<String, String> report) {
|
||||||
|
sendMessage(new Message(MessageType.REPORT, workflowId, report));
|
||||||
|
}
|
||||||
|
|
||||||
|
private Message createOngoingMessage(final Long current, final Long total) {
|
||||||
|
final Message m = new Message(MessageType.ONGOING, workflowId);
|
||||||
|
m.getBody().put(Message.CURRENT_PARAM, current.toString());
|
||||||
|
if (total != null) {
|
||||||
|
m.getBody().put(Message.TOTAL_PARAM, total.toString());
|
||||||
|
}
|
||||||
|
return m;
|
||||||
|
}
|
||||||
|
|
||||||
|
private void _sendMessage(final Message message) {
|
||||||
|
try {
|
||||||
|
final String json = objectMapper.writeValueAsString(message);
|
||||||
|
|
||||||
|
final HttpPut req = new HttpPut(dnetMessageEndpoint);
|
||||||
|
req.setEntity(new StringEntity(json, ContentType.APPLICATION_JSON));
|
||||||
|
|
||||||
|
final RequestConfig requestConfig = RequestConfig
|
||||||
|
.custom()
|
||||||
|
.setConnectTimeout(CONNTECTION_TIMEOUT_MS)
|
||||||
|
.setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT_MS)
|
||||||
|
.setSocketTimeout(SOCKET_TIMEOUT_MS)
|
||||||
|
.build();
|
||||||
|
|
||||||
|
try (final CloseableHttpClient client = HttpClients
|
||||||
|
.custom()
|
||||||
|
.setDefaultRequestConfig(requestConfig)
|
||||||
|
.build();
|
||||||
|
final CloseableHttpResponse response = client.execute(req)) {
|
||||||
|
log.debug("Sent Message to " + dnetMessageEndpoint);
|
||||||
|
log.debug("MESSAGE:" + message);
|
||||||
|
} catch (final Throwable e) {
|
||||||
|
log.error("Error sending message to " + dnetMessageEndpoint + ", message content: " + message, e);
|
||||||
|
}
|
||||||
|
} catch (final JsonProcessingException e) {
|
||||||
|
log.error("Error sending message to " + dnetMessageEndpoint + ", message content: " + message, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,21 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.message;
|
||||||
|
|
||||||
|
import java.io.Serializable;
|
||||||
|
import java.util.Optional;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
|
public enum MessageType implements Serializable {
|
||||||
|
|
||||||
|
ONGOING, REPORT;
|
||||||
|
|
||||||
|
public MessageType from(String value) {
|
||||||
|
return Optional
|
||||||
|
.ofNullable(value)
|
||||||
|
.map(StringUtils::upperCase)
|
||||||
|
.map(MessageType::valueOf)
|
||||||
|
.orElseThrow(() -> new IllegalArgumentException("unknown message type: " + value));
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -1,121 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.model.mdstore;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.utils.DHPUtils;
|
|
||||||
|
|
||||||
/** This class models a record inside the new Metadata store collection on HDFS * */
|
|
||||||
public class MetadataRecord implements Serializable {
|
|
||||||
|
|
||||||
/** The D-Net Identifier associated to the record */
|
|
||||||
private String id;
|
|
||||||
|
|
||||||
/** The original Identifier of the record */
|
|
||||||
private String originalId;
|
|
||||||
|
|
||||||
/** The encoding of the record, should be JSON or XML */
|
|
||||||
private String encoding;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* The information about the provenance of the record see @{@link Provenance} for the model of this information
|
|
||||||
*/
|
|
||||||
private Provenance provenance;
|
|
||||||
|
|
||||||
/** The content of the metadata */
|
|
||||||
private String body;
|
|
||||||
|
|
||||||
/** the date when the record has been stored */
|
|
||||||
private long dateOfCollection;
|
|
||||||
|
|
||||||
/** the date when the record has been stored */
|
|
||||||
private long dateOfTransformation;
|
|
||||||
|
|
||||||
public MetadataRecord() {
|
|
||||||
this.dateOfCollection = System.currentTimeMillis();
|
|
||||||
}
|
|
||||||
|
|
||||||
public MetadataRecord(
|
|
||||||
String originalId,
|
|
||||||
String encoding,
|
|
||||||
Provenance provenance,
|
|
||||||
String body,
|
|
||||||
long dateOfCollection) {
|
|
||||||
|
|
||||||
this.originalId = originalId;
|
|
||||||
this.encoding = encoding;
|
|
||||||
this.provenance = provenance;
|
|
||||||
this.body = body;
|
|
||||||
this.dateOfCollection = dateOfCollection;
|
|
||||||
this.id = DHPUtils.generateIdentifier(originalId, this.provenance.getNsPrefix());
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getId() {
|
|
||||||
return id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setId(String id) {
|
|
||||||
this.id = id;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getOriginalId() {
|
|
||||||
return originalId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setOriginalId(String originalId) {
|
|
||||||
this.originalId = originalId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getEncoding() {
|
|
||||||
return encoding;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setEncoding(String encoding) {
|
|
||||||
this.encoding = encoding;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Provenance getProvenance() {
|
|
||||||
return provenance;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setProvenance(Provenance provenance) {
|
|
||||||
this.provenance = provenance;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getBody() {
|
|
||||||
return body;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setBody(String body) {
|
|
||||||
this.body = body;
|
|
||||||
}
|
|
||||||
|
|
||||||
public long getDateOfCollection() {
|
|
||||||
return dateOfCollection;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDateOfCollection(long dateOfCollection) {
|
|
||||||
this.dateOfCollection = dateOfCollection;
|
|
||||||
}
|
|
||||||
|
|
||||||
public long getDateOfTransformation() {
|
|
||||||
return dateOfTransformation;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDateOfTransformation(long dateOfTransformation) {
|
|
||||||
this.dateOfTransformation = dateOfTransformation;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public boolean equals(Object o) {
|
|
||||||
if (!(o instanceof MetadataRecord)) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return ((MetadataRecord) o).getId().equalsIgnoreCase(id);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public int hashCode() {
|
|
||||||
return id.hashCode();
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,52 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.model.mdstore;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* @author Sandro La Bruzzo
|
|
||||||
* <p>
|
|
||||||
* Provenace class models the provenance of the record in the metadataStore It contains the identifier and the
|
|
||||||
* name of the datasource that gives the record
|
|
||||||
*/
|
|
||||||
public class Provenance implements Serializable {
|
|
||||||
|
|
||||||
private String datasourceId;
|
|
||||||
|
|
||||||
private String datasourceName;
|
|
||||||
|
|
||||||
private String nsPrefix;
|
|
||||||
|
|
||||||
public Provenance() {
|
|
||||||
}
|
|
||||||
|
|
||||||
public Provenance(String datasourceId, String datasourceName, String nsPrefix) {
|
|
||||||
this.datasourceId = datasourceId;
|
|
||||||
this.datasourceName = datasourceName;
|
|
||||||
this.nsPrefix = nsPrefix;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getDatasourceId() {
|
|
||||||
return datasourceId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDatasourceId(String datasourceId) {
|
|
||||||
this.datasourceId = datasourceId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getDatasourceName() {
|
|
||||||
return datasourceName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setDatasourceName(String datasourceName) {
|
|
||||||
this.datasourceName = datasourceName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getNsPrefix() {
|
|
||||||
return nsPrefix;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setNsPrefix(String nsPrefix) {
|
|
||||||
this.nsPrefix = nsPrefix;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -18,6 +18,9 @@ public class AuthorMerger {
|
||||||
|
|
||||||
private static final Double THRESHOLD = 0.95;
|
private static final Double THRESHOLD = 0.95;
|
||||||
|
|
||||||
|
private AuthorMerger() {
|
||||||
|
}
|
||||||
|
|
||||||
public static List<Author> merge(List<List<Author>> authors) {
|
public static List<Author> merge(List<List<Author>> authors) {
|
||||||
|
|
||||||
authors.sort((o1, o2) -> -Integer.compare(countAuthorsPids(o1), countAuthorsPids(o2)));
|
authors.sort((o1, o2) -> -Integer.compare(countAuthorsPids(o1), countAuthorsPids(o2)));
|
||||||
|
@ -32,44 +35,54 @@ public class AuthorMerger {
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static List<Author> mergeAuthor(final List<Author> a, final List<Author> b) {
|
public static List<Author> mergeAuthor(final List<Author> a, final List<Author> b, Double threshold) {
|
||||||
int pa = countAuthorsPids(a);
|
int pa = countAuthorsPids(a);
|
||||||
int pb = countAuthorsPids(b);
|
int pb = countAuthorsPids(b);
|
||||||
List<Author> base, enrich;
|
List<Author> base;
|
||||||
|
List<Author> enrich;
|
||||||
int sa = authorsSize(a);
|
int sa = authorsSize(a);
|
||||||
int sb = authorsSize(b);
|
int sb = authorsSize(b);
|
||||||
|
|
||||||
if (pa == pb) {
|
if (sa == sb) {
|
||||||
base = sa > sb ? a : b;
|
|
||||||
enrich = sa > sb ? b : a;
|
|
||||||
} else {
|
|
||||||
base = pa > pb ? a : b;
|
base = pa > pb ? a : b;
|
||||||
enrich = pa > pb ? b : a;
|
enrich = pa > pb ? b : a;
|
||||||
|
} else {
|
||||||
|
base = sa > sb ? a : b;
|
||||||
|
enrich = sa > sb ? b : a;
|
||||||
}
|
}
|
||||||
enrichPidFromList(base, enrich);
|
enrichPidFromList(base, enrich, threshold);
|
||||||
return base;
|
return base;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void enrichPidFromList(List<Author> base, List<Author> enrich) {
|
public static List<Author> mergeAuthor(final List<Author> a, final List<Author> b) {
|
||||||
|
return mergeAuthor(a, b, THRESHOLD);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void enrichPidFromList(List<Author> base, List<Author> enrich, Double threshold) {
|
||||||
if (base == null || enrich == null)
|
if (base == null || enrich == null)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
// <pidComparableString, Author> (if an Author has more than 1 pid, it appears 2 times in the list)
|
||||||
final Map<String, Author> basePidAuthorMap = base
|
final Map<String, Author> basePidAuthorMap = base
|
||||||
.stream()
|
.stream()
|
||||||
.filter(a -> a.getPid() != null && a.getPid().size() > 0)
|
.filter(a -> a.getPid() != null && !a.getPid().isEmpty())
|
||||||
.flatMap(
|
.flatMap(
|
||||||
a -> a
|
a -> a
|
||||||
.getPid()
|
.getPid()
|
||||||
.stream()
|
.stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
.map(p -> new Tuple2<>(pidToComparableString(p), a)))
|
.map(p -> new Tuple2<>(pidToComparableString(p), a)))
|
||||||
.collect(Collectors.toMap(Tuple2::_1, Tuple2::_2, (x1, x2) -> x1));
|
.collect(Collectors.toMap(Tuple2::_1, Tuple2::_2, (x1, x2) -> x1));
|
||||||
|
|
||||||
|
// <pid, Author> (list of pid that are missing in the other list)
|
||||||
final List<Tuple2<StructuredProperty, Author>> pidToEnrich = enrich
|
final List<Tuple2<StructuredProperty, Author>> pidToEnrich = enrich
|
||||||
.stream()
|
.stream()
|
||||||
.filter(a -> a.getPid() != null && a.getPid().size() > 0)
|
.filter(a -> a.getPid() != null && !a.getPid().isEmpty())
|
||||||
.flatMap(
|
.flatMap(
|
||||||
a -> a
|
a -> a
|
||||||
.getPid()
|
.getPid()
|
||||||
.stream()
|
.stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
.filter(p -> !basePidAuthorMap.containsKey(pidToComparableString(p)))
|
.filter(p -> !basePidAuthorMap.containsKey(pidToComparableString(p)))
|
||||||
.map(p -> new Tuple2<>(p, a)))
|
.map(p -> new Tuple2<>(p, a)))
|
||||||
.collect(Collectors.toList());
|
.collect(Collectors.toList());
|
||||||
|
@ -83,10 +96,10 @@ public class AuthorMerger {
|
||||||
.max(Comparator.comparing(Tuple2::_1));
|
.max(Comparator.comparing(Tuple2::_1));
|
||||||
|
|
||||||
if (simAuthor.isPresent()) {
|
if (simAuthor.isPresent()) {
|
||||||
double th = THRESHOLD;
|
double th = threshold;
|
||||||
// increase the threshold if the surname is too short
|
// increase the threshold if the surname is too short
|
||||||
if (simAuthor.get()._2().getSurname() != null
|
if (simAuthor.get()._2().getSurname() != null
|
||||||
&& simAuthor.get()._2().getSurname().length() <= 3)
|
&& simAuthor.get()._2().getSurname().length() <= 3 && threshold > 0.0)
|
||||||
th = 0.99;
|
th = 0.99;
|
||||||
|
|
||||||
if (simAuthor.get()._1() > th) {
|
if (simAuthor.get()._1() > th) {
|
||||||
|
@ -107,9 +120,9 @@ public class AuthorMerger {
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String pidToComparableString(StructuredProperty pid) {
|
public static String pidToComparableString(StructuredProperty pid) {
|
||||||
return (pid.getQualifier() != null
|
final String classid = pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase()
|
||||||
? pid.getQualifier().getClassid() != null ? pid.getQualifier().getClassid().toLowerCase() : ""
|
: "";
|
||||||
: "")
|
return (pid.getQualifier() != null ? classid : "")
|
||||||
+ (pid.getValue() != null ? pid.getValue().toLowerCase() : "");
|
+ (pid.getValue() != null ? pid.getValue().toLowerCase() : "");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -142,7 +155,7 @@ public class AuthorMerger {
|
||||||
}
|
}
|
||||||
|
|
||||||
private static boolean hasPid(Author a) {
|
private static boolean hasPid(Author a) {
|
||||||
if (a == null || a.getPid() == null || a.getPid().size() == 0)
|
if (a == null || a.getPid() == null || a.getPid().isEmpty())
|
||||||
return false;
|
return false;
|
||||||
return a.getPid().stream().anyMatch(p -> p != null && StringUtils.isNotBlank(p.getValue()));
|
return a.getPid().stream().anyMatch(p -> p != null && StringUtils.isNotBlank(p.getValue()));
|
||||||
}
|
}
|
||||||
|
@ -151,12 +164,15 @@ public class AuthorMerger {
|
||||||
if (StringUtils.isNotBlank(author.getSurname())) {
|
if (StringUtils.isNotBlank(author.getSurname())) {
|
||||||
return new Person(author.getSurname() + ", " + author.getName(), false);
|
return new Person(author.getSurname() + ", " + author.getName(), false);
|
||||||
} else {
|
} else {
|
||||||
|
if (StringUtils.isNotBlank(author.getFullname()))
|
||||||
return new Person(author.getFullname(), false);
|
return new Person(author.getFullname(), false);
|
||||||
|
else
|
||||||
|
return new Person("", false);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String normalize(final String s) {
|
private static String normalize(final String s) {
|
||||||
return nfd(s)
|
String[] normalized = nfd(s)
|
||||||
.toLowerCase()
|
.toLowerCase()
|
||||||
// do not compact the regexes in a single expression, would cause StackOverflowError
|
// do not compact the regexes in a single expression, would cause StackOverflowError
|
||||||
// in case
|
// in case
|
||||||
|
@ -166,7 +182,12 @@ public class AuthorMerger {
|
||||||
.replaceAll("(\\p{Punct})+", " ")
|
.replaceAll("(\\p{Punct})+", " ")
|
||||||
.replaceAll("(\\d)+", " ")
|
.replaceAll("(\\d)+", " ")
|
||||||
.replaceAll("(\\n)+", " ")
|
.replaceAll("(\\n)+", " ")
|
||||||
.trim();
|
.trim()
|
||||||
|
.split(" ");
|
||||||
|
|
||||||
|
Arrays.sort(normalized);
|
||||||
|
|
||||||
|
return String.join(" ", normalized);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static String nfd(final String s) {
|
private static String nfd(final String s) {
|
||||||
|
|
|
@ -12,6 +12,9 @@ import com.ximpleware.VTDNav;
|
||||||
/** Created by sandro on 9/29/16. */
|
/** Created by sandro on 9/29/16. */
|
||||||
public class VtdUtilityParser {
|
public class VtdUtilityParser {
|
||||||
|
|
||||||
|
private VtdUtilityParser() {
|
||||||
|
}
|
||||||
|
|
||||||
public static List<Node> getTextValuesWithAttributes(
|
public static List<Node> getTextValuesWithAttributes(
|
||||||
final AutoPilot ap, final VTDNav vn, final String xpath, final List<String> attributes)
|
final AutoPilot ap, final VTDNav vn, final String xpath, final List<String> attributes)
|
||||||
throws VtdException {
|
throws VtdException {
|
||||||
|
|
|
@ -1,238 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.oaf;
|
|
||||||
|
|
||||||
import java.util.LinkedHashMap;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
|
|
||||||
import com.clearspring.analytics.util.Lists;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
|
||||||
|
|
||||||
public class CleaningFunctions {
|
|
||||||
|
|
||||||
public static final String DOI_URL_PREFIX_REGEX = "(^http(s?):\\/\\/)(((dx\\.)?doi\\.org)|(handle\\.test\\.datacite\\.org))\\/";
|
|
||||||
public static final String ORCID_PREFIX_REGEX = "^http(s?):\\/\\/orcid\\.org\\/";
|
|
||||||
public static final String NONE = "none";
|
|
||||||
|
|
||||||
public static <T extends Oaf> T fixVocabularyNames(T value) {
|
|
||||||
if (value instanceof Datasource) {
|
|
||||||
// nothing to clean here
|
|
||||||
} else if (value instanceof Project) {
|
|
||||||
// nothing to clean here
|
|
||||||
} else if (value instanceof Organization) {
|
|
||||||
Organization o = (Organization) value;
|
|
||||||
if (Objects.nonNull(o.getCountry())) {
|
|
||||||
fixVocabName(o.getCountry(), ModelConstants.DNET_COUNTRY_TYPE);
|
|
||||||
}
|
|
||||||
} else if (value instanceof Relation) {
|
|
||||||
// nothing to clean here
|
|
||||||
} else if (value instanceof Result) {
|
|
||||||
|
|
||||||
Result r = (Result) value;
|
|
||||||
|
|
||||||
fixVocabName(r.getLanguage(), ModelConstants.DNET_LANGUAGES);
|
|
||||||
fixVocabName(r.getResourcetype(), ModelConstants.DNET_DATA_CITE_RESOURCE);
|
|
||||||
fixVocabName(r.getBestaccessright(), ModelConstants.DNET_ACCESS_MODES);
|
|
||||||
|
|
||||||
if (Objects.nonNull(r.getSubject())) {
|
|
||||||
r.getSubject().forEach(s -> fixVocabName(s.getQualifier(), ModelConstants.DNET_SUBJECT_TYPOLOGIES));
|
|
||||||
}
|
|
||||||
if (Objects.nonNull(r.getInstance())) {
|
|
||||||
for (Instance i : r.getInstance()) {
|
|
||||||
fixVocabName(i.getAccessright(), ModelConstants.DNET_ACCESS_MODES);
|
|
||||||
fixVocabName(i.getRefereed(), ModelConstants.DNET_REVIEW_LEVELS);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (Objects.nonNull(r.getAuthor())) {
|
|
||||||
r.getAuthor().forEach(a -> {
|
|
||||||
if (Objects.nonNull(a.getPid())) {
|
|
||||||
a.getPid().forEach(p -> {
|
|
||||||
fixVocabName(p.getQualifier(), ModelConstants.DNET_PID_TYPES);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
if (value instanceof Publication) {
|
|
||||||
|
|
||||||
} else if (value instanceof eu.dnetlib.dhp.schema.oaf.Dataset) {
|
|
||||||
|
|
||||||
} else if (value instanceof OtherResearchProduct) {
|
|
||||||
|
|
||||||
} else if (value instanceof Software) {
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static <T extends Oaf> T fixDefaults(T value) {
|
|
||||||
if (value instanceof Datasource) {
|
|
||||||
// nothing to clean here
|
|
||||||
} else if (value instanceof Project) {
|
|
||||||
// nothing to clean here
|
|
||||||
} else if (value instanceof Organization) {
|
|
||||||
Organization o = (Organization) value;
|
|
||||||
if (Objects.isNull(o.getCountry()) || StringUtils.isBlank(o.getCountry().getClassid())) {
|
|
||||||
o.setCountry(qualifier("UNKNOWN", "Unknown", ModelConstants.DNET_COUNTRY_TYPE));
|
|
||||||
}
|
|
||||||
} else if (value instanceof Relation) {
|
|
||||||
// nothing to clean here
|
|
||||||
} else if (value instanceof Result) {
|
|
||||||
|
|
||||||
Result r = (Result) value;
|
|
||||||
if (Objects.nonNull(r.getPublisher()) && StringUtils.isBlank(r.getPublisher().getValue())) {
|
|
||||||
r.setPublisher(null);
|
|
||||||
}
|
|
||||||
if (Objects.isNull(r.getLanguage()) || StringUtils.isBlank(r.getLanguage().getClassid())) {
|
|
||||||
r
|
|
||||||
.setLanguage(
|
|
||||||
qualifier("und", "Undetermined", ModelConstants.DNET_LANGUAGES));
|
|
||||||
}
|
|
||||||
if (Objects.nonNull(r.getSubject())) {
|
|
||||||
r
|
|
||||||
.setSubject(
|
|
||||||
r
|
|
||||||
.getSubject()
|
|
||||||
.stream()
|
|
||||||
.filter(Objects::nonNull)
|
|
||||||
.filter(sp -> StringUtils.isNotBlank(sp.getValue()))
|
|
||||||
.filter(sp -> Objects.nonNull(sp.getQualifier()))
|
|
||||||
.filter(sp -> StringUtils.isNotBlank(sp.getQualifier().getClassid()))
|
|
||||||
.collect(Collectors.toList()));
|
|
||||||
}
|
|
||||||
if (Objects.nonNull(r.getPid())) {
|
|
||||||
r
|
|
||||||
.setPid(
|
|
||||||
r
|
|
||||||
.getPid()
|
|
||||||
.stream()
|
|
||||||
.filter(Objects::nonNull)
|
|
||||||
.filter(sp -> StringUtils.isNotBlank(StringUtils.trim(sp.getValue())))
|
|
||||||
.filter(sp -> NONE.equalsIgnoreCase(sp.getValue()))
|
|
||||||
.filter(sp -> Objects.nonNull(sp.getQualifier()))
|
|
||||||
.filter(sp -> StringUtils.isNotBlank(sp.getQualifier().getClassid()))
|
|
||||||
.map(CleaningFunctions::normalizePidValue)
|
|
||||||
.collect(Collectors.toList()));
|
|
||||||
}
|
|
||||||
if (Objects.isNull(r.getResourcetype()) || StringUtils.isBlank(r.getResourcetype().getClassid())) {
|
|
||||||
r
|
|
||||||
.setResourcetype(
|
|
||||||
qualifier("UNKNOWN", "Unknown", ModelConstants.DNET_DATA_CITE_RESOURCE));
|
|
||||||
}
|
|
||||||
if (Objects.nonNull(r.getInstance())) {
|
|
||||||
for (Instance i : r.getInstance()) {
|
|
||||||
if (Objects.isNull(i.getAccessright()) || StringUtils.isBlank(i.getAccessright().getClassid())) {
|
|
||||||
i.setAccessright(qualifier("UNKNOWN", "not available", ModelConstants.DNET_ACCESS_MODES));
|
|
||||||
}
|
|
||||||
if (Objects.isNull(i.getHostedby()) || StringUtils.isBlank(i.getHostedby().getKey())) {
|
|
||||||
i.setHostedby(ModelConstants.UNKNOWN_REPOSITORY);
|
|
||||||
}
|
|
||||||
if (Objects.isNull(i.getRefereed())) {
|
|
||||||
i.setRefereed(qualifier("0000", "Unknown", ModelConstants.DNET_REVIEW_LEVELS));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (Objects.isNull(r.getBestaccessright()) || StringUtils.isBlank(r.getBestaccessright().getClassid())) {
|
|
||||||
Qualifier bestaccessrights = OafMapperUtils.createBestAccessRights(r.getInstance());
|
|
||||||
if (Objects.isNull(bestaccessrights)) {
|
|
||||||
r
|
|
||||||
.setBestaccessright(
|
|
||||||
qualifier("UNKNOWN", "not available", ModelConstants.DNET_ACCESS_MODES));
|
|
||||||
} else {
|
|
||||||
r.setBestaccessright(bestaccessrights);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (Objects.nonNull(r.getAuthor())) {
|
|
||||||
boolean nullRank = r
|
|
||||||
.getAuthor()
|
|
||||||
.stream()
|
|
||||||
.anyMatch(a -> Objects.isNull(a.getRank()));
|
|
||||||
if (nullRank) {
|
|
||||||
int i = 1;
|
|
||||||
for (Author author : r.getAuthor()) {
|
|
||||||
author.setRank(i++);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (Author a : r.getAuthor()) {
|
|
||||||
if (Objects.isNull(a.getPid())) {
|
|
||||||
a.setPid(Lists.newArrayList());
|
|
||||||
} else {
|
|
||||||
a
|
|
||||||
.setPid(
|
|
||||||
a
|
|
||||||
.getPid()
|
|
||||||
.stream()
|
|
||||||
.filter(p -> Objects.nonNull(p.getQualifier()))
|
|
||||||
.filter(p -> StringUtils.isNotBlank(p.getValue()))
|
|
||||||
.map(p -> {
|
|
||||||
p.setValue(p.getValue().trim().replaceAll(ORCID_PREFIX_REGEX, ""));
|
|
||||||
return p;
|
|
||||||
})
|
|
||||||
.collect(
|
|
||||||
Collectors
|
|
||||||
.toMap(
|
|
||||||
StructuredProperty::getValue, Function.identity(), (p1, p2) -> p1,
|
|
||||||
LinkedHashMap::new))
|
|
||||||
.values()
|
|
||||||
.stream()
|
|
||||||
.collect(Collectors.toList()));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
if (value instanceof Publication) {
|
|
||||||
|
|
||||||
} else if (value instanceof eu.dnetlib.dhp.schema.oaf.Dataset) {
|
|
||||||
|
|
||||||
} else if (value instanceof OtherResearchProduct) {
|
|
||||||
|
|
||||||
} else if (value instanceof Software) {
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
// HELPERS
|
|
||||||
|
|
||||||
private static void fixVocabName(Qualifier q, String vocabularyName) {
|
|
||||||
if (Objects.nonNull(q) && StringUtils.isBlank(q.getSchemeid())) {
|
|
||||||
q.setSchemeid(vocabularyName);
|
|
||||||
q.setSchemename(vocabularyName);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static Qualifier qualifier(String classid, String classname, String scheme) {
|
|
||||||
return OafMapperUtils
|
|
||||||
.qualifier(
|
|
||||||
classid, classname, scheme, scheme);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Utility method that normalises PID values on a per-type basis.
|
|
||||||
* @param pid the PID whose value will be normalised.
|
|
||||||
* @return the PID containing the normalised value.
|
|
||||||
*/
|
|
||||||
public static StructuredProperty normalizePidValue(StructuredProperty pid) {
|
|
||||||
String value = Optional
|
|
||||||
.ofNullable(pid.getValue())
|
|
||||||
.map(String::trim)
|
|
||||||
.orElseThrow(() -> new IllegalArgumentException("PID value cannot be empty"));
|
|
||||||
switch (pid.getQualifier().getClassid()) {
|
|
||||||
|
|
||||||
// TODO add cleaning for more PID types as needed
|
|
||||||
case "doi":
|
|
||||||
pid.setValue(value.toLowerCase().replaceAll(DOI_URL_PREFIX_REGEX, ""));
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
return pid;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,14 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.oaf;
|
|
||||||
|
|
||||||
public class ModelHardLimits {
|
|
||||||
|
|
||||||
public static final int MAX_EXTERNAL_ENTITIES = 50;
|
|
||||||
public static final int MAX_AUTHORS = 200;
|
|
||||||
public static final int MAX_AUTHOR_FULLNAME_LENGTH = 1000;
|
|
||||||
public static final int MAX_TITLE_LENGTH = 5000;
|
|
||||||
public static final int MAX_TITLES = 10;
|
|
||||||
public static final int MAX_ABSTRACT_LENGTH = 150000;
|
|
||||||
public static final int MAX_INSTANCES = 10;
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,296 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.oaf;
|
|
||||||
|
|
||||||
import static eu.dnetlib.dhp.schema.common.ModelConstants.*;
|
|
||||||
import static eu.dnetlib.dhp.schema.common.ModelConstants.DNET_ACCESS_MODES;
|
|
||||||
|
|
||||||
import java.util.*;
|
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
|
||||||
import java.util.function.Function;
|
|
||||||
import java.util.function.Predicate;
|
|
||||||
import java.util.stream.Collectors;
|
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.common.LicenseComparator;
|
|
||||||
import eu.dnetlib.dhp.utils.DHPUtils;
|
|
||||||
|
|
||||||
public class OafMapperUtils {
|
|
||||||
|
|
||||||
public static KeyValue keyValue(final String k, final String v) {
|
|
||||||
final KeyValue kv = new KeyValue();
|
|
||||||
kv.setKey(k);
|
|
||||||
kv.setValue(v);
|
|
||||||
return kv;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static List<KeyValue> listKeyValues(final String... s) {
|
|
||||||
if (s.length % 2 > 0) {
|
|
||||||
throw new RuntimeException("Invalid number of parameters (k,v,k,v,....)");
|
|
||||||
}
|
|
||||||
|
|
||||||
final List<KeyValue> list = new ArrayList<>();
|
|
||||||
for (int i = 0; i < s.length; i += 2) {
|
|
||||||
list.add(keyValue(s[i], s[i + 1]));
|
|
||||||
}
|
|
||||||
return list;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static <T> Field<T> field(final T value, final DataInfo info) {
|
|
||||||
if (value == null || StringUtils.isBlank(value.toString())) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
final Field<T> field = new Field<>();
|
|
||||||
field.setValue(value);
|
|
||||||
field.setDataInfo(info);
|
|
||||||
return field;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static List<Field<String>> listFields(final DataInfo info, final String... values) {
|
|
||||||
return Arrays
|
|
||||||
.stream(values)
|
|
||||||
.map(v -> field(v, info))
|
|
||||||
.filter(Objects::nonNull)
|
|
||||||
.filter(distinctByKey(f -> f.getValue()))
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static List<Field<String>> listFields(final DataInfo info, final List<String> values) {
|
|
||||||
return values
|
|
||||||
.stream()
|
|
||||||
.map(v -> field(v, info))
|
|
||||||
.filter(Objects::nonNull)
|
|
||||||
.filter(distinctByKey(f -> f.getValue()))
|
|
||||||
.collect(Collectors.toList());
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Qualifier unknown(final String schemeid, final String schemename) {
|
|
||||||
return qualifier("UNKNOWN", "Unknown", schemeid, schemename);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Qualifier qualifier(
|
|
||||||
final String classid,
|
|
||||||
final String classname,
|
|
||||||
final String schemeid,
|
|
||||||
final String schemename) {
|
|
||||||
final Qualifier q = new Qualifier();
|
|
||||||
q.setClassid(classid);
|
|
||||||
q.setClassname(classname);
|
|
||||||
q.setSchemeid(schemeid);
|
|
||||||
q.setSchemename(schemename);
|
|
||||||
return q;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static StructuredProperty structuredProperty(
|
|
||||||
final String value,
|
|
||||||
final String classid,
|
|
||||||
final String classname,
|
|
||||||
final String schemeid,
|
|
||||||
final String schemename,
|
|
||||||
final DataInfo dataInfo) {
|
|
||||||
|
|
||||||
return structuredProperty(value, qualifier(classid, classname, schemeid, schemename), dataInfo);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static StructuredProperty structuredProperty(
|
|
||||||
final String value,
|
|
||||||
final Qualifier qualifier,
|
|
||||||
final DataInfo dataInfo) {
|
|
||||||
if (value == null) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
final StructuredProperty sp = new StructuredProperty();
|
|
||||||
sp.setValue(value);
|
|
||||||
sp.setQualifier(qualifier);
|
|
||||||
sp.setDataInfo(dataInfo);
|
|
||||||
return sp;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static ExtraInfo extraInfo(
|
|
||||||
final String name,
|
|
||||||
final String value,
|
|
||||||
final String typology,
|
|
||||||
final String provenance,
|
|
||||||
final String trust) {
|
|
||||||
final ExtraInfo info = new ExtraInfo();
|
|
||||||
info.setName(name);
|
|
||||||
info.setValue(value);
|
|
||||||
info.setTypology(typology);
|
|
||||||
info.setProvenance(provenance);
|
|
||||||
info.setTrust(trust);
|
|
||||||
return info;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static OAIProvenance oaiIProvenance(
|
|
||||||
final String identifier,
|
|
||||||
final String baseURL,
|
|
||||||
final String metadataNamespace,
|
|
||||||
final Boolean altered,
|
|
||||||
final String datestamp,
|
|
||||||
final String harvestDate) {
|
|
||||||
|
|
||||||
final OriginDescription desc = new OriginDescription();
|
|
||||||
desc.setIdentifier(identifier);
|
|
||||||
desc.setBaseURL(baseURL);
|
|
||||||
desc.setMetadataNamespace(metadataNamespace);
|
|
||||||
desc.setAltered(altered);
|
|
||||||
desc.setDatestamp(datestamp);
|
|
||||||
desc.setHarvestDate(harvestDate);
|
|
||||||
|
|
||||||
final OAIProvenance p = new OAIProvenance();
|
|
||||||
p.setOriginDescription(desc);
|
|
||||||
|
|
||||||
return p;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Journal journal(
|
|
||||||
final String name,
|
|
||||||
final String issnPrinted,
|
|
||||||
final String issnOnline,
|
|
||||||
final String issnLinking,
|
|
||||||
final DataInfo dataInfo) {
|
|
||||||
return journal(
|
|
||||||
name,
|
|
||||||
issnPrinted,
|
|
||||||
issnOnline,
|
|
||||||
issnLinking,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
null,
|
|
||||||
dataInfo);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Journal journal(
|
|
||||||
final String name,
|
|
||||||
final String issnPrinted,
|
|
||||||
final String issnOnline,
|
|
||||||
final String issnLinking,
|
|
||||||
final String ep,
|
|
||||||
final String iss,
|
|
||||||
final String sp,
|
|
||||||
final String vol,
|
|
||||||
final String edition,
|
|
||||||
final String conferenceplace,
|
|
||||||
final String conferencedate,
|
|
||||||
final DataInfo dataInfo) {
|
|
||||||
|
|
||||||
if (StringUtils.isNotBlank(name)
|
|
||||||
|| StringUtils.isNotBlank(issnPrinted)
|
|
||||||
|| StringUtils.isNotBlank(issnOnline)
|
|
||||||
|| StringUtils.isNotBlank(issnLinking)) {
|
|
||||||
final Journal j = new Journal();
|
|
||||||
j.setName(name);
|
|
||||||
j.setIssnPrinted(issnPrinted);
|
|
||||||
j.setIssnOnline(issnOnline);
|
|
||||||
j.setIssnLinking(issnLinking);
|
|
||||||
j.setEp(ep);
|
|
||||||
j.setIss(iss);
|
|
||||||
j.setSp(sp);
|
|
||||||
j.setVol(vol);
|
|
||||||
j.setEdition(edition);
|
|
||||||
j.setConferenceplace(conferenceplace);
|
|
||||||
j.setConferencedate(conferencedate);
|
|
||||||
j.setDataInfo(dataInfo);
|
|
||||||
return j;
|
|
||||||
} else {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static DataInfo dataInfo(
|
|
||||||
final Boolean deletedbyinference,
|
|
||||||
final String inferenceprovenance,
|
|
||||||
final Boolean inferred,
|
|
||||||
final Boolean invisible,
|
|
||||||
final Qualifier provenanceaction,
|
|
||||||
final String trust) {
|
|
||||||
final DataInfo d = new DataInfo();
|
|
||||||
d.setDeletedbyinference(deletedbyinference);
|
|
||||||
d.setInferenceprovenance(inferenceprovenance);
|
|
||||||
d.setInferred(inferred);
|
|
||||||
d.setInvisible(invisible);
|
|
||||||
d.setProvenanceaction(provenanceaction);
|
|
||||||
d.setTrust(trust);
|
|
||||||
return d;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String createOpenaireId(
|
|
||||||
final int prefix,
|
|
||||||
final String originalId,
|
|
||||||
final boolean to_md5) {
|
|
||||||
if (StringUtils.isBlank(originalId)) {
|
|
||||||
return null;
|
|
||||||
} else if (to_md5) {
|
|
||||||
final String nsPrefix = StringUtils.substringBefore(originalId, "::");
|
|
||||||
final String rest = StringUtils.substringAfter(originalId, "::");
|
|
||||||
return String.format("%s|%s::%s", prefix, nsPrefix, DHPUtils.md5(rest));
|
|
||||||
} else {
|
|
||||||
return String.format("%s|%s", prefix, originalId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String createOpenaireId(
|
|
||||||
final String type,
|
|
||||||
final String originalId,
|
|
||||||
final boolean to_md5) {
|
|
||||||
switch (type) {
|
|
||||||
case "datasource":
|
|
||||||
return createOpenaireId(10, originalId, to_md5);
|
|
||||||
case "organization":
|
|
||||||
return createOpenaireId(20, originalId, to_md5);
|
|
||||||
case "person":
|
|
||||||
return createOpenaireId(30, originalId, to_md5);
|
|
||||||
case "project":
|
|
||||||
return createOpenaireId(40, originalId, to_md5);
|
|
||||||
default:
|
|
||||||
return createOpenaireId(50, originalId, to_md5);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String asString(final Object o) {
|
|
||||||
return o == null ? "" : o.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
public static <T> Predicate<T> distinctByKey(
|
|
||||||
final Function<? super T, ?> keyExtractor) {
|
|
||||||
final Map<Object, Boolean> seen = new ConcurrentHashMap<>();
|
|
||||||
return t -> seen.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
|
|
||||||
}
|
|
||||||
|
|
||||||
public static Qualifier createBestAccessRights(final List<Instance> instanceList) {
|
|
||||||
return getBestAccessRights(instanceList);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected static Qualifier getBestAccessRights(final List<Instance> instanceList) {
|
|
||||||
if (instanceList != null) {
|
|
||||||
final Optional<Qualifier> min = instanceList
|
|
||||||
.stream()
|
|
||||||
.map(i -> i.getAccessright())
|
|
||||||
.min(new LicenseComparator());
|
|
||||||
|
|
||||||
final Qualifier rights = min.isPresent() ? min.get() : new Qualifier();
|
|
||||||
|
|
||||||
if (StringUtils.isBlank(rights.getClassid())) {
|
|
||||||
rights.setClassid(UNKNOWN);
|
|
||||||
}
|
|
||||||
if (StringUtils.isBlank(rights.getClassname())
|
|
||||||
|| UNKNOWN.equalsIgnoreCase(rights.getClassname())) {
|
|
||||||
rights.setClassname(NOT_AVAILABLE);
|
|
||||||
}
|
|
||||||
if (StringUtils.isBlank(rights.getSchemeid())) {
|
|
||||||
rights.setSchemeid(DNET_ACCESS_MODES);
|
|
||||||
}
|
|
||||||
if (StringUtils.isBlank(rights.getSchemename())) {
|
|
||||||
rights.setSchemename(DNET_ACCESS_MODES);
|
|
||||||
}
|
|
||||||
|
|
||||||
return rights;
|
|
||||||
}
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,49 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.oaf;
|
|
||||||
|
|
||||||
import java.util.Comparator;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
|
||||||
|
|
||||||
public class ResultTypeComparator implements Comparator<Result> {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public int compare(Result left, Result right) {
|
|
||||||
|
|
||||||
if (left == null && right == null)
|
|
||||||
return 0;
|
|
||||||
if (left == null)
|
|
||||||
return 1;
|
|
||||||
if (right == null)
|
|
||||||
return -1;
|
|
||||||
|
|
||||||
String lClass = left.getResulttype().getClassid();
|
|
||||||
String rClass = right.getResulttype().getClassid();
|
|
||||||
|
|
||||||
if (lClass.equals(rClass))
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
if (lClass.equals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID))
|
|
||||||
return -1;
|
|
||||||
if (rClass.equals(ModelConstants.PUBLICATION_RESULTTYPE_CLASSID))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (lClass.equals(ModelConstants.DATASET_RESULTTYPE_CLASSID))
|
|
||||||
return -1;
|
|
||||||
if (rClass.equals(ModelConstants.DATASET_RESULTTYPE_CLASSID))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (lClass.equals(ModelConstants.SOFTWARE_RESULTTYPE_CLASSID))
|
|
||||||
return -1;
|
|
||||||
if (rClass.equals(ModelConstants.SOFTWARE_RESULTTYPE_CLASSID))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (lClass.equals(ModelConstants.ORP_RESULTTYPE_CLASSID))
|
|
||||||
return -1;
|
|
||||||
if (rClass.equals(ModelConstants.ORP_RESULTTYPE_CLASSID))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
// Else (but unlikely), lexicographical ordering will do.
|
|
||||||
return lClass.compareTo(rClass);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -0,0 +1,528 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.schema.oaf.utils;
|
||||||
|
|
||||||
|
import java.time.LocalDate;
|
||||||
|
import java.time.ZoneId;
|
||||||
|
import java.time.format.DateTimeFormatter;
|
||||||
|
import java.time.format.DateTimeParseException;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.function.Function;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
import java.util.stream.Stream;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
|
import com.github.sisyphsu.dateparser.DateParserUtils;
|
||||||
|
import com.google.common.collect.Lists;
|
||||||
|
import com.google.common.collect.Sets;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.common.vocabulary.Vocabulary;
|
||||||
|
import eu.dnetlib.dhp.common.vocabulary.VocabularyGroup;
|
||||||
|
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||||
|
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.*;
|
||||||
|
import me.xuender.unidecode.Unidecode;
|
||||||
|
|
||||||
|
public class GraphCleaningFunctions extends CleaningFunctions {
|
||||||
|
|
||||||
|
public static final String ORCID_CLEANING_REGEX = ".*([0-9]{4}).*[-–—−=].*([0-9]{4}).*[-–—−=].*([0-9]{4}).*[-–—−=].*([0-9x]{4})";
|
||||||
|
public static final int ORCID_LEN = 19;
|
||||||
|
public static final String CLEANING_REGEX = "(?:\\n|\\r|\\t)";
|
||||||
|
public static final String INVALID_AUTHOR_REGEX = ".*deactivated.*";
|
||||||
|
|
||||||
|
public static final String TITLE_TEST = "test";
|
||||||
|
public static final String TITLE_FILTER_REGEX = String.format("(%s)|\\W|\\d", TITLE_TEST);
|
||||||
|
|
||||||
|
public static final int TITLE_FILTER_RESIDUAL_LENGTH = 5;
|
||||||
|
|
||||||
|
public static <T extends Oaf> T fixVocabularyNames(T value) {
|
||||||
|
if (value instanceof Datasource) {
|
||||||
|
// nothing to clean here
|
||||||
|
} else if (value instanceof Project) {
|
||||||
|
// nothing to clean here
|
||||||
|
} else if (value instanceof Organization) {
|
||||||
|
Organization o = (Organization) value;
|
||||||
|
if (Objects.nonNull(o.getCountry())) {
|
||||||
|
fixVocabName(o.getCountry(), ModelConstants.DNET_COUNTRY_TYPE);
|
||||||
|
}
|
||||||
|
} else if (value instanceof Relation) {
|
||||||
|
// nothing to clean here
|
||||||
|
} else if (value instanceof Result) {
|
||||||
|
|
||||||
|
Result r = (Result) value;
|
||||||
|
|
||||||
|
fixVocabName(r.getLanguage(), ModelConstants.DNET_LANGUAGES);
|
||||||
|
fixVocabName(r.getResourcetype(), ModelConstants.DNET_DATA_CITE_RESOURCE);
|
||||||
|
fixVocabName(r.getBestaccessright(), ModelConstants.DNET_ACCESS_MODES);
|
||||||
|
|
||||||
|
if (Objects.nonNull(r.getSubject())) {
|
||||||
|
r.getSubject().forEach(s -> fixVocabName(s.getQualifier(), ModelConstants.DNET_SUBJECT_TYPOLOGIES));
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(r.getInstance())) {
|
||||||
|
for (Instance i : r.getInstance()) {
|
||||||
|
fixVocabName(i.getAccessright(), ModelConstants.DNET_ACCESS_MODES);
|
||||||
|
fixVocabName(i.getRefereed(), ModelConstants.DNET_REVIEW_LEVELS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(r.getAuthor())) {
|
||||||
|
r.getAuthor().stream().filter(Objects::nonNull).forEach(a -> {
|
||||||
|
if (Objects.nonNull(a.getPid())) {
|
||||||
|
a.getPid().stream().filter(Objects::nonNull).forEach(p -> {
|
||||||
|
fixVocabName(p.getQualifier(), ModelConstants.DNET_PID_TYPES);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (value instanceof Publication) {
|
||||||
|
|
||||||
|
} else if (value instanceof Dataset) {
|
||||||
|
|
||||||
|
} else if (value instanceof OtherResearchProduct) {
|
||||||
|
|
||||||
|
} else if (value instanceof Software) {
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T extends Oaf> boolean filter(T value) {
|
||||||
|
if (Boolean.TRUE
|
||||||
|
.equals(
|
||||||
|
Optional
|
||||||
|
.ofNullable(value)
|
||||||
|
.map(
|
||||||
|
o -> Optional
|
||||||
|
.ofNullable(o.getDataInfo())
|
||||||
|
.map(
|
||||||
|
d -> Optional
|
||||||
|
.ofNullable(d.getInvisible())
|
||||||
|
.orElse(true))
|
||||||
|
.orElse(true))
|
||||||
|
.orElse(true))) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (value instanceof Datasource) {
|
||||||
|
// nothing to evaluate here
|
||||||
|
} else if (value instanceof Project) {
|
||||||
|
// nothing to evaluate here
|
||||||
|
} else if (value instanceof Organization) {
|
||||||
|
// nothing to evaluate here
|
||||||
|
} else if (value instanceof Relation) {
|
||||||
|
// nothing to clean here
|
||||||
|
} else if (value instanceof Result) {
|
||||||
|
|
||||||
|
Result r = (Result) value;
|
||||||
|
|
||||||
|
if (Objects.isNull(r.getTitle()) || r.getTitle().isEmpty()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (value instanceof Publication) {
|
||||||
|
|
||||||
|
} else if (value instanceof Dataset) {
|
||||||
|
|
||||||
|
} else if (value instanceof OtherResearchProduct) {
|
||||||
|
|
||||||
|
} else if (value instanceof Software) {
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T extends Oaf> T cleanup(T value, VocabularyGroup vocs) {
|
||||||
|
if (value instanceof Datasource) {
|
||||||
|
// nothing to clean here
|
||||||
|
} else if (value instanceof Project) {
|
||||||
|
// nothing to clean here
|
||||||
|
} else if (value instanceof Organization) {
|
||||||
|
Organization o = (Organization) value;
|
||||||
|
if (Objects.isNull(o.getCountry()) || StringUtils.isBlank(o.getCountry().getClassid())) {
|
||||||
|
o.setCountry(ModelConstants.UNKNOWN_COUNTRY);
|
||||||
|
}
|
||||||
|
} else if (value instanceof Relation) {
|
||||||
|
Relation r = (Relation) value;
|
||||||
|
|
||||||
|
Optional<String> validationDate = doCleanDate(r.getValidationDate());
|
||||||
|
if (validationDate.isPresent()) {
|
||||||
|
r.setValidationDate(validationDate.get());
|
||||||
|
r.setValidated(true);
|
||||||
|
} else {
|
||||||
|
r.setValidationDate(null);
|
||||||
|
r.setValidated(false);
|
||||||
|
}
|
||||||
|
} else if (value instanceof Result) {
|
||||||
|
|
||||||
|
Result r = (Result) value;
|
||||||
|
|
||||||
|
if (Objects.nonNull(r.getDateofacceptance())) {
|
||||||
|
Optional<String> date = cleanDateField(r.getDateofacceptance());
|
||||||
|
if (date.isPresent()) {
|
||||||
|
r.getDateofacceptance().setValue(date.get());
|
||||||
|
} else {
|
||||||
|
r.setDateofacceptance(null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(r.getRelevantdate())) {
|
||||||
|
r
|
||||||
|
.setRelevantdate(
|
||||||
|
r
|
||||||
|
.getRelevantdate()
|
||||||
|
.stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.filter(sp -> Objects.nonNull(sp.getQualifier()))
|
||||||
|
.filter(sp -> StringUtils.isNotBlank(sp.getQualifier().getClassid()))
|
||||||
|
.map(sp -> {
|
||||||
|
sp.setValue(GraphCleaningFunctions.cleanDate(sp.getValue()));
|
||||||
|
return sp;
|
||||||
|
})
|
||||||
|
.filter(sp -> StringUtils.isNotBlank(sp.getValue()))
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(r.getPublisher()) && StringUtils.isBlank(r.getPublisher().getValue())) {
|
||||||
|
r.setPublisher(null);
|
||||||
|
}
|
||||||
|
if (Objects.isNull(r.getLanguage()) || StringUtils.isBlank(r.getLanguage().getClassid())) {
|
||||||
|
r
|
||||||
|
.setLanguage(
|
||||||
|
qualifier("und", "Undetermined", ModelConstants.DNET_LANGUAGES));
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(r.getSubject())) {
|
||||||
|
r
|
||||||
|
.setSubject(
|
||||||
|
r
|
||||||
|
.getSubject()
|
||||||
|
.stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.filter(sp -> StringUtils.isNotBlank(sp.getValue()))
|
||||||
|
.filter(sp -> Objects.nonNull(sp.getQualifier()))
|
||||||
|
.filter(sp -> StringUtils.isNotBlank(sp.getQualifier().getClassid()))
|
||||||
|
.map(GraphCleaningFunctions::cleanValue)
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(r.getTitle())) {
|
||||||
|
r
|
||||||
|
.setTitle(
|
||||||
|
r
|
||||||
|
.getTitle()
|
||||||
|
.stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.filter(sp -> StringUtils.isNotBlank(sp.getValue()))
|
||||||
|
.filter(
|
||||||
|
sp -> {
|
||||||
|
final String title = sp
|
||||||
|
.getValue()
|
||||||
|
.toLowerCase();
|
||||||
|
final String decoded = Unidecode.decode(title);
|
||||||
|
|
||||||
|
if (StringUtils.contains(decoded, TITLE_TEST)) {
|
||||||
|
return decoded
|
||||||
|
.replaceAll(TITLE_FILTER_REGEX, "")
|
||||||
|
.length() > TITLE_FILTER_RESIDUAL_LENGTH;
|
||||||
|
}
|
||||||
|
return !decoded
|
||||||
|
.replaceAll("\\W|\\d", "")
|
||||||
|
.isEmpty();
|
||||||
|
})
|
||||||
|
.map(GraphCleaningFunctions::cleanValue)
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(r.getFormat())) {
|
||||||
|
r
|
||||||
|
.setFormat(
|
||||||
|
r
|
||||||
|
.getFormat()
|
||||||
|
.stream()
|
||||||
|
.map(GraphCleaningFunctions::cleanValue)
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(r.getDescription())) {
|
||||||
|
r
|
||||||
|
.setDescription(
|
||||||
|
r
|
||||||
|
.getDescription()
|
||||||
|
.stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.filter(sp -> StringUtils.isNotBlank(sp.getValue()))
|
||||||
|
.map(GraphCleaningFunctions::cleanValue)
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(r.getPid())) {
|
||||||
|
r.setPid(processPidCleaning(r.getPid()));
|
||||||
|
}
|
||||||
|
if (Objects.isNull(r.getResourcetype()) || StringUtils.isBlank(r.getResourcetype().getClassid())) {
|
||||||
|
r
|
||||||
|
.setResourcetype(
|
||||||
|
qualifier(ModelConstants.UNKNOWN, "Unknown", ModelConstants.DNET_DATA_CITE_RESOURCE));
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(r.getInstance())) {
|
||||||
|
|
||||||
|
for (Instance i : r.getInstance()) {
|
||||||
|
if (!vocs.termExists(ModelConstants.DNET_PUBLICATION_RESOURCE, i.getInstancetype().getClassid())) {
|
||||||
|
if (r instanceof Publication) {
|
||||||
|
i
|
||||||
|
.setInstancetype(
|
||||||
|
OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
"0038", "Other literature type", ModelConstants.DNET_PUBLICATION_RESOURCE,
|
||||||
|
ModelConstants.DNET_PUBLICATION_RESOURCE));
|
||||||
|
} else if (r instanceof Dataset) {
|
||||||
|
i
|
||||||
|
.setInstancetype(
|
||||||
|
OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
"0039", "Other dataset type", ModelConstants.DNET_PUBLICATION_RESOURCE,
|
||||||
|
ModelConstants.DNET_PUBLICATION_RESOURCE));
|
||||||
|
} else if (r instanceof Software) {
|
||||||
|
i
|
||||||
|
.setInstancetype(
|
||||||
|
OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
"0040", "Other software type", ModelConstants.DNET_PUBLICATION_RESOURCE,
|
||||||
|
ModelConstants.DNET_PUBLICATION_RESOURCE));
|
||||||
|
} else if (r instanceof OtherResearchProduct) {
|
||||||
|
i
|
||||||
|
.setInstancetype(
|
||||||
|
OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
"0020", "Other ORP type", ModelConstants.DNET_PUBLICATION_RESOURCE,
|
||||||
|
ModelConstants.DNET_PUBLICATION_RESOURCE));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (Objects.nonNull(i.getPid())) {
|
||||||
|
i.setPid(processPidCleaning(i.getPid()));
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(i.getAlternateIdentifier())) {
|
||||||
|
i.setAlternateIdentifier(processPidCleaning(i.getAlternateIdentifier()));
|
||||||
|
}
|
||||||
|
Optional
|
||||||
|
.ofNullable(i.getPid())
|
||||||
|
.ifPresent(pid -> {
|
||||||
|
final Set<StructuredProperty> pids = Sets.newHashSet(pid);
|
||||||
|
Optional
|
||||||
|
.ofNullable(i.getAlternateIdentifier())
|
||||||
|
.ifPresent(altId -> {
|
||||||
|
final Set<StructuredProperty> altIds = Sets.newHashSet(altId);
|
||||||
|
i.setAlternateIdentifier(Lists.newArrayList(Sets.difference(altIds, pids)));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
if (Objects.isNull(i.getAccessright()) || StringUtils.isBlank(i.getAccessright().getClassid())) {
|
||||||
|
i
|
||||||
|
.setAccessright(
|
||||||
|
accessRight(
|
||||||
|
ModelConstants.UNKNOWN, ModelConstants.NOT_AVAILABLE,
|
||||||
|
ModelConstants.DNET_ACCESS_MODES));
|
||||||
|
}
|
||||||
|
if (Objects.isNull(i.getHostedby()) || StringUtils.isBlank(i.getHostedby().getKey())) {
|
||||||
|
i.setHostedby(ModelConstants.UNKNOWN_REPOSITORY);
|
||||||
|
}
|
||||||
|
if (Objects.isNull(i.getRefereed())) {
|
||||||
|
i.setRefereed(qualifier("0000", "Unknown", ModelConstants.DNET_REVIEW_LEVELS));
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(i.getDateofacceptance())) {
|
||||||
|
Optional<String> date = cleanDateField(i.getDateofacceptance());
|
||||||
|
if (date.isPresent()) {
|
||||||
|
i.getDateofacceptance().setValue(date.get());
|
||||||
|
} else {
|
||||||
|
i.setDateofacceptance(null);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Objects.isNull(r.getBestaccessright()) || StringUtils.isBlank(r.getBestaccessright().getClassid())) {
|
||||||
|
Qualifier bestaccessrights = OafMapperUtils.createBestAccessRights(r.getInstance());
|
||||||
|
if (Objects.isNull(bestaccessrights)) {
|
||||||
|
r
|
||||||
|
.setBestaccessright(
|
||||||
|
qualifier(
|
||||||
|
ModelConstants.UNKNOWN, ModelConstants.NOT_AVAILABLE,
|
||||||
|
ModelConstants.DNET_ACCESS_MODES));
|
||||||
|
} else {
|
||||||
|
r.setBestaccessright(bestaccessrights);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (Objects.nonNull(r.getAuthor())) {
|
||||||
|
r
|
||||||
|
.setAuthor(
|
||||||
|
r
|
||||||
|
.getAuthor()
|
||||||
|
.stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.filter(a -> StringUtils.isNotBlank(a.getFullname()))
|
||||||
|
.filter(a -> StringUtils.isNotBlank(a.getFullname().replaceAll("[\\W]", "")))
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
|
||||||
|
boolean nullRank = r
|
||||||
|
.getAuthor()
|
||||||
|
.stream()
|
||||||
|
.anyMatch(a -> Objects.isNull(a.getRank()));
|
||||||
|
if (nullRank) {
|
||||||
|
int i = 1;
|
||||||
|
for (Author author : r.getAuthor()) {
|
||||||
|
author.setRank(i++);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (Author a : r.getAuthor()) {
|
||||||
|
if (Objects.isNull(a.getPid())) {
|
||||||
|
a.setPid(Lists.newArrayList());
|
||||||
|
} else {
|
||||||
|
a
|
||||||
|
.setPid(
|
||||||
|
a
|
||||||
|
.getPid()
|
||||||
|
.stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.filter(p -> Objects.nonNull(p.getQualifier()))
|
||||||
|
.filter(p -> StringUtils.isNotBlank(p.getValue()))
|
||||||
|
.map(p -> {
|
||||||
|
// hack to distinguish orcid from orcid_pending
|
||||||
|
String pidProvenance = Optional
|
||||||
|
.ofNullable(p.getDataInfo())
|
||||||
|
.map(
|
||||||
|
d -> Optional
|
||||||
|
.ofNullable(d.getProvenanceaction())
|
||||||
|
.map(Qualifier::getClassid)
|
||||||
|
.orElse(""))
|
||||||
|
.orElse("");
|
||||||
|
if (p
|
||||||
|
.getQualifier()
|
||||||
|
.getClassid()
|
||||||
|
.toLowerCase()
|
||||||
|
.contains(ModelConstants.ORCID)) {
|
||||||
|
if (pidProvenance
|
||||||
|
.equals(ModelConstants.SYSIMPORT_CROSSWALK_ENTITYREGISTRY)) {
|
||||||
|
p.getQualifier().setClassid(ModelConstants.ORCID);
|
||||||
|
} else {
|
||||||
|
p.getQualifier().setClassid(ModelConstants.ORCID_PENDING);
|
||||||
|
}
|
||||||
|
final String orcid = p
|
||||||
|
.getValue()
|
||||||
|
.trim()
|
||||||
|
.toLowerCase()
|
||||||
|
.replaceAll(ORCID_CLEANING_REGEX, "$1-$2-$3-$4");
|
||||||
|
if (orcid.length() == ORCID_LEN) {
|
||||||
|
p.setValue(orcid);
|
||||||
|
} else {
|
||||||
|
p.setValue("");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return p;
|
||||||
|
})
|
||||||
|
.filter(p -> StringUtils.isNotBlank(p.getValue()))
|
||||||
|
.collect(
|
||||||
|
Collectors
|
||||||
|
.toMap(
|
||||||
|
p -> p.getQualifier().getClassid() + p.getValue(),
|
||||||
|
Function.identity(),
|
||||||
|
(p1, p2) -> p1,
|
||||||
|
LinkedHashMap::new))
|
||||||
|
.values()
|
||||||
|
.stream()
|
||||||
|
.collect(Collectors.toList()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (value instanceof Publication) {
|
||||||
|
|
||||||
|
} else if (value instanceof Dataset) {
|
||||||
|
|
||||||
|
} else if (value instanceof OtherResearchProduct) {
|
||||||
|
|
||||||
|
} else if (value instanceof Software) {
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return value;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Optional<String> cleanDateField(Field<String> dateofacceptance) {
|
||||||
|
return Optional
|
||||||
|
.ofNullable(dateofacceptance)
|
||||||
|
.map(Field::getValue)
|
||||||
|
.map(GraphCleaningFunctions::cleanDate)
|
||||||
|
.filter(Objects::nonNull);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static Optional<String> doCleanDate(String date) {
|
||||||
|
return Optional.ofNullable(cleanDate(date));
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String cleanDate(final String inputDate) {
|
||||||
|
|
||||||
|
if (StringUtils.isBlank(inputDate)) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
final LocalDate date = DateParserUtils
|
||||||
|
.parseDate(inputDate.trim())
|
||||||
|
.toInstant()
|
||||||
|
.atZone(ZoneId.systemDefault())
|
||||||
|
.toLocalDate();
|
||||||
|
return DateTimeFormatter.ofPattern(ModelSupport.DATE_FORMAT).format(date);
|
||||||
|
} catch (DateTimeParseException e) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// HELPERS
|
||||||
|
|
||||||
|
private static boolean isValidAuthorName(Author a) {
|
||||||
|
return !Stream
|
||||||
|
.of(a.getFullname(), a.getName(), a.getSurname())
|
||||||
|
.filter(s -> s != null && !s.isEmpty())
|
||||||
|
.collect(Collectors.joining(""))
|
||||||
|
.toLowerCase()
|
||||||
|
.matches(INVALID_AUTHOR_REGEX);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static List<StructuredProperty> processPidCleaning(List<StructuredProperty> pids) {
|
||||||
|
return pids
|
||||||
|
.stream()
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.filter(sp -> StringUtils.isNotBlank(StringUtils.trim(sp.getValue())))
|
||||||
|
.filter(sp -> !PID_BLACKLIST.contains(sp.getValue().trim().toLowerCase()))
|
||||||
|
.filter(sp -> Objects.nonNull(sp.getQualifier()))
|
||||||
|
.filter(sp -> StringUtils.isNotBlank(sp.getQualifier().getClassid()))
|
||||||
|
.map(CleaningFunctions::normalizePidValue)
|
||||||
|
.filter(CleaningFunctions::pidFilter)
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
private static void fixVocabName(Qualifier q, String vocabularyName) {
|
||||||
|
if (Objects.nonNull(q) && StringUtils.isBlank(q.getSchemeid())) {
|
||||||
|
q.setSchemeid(vocabularyName);
|
||||||
|
q.setSchemename(vocabularyName);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static AccessRight accessRight(String classid, String classname, String scheme) {
|
||||||
|
return OafMapperUtils
|
||||||
|
.accessRight(
|
||||||
|
classid, classname, scheme, scheme);
|
||||||
|
}
|
||||||
|
|
||||||
|
private static Qualifier qualifier(String classid, String classname, String scheme) {
|
||||||
|
return OafMapperUtils
|
||||||
|
.qualifier(
|
||||||
|
classid, classname, scheme, scheme);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static StructuredProperty cleanValue(StructuredProperty s) {
|
||||||
|
s.setValue(s.getValue().replaceAll(CLEANING_REGEX, " "));
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static Field<String> cleanValue(Field<String> s) {
|
||||||
|
s.setValue(s.getValue().replaceAll(CLEANING_REGEX, " "));
|
||||||
|
return s;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -1,102 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.oaf.utils;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
import java.util.Objects;
|
|
||||||
import java.util.Optional;
|
|
||||||
import java.util.regex.Pattern;
|
|
||||||
|
|
||||||
import org.apache.commons.lang.StringUtils;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.CleaningFunctions;
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.OafEntity;
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
|
|
||||||
import eu.dnetlib.dhp.utils.DHPUtils;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Factory class for OpenAIRE identifiers in the Graph
|
|
||||||
*/
|
|
||||||
public class IdentifierFactory implements Serializable {
|
|
||||||
|
|
||||||
public static final String ID_SEPARATOR = "::";
|
|
||||||
public static final String ID_PREFIX_SEPARATOR = "|";
|
|
||||||
public final static String ID_REGEX = "^[0-9][0-9]\\" + ID_PREFIX_SEPARATOR + ".{12}" + ID_SEPARATOR
|
|
||||||
+ "[a-zA-Z0-9]{32}$";
|
|
||||||
|
|
||||||
public final static String DOI_REGEX = "(^10\\.[0-9]{4,9}\\/[-._;()\\/:a-zA-Z0-9]+$)|" +
|
|
||||||
"(^10\\.1002\\/[^\\s]+$)|" +
|
|
||||||
"(^10\\.1021\\/[a-zA-Z0-9_][a-zA-Z0-9_][0-9]++$)|" +
|
|
||||||
"(^10\\.1207\\/[a-zA-Z0-9_]+\\&[0-9]+_[0-9]+$)";
|
|
||||||
|
|
||||||
public static final int ID_PREFIX_LEN = 12;
|
|
||||||
public static final String NONE = "none";
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Creates an identifier from the most relevant PID (if available) in the given entity T. Returns entity.id
|
|
||||||
* when no PID is available
|
|
||||||
* @param entity the entity providing PIDs and a default ID.
|
|
||||||
* @param <T> the specific entity type. Currently Organization and Result subclasses are supported.
|
|
||||||
* @return an identifier from the most relevant PID, entity.id otherwise
|
|
||||||
*/
|
|
||||||
public static <T extends OafEntity> String createIdentifier(T entity) {
|
|
||||||
|
|
||||||
if (Objects.isNull(entity.getPid()) || entity.getPid().isEmpty()) {
|
|
||||||
return entity.getId();
|
|
||||||
}
|
|
||||||
|
|
||||||
return entity
|
|
||||||
.getPid()
|
|
||||||
.stream()
|
|
||||||
.filter(s -> pidFilter(s))
|
|
||||||
.min(new PidComparator<>(entity))
|
|
||||||
.map(s -> idFromPid(entity, s))
|
|
||||||
.map(IdentifierFactory::verifyIdSyntax)
|
|
||||||
.orElseGet(entity::getId);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected static boolean pidFilter(StructuredProperty s) {
|
|
||||||
if (Objects.isNull(s.getQualifier()) ||
|
|
||||||
StringUtils.isBlank(StringUtils.trim(s.getValue()))) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
try {
|
|
||||||
switch (PidType.valueOf(s.getQualifier().getClassid())) {
|
|
||||||
case doi:
|
|
||||||
final String doi = StringUtils.trim(StringUtils.lowerCase(s.getValue()));
|
|
||||||
return doi.matches(DOI_REGEX);
|
|
||||||
default:
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
} catch (IllegalArgumentException e) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static String verifyIdSyntax(String s) {
|
|
||||||
if (StringUtils.isBlank(s) || !s.matches(ID_REGEX)) {
|
|
||||||
throw new RuntimeException(String.format("malformed id: '%s'", s));
|
|
||||||
} else {
|
|
||||||
return s;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static <T extends OafEntity> String idFromPid(T entity, StructuredProperty s) {
|
|
||||||
return new StringBuilder()
|
|
||||||
.append(StringUtils.substringBefore(entity.getId(), ID_PREFIX_SEPARATOR))
|
|
||||||
.append(ID_PREFIX_SEPARATOR)
|
|
||||||
.append(createPrefix(s.getQualifier().getClassid()))
|
|
||||||
.append(ID_SEPARATOR)
|
|
||||||
.append(DHPUtils.md5(CleaningFunctions.normalizePidValue(s).getValue()))
|
|
||||||
.toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
// create the prefix (length = 12)
|
|
||||||
private static String createPrefix(String pidType) {
|
|
||||||
StringBuilder prefix = new StringBuilder(StringUtils.left(pidType, ID_PREFIX_LEN));
|
|
||||||
while (prefix.length() < ID_PREFIX_LEN) {
|
|
||||||
prefix.append("_");
|
|
||||||
}
|
|
||||||
return prefix.substring(0, ID_PREFIX_LEN);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -0,0 +1,371 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.schema.oaf.utils;
|
||||||
|
|
||||||
|
import static eu.dnetlib.dhp.schema.common.ModelConstants.*;
|
||||||
|
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.function.Function;
|
||||||
|
import java.util.function.Predicate;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.schema.common.AccessRightComparator;
|
||||||
|
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.*;
|
||||||
|
|
||||||
|
public class OafMapperUtils {
|
||||||
|
|
||||||
|
private OafMapperUtils() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Oaf merge(final Oaf left, final Oaf right) {
|
||||||
|
if (ModelSupport.isSubClass(left, OafEntity.class)) {
|
||||||
|
return mergeEntities((OafEntity) left, (OafEntity) right);
|
||||||
|
} else if (ModelSupport.isSubClass(left, Relation.class)) {
|
||||||
|
((Relation) left).mergeFrom((Relation) right);
|
||||||
|
} else {
|
||||||
|
throw new IllegalArgumentException("invalid Oaf type:" + left.getClass().getCanonicalName());
|
||||||
|
}
|
||||||
|
return left;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static OafEntity mergeEntities(OafEntity left, OafEntity right) {
|
||||||
|
if (ModelSupport.isSubClass(left, Result.class)) {
|
||||||
|
return mergeResults((Result) left, (Result) right);
|
||||||
|
} else if (ModelSupport.isSubClass(left, Datasource.class)) {
|
||||||
|
left.mergeFrom(right);
|
||||||
|
} else if (ModelSupport.isSubClass(left, Organization.class)) {
|
||||||
|
left.mergeFrom(right);
|
||||||
|
} else if (ModelSupport.isSubClass(left, Project.class)) {
|
||||||
|
left.mergeFrom(right);
|
||||||
|
} else {
|
||||||
|
throw new IllegalArgumentException("invalid OafEntity subtype:" + left.getClass().getCanonicalName());
|
||||||
|
}
|
||||||
|
return left;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Result mergeResults(Result left, Result right) {
|
||||||
|
if (new ResultTypeComparator().compare(left, right) < 0) {
|
||||||
|
left.mergeFrom(right);
|
||||||
|
return left;
|
||||||
|
} else {
|
||||||
|
right.mergeFrom(left);
|
||||||
|
return right;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static KeyValue keyValue(final String k, final String v) {
|
||||||
|
final KeyValue kv = new KeyValue();
|
||||||
|
kv.setKey(k);
|
||||||
|
kv.setValue(v);
|
||||||
|
return kv;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<KeyValue> listKeyValues(final String... s) {
|
||||||
|
if (s.length % 2 > 0) {
|
||||||
|
throw new IllegalArgumentException("Invalid number of parameters (k,v,k,v,....)");
|
||||||
|
}
|
||||||
|
|
||||||
|
final List<KeyValue> list = new ArrayList<>();
|
||||||
|
for (int i = 0; i < s.length; i += 2) {
|
||||||
|
list.add(keyValue(s[i], s[i + 1]));
|
||||||
|
}
|
||||||
|
return list;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> Field<T> field(final T value, final DataInfo info) {
|
||||||
|
if (value == null || StringUtils.isBlank(value.toString())) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
final Field<T> field = new Field<>();
|
||||||
|
field.setValue(value);
|
||||||
|
field.setDataInfo(info);
|
||||||
|
return field;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<Field<String>> listFields(final DataInfo info, final String... values) {
|
||||||
|
return Arrays
|
||||||
|
.stream(values)
|
||||||
|
.map(v -> field(v, info))
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.filter(distinctByKey(Field::getValue))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static List<Field<String>> listFields(final DataInfo info, final List<String> values) {
|
||||||
|
return values
|
||||||
|
.stream()
|
||||||
|
.map(v -> field(v, info))
|
||||||
|
.filter(Objects::nonNull)
|
||||||
|
.filter(distinctByKey(Field::getValue))
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Qualifier unknown(final String schemeid, final String schemename) {
|
||||||
|
return qualifier("UNKNOWN", "Unknown", schemeid, schemename);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static AccessRight accessRight(
|
||||||
|
final String classid,
|
||||||
|
final String classname,
|
||||||
|
final String schemeid,
|
||||||
|
final String schemename) {
|
||||||
|
return accessRight(classid, classname, schemeid, schemename, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static AccessRight accessRight(
|
||||||
|
final String classid,
|
||||||
|
final String classname,
|
||||||
|
final String schemeid,
|
||||||
|
final String schemename,
|
||||||
|
final OpenAccessRoute openAccessRoute) {
|
||||||
|
final AccessRight accessRight = new AccessRight();
|
||||||
|
accessRight.setClassid(classid);
|
||||||
|
accessRight.setClassname(classname);
|
||||||
|
accessRight.setSchemeid(schemeid);
|
||||||
|
accessRight.setSchemename(schemename);
|
||||||
|
accessRight.setOpenAccessRoute(openAccessRoute);
|
||||||
|
return accessRight;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Qualifier qualifier(
|
||||||
|
final String classid,
|
||||||
|
final String classname,
|
||||||
|
final String schemeid,
|
||||||
|
final String schemename) {
|
||||||
|
final Qualifier q = new Qualifier();
|
||||||
|
q.setClassid(classid);
|
||||||
|
q.setClassname(classname);
|
||||||
|
q.setSchemeid(schemeid);
|
||||||
|
q.setSchemename(schemename);
|
||||||
|
return q;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Qualifier qualifier(final Qualifier qualifier) {
|
||||||
|
final Qualifier q = new Qualifier();
|
||||||
|
q.setClassid(qualifier.getClassid());
|
||||||
|
q.setClassname(qualifier.getClassname());
|
||||||
|
q.setSchemeid(qualifier.getSchemeid());
|
||||||
|
q.setSchemename(qualifier.getSchemename());
|
||||||
|
return q;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static StructuredProperty structuredProperty(
|
||||||
|
final String value,
|
||||||
|
final String classid,
|
||||||
|
final String classname,
|
||||||
|
final String schemeid,
|
||||||
|
final String schemename,
|
||||||
|
final DataInfo dataInfo) {
|
||||||
|
|
||||||
|
return structuredProperty(value, qualifier(classid, classname, schemeid, schemename), dataInfo);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static StructuredProperty structuredProperty(
|
||||||
|
final String value,
|
||||||
|
final Qualifier qualifier,
|
||||||
|
final DataInfo dataInfo) {
|
||||||
|
if (value == null) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
final StructuredProperty sp = new StructuredProperty();
|
||||||
|
sp.setValue(value);
|
||||||
|
sp.setQualifier(qualifier);
|
||||||
|
sp.setDataInfo(dataInfo);
|
||||||
|
return sp;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static ExtraInfo extraInfo(
|
||||||
|
final String name,
|
||||||
|
final String value,
|
||||||
|
final String typology,
|
||||||
|
final String provenance,
|
||||||
|
final String trust) {
|
||||||
|
final ExtraInfo info = new ExtraInfo();
|
||||||
|
info.setName(name);
|
||||||
|
info.setValue(value);
|
||||||
|
info.setTypology(typology);
|
||||||
|
info.setProvenance(provenance);
|
||||||
|
info.setTrust(trust);
|
||||||
|
return info;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static OAIProvenance oaiIProvenance(
|
||||||
|
final String identifier,
|
||||||
|
final String baseURL,
|
||||||
|
final String metadataNamespace,
|
||||||
|
final Boolean altered,
|
||||||
|
final String datestamp,
|
||||||
|
final String harvestDate) {
|
||||||
|
|
||||||
|
final OriginDescription desc = new OriginDescription();
|
||||||
|
desc.setIdentifier(identifier);
|
||||||
|
desc.setBaseURL(baseURL);
|
||||||
|
desc.setMetadataNamespace(metadataNamespace);
|
||||||
|
desc.setAltered(altered);
|
||||||
|
desc.setDatestamp(datestamp);
|
||||||
|
desc.setHarvestDate(harvestDate);
|
||||||
|
|
||||||
|
final OAIProvenance p = new OAIProvenance();
|
||||||
|
p.setOriginDescription(desc);
|
||||||
|
|
||||||
|
return p;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Journal journal(
|
||||||
|
final String name,
|
||||||
|
final String issnPrinted,
|
||||||
|
final String issnOnline,
|
||||||
|
final String issnLinking,
|
||||||
|
final DataInfo dataInfo) {
|
||||||
|
|
||||||
|
return hasIssn(issnPrinted, issnOnline, issnLinking) ? journal(
|
||||||
|
name,
|
||||||
|
issnPrinted,
|
||||||
|
issnOnline,
|
||||||
|
issnLinking,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
null,
|
||||||
|
dataInfo) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Journal journal(
|
||||||
|
final String name,
|
||||||
|
final String issnPrinted,
|
||||||
|
final String issnOnline,
|
||||||
|
final String issnLinking,
|
||||||
|
final String ep,
|
||||||
|
final String iss,
|
||||||
|
final String sp,
|
||||||
|
final String vol,
|
||||||
|
final String edition,
|
||||||
|
final String conferenceplace,
|
||||||
|
final String conferencedate,
|
||||||
|
final DataInfo dataInfo) {
|
||||||
|
|
||||||
|
if (StringUtils.isNotBlank(name) || hasIssn(issnPrinted, issnOnline, issnLinking)) {
|
||||||
|
final Journal j = new Journal();
|
||||||
|
j.setName(name);
|
||||||
|
j.setIssnPrinted(issnPrinted);
|
||||||
|
j.setIssnOnline(issnOnline);
|
||||||
|
j.setIssnLinking(issnLinking);
|
||||||
|
j.setEp(ep);
|
||||||
|
j.setIss(iss);
|
||||||
|
j.setSp(sp);
|
||||||
|
j.setVol(vol);
|
||||||
|
j.setEdition(edition);
|
||||||
|
j.setConferenceplace(conferenceplace);
|
||||||
|
j.setConferencedate(conferencedate);
|
||||||
|
j.setDataInfo(dataInfo);
|
||||||
|
return j;
|
||||||
|
} else {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static boolean hasIssn(String issnPrinted, String issnOnline, String issnLinking) {
|
||||||
|
return StringUtils.isNotBlank(issnPrinted)
|
||||||
|
|| StringUtils.isNotBlank(issnOnline)
|
||||||
|
|| StringUtils.isNotBlank(issnLinking);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static DataInfo dataInfo(
|
||||||
|
final Boolean deletedbyinference,
|
||||||
|
final String inferenceprovenance,
|
||||||
|
final Boolean inferred,
|
||||||
|
final Boolean invisible,
|
||||||
|
final Qualifier provenanceaction,
|
||||||
|
final String trust) {
|
||||||
|
final DataInfo d = new DataInfo();
|
||||||
|
d.setDeletedbyinference(deletedbyinference);
|
||||||
|
d.setInferenceprovenance(inferenceprovenance);
|
||||||
|
d.setInferred(inferred);
|
||||||
|
d.setInvisible(invisible);
|
||||||
|
d.setProvenanceaction(provenanceaction);
|
||||||
|
d.setTrust(trust);
|
||||||
|
return d;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String createOpenaireId(
|
||||||
|
final int prefix,
|
||||||
|
final String originalId,
|
||||||
|
final boolean to_md5) {
|
||||||
|
if (StringUtils.isBlank(originalId)) {
|
||||||
|
return null;
|
||||||
|
} else if (to_md5) {
|
||||||
|
final String nsPrefix = StringUtils.substringBefore(originalId, "::");
|
||||||
|
final String rest = StringUtils.substringAfter(originalId, "::");
|
||||||
|
return String.format("%s|%s::%s", prefix, nsPrefix, IdentifierFactory.md5(rest));
|
||||||
|
} else {
|
||||||
|
return String.format("%s|%s", prefix, originalId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String createOpenaireId(
|
||||||
|
final String type,
|
||||||
|
final String originalId,
|
||||||
|
final boolean to_md5) {
|
||||||
|
switch (type) {
|
||||||
|
case "datasource":
|
||||||
|
return createOpenaireId(10, originalId, to_md5);
|
||||||
|
case "organization":
|
||||||
|
return createOpenaireId(20, originalId, to_md5);
|
||||||
|
case "person":
|
||||||
|
return createOpenaireId(30, originalId, to_md5);
|
||||||
|
case "project":
|
||||||
|
return createOpenaireId(40, originalId, to_md5);
|
||||||
|
default:
|
||||||
|
return createOpenaireId(50, originalId, to_md5);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String asString(final Object o) {
|
||||||
|
return o == null ? "" : o.toString();
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> Predicate<T> distinctByKey(
|
||||||
|
final Function<? super T, ?> keyExtractor) {
|
||||||
|
final Map<Object, Boolean> seen = new ConcurrentHashMap<>();
|
||||||
|
return t -> seen.putIfAbsent(keyExtractor.apply(t), Boolean.TRUE) == null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Qualifier createBestAccessRights(final List<Instance> instanceList) {
|
||||||
|
return getBestAccessRights(instanceList);
|
||||||
|
}
|
||||||
|
|
||||||
|
protected static Qualifier getBestAccessRights(final List<Instance> instanceList) {
|
||||||
|
if (instanceList != null) {
|
||||||
|
final Optional<AccessRight> min = instanceList
|
||||||
|
.stream()
|
||||||
|
.map(Instance::getAccessright)
|
||||||
|
.min(new AccessRightComparator<>());
|
||||||
|
|
||||||
|
final Qualifier rights = min.map(OafMapperUtils::qualifier).orElseGet(Qualifier::new);
|
||||||
|
|
||||||
|
if (StringUtils.isBlank(rights.getClassid())) {
|
||||||
|
rights.setClassid(UNKNOWN);
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(rights.getClassname())
|
||||||
|
|| UNKNOWN.equalsIgnoreCase(rights.getClassname())) {
|
||||||
|
rights.setClassname(NOT_AVAILABLE);
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(rights.getSchemeid())) {
|
||||||
|
rights.setSchemeid(DNET_ACCESS_MODES);
|
||||||
|
}
|
||||||
|
if (StringUtils.isBlank(rights.getSchemename())) {
|
||||||
|
rights.setSchemename(DNET_ACCESS_MODES);
|
||||||
|
}
|
||||||
|
|
||||||
|
return rights;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,27 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.oaf.utils;
|
|
||||||
|
|
||||||
import java.util.Comparator;
|
|
||||||
|
|
||||||
public class OrganizationPidComparator implements Comparator<PidType> {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public int compare(PidType pLeft, PidType pRight) {
|
|
||||||
if (pLeft.equals(PidType.GRID))
|
|
||||||
return -1;
|
|
||||||
if (pRight.equals(PidType.GRID))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (pLeft.equals(PidType.mag_id))
|
|
||||||
return -1;
|
|
||||||
if (pRight.equals(PidType.mag_id))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (pLeft.equals(PidType.urn))
|
|
||||||
return -1;
|
|
||||||
if (pRight.equals(PidType.urn))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,54 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.oaf.utils;
|
|
||||||
|
|
||||||
import java.util.Comparator;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.common.ModelSupport;
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.OafEntity;
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.Organization;
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.Result;
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
|
|
||||||
|
|
||||||
public class PidComparator<T extends OafEntity> implements Comparator<StructuredProperty> {
|
|
||||||
|
|
||||||
private T entity;
|
|
||||||
|
|
||||||
public PidComparator(T entity) {
|
|
||||||
this.entity = entity;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public int compare(StructuredProperty left, StructuredProperty right) {
|
|
||||||
|
|
||||||
if (left == null && right == null)
|
|
||||||
return 0;
|
|
||||||
if (left == null)
|
|
||||||
return 1;
|
|
||||||
if (right == null)
|
|
||||||
return -1;
|
|
||||||
|
|
||||||
PidType lClass = PidType.valueOf(left.getQualifier().getClassid());
|
|
||||||
PidType rClass = PidType.valueOf(right.getQualifier().getClassid());
|
|
||||||
|
|
||||||
if (lClass.equals(rClass))
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
if (ModelSupport.isSubClass(entity, Result.class)) {
|
|
||||||
return compareResultPids(lClass, rClass);
|
|
||||||
}
|
|
||||||
if (ModelSupport.isSubClass(entity, Organization.class)) {
|
|
||||||
return compareOrganizationtPids(lClass, rClass);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Else (but unlikely), lexicographical ordering will do.
|
|
||||||
return lClass.compareTo(rClass);
|
|
||||||
}
|
|
||||||
|
|
||||||
private int compareResultPids(PidType lClass, PidType rClass) {
|
|
||||||
return new ResultPidComparator().compare(lClass, rClass);
|
|
||||||
}
|
|
||||||
|
|
||||||
private int compareOrganizationtPids(PidType lClass, PidType rClass) {
|
|
||||||
return new OrganizationPidComparator().compare(lClass, rClass);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,29 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.oaf.utils;
|
|
||||||
|
|
||||||
import org.apache.commons.lang3.EnumUtils;
|
|
||||||
|
|
||||||
public enum PidType {
|
|
||||||
|
|
||||||
// Result
|
|
||||||
doi, pmid, pmc, handle, arXiv, NCID, GBIF, nct, pdb,
|
|
||||||
|
|
||||||
// Organization
|
|
||||||
GRID, mag_id, urn,
|
|
||||||
|
|
||||||
// Used by dedup
|
|
||||||
undefined, original;
|
|
||||||
|
|
||||||
public static boolean isValid(String type) {
|
|
||||||
return EnumUtils.isValidEnum(PidType.class, type);
|
|
||||||
}
|
|
||||||
|
|
||||||
public static PidType tryValueOf(String s) {
|
|
||||||
try {
|
|
||||||
return PidType.valueOf(s);
|
|
||||||
} catch (Exception e) {
|
|
||||||
return PidType.original;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,57 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.oaf.utils;
|
|
||||||
|
|
||||||
import java.util.Comparator;
|
|
||||||
|
|
||||||
public class ResultPidComparator implements Comparator<PidType> {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public int compare(PidType pLeft, PidType pRight) {
|
|
||||||
if (pLeft.equals(PidType.doi))
|
|
||||||
return -1;
|
|
||||||
if (pRight.equals(PidType.doi))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (pLeft.equals(PidType.pmid))
|
|
||||||
return -1;
|
|
||||||
if (pRight.equals(PidType.pmid))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (pLeft.equals(PidType.pmc))
|
|
||||||
return -1;
|
|
||||||
if (pRight.equals(PidType.pmc))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (pLeft.equals(PidType.handle))
|
|
||||||
return -1;
|
|
||||||
if (pRight.equals(PidType.handle))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (pLeft.equals(PidType.arXiv))
|
|
||||||
return -1;
|
|
||||||
if (pRight.equals(PidType.arXiv))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (pLeft.equals(PidType.NCID))
|
|
||||||
return -1;
|
|
||||||
if (pRight.equals(PidType.NCID))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (pLeft.equals(PidType.GBIF))
|
|
||||||
return -1;
|
|
||||||
if (pRight.equals(PidType.GBIF))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (pLeft.equals(PidType.nct))
|
|
||||||
return -1;
|
|
||||||
if (pRight.equals(PidType.nct))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
if (pLeft.equals(PidType.urn))
|
|
||||||
return -1;
|
|
||||||
if (pRight.equals(PidType.urn))
|
|
||||||
return 1;
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,63 +1,107 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.utils;
|
package eu.dnetlib.dhp.utils;
|
||||||
|
|
||||||
import java.io.ByteArrayInputStream;
|
import java.io.*;
|
||||||
import java.io.ByteArrayOutputStream;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
import java.nio.charset.StandardCharsets;
|
||||||
import java.security.MessageDigest;
|
import java.security.MessageDigest;
|
||||||
import java.util.zip.GZIPInputStream;
|
import java.util.*;
|
||||||
import java.util.zip.GZIPOutputStream;
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
import org.apache.commons.codec.binary.Base64;
|
|
||||||
import org.apache.commons.codec.binary.Base64OutputStream;
|
|
||||||
import org.apache.commons.codec.binary.Hex;
|
import org.apache.commons.codec.binary.Hex;
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
import org.apache.hadoop.conf.Configuration;
|
||||||
|
import org.apache.hadoop.fs.FileSystem;
|
||||||
|
import org.apache.hadoop.fs.Path;
|
||||||
|
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||||
|
import org.apache.http.client.methods.HttpGet;
|
||||||
|
import org.apache.http.impl.client.CloseableHttpClient;
|
||||||
|
import org.apache.http.impl.client.HttpClients;
|
||||||
|
import org.apache.spark.sql.Dataset;
|
||||||
|
import org.apache.spark.sql.SaveMode;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
import com.google.common.collect.Maps;
|
||||||
import com.jayway.jsonpath.JsonPath;
|
import com.jayway.jsonpath.JsonPath;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.schema.mdstore.MDStoreWithInfo;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.utils.CleaningFunctions;
|
||||||
import net.minidev.json.JSONArray;
|
import net.minidev.json.JSONArray;
|
||||||
|
import scala.collection.JavaConverters;
|
||||||
|
import scala.collection.Seq;
|
||||||
|
|
||||||
public class DHPUtils {
|
public class DHPUtils {
|
||||||
|
|
||||||
|
private static final Logger log = LoggerFactory.getLogger(DHPUtils.class);
|
||||||
|
|
||||||
|
private DHPUtils() {
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Seq<String> toSeq(List<String> list) {
|
||||||
|
return JavaConverters.asScalaIteratorConverter(list.iterator()).asScala().toSeq();
|
||||||
|
}
|
||||||
|
|
||||||
public static String md5(final String s) {
|
public static String md5(final String s) {
|
||||||
try {
|
try {
|
||||||
final MessageDigest md = MessageDigest.getInstance("MD5");
|
final MessageDigest md = MessageDigest.getInstance("MD5");
|
||||||
md.update(s.getBytes(StandardCharsets.UTF_8));
|
md.update(s.getBytes(StandardCharsets.UTF_8));
|
||||||
return new String(Hex.encodeHex(md.digest()));
|
return new String(Hex.encodeHex(md.digest()));
|
||||||
} catch (final Exception e) {
|
} catch (final Exception e) {
|
||||||
System.err.println("Error creating id");
|
log.error("Error creating id from {}", s);
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Retrieves from the metadata store manager application the list of paths associated with mdstores characterized
|
||||||
|
* by he given format, layout, interpretation
|
||||||
|
* @param mdstoreManagerUrl the URL of the mdstore manager service
|
||||||
|
* @param format the mdstore format
|
||||||
|
* @param layout the mdstore layout
|
||||||
|
* @param interpretation the mdstore interpretation
|
||||||
|
* @param includeEmpty include Empty mdstores
|
||||||
|
* @return the set of hdfs paths
|
||||||
|
* @throws IOException in case of HTTP communication issues
|
||||||
|
*/
|
||||||
|
public static Set<String> mdstorePaths(final String mdstoreManagerUrl,
|
||||||
|
final String format,
|
||||||
|
final String layout,
|
||||||
|
final String interpretation,
|
||||||
|
boolean includeEmpty) throws IOException {
|
||||||
|
final String url = mdstoreManagerUrl + "/mdstores/";
|
||||||
|
final ObjectMapper objectMapper = new ObjectMapper();
|
||||||
|
|
||||||
|
final HttpGet req = new HttpGet(url);
|
||||||
|
|
||||||
|
try (final CloseableHttpClient client = HttpClients.createDefault()) {
|
||||||
|
try (final CloseableHttpResponse response = client.execute(req)) {
|
||||||
|
final String json = IOUtils.toString(response.getEntity().getContent());
|
||||||
|
final MDStoreWithInfo[] mdstores = objectMapper.readValue(json, MDStoreWithInfo[].class);
|
||||||
|
return Arrays
|
||||||
|
.stream(mdstores)
|
||||||
|
.filter(md -> md.getFormat().equalsIgnoreCase(format))
|
||||||
|
.filter(md -> md.getLayout().equalsIgnoreCase(layout))
|
||||||
|
.filter(md -> md.getInterpretation().equalsIgnoreCase(interpretation))
|
||||||
|
.filter(md -> StringUtils.isNotBlank(md.getHdfsPath()))
|
||||||
|
.filter(md -> StringUtils.isNotBlank(md.getCurrentVersion()))
|
||||||
|
.filter(md -> includeEmpty || md.getSize() > 0)
|
||||||
|
.map(md -> md.getHdfsPath() + "/" + md.getCurrentVersion() + "/store")
|
||||||
|
.collect(Collectors.toSet());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public static String generateIdentifier(final String originalId, final String nsPrefix) {
|
public static String generateIdentifier(final String originalId, final String nsPrefix) {
|
||||||
return String.format("%s::%s", nsPrefix, DHPUtils.md5(originalId));
|
return String.format("%s::%s", nsPrefix, DHPUtils.md5(originalId));
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String compressString(final String input) {
|
public static String generateUnresolvedIdentifier(final String pid, final String pidType) {
|
||||||
try (ByteArrayOutputStream out = new ByteArrayOutputStream();
|
|
||||||
Base64OutputStream b64os = new Base64OutputStream(out)) {
|
|
||||||
GZIPOutputStream gzip = new GZIPOutputStream(b64os);
|
|
||||||
gzip.write(input.getBytes(StandardCharsets.UTF_8));
|
|
||||||
gzip.close();
|
|
||||||
return out.toString();
|
|
||||||
} catch (Throwable e) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public static String decompressString(final String input) {
|
final String cleanedPid = CleaningFunctions.normalizePidValue(pidType, pid);
|
||||||
byte[] byteArray = Base64.decodeBase64(input.getBytes());
|
|
||||||
int len;
|
return String.format("unresolved::%s::%s", cleanedPid, pidType.toLowerCase().trim());
|
||||||
try (GZIPInputStream gis = new GZIPInputStream(new ByteArrayInputStream((byteArray)));
|
|
||||||
ByteArrayOutputStream bos = new ByteArrayOutputStream(byteArray.length)) {
|
|
||||||
byte[] buffer = new byte[1024];
|
|
||||||
while ((len = gis.read(buffer)) != -1) {
|
|
||||||
bos.write(buffer, 0, len);
|
|
||||||
}
|
|
||||||
return bos.toString();
|
|
||||||
} catch (Exception e) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static String getJPathString(final String jsonPath, final String json) {
|
public static String getJPathString(final String jsonPath, final String json) {
|
||||||
|
@ -72,4 +116,72 @@ public class DHPUtils {
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static final ObjectMapper MAPPER = new ObjectMapper();
|
||||||
|
|
||||||
|
public static void writeHdfsFile(final Configuration conf, final String content, final String path)
|
||||||
|
throws IOException {
|
||||||
|
|
||||||
|
log.info("writing file {}, size {}", path, content.length());
|
||||||
|
try (FileSystem fs = FileSystem.get(conf);
|
||||||
|
BufferedOutputStream os = new BufferedOutputStream(fs.create(new Path(path)))) {
|
||||||
|
os.write(content.getBytes(StandardCharsets.UTF_8));
|
||||||
|
os.flush();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static String readHdfsFile(Configuration conf, String path) throws IOException {
|
||||||
|
log.info("reading file {}", path);
|
||||||
|
|
||||||
|
try (FileSystem fs = FileSystem.get(conf)) {
|
||||||
|
final Path p = new Path(path);
|
||||||
|
if (!fs.exists(p)) {
|
||||||
|
throw new FileNotFoundException(path);
|
||||||
|
}
|
||||||
|
return IOUtils.toString(fs.open(p));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> T readHdfsFileAs(Configuration conf, String path, Class<T> clazz) throws IOException {
|
||||||
|
return MAPPER.readValue(readHdfsFile(conf, path), clazz);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static <T> void saveDataset(final Dataset<T> mdstore, final String targetPath) {
|
||||||
|
log.info("saving dataset in: {}", targetPath);
|
||||||
|
mdstore
|
||||||
|
.write()
|
||||||
|
.mode(SaveMode.Overwrite)
|
||||||
|
.format("parquet")
|
||||||
|
.save(targetPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
public static Configuration getHadoopConfiguration(String nameNode) {
|
||||||
|
// ====== Init HDFS File System Object
|
||||||
|
Configuration conf = new Configuration();
|
||||||
|
// Set FileSystem URI
|
||||||
|
conf.set("fs.defaultFS", nameNode);
|
||||||
|
// Because of Maven
|
||||||
|
conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
|
||||||
|
conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
|
||||||
|
|
||||||
|
System.setProperty("hadoop.home.dir", "/");
|
||||||
|
return conf;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void populateOOZIEEnv(final Map<String, String> report) throws IOException {
|
||||||
|
File file = new File(System.getProperty("oozie.action.output.properties"));
|
||||||
|
Properties props = new Properties();
|
||||||
|
report.forEach((k, v) -> props.setProperty(k, v));
|
||||||
|
|
||||||
|
try (OutputStream os = new FileOutputStream(file)) {
|
||||||
|
props.store(os, "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public static void populateOOZIEEnv(final String paramName, String value) throws IOException {
|
||||||
|
Map<String, String> report = Maps.newHashMap();
|
||||||
|
report.put(paramName, value);
|
||||||
|
|
||||||
|
populateOOZIEEnv(report);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,11 +1,11 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.utils;
|
package eu.dnetlib.dhp.utils;
|
||||||
|
|
||||||
import java.util.Map;
|
import org.apache.cxf.endpoint.Client;
|
||||||
|
import org.apache.cxf.frontend.ClientProxy;
|
||||||
import javax.xml.ws.BindingProvider;
|
|
||||||
|
|
||||||
import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
|
import org.apache.cxf.jaxws.JaxWsProxyFactoryBean;
|
||||||
|
import org.apache.cxf.transport.http.HTTPConduit;
|
||||||
|
import org.apache.cxf.transports.http.configuration.HTTPClientPolicy;
|
||||||
import org.slf4j.Logger;
|
import org.slf4j.Logger;
|
||||||
import org.slf4j.LoggerFactory;
|
import org.slf4j.LoggerFactory;
|
||||||
|
|
||||||
|
@ -15,8 +15,11 @@ public class ISLookupClientFactory {
|
||||||
|
|
||||||
private static final Logger log = LoggerFactory.getLogger(ISLookupClientFactory.class);
|
private static final Logger log = LoggerFactory.getLogger(ISLookupClientFactory.class);
|
||||||
|
|
||||||
private static int requestTimeout = 60000 * 10;
|
private static final int requestTimeout = 60000 * 10;
|
||||||
private static int connectTimeout = 60000 * 10;
|
private static final int connectTimeout = 60000 * 10;
|
||||||
|
|
||||||
|
private ISLookupClientFactory() {
|
||||||
|
}
|
||||||
|
|
||||||
public static ISLookUpService getLookUpService(final String isLookupUrl) {
|
public static ISLookUpService getLookUpService(final String isLookupUrl) {
|
||||||
return getServiceStub(ISLookUpService.class, isLookupUrl);
|
return getServiceStub(ISLookUpService.class, isLookupUrl);
|
||||||
|
@ -24,27 +27,28 @@ public class ISLookupClientFactory {
|
||||||
|
|
||||||
@SuppressWarnings("unchecked")
|
@SuppressWarnings("unchecked")
|
||||||
private static <T> T getServiceStub(final Class<T> clazz, final String endpoint) {
|
private static <T> T getServiceStub(final Class<T> clazz, final String endpoint) {
|
||||||
log.info(String.format("creating %s stub from %s", clazz.getName(), endpoint));
|
log.info("creating {} stub from {}", clazz.getName(), endpoint);
|
||||||
final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean();
|
final JaxWsProxyFactoryBean jaxWsProxyFactory = new JaxWsProxyFactoryBean();
|
||||||
jaxWsProxyFactory.setServiceClass(clazz);
|
jaxWsProxyFactory.setServiceClass(clazz);
|
||||||
jaxWsProxyFactory.setAddress(endpoint);
|
jaxWsProxyFactory.setAddress(endpoint);
|
||||||
|
|
||||||
final T service = (T) jaxWsProxyFactory.create();
|
final T service = (T) jaxWsProxyFactory.create();
|
||||||
|
|
||||||
if (service instanceof BindingProvider) {
|
Client client = ClientProxy.getClient(service);
|
||||||
|
if (client != null) {
|
||||||
|
HTTPConduit conduit = (HTTPConduit) client.getConduit();
|
||||||
|
HTTPClientPolicy policy = new HTTPClientPolicy();
|
||||||
|
|
||||||
log
|
log
|
||||||
.info(
|
.info(
|
||||||
"setting timeouts for {} to requestTimeout: {}, connectTimeout: {}",
|
"setting connectTimeout to {}, requestTimeout to {} for service {}",
|
||||||
BindingProvider.class.getName(), requestTimeout, connectTimeout);
|
connectTimeout,
|
||||||
|
requestTimeout,
|
||||||
|
clazz.getCanonicalName());
|
||||||
|
|
||||||
Map<String, Object> requestContext = ((BindingProvider) service).getRequestContext();
|
policy.setConnectionTimeout(connectTimeout);
|
||||||
|
policy.setReceiveTimeout(requestTimeout);
|
||||||
requestContext.put("com.sun.xml.internal.ws.request.timeout", requestTimeout);
|
conduit.setClient(policy);
|
||||||
requestContext.put("com.sun.xml.internal.ws.connect.timeout", connectTimeout);
|
|
||||||
requestContext.put("com.sun.xml.ws.request.timeout", requestTimeout);
|
|
||||||
requestContext.put("com.sun.xml.ws.connect.timeout", connectTimeout);
|
|
||||||
requestContext.put("javax.xml.ws.client.receiveTimeout", requestTimeout);
|
|
||||||
requestContext.put("javax.xml.ws.client.connectionTimeout", connectTimeout);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return service;
|
return service;
|
||||||
|
|
|
@ -10,7 +10,7 @@ import net.sf.saxon.trans.XPathException;
|
||||||
|
|
||||||
public abstract class AbstractExtensionFunction extends ExtensionFunctionDefinition {
|
public abstract class AbstractExtensionFunction extends ExtensionFunctionDefinition {
|
||||||
|
|
||||||
public static String DEFAULT_SAXON_EXT_NS_URI = "http://www.d-net.research-infrastructures.eu/saxon-extension";
|
public static final String DEFAULT_SAXON_EXT_NS_URI = "http://www.d-net.research-infrastructures.eu/saxon-extension";
|
||||||
|
|
||||||
public abstract String getName();
|
public abstract String getName();
|
||||||
|
|
||||||
|
|
|
@ -26,7 +26,7 @@ public class ExtractYear extends AbstractExtensionFunction {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Sequence doCall(XPathContext context, Sequence[] arguments) throws XPathException {
|
public Sequence doCall(XPathContext context, Sequence[] arguments) throws XPathException {
|
||||||
if (arguments == null | arguments.length == 0) {
|
if (arguments == null || arguments.length == 0) {
|
||||||
return new StringValue("");
|
return new StringValue("");
|
||||||
}
|
}
|
||||||
final Item item = arguments[0].head();
|
final Item item = arguments[0].head();
|
||||||
|
@ -63,8 +63,7 @@ public class ExtractYear extends AbstractExtensionFunction {
|
||||||
for (String format : dateFormats) {
|
for (String format : dateFormats) {
|
||||||
try {
|
try {
|
||||||
c.setTime(new SimpleDateFormat(format).parse(s));
|
c.setTime(new SimpleDateFormat(format).parse(s));
|
||||||
String year = String.valueOf(c.get(Calendar.YEAR));
|
return String.valueOf(c.get(Calendar.YEAR));
|
||||||
return year;
|
|
||||||
} catch (ParseException e) {
|
} catch (ParseException e) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -30,7 +30,7 @@ public class NormalizeDate extends AbstractExtensionFunction {
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Sequence doCall(XPathContext context, Sequence[] arguments) throws XPathException {
|
public Sequence doCall(XPathContext context, Sequence[] arguments) throws XPathException {
|
||||||
if (arguments == null | arguments.length == 0) {
|
if (arguments == null || arguments.length == 0) {
|
||||||
return new StringValue(BLANK);
|
return new StringValue(BLANK);
|
||||||
}
|
}
|
||||||
String s = arguments[0].head().getStringValue();
|
String s = arguments[0].head().getStringValue();
|
||||||
|
|
|
@ -1,6 +1,8 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.utils.saxon;
|
package eu.dnetlib.dhp.utils.saxon;
|
||||||
|
|
||||||
|
import static org.apache.commons.lang3.StringUtils.isNotBlank;
|
||||||
|
|
||||||
import org.apache.commons.lang3.StringUtils;
|
import org.apache.commons.lang3.StringUtils;
|
||||||
|
|
||||||
import net.sf.saxon.expr.XPathContext;
|
import net.sf.saxon.expr.XPathContext;
|
||||||
|
@ -26,7 +28,8 @@ public class PickFirst extends AbstractExtensionFunction {
|
||||||
final String s1 = getValue(arguments[0]);
|
final String s1 = getValue(arguments[0]);
|
||||||
final String s2 = getValue(arguments[1]);
|
final String s2 = getValue(arguments[1]);
|
||||||
|
|
||||||
return new StringValue(StringUtils.isNotBlank(s1) ? s1 : StringUtils.isNotBlank(s2) ? s2 : "");
|
final String value = isNotBlank(s1) ? s1 : isNotBlank(s2) ? s2 : "";
|
||||||
|
return new StringValue(value);
|
||||||
}
|
}
|
||||||
|
|
||||||
private String getValue(final Sequence arg) throws XPathException {
|
private String getValue(final Sequence arg) throws XPathException {
|
||||||
|
|
|
@ -12,6 +12,9 @@ import net.sf.saxon.TransformerFactoryImpl;
|
||||||
|
|
||||||
public class SaxonTransformerFactory {
|
public class SaxonTransformerFactory {
|
||||||
|
|
||||||
|
private SaxonTransformerFactory() {
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Creates the index record transformer from the given XSLT
|
* Creates the index record transformer from the given XSLT
|
||||||
*
|
*
|
||||||
|
|
|
@ -1,76 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.message;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
|
|
||||||
public class Message {
|
|
||||||
|
|
||||||
private String workflowId;
|
|
||||||
|
|
||||||
private String jobName;
|
|
||||||
|
|
||||||
private MessageType type;
|
|
||||||
|
|
||||||
private Map<String, String> body;
|
|
||||||
|
|
||||||
public static Message fromJson(final String json) throws IOException {
|
|
||||||
final ObjectMapper jsonMapper = new ObjectMapper();
|
|
||||||
return jsonMapper.readValue(json, Message.class);
|
|
||||||
}
|
|
||||||
|
|
||||||
public Message() {
|
|
||||||
}
|
|
||||||
|
|
||||||
public Message(String workflowId, String jobName, MessageType type, Map<String, String> body) {
|
|
||||||
this.workflowId = workflowId;
|
|
||||||
this.jobName = jobName;
|
|
||||||
this.type = type;
|
|
||||||
this.body = body;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getWorkflowId() {
|
|
||||||
return workflowId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setWorkflowId(String workflowId) {
|
|
||||||
this.workflowId = workflowId;
|
|
||||||
}
|
|
||||||
|
|
||||||
public String getJobName() {
|
|
||||||
return jobName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setJobName(String jobName) {
|
|
||||||
this.jobName = jobName;
|
|
||||||
}
|
|
||||||
|
|
||||||
public MessageType getType() {
|
|
||||||
return type;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setType(MessageType type) {
|
|
||||||
this.type = type;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Map<String, String> getBody() {
|
|
||||||
return body;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setBody(Map<String, String> body) {
|
|
||||||
this.body = body;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String toString() {
|
|
||||||
final ObjectMapper jsonMapper = new ObjectMapper();
|
|
||||||
try {
|
|
||||||
return jsonMapper.writeValueAsString(this);
|
|
||||||
} catch (JsonProcessingException e) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,47 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.message;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.nio.charset.StandardCharsets;
|
|
||||||
import java.util.concurrent.LinkedBlockingQueue;
|
|
||||||
|
|
||||||
import com.rabbitmq.client.AMQP;
|
|
||||||
import com.rabbitmq.client.Channel;
|
|
||||||
import com.rabbitmq.client.DefaultConsumer;
|
|
||||||
import com.rabbitmq.client.Envelope;
|
|
||||||
|
|
||||||
public class MessageConsumer extends DefaultConsumer {
|
|
||||||
|
|
||||||
final LinkedBlockingQueue<Message> queueMessages;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Constructs a new instance and records its association to the passed-in channel.
|
|
||||||
*
|
|
||||||
* @param channel the channel to which this consumer is attached
|
|
||||||
* @param queueMessages
|
|
||||||
*/
|
|
||||||
public MessageConsumer(Channel channel, LinkedBlockingQueue<Message> queueMessages) {
|
|
||||||
super(channel);
|
|
||||||
this.queueMessages = queueMessages;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void handleDelivery(
|
|
||||||
String consumerTag, Envelope envelope, AMQP.BasicProperties properties, byte[] body)
|
|
||||||
throws IOException {
|
|
||||||
final String json = new String(body, StandardCharsets.UTF_8);
|
|
||||||
Message message = Message.fromJson(json);
|
|
||||||
try {
|
|
||||||
this.queueMessages.put(message);
|
|
||||||
System.out.println("Receiving Message " + message);
|
|
||||||
} catch (InterruptedException e) {
|
|
||||||
if (message.getType() == MessageType.REPORT)
|
|
||||||
throw new RuntimeException("Error on sending message");
|
|
||||||
else {
|
|
||||||
// TODO LOGGING EXCEPTION
|
|
||||||
}
|
|
||||||
} finally {
|
|
||||||
getChannel().basicAck(envelope.getDeliveryTag(), false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,136 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.message;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.concurrent.LinkedBlockingQueue;
|
|
||||||
import java.util.concurrent.TimeoutException;
|
|
||||||
|
|
||||||
import com.rabbitmq.client.Channel;
|
|
||||||
import com.rabbitmq.client.Connection;
|
|
||||||
import com.rabbitmq.client.ConnectionFactory;
|
|
||||||
|
|
||||||
public class MessageManager {
|
|
||||||
|
|
||||||
private final String messageHost;
|
|
||||||
|
|
||||||
private final String username;
|
|
||||||
|
|
||||||
private final String password;
|
|
||||||
|
|
||||||
private Connection connection;
|
|
||||||
|
|
||||||
private final Map<String, Channel> channels = new HashMap<>();
|
|
||||||
|
|
||||||
private boolean durable;
|
|
||||||
|
|
||||||
private boolean autodelete;
|
|
||||||
|
|
||||||
private final LinkedBlockingQueue<Message> queueMessages;
|
|
||||||
|
|
||||||
public MessageManager(
|
|
||||||
String messageHost,
|
|
||||||
String username,
|
|
||||||
String password,
|
|
||||||
final LinkedBlockingQueue<Message> queueMessages) {
|
|
||||||
this.queueMessages = queueMessages;
|
|
||||||
this.messageHost = messageHost;
|
|
||||||
this.username = username;
|
|
||||||
this.password = password;
|
|
||||||
}
|
|
||||||
|
|
||||||
public MessageManager(
|
|
||||||
String messageHost,
|
|
||||||
String username,
|
|
||||||
String password,
|
|
||||||
boolean durable,
|
|
||||||
boolean autodelete,
|
|
||||||
final LinkedBlockingQueue<Message> queueMessages) {
|
|
||||||
this.queueMessages = queueMessages;
|
|
||||||
this.messageHost = messageHost;
|
|
||||||
this.username = username;
|
|
||||||
this.password = password;
|
|
||||||
|
|
||||||
this.durable = durable;
|
|
||||||
this.autodelete = autodelete;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Connection createConnection() throws IOException, TimeoutException {
|
|
||||||
ConnectionFactory factory = new ConnectionFactory();
|
|
||||||
factory.setHost(this.messageHost);
|
|
||||||
factory.setUsername(this.username);
|
|
||||||
factory.setPassword(this.password);
|
|
||||||
return factory.newConnection();
|
|
||||||
}
|
|
||||||
|
|
||||||
private Channel createChannel(
|
|
||||||
final Connection connection,
|
|
||||||
final String queueName,
|
|
||||||
final boolean durable,
|
|
||||||
final boolean autodelete)
|
|
||||||
throws Exception {
|
|
||||||
Map<String, Object> args = new HashMap<>();
|
|
||||||
args.put("x-message-ttl", 10000);
|
|
||||||
Channel channel = connection.createChannel();
|
|
||||||
channel.queueDeclare(queueName, durable, false, this.autodelete, args);
|
|
||||||
return channel;
|
|
||||||
}
|
|
||||||
|
|
||||||
private Channel getOrCreateChannel(final String queueName, boolean durable, boolean autodelete)
|
|
||||||
throws Exception {
|
|
||||||
if (channels.containsKey(queueName)) {
|
|
||||||
return channels.get(queueName);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (this.connection == null) {
|
|
||||||
this.connection = createConnection();
|
|
||||||
}
|
|
||||||
channels.put(queueName, createChannel(this.connection, queueName, durable, autodelete));
|
|
||||||
return channels.get(queueName);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void close() throws IOException {
|
|
||||||
channels
|
|
||||||
.values()
|
|
||||||
.forEach(
|
|
||||||
ch -> {
|
|
||||||
try {
|
|
||||||
ch.close();
|
|
||||||
} catch (Exception e) {
|
|
||||||
// TODO LOG
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
this.connection.close();
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean sendMessage(final Message message, String queueName) throws Exception {
|
|
||||||
try {
|
|
||||||
Channel channel = getOrCreateChannel(queueName, this.durable, this.autodelete);
|
|
||||||
channel.basicPublish("", queueName, null, message.toString().getBytes());
|
|
||||||
return true;
|
|
||||||
} catch (Throwable e) {
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public boolean sendMessage(
|
|
||||||
final Message message, String queueName, boolean durable_var, boolean autodelete_var)
|
|
||||||
throws Exception {
|
|
||||||
try {
|
|
||||||
Channel channel = getOrCreateChannel(queueName, durable_var, autodelete_var);
|
|
||||||
channel.basicPublish("", queueName, null, message.toString().getBytes());
|
|
||||||
return true;
|
|
||||||
} catch (Throwable e) {
|
|
||||||
throw new RuntimeException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public void startConsumingMessage(
|
|
||||||
final String queueName, final boolean durable, final boolean autodelete) throws Exception {
|
|
||||||
|
|
||||||
Channel channel = createChannel(createConnection(), queueName, durable, autodelete);
|
|
||||||
channel.basicConsume(queueName, false, new MessageConsumer(channel, queueMessages));
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,6 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.message;
|
|
||||||
|
|
||||||
public enum MessageType {
|
|
||||||
ONGOING, REPORT
|
|
||||||
}
|
|
File diff suppressed because one or more lines are too long
|
@ -0,0 +1,73 @@
|
||||||
|
package eu.dnetlib.dhp.application
|
||||||
|
|
||||||
|
import scala.io.Source
|
||||||
|
|
||||||
|
/** This is the main Interface SparkApplication
|
||||||
|
* where all the Spark Scala class should inherit
|
||||||
|
*/
|
||||||
|
trait SparkScalaApplication {
|
||||||
|
|
||||||
|
/** This is the path in the classpath of the json
|
||||||
|
* describes all the argument needed to run
|
||||||
|
*/
|
||||||
|
val propertyPath: String
|
||||||
|
|
||||||
|
/** Utility to parse the arguments using the
|
||||||
|
* property json in the classpath identified from
|
||||||
|
* the variable propertyPath
|
||||||
|
*
|
||||||
|
* @param args the list of arguments
|
||||||
|
*/
|
||||||
|
def parseArguments(args: Array[String]): ArgumentApplicationParser = {
|
||||||
|
val parser = new ArgumentApplicationParser(
|
||||||
|
Source.fromInputStream(getClass.getResourceAsStream(propertyPath)).mkString
|
||||||
|
)
|
||||||
|
parser.parseArgument(args)
|
||||||
|
parser
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Here all the spark applications runs this method
|
||||||
|
* where the whole logic of the spark node is defined
|
||||||
|
*/
|
||||||
|
def run(): Unit
|
||||||
|
}
|
||||||
|
|
||||||
|
import org.apache.spark.SparkConf
|
||||||
|
import org.apache.spark.sql.SparkSession
|
||||||
|
import org.slf4j.Logger
|
||||||
|
|
||||||
|
abstract class AbstractScalaApplication(
|
||||||
|
val propertyPath: String,
|
||||||
|
val args: Array[String],
|
||||||
|
log: Logger
|
||||||
|
) extends SparkScalaApplication {
|
||||||
|
|
||||||
|
var parser: ArgumentApplicationParser = null
|
||||||
|
|
||||||
|
var spark: SparkSession = null
|
||||||
|
|
||||||
|
def initialize(): SparkScalaApplication = {
|
||||||
|
parser = parseArguments(args)
|
||||||
|
spark = createSparkSession()
|
||||||
|
this
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Utility for creating a spark session starting from parser
|
||||||
|
*
|
||||||
|
* @return a spark Session
|
||||||
|
*/
|
||||||
|
private def createSparkSession(): SparkSession = {
|
||||||
|
require(parser != null)
|
||||||
|
|
||||||
|
val conf: SparkConf = new SparkConf()
|
||||||
|
val master = parser.get("master")
|
||||||
|
log.info(s"Creating Spark session: Master: $master")
|
||||||
|
SparkSession
|
||||||
|
.builder()
|
||||||
|
.config(conf)
|
||||||
|
.appName(getClass.getSimpleName)
|
||||||
|
.master(master)
|
||||||
|
.getOrCreate()
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -0,0 +1,442 @@
|
||||||
|
package eu.dnetlib.dhp.sx.graph.scholix
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.{Publication, Relation, Result, StructuredProperty}
|
||||||
|
import eu.dnetlib.dhp.schema.sx.scholix._
|
||||||
|
import eu.dnetlib.dhp.schema.sx.summary.{CollectedFromType, SchemeValue, ScholixSummary, Typology}
|
||||||
|
import eu.dnetlib.dhp.utils.DHPUtils
|
||||||
|
import org.apache.spark.sql.expressions.Aggregator
|
||||||
|
import org.apache.spark.sql.{Encoder, Encoders}
|
||||||
|
import org.json4s
|
||||||
|
import org.json4s.DefaultFormats
|
||||||
|
import org.json4s.jackson.JsonMethods.parse
|
||||||
|
import scala.collection.JavaConverters._
|
||||||
|
import scala.io.Source
|
||||||
|
|
||||||
|
object ScholixUtils extends Serializable {
|
||||||
|
|
||||||
|
val DNET_IDENTIFIER_SCHEMA: String = "DNET Identifier"
|
||||||
|
|
||||||
|
val DATE_RELATION_KEY: String = "RelationDate"
|
||||||
|
|
||||||
|
case class RelationVocabulary(original: String, inverse: String) {}
|
||||||
|
|
||||||
|
case class RelatedEntities(id: String, relatedDataset: Long, relatedPublication: Long) {}
|
||||||
|
|
||||||
|
val relations: Map[String, RelationVocabulary] = {
|
||||||
|
val input = Source
|
||||||
|
.fromInputStream(
|
||||||
|
getClass.getResourceAsStream("/eu/dnetlib/scholexplorer/relation/relations.json")
|
||||||
|
)
|
||||||
|
.mkString
|
||||||
|
implicit lazy val formats: DefaultFormats.type = org.json4s.DefaultFormats
|
||||||
|
|
||||||
|
lazy val json: json4s.JValue = parse(input)
|
||||||
|
|
||||||
|
json.extract[Map[String, RelationVocabulary]]
|
||||||
|
}
|
||||||
|
|
||||||
|
def extractRelationDate(relation: Relation): String = {
|
||||||
|
|
||||||
|
if (relation.getProperties == null || !relation.getProperties.isEmpty)
|
||||||
|
null
|
||||||
|
else {
|
||||||
|
val date = relation.getProperties.asScala
|
||||||
|
.find(p => DATE_RELATION_KEY.equalsIgnoreCase(p.getKey))
|
||||||
|
.map(p => p.getValue)
|
||||||
|
if (date.isDefined)
|
||||||
|
date.get
|
||||||
|
else
|
||||||
|
null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def extractRelationDate(summary: ScholixSummary): String = {
|
||||||
|
|
||||||
|
if (summary.getDate == null || summary.getDate.isEmpty)
|
||||||
|
null
|
||||||
|
else {
|
||||||
|
summary.getDate.get(0)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def inverseRelationShip(rel: ScholixRelationship): ScholixRelationship = {
|
||||||
|
new ScholixRelationship(rel.getInverse, rel.getSchema, rel.getName)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def generateScholixResourceFromResult(r: Result): ScholixResource = {
|
||||||
|
generateScholixResourceFromSummary(ScholixUtils.resultToSummary(r))
|
||||||
|
}
|
||||||
|
|
||||||
|
val statsAggregator: Aggregator[(String, String, Long), RelatedEntities, RelatedEntities] =
|
||||||
|
new Aggregator[(String, String, Long), RelatedEntities, RelatedEntities] with Serializable {
|
||||||
|
override def zero: RelatedEntities = null
|
||||||
|
|
||||||
|
override def reduce(b: RelatedEntities, a: (String, String, Long)): RelatedEntities = {
|
||||||
|
val relatedDataset = if ("dataset".equalsIgnoreCase(a._2)) a._3 else 0
|
||||||
|
val relatedPublication = if ("publication".equalsIgnoreCase(a._2)) a._3 else 0
|
||||||
|
|
||||||
|
if (b == null)
|
||||||
|
RelatedEntities(a._1, relatedDataset, relatedPublication)
|
||||||
|
else
|
||||||
|
RelatedEntities(
|
||||||
|
a._1,
|
||||||
|
b.relatedDataset + relatedDataset,
|
||||||
|
b.relatedPublication + relatedPublication
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
override def merge(b1: RelatedEntities, b2: RelatedEntities): RelatedEntities = {
|
||||||
|
if (b1 != null && b2 != null)
|
||||||
|
RelatedEntities(
|
||||||
|
b1.id,
|
||||||
|
b1.relatedDataset + b2.relatedDataset,
|
||||||
|
b1.relatedPublication + b2.relatedPublication
|
||||||
|
)
|
||||||
|
else if (b1 != null)
|
||||||
|
b1
|
||||||
|
else
|
||||||
|
b2
|
||||||
|
}
|
||||||
|
|
||||||
|
override def finish(reduction: RelatedEntities): RelatedEntities = reduction
|
||||||
|
|
||||||
|
override def bufferEncoder: Encoder[RelatedEntities] = Encoders.bean(classOf[RelatedEntities])
|
||||||
|
|
||||||
|
override def outputEncoder: Encoder[RelatedEntities] = Encoders.bean(classOf[RelatedEntities])
|
||||||
|
}
|
||||||
|
|
||||||
|
val scholixAggregator: Aggregator[(String, Scholix), Scholix, Scholix] =
|
||||||
|
new Aggregator[(String, Scholix), Scholix, Scholix] with Serializable {
|
||||||
|
override def zero: Scholix = null
|
||||||
|
|
||||||
|
def scholix_complete(s: Scholix): Boolean = {
|
||||||
|
if (s == null || s.getIdentifier == null) {
|
||||||
|
false
|
||||||
|
} else if (s.getSource == null || s.getTarget == null) {
|
||||||
|
false
|
||||||
|
} else if (s.getLinkprovider == null || s.getLinkprovider.isEmpty)
|
||||||
|
false
|
||||||
|
else
|
||||||
|
true
|
||||||
|
}
|
||||||
|
|
||||||
|
override def reduce(b: Scholix, a: (String, Scholix)): Scholix = {
|
||||||
|
if (scholix_complete(b)) b else a._2
|
||||||
|
}
|
||||||
|
|
||||||
|
override def merge(b1: Scholix, b2: Scholix): Scholix = {
|
||||||
|
if (scholix_complete(b1)) b1 else b2
|
||||||
|
}
|
||||||
|
|
||||||
|
override def finish(reduction: Scholix): Scholix = reduction
|
||||||
|
|
||||||
|
override def bufferEncoder: Encoder[Scholix] = Encoders.kryo[Scholix]
|
||||||
|
|
||||||
|
override def outputEncoder: Encoder[Scholix] = Encoders.kryo[Scholix]
|
||||||
|
}
|
||||||
|
|
||||||
|
def createInverseScholixRelation(scholix: Scholix): Scholix = {
|
||||||
|
val s = new Scholix
|
||||||
|
s.setPublicationDate(scholix.getPublicationDate)
|
||||||
|
s.setPublisher(scholix.getPublisher)
|
||||||
|
s.setLinkprovider(scholix.getLinkprovider)
|
||||||
|
s.setRelationship(inverseRelationShip(scholix.getRelationship))
|
||||||
|
s.setSource(scholix.getTarget)
|
||||||
|
s.setTarget(scholix.getSource)
|
||||||
|
s.setIdentifier(
|
||||||
|
DHPUtils.md5(
|
||||||
|
s"${s.getSource.getIdentifier}::${s.getRelationship.getName}::${s.getTarget.getIdentifier}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
s
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def extractCollectedFrom(summary: ScholixResource): List[ScholixEntityId] = {
|
||||||
|
if (summary.getCollectedFrom != null && !summary.getCollectedFrom.isEmpty) {
|
||||||
|
val l: List[ScholixEntityId] = summary.getCollectedFrom.asScala.map { d =>
|
||||||
|
new ScholixEntityId(d.getProvider.getName, d.getProvider.getIdentifiers)
|
||||||
|
}(collection.breakOut)
|
||||||
|
l
|
||||||
|
} else List()
|
||||||
|
}
|
||||||
|
|
||||||
|
def extractCollectedFrom(summary: ScholixSummary): List[ScholixEntityId] = {
|
||||||
|
if (summary.getDatasources != null && !summary.getDatasources.isEmpty) {
|
||||||
|
val l: List[ScholixEntityId] = summary.getDatasources.asScala.map { d =>
|
||||||
|
new ScholixEntityId(
|
||||||
|
d.getDatasourceName,
|
||||||
|
List(new ScholixIdentifier(d.getDatasourceId, "DNET Identifier", null)).asJava
|
||||||
|
)
|
||||||
|
}(collection.breakOut)
|
||||||
|
l
|
||||||
|
} else List()
|
||||||
|
}
|
||||||
|
|
||||||
|
def extractCollectedFrom(relation: Relation): List[ScholixEntityId] = {
|
||||||
|
if (relation.getCollectedfrom != null && !relation.getCollectedfrom.isEmpty) {
|
||||||
|
|
||||||
|
val l: List[ScholixEntityId] = relation.getCollectedfrom.asScala.map { c =>
|
||||||
|
new ScholixEntityId(
|
||||||
|
c.getValue,
|
||||||
|
List(new ScholixIdentifier(c.getKey, DNET_IDENTIFIER_SCHEMA, null)).asJava
|
||||||
|
)
|
||||||
|
}.toList
|
||||||
|
l
|
||||||
|
} else List()
|
||||||
|
}
|
||||||
|
|
||||||
|
def generateCompleteScholix(scholix: Scholix, target: ScholixSummary): Scholix = {
|
||||||
|
val s = new Scholix
|
||||||
|
s.setPublicationDate(scholix.getPublicationDate)
|
||||||
|
s.setPublisher(scholix.getPublisher)
|
||||||
|
s.setLinkprovider(scholix.getLinkprovider)
|
||||||
|
s.setRelationship(scholix.getRelationship)
|
||||||
|
s.setSource(scholix.getSource)
|
||||||
|
s.setTarget(generateScholixResourceFromSummary(target))
|
||||||
|
s.setIdentifier(
|
||||||
|
DHPUtils.md5(
|
||||||
|
s"${s.getSource.getIdentifier}::${s.getRelationship.getName}::${s.getTarget.getIdentifier}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
def generateCompleteScholix(scholix: Scholix, target: ScholixResource): Scholix = {
|
||||||
|
val s = new Scholix
|
||||||
|
s.setPublicationDate(scholix.getPublicationDate)
|
||||||
|
s.setPublisher(scholix.getPublisher)
|
||||||
|
s.setLinkprovider(scholix.getLinkprovider)
|
||||||
|
s.setRelationship(scholix.getRelationship)
|
||||||
|
s.setSource(scholix.getSource)
|
||||||
|
s.setTarget(target)
|
||||||
|
s.setIdentifier(
|
||||||
|
DHPUtils.md5(
|
||||||
|
s"${s.getSource.getIdentifier}::${s.getRelationship.getName}::${s.getTarget.getIdentifier}"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
def generateScholixResourceFromSummary(summaryObject: ScholixSummary): ScholixResource = {
|
||||||
|
val r = new ScholixResource
|
||||||
|
r.setIdentifier(summaryObject.getLocalIdentifier)
|
||||||
|
r.setDnetIdentifier(summaryObject.getId)
|
||||||
|
|
||||||
|
r.setObjectType(summaryObject.getTypology.toString)
|
||||||
|
r.setObjectSubType(summaryObject.getSubType)
|
||||||
|
|
||||||
|
if (summaryObject.getTitle != null && !summaryObject.getTitle.isEmpty)
|
||||||
|
r.setTitle(summaryObject.getTitle.get(0))
|
||||||
|
|
||||||
|
if (summaryObject.getAuthor != null && !summaryObject.getAuthor.isEmpty) {
|
||||||
|
val l: List[ScholixEntityId] =
|
||||||
|
summaryObject.getAuthor.asScala.map(a => new ScholixEntityId(a, null)).toList
|
||||||
|
if (l.nonEmpty)
|
||||||
|
r.setCreator(l.asJava)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (summaryObject.getDate != null && !summaryObject.getDate.isEmpty)
|
||||||
|
r.setPublicationDate(summaryObject.getDate.get(0))
|
||||||
|
if (summaryObject.getPublisher != null && !summaryObject.getPublisher.isEmpty) {
|
||||||
|
val plist: List[ScholixEntityId] =
|
||||||
|
summaryObject.getPublisher.asScala.map(p => new ScholixEntityId(p, null)).toList
|
||||||
|
|
||||||
|
if (plist.nonEmpty)
|
||||||
|
r.setPublisher(plist.asJava)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (summaryObject.getDatasources != null && !summaryObject.getDatasources.isEmpty) {
|
||||||
|
|
||||||
|
val l: List[ScholixCollectedFrom] = summaryObject.getDatasources.asScala
|
||||||
|
.map(c =>
|
||||||
|
new ScholixCollectedFrom(
|
||||||
|
new ScholixEntityId(
|
||||||
|
c.getDatasourceName,
|
||||||
|
List(new ScholixIdentifier(c.getDatasourceId, DNET_IDENTIFIER_SCHEMA, null)).asJava
|
||||||
|
),
|
||||||
|
"collected",
|
||||||
|
"complete"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.toList
|
||||||
|
|
||||||
|
if (l.nonEmpty)
|
||||||
|
r.setCollectedFrom(l.asJava)
|
||||||
|
|
||||||
|
}
|
||||||
|
r
|
||||||
|
}
|
||||||
|
|
||||||
|
def scholixFromSource(relation: Relation, source: ScholixResource): Scholix = {
|
||||||
|
if (relation == null || source == null)
|
||||||
|
return null
|
||||||
|
val s = new Scholix
|
||||||
|
var l: List[ScholixEntityId] = extractCollectedFrom(relation)
|
||||||
|
if (l.isEmpty)
|
||||||
|
l = extractCollectedFrom(source)
|
||||||
|
if (l.isEmpty)
|
||||||
|
return null
|
||||||
|
s.setLinkprovider(l.asJava)
|
||||||
|
var d = extractRelationDate(relation)
|
||||||
|
if (d == null)
|
||||||
|
d = source.getPublicationDate
|
||||||
|
|
||||||
|
s.setPublicationDate(d)
|
||||||
|
|
||||||
|
if (source.getPublisher != null && !source.getPublisher.isEmpty) {
|
||||||
|
s.setPublisher(source.getPublisher)
|
||||||
|
}
|
||||||
|
|
||||||
|
val semanticRelation = relations.getOrElse(relation.getRelClass.toLowerCase, null)
|
||||||
|
if (semanticRelation == null)
|
||||||
|
return null
|
||||||
|
s.setRelationship(
|
||||||
|
new ScholixRelationship(semanticRelation.original, "datacite", semanticRelation.inverse)
|
||||||
|
)
|
||||||
|
s.setSource(source)
|
||||||
|
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
def scholixFromSource(relation: Relation, source: ScholixSummary): Scholix = {
|
||||||
|
|
||||||
|
if (relation == null || source == null)
|
||||||
|
return null
|
||||||
|
|
||||||
|
val s = new Scholix
|
||||||
|
|
||||||
|
var l: List[ScholixEntityId] = extractCollectedFrom(relation)
|
||||||
|
if (l.isEmpty)
|
||||||
|
l = extractCollectedFrom(source)
|
||||||
|
if (l.isEmpty)
|
||||||
|
return null
|
||||||
|
|
||||||
|
s.setLinkprovider(l.asJava)
|
||||||
|
|
||||||
|
var d = extractRelationDate(relation)
|
||||||
|
if (d == null)
|
||||||
|
d = extractRelationDate(source)
|
||||||
|
|
||||||
|
s.setPublicationDate(d)
|
||||||
|
|
||||||
|
if (source.getPublisher != null && !source.getPublisher.isEmpty) {
|
||||||
|
val l: List[ScholixEntityId] = source.getPublisher.asScala
|
||||||
|
.map { p =>
|
||||||
|
new ScholixEntityId(p, null)
|
||||||
|
}(collection.breakOut)
|
||||||
|
|
||||||
|
if (l.nonEmpty)
|
||||||
|
s.setPublisher(l.asJava)
|
||||||
|
}
|
||||||
|
|
||||||
|
val semanticRelation = relations.getOrElse(relation.getRelClass.toLowerCase, null)
|
||||||
|
if (semanticRelation == null)
|
||||||
|
return null
|
||||||
|
s.setRelationship(
|
||||||
|
new ScholixRelationship(semanticRelation.original, "datacite", semanticRelation.inverse)
|
||||||
|
)
|
||||||
|
s.setSource(generateScholixResourceFromSummary(source))
|
||||||
|
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
def findURLForPID(
|
||||||
|
pidValue: List[StructuredProperty],
|
||||||
|
urls: List[String]
|
||||||
|
): List[(StructuredProperty, String)] = {
|
||||||
|
pidValue.map { p =>
|
||||||
|
val pv = p.getValue
|
||||||
|
|
||||||
|
val r = urls.find(u => u.toLowerCase.contains(pv.toLowerCase))
|
||||||
|
(p, r.orNull)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def extractTypedIdentifierFromInstance(r: Result): List[ScholixIdentifier] = {
|
||||||
|
if (r.getInstance() == null || r.getInstance().isEmpty)
|
||||||
|
return List()
|
||||||
|
r.getInstance()
|
||||||
|
.asScala
|
||||||
|
.filter(i => i.getUrl != null && !i.getUrl.isEmpty)
|
||||||
|
.filter(i => i.getPid != null && i.getUrl != null)
|
||||||
|
.flatMap(i => findURLForPID(i.getPid.asScala.toList, i.getUrl.asScala.toList))
|
||||||
|
.map(i => new ScholixIdentifier(i._1.getValue, i._1.getQualifier.getClassid, i._2))
|
||||||
|
.distinct
|
||||||
|
.toList
|
||||||
|
}
|
||||||
|
|
||||||
|
def resultToSummary(r: Result): ScholixSummary = {
|
||||||
|
val s = new ScholixSummary
|
||||||
|
s.setId(r.getId)
|
||||||
|
if (r.getPid == null || r.getPid.isEmpty)
|
||||||
|
return null
|
||||||
|
|
||||||
|
val persistentIdentifiers: List[ScholixIdentifier] = extractTypedIdentifierFromInstance(r)
|
||||||
|
if (persistentIdentifiers.isEmpty)
|
||||||
|
return null
|
||||||
|
s.setLocalIdentifier(persistentIdentifiers.asJava)
|
||||||
|
if (r.isInstanceOf[Publication])
|
||||||
|
s.setTypology(Typology.publication)
|
||||||
|
else
|
||||||
|
s.setTypology(Typology.dataset)
|
||||||
|
|
||||||
|
s.setSubType(r.getInstance().get(0).getInstancetype.getClassname)
|
||||||
|
|
||||||
|
if (r.getTitle != null && r.getTitle.asScala.nonEmpty) {
|
||||||
|
val titles: List[String] = r.getTitle.asScala.map(t => t.getValue).toList
|
||||||
|
if (titles.nonEmpty)
|
||||||
|
s.setTitle(titles.asJava)
|
||||||
|
else
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
if (r.getAuthor != null && !r.getAuthor.isEmpty) {
|
||||||
|
val authors: List[String] = r.getAuthor.asScala.map(a => a.getFullname).toList
|
||||||
|
if (authors.nonEmpty)
|
||||||
|
s.setAuthor(authors.asJava)
|
||||||
|
}
|
||||||
|
if (r.getInstance() != null) {
|
||||||
|
val dt: List[String] = r
|
||||||
|
.getInstance()
|
||||||
|
.asScala
|
||||||
|
.filter(i => i.getDateofacceptance != null)
|
||||||
|
.map(i => i.getDateofacceptance.getValue)
|
||||||
|
.toList
|
||||||
|
if (dt.nonEmpty)
|
||||||
|
s.setDate(dt.distinct.asJava)
|
||||||
|
}
|
||||||
|
if (r.getDescription != null && !r.getDescription.isEmpty) {
|
||||||
|
val d = r.getDescription.asScala.find(f => f != null && f.getValue != null)
|
||||||
|
if (d.isDefined)
|
||||||
|
s.setDescription(d.get.getValue)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (r.getSubject != null && !r.getSubject.isEmpty) {
|
||||||
|
val subjects: List[SchemeValue] = r.getSubject.asScala
|
||||||
|
.map(s => new SchemeValue(s.getQualifier.getClassname, s.getValue))
|
||||||
|
.toList
|
||||||
|
if (subjects.nonEmpty)
|
||||||
|
s.setSubject(subjects.asJava)
|
||||||
|
}
|
||||||
|
|
||||||
|
if (r.getPublisher != null)
|
||||||
|
s.setPublisher(List(r.getPublisher.getValue).asJava)
|
||||||
|
|
||||||
|
if (r.getCollectedfrom != null && !r.getCollectedfrom.isEmpty) {
|
||||||
|
val cf: List[CollectedFromType] = r.getCollectedfrom.asScala
|
||||||
|
.map(c => new CollectedFromType(c.getValue, c.getKey, "complete"))
|
||||||
|
.toList
|
||||||
|
if (cf.nonEmpty)
|
||||||
|
s.setDatasources(cf.distinct.asJava)
|
||||||
|
}
|
||||||
|
|
||||||
|
s.setRelatedDatasets(0)
|
||||||
|
s.setRelatedPublications(0)
|
||||||
|
s.setRelatedUnknown(0)
|
||||||
|
|
||||||
|
s
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -7,10 +7,10 @@ import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||||
import org.apache.commons.io.IOUtils;
|
import org.apache.commons.io.IOUtils;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
public class ArgumentApplicationParserTest {
|
class ArgumentApplicationParserTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testParseParameter() throws Exception {
|
void testParseParameter() throws Exception {
|
||||||
final String jsonConfiguration = IOUtils
|
final String jsonConfiguration = IOUtils
|
||||||
.toString(
|
.toString(
|
||||||
this.getClass().getResourceAsStream("/eu/dnetlib/application/parameters.json"));
|
this.getClass().getResourceAsStream("/eu/dnetlib/application/parameters.json"));
|
||||||
|
|
|
@ -21,13 +21,13 @@ public class HdfsSupportTest {
|
||||||
class Remove {
|
class Remove {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldThrowARuntimeExceptionOnError() {
|
void shouldThrowARuntimeExceptionOnError() {
|
||||||
// when
|
// when
|
||||||
assertThrows(RuntimeException.class, () -> HdfsSupport.remove(null, new Configuration()));
|
assertThrows(RuntimeException.class, () -> HdfsSupport.remove(null, new Configuration()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldRemoveADirFromHDFS(@TempDir Path tempDir) {
|
void shouldRemoveADirFromHDFS(@TempDir Path tempDir) {
|
||||||
// when
|
// when
|
||||||
HdfsSupport.remove(tempDir.toString(), new Configuration());
|
HdfsSupport.remove(tempDir.toString(), new Configuration());
|
||||||
|
|
||||||
|
@ -36,7 +36,7 @@ public class HdfsSupportTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldRemoveAFileFromHDFS(@TempDir Path tempDir) throws IOException {
|
void shouldRemoveAFileFromHDFS(@TempDir Path tempDir) throws IOException {
|
||||||
// given
|
// given
|
||||||
Path file = Files.createTempFile(tempDir, "p", "s");
|
Path file = Files.createTempFile(tempDir, "p", "s");
|
||||||
|
|
||||||
|
@ -52,13 +52,13 @@ public class HdfsSupportTest {
|
||||||
class ListFiles {
|
class ListFiles {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldThrowARuntimeExceptionOnError() {
|
void shouldThrowARuntimeExceptionOnError() {
|
||||||
// when
|
// when
|
||||||
assertThrows(RuntimeException.class, () -> HdfsSupport.listFiles(null, new Configuration()));
|
assertThrows(RuntimeException.class, () -> HdfsSupport.listFiles(null, new Configuration()));
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldListFilesLocatedInPath(@TempDir Path tempDir) throws IOException {
|
void shouldListFilesLocatedInPath(@TempDir Path tempDir) throws IOException {
|
||||||
Path subDir1 = Files.createTempDirectory(tempDir, "list_me");
|
Path subDir1 = Files.createTempDirectory(tempDir, "list_me");
|
||||||
Path subDir2 = Files.createTempDirectory(tempDir, "list_me");
|
Path subDir2 = Files.createTempDirectory(tempDir, "list_me");
|
||||||
|
|
||||||
|
|
|
@ -5,10 +5,10 @@ import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
public class PacePersonTest {
|
class PacePersonTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void pacePersonTest1() {
|
void pacePersonTest1() {
|
||||||
|
|
||||||
PacePerson p = new PacePerson("Artini, Michele", false);
|
PacePerson p = new PacePerson("Artini, Michele", false);
|
||||||
assertEquals("Artini", p.getSurnameString());
|
assertEquals("Artini", p.getSurnameString());
|
||||||
|
@ -17,7 +17,7 @@ public class PacePersonTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void pacePersonTest2() {
|
void pacePersonTest2() {
|
||||||
PacePerson p = new PacePerson("Michele G. Artini", false);
|
PacePerson p = new PacePerson("Michele G. Artini", false);
|
||||||
assertEquals("Artini, Michele G.", p.getNormalisedFullname());
|
assertEquals("Artini, Michele G.", p.getNormalisedFullname());
|
||||||
assertEquals("Michele G", p.getNameString());
|
assertEquals("Michele G", p.getNameString());
|
||||||
|
|
|
@ -18,7 +18,8 @@ public class SparkSessionSupportTest {
|
||||||
class RunWithSparkSession {
|
class RunWithSparkSession {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldExecuteFunctionAndNotStopSparkSessionWhenSparkSessionIsNotManaged()
|
@SuppressWarnings("unchecked")
|
||||||
|
void shouldExecuteFunctionAndNotStopSparkSessionWhenSparkSessionIsNotManaged()
|
||||||
throws Exception {
|
throws Exception {
|
||||||
// given
|
// given
|
||||||
SparkSession spark = mock(SparkSession.class);
|
SparkSession spark = mock(SparkSession.class);
|
||||||
|
@ -37,7 +38,8 @@ public class SparkSessionSupportTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void shouldExecuteFunctionAndStopSparkSessionWhenSparkSessionIsManaged()
|
@SuppressWarnings("unchecked")
|
||||||
|
void shouldExecuteFunctionAndStopSparkSessionWhenSparkSessionIsManaged()
|
||||||
throws Exception {
|
throws Exception {
|
||||||
// given
|
// given
|
||||||
SparkSession spark = mock(SparkSession.class);
|
SparkSession spark = mock(SparkSession.class);
|
||||||
|
|
|
@ -12,15 +12,39 @@ import org.junit.jupiter.api.Disabled;
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
@Disabled
|
@Disabled
|
||||||
public class ZenodoAPIClientTest {
|
class ZenodoAPIClientTest {
|
||||||
|
|
||||||
private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions";
|
private final String URL_STRING = "https://sandbox.zenodo.org/api/deposit/depositions";
|
||||||
private final String ACCESS_TOKEN = "";
|
private final String ACCESS_TOKEN = "";
|
||||||
|
|
||||||
private final String CONCEPT_REC_ID = "657113";
|
private final String CONCEPT_REC_ID = "657113";
|
||||||
|
|
||||||
|
private final String depositionId = "674915";
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testNewDeposition() throws IOException {
|
void testUploadOldDeposition() throws IOException, MissingConceptDoiException {
|
||||||
|
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
||||||
|
ACCESS_TOKEN);
|
||||||
|
Assertions.assertEquals(200, client.uploadOpenDeposition(depositionId));
|
||||||
|
|
||||||
|
File file = new File(getClass()
|
||||||
|
.getResource("/eu/dnetlib/dhp/common/api/COVID-19.json.gz")
|
||||||
|
.getPath());
|
||||||
|
|
||||||
|
InputStream is = new FileInputStream(file);
|
||||||
|
|
||||||
|
Assertions.assertEquals(200, client.uploadIS(is, "COVID-19.json.gz", file.length()));
|
||||||
|
|
||||||
|
String metadata = IOUtils.toString(getClass().getResourceAsStream("/eu/dnetlib/dhp/common/api/metadata.json"));
|
||||||
|
|
||||||
|
Assertions.assertEquals(200, client.sendMretadata(metadata));
|
||||||
|
|
||||||
|
Assertions.assertEquals(202, client.publish());
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testNewDeposition() throws IOException {
|
||||||
|
|
||||||
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
||||||
ACCESS_TOKEN);
|
ACCESS_TOKEN);
|
||||||
|
@ -43,7 +67,7 @@ public class ZenodoAPIClientTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testNewVersionNewName() throws IOException, MissingConceptDoiException {
|
void testNewVersionNewName() throws IOException, MissingConceptDoiException {
|
||||||
|
|
||||||
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
||||||
ACCESS_TOKEN);
|
ACCESS_TOKEN);
|
||||||
|
@ -63,7 +87,7 @@ public class ZenodoAPIClientTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testNewVersionOldName() throws IOException, MissingConceptDoiException {
|
void testNewVersionOldName() throws IOException, MissingConceptDoiException {
|
||||||
|
|
||||||
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
ZenodoAPIClient client = new ZenodoAPIClient(URL_STRING,
|
||||||
ACCESS_TOKEN);
|
ACCESS_TOKEN);
|
||||||
|
|
|
@ -1,16 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.model.mdstore;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
|
||||||
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
public class MetadataRecordTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void getTimestamp() {
|
|
||||||
|
|
||||||
MetadataRecord r = new MetadataRecord();
|
|
||||||
assertTrue(r.getDateOfCollection() > 0);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -0,0 +1,100 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.oa.merge;
|
||||||
|
|
||||||
|
import java.io.BufferedReader;
|
||||||
|
import java.io.FileReader;
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.nio.file.Paths;
|
||||||
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.junit.jupiter.api.Assertions;
|
||||||
|
import org.junit.jupiter.api.BeforeEach;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Author;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Publication;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.StructuredProperty;
|
||||||
|
import eu.dnetlib.pace.util.MapDocumentUtil;
|
||||||
|
import scala.Tuple2;
|
||||||
|
|
||||||
|
class AuthorMergerTest {
|
||||||
|
|
||||||
|
private String publicationsBasePath;
|
||||||
|
|
||||||
|
private List<List<Author>> authors;
|
||||||
|
|
||||||
|
@BeforeEach
|
||||||
|
public void setUp() throws Exception {
|
||||||
|
|
||||||
|
publicationsBasePath = Paths
|
||||||
|
.get(AuthorMergerTest.class.getResource("/eu/dnetlib/dhp/oa/merge").toURI())
|
||||||
|
.toFile()
|
||||||
|
.getAbsolutePath();
|
||||||
|
|
||||||
|
authors = readSample(publicationsBasePath + "/publications_with_authors.json", Publication.class)
|
||||||
|
.stream()
|
||||||
|
.map(p -> p._2().getAuthor())
|
||||||
|
.collect(Collectors.toList());
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void mergeTest() { // used in the dedup: threshold set to 0.95
|
||||||
|
|
||||||
|
for (List<Author> authors1 : authors) {
|
||||||
|
System.out.println("List " + (authors.indexOf(authors1) + 1));
|
||||||
|
for (Author author : authors1) {
|
||||||
|
System.out.println(authorToString(author));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
List<Author> merge = AuthorMerger.merge(authors);
|
||||||
|
|
||||||
|
System.out.println("Merge ");
|
||||||
|
for (Author author : merge) {
|
||||||
|
System.out.println(authorToString(author));
|
||||||
|
}
|
||||||
|
|
||||||
|
Assertions.assertEquals(7, merge.size());
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
public <T> List<Tuple2<String, T>> readSample(String path, Class<T> clazz) {
|
||||||
|
List<Tuple2<String, T>> res = new ArrayList<>();
|
||||||
|
BufferedReader reader;
|
||||||
|
try {
|
||||||
|
reader = new BufferedReader(new FileReader(path));
|
||||||
|
String line = reader.readLine();
|
||||||
|
while (line != null) {
|
||||||
|
res
|
||||||
|
.add(
|
||||||
|
new Tuple2<>(
|
||||||
|
MapDocumentUtil.getJPathString("$.id", line),
|
||||||
|
new ObjectMapper().readValue(line, clazz)));
|
||||||
|
// read next line
|
||||||
|
line = reader.readLine();
|
||||||
|
}
|
||||||
|
reader.close();
|
||||||
|
} catch (IOException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
}
|
||||||
|
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String authorToString(Author a) {
|
||||||
|
|
||||||
|
String print = "Fullname = ";
|
||||||
|
print += a.getFullname() + " pid = [";
|
||||||
|
if (a.getPid() != null)
|
||||||
|
for (StructuredProperty sp : a.getPid()) {
|
||||||
|
print += sp.toComparableString() + " ";
|
||||||
|
}
|
||||||
|
print += "]";
|
||||||
|
return print;
|
||||||
|
}
|
||||||
|
}
|
|
@ -1,47 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.oaf.utils;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import org.apache.commons.io.IOUtils;
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.DeserializationFeature;
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.Publication;
|
|
||||||
import eu.dnetlib.dhp.utils.DHPUtils;
|
|
||||||
|
|
||||||
public class IdentifierFactoryTest {
|
|
||||||
|
|
||||||
private static ObjectMapper OBJECT_MAPPER = new ObjectMapper()
|
|
||||||
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void testCreateIdentifierForPublication() throws IOException {
|
|
||||||
|
|
||||||
verifyIdentifier("publication_doi.json", "50|doi_________::" + DHPUtils.md5("10.1016/j.cmet.2011.03.013"));
|
|
||||||
verifyIdentifier("publication_pmc.json", "50|pmc_________::" + DHPUtils.md5("21459329"));
|
|
||||||
verifyIdentifier(
|
|
||||||
"publication_urn.json",
|
|
||||||
"50|urn_________::" + DHPUtils.md5("urn:nbn:nl:ui:29-f3ed5f9e-edf6-457e-8848-61b58a4075e2"));
|
|
||||||
|
|
||||||
final String defaultID = "50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1f";
|
|
||||||
verifyIdentifier("publication_3.json", defaultID);
|
|
||||||
verifyIdentifier("publication_4.json", defaultID);
|
|
||||||
verifyIdentifier("publication_5.json", defaultID);
|
|
||||||
}
|
|
||||||
|
|
||||||
protected void verifyIdentifier(String filename, String expectedID) throws IOException {
|
|
||||||
final String json = IOUtils.toString(getClass().getResourceAsStream(filename));
|
|
||||||
final Publication pub = OBJECT_MAPPER.readValue(json, Publication.class);
|
|
||||||
|
|
||||||
String id = IdentifierFactory.createIdentifier(pub);
|
|
||||||
|
|
||||||
assertNotNull(id);
|
|
||||||
assertEquals(expectedID, id);
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -0,0 +1,197 @@
|
||||||
|
|
||||||
|
package eu.dnetlib.dhp.schema.oaf.utils;
|
||||||
|
|
||||||
|
import static org.junit.jupiter.api.Assertions.*;
|
||||||
|
|
||||||
|
import java.io.IOException;
|
||||||
|
import java.util.HashSet;
|
||||||
|
import java.util.List;
|
||||||
|
import java.util.stream.Collectors;
|
||||||
|
|
||||||
|
import org.apache.commons.io.IOUtils;
|
||||||
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
|
import com.fasterxml.jackson.databind.DeserializationFeature;
|
||||||
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
|
|
||||||
|
import eu.dnetlib.dhp.schema.common.ModelConstants;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Dataset;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.KeyValue;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Publication;
|
||||||
|
import eu.dnetlib.dhp.schema.oaf.Result;
|
||||||
|
import me.xuender.unidecode.Unidecode;
|
||||||
|
|
||||||
|
class OafMapperUtilsTest {
|
||||||
|
|
||||||
|
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper()
|
||||||
|
.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false);
|
||||||
|
|
||||||
|
@Test
|
||||||
|
public void testUnidecode() {
|
||||||
|
|
||||||
|
assertEquals("Liu Ben Mu hiruzuSen tawa", Unidecode.decode("六本木ヒルズ森タワ"));
|
||||||
|
assertEquals("Nan Wu A Mi Tuo Fo", Unidecode.decode("南无阿弥陀佛"));
|
||||||
|
assertEquals("Yi Tiao Hui Zou Lu De Yu", Unidecode.decode("一条会走路的鱼"));
|
||||||
|
assertEquals("amidaniyorai", Unidecode.decode("あみだにょらい"));
|
||||||
|
assertEquals("T`owrk`iayi", Unidecode.decode("Թուրքիայի"));
|
||||||
|
assertEquals("Obzor tematiki", Unidecode.decode("Обзор тематики"));
|
||||||
|
assertEquals("GERMANSKIE IaZYKI", Unidecode.decode("ГЕРМАНСКИЕ ЯЗЫКИ"));
|
||||||
|
assertEquals("Diereunese tes ikanopoieses", Unidecode.decode("Διερεύνηση της ικανοποίησης"));
|
||||||
|
assertEquals("lqDy l'wly@", Unidecode.decode("القضايا الأولية"));
|
||||||
|
assertEquals("abc def ghi", Unidecode.decode("abc def ghi"));
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testDateValidation() {
|
||||||
|
|
||||||
|
assertTrue(GraphCleaningFunctions.doCleanDate("2016-05-07T12:41:19.202Z ").isPresent());
|
||||||
|
assertTrue(GraphCleaningFunctions.doCleanDate("2020-09-10 11:08:52 ").isPresent());
|
||||||
|
assertTrue(GraphCleaningFunctions.doCleanDate(" 2016-04-05").isPresent());
|
||||||
|
|
||||||
|
assertEquals("2016-04-05", GraphCleaningFunctions.doCleanDate("2016 Apr 05").get());
|
||||||
|
|
||||||
|
assertEquals("2009-05-08", GraphCleaningFunctions.doCleanDate("May 8, 2009 5:57:51 PM").get());
|
||||||
|
assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("oct 7, 1970").get());
|
||||||
|
assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("oct 7, '70").get());
|
||||||
|
assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("oct. 7, 1970").get());
|
||||||
|
assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("oct. 7, 70").get());
|
||||||
|
assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("Mon Jan 2 15:04:05 2006").get());
|
||||||
|
assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("Mon Jan 2 15:04:05 MST 2006").get());
|
||||||
|
assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("Mon Jan 02 15:04:05 -0700 2006").get());
|
||||||
|
assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("Monday, 02-Jan-06 15:04:05 MST").get());
|
||||||
|
assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("Mon, 02 Jan 2006 15:04:05 MST").get());
|
||||||
|
assertEquals("2017-07-11", GraphCleaningFunctions.doCleanDate("Tue, 11 Jul 2017 16:28:13 +0200 (CEST)").get());
|
||||||
|
assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("Mon, 02 Jan 2006 15:04:05 -0700").get());
|
||||||
|
assertEquals("2018-01-04", GraphCleaningFunctions.doCleanDate("Thu, 4 Jan 2018 17:53:36 +0000").get());
|
||||||
|
assertEquals("2015-08-10", GraphCleaningFunctions.doCleanDate("Mon Aug 10 15:44:11 UTC+0100 2015").get());
|
||||||
|
assertEquals(
|
||||||
|
"2015-07-03",
|
||||||
|
GraphCleaningFunctions.doCleanDate("Fri Jul 03 2015 18:04:07 GMT+0100 (GMT Daylight Time)").get());
|
||||||
|
assertEquals("2012-09-17", GraphCleaningFunctions.doCleanDate("September 17, 2012 10:09am").get());
|
||||||
|
assertEquals("2012-09-17", GraphCleaningFunctions.doCleanDate("September 17, 2012 at 10:09am PST-08").get());
|
||||||
|
assertEquals("2012-09-17", GraphCleaningFunctions.doCleanDate("September 17, 2012, 10:10:09").get());
|
||||||
|
assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("October 7, 1970").get());
|
||||||
|
assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("October 7th, 1970").get());
|
||||||
|
assertEquals("2006-02-12", GraphCleaningFunctions.doCleanDate("12 Feb 2006, 19:17").get());
|
||||||
|
assertEquals("2006-02-12", GraphCleaningFunctions.doCleanDate("12 Feb 2006 19:17").get());
|
||||||
|
assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("7 oct 70").get());
|
||||||
|
assertEquals("1970-10-07", GraphCleaningFunctions.doCleanDate("7 oct 1970").get());
|
||||||
|
assertEquals("2013-02-03", GraphCleaningFunctions.doCleanDate("03 February 2013").get());
|
||||||
|
assertEquals("2013-07-01", GraphCleaningFunctions.doCleanDate("1 July 2013").get());
|
||||||
|
assertEquals("2013-02-03", GraphCleaningFunctions.doCleanDate("2013-Feb-03").get());
|
||||||
|
assertEquals("2014-03-31", GraphCleaningFunctions.doCleanDate("3/31/2014").get());
|
||||||
|
assertEquals("2014-03-31", GraphCleaningFunctions.doCleanDate("03/31/2014").get());
|
||||||
|
assertEquals("1971-08-21", GraphCleaningFunctions.doCleanDate("08/21/71").get());
|
||||||
|
assertEquals("1971-01-08", GraphCleaningFunctions.doCleanDate("8/1/71").get());
|
||||||
|
assertEquals("2014-08-04", GraphCleaningFunctions.doCleanDate("4/8/2014 22:05").get());
|
||||||
|
assertEquals("2014-08-04", GraphCleaningFunctions.doCleanDate("04/08/2014 22:05").get());
|
||||||
|
assertEquals("2014-08-04", GraphCleaningFunctions.doCleanDate("4/8/14 22:05").get());
|
||||||
|
assertEquals("2014-02-04", GraphCleaningFunctions.doCleanDate("04/2/2014 03:00:51").get());
|
||||||
|
assertEquals("1965-08-08", GraphCleaningFunctions.doCleanDate("8/8/1965 12:00:00 AM").get());
|
||||||
|
assertEquals("1965-08-08", GraphCleaningFunctions.doCleanDate("8/8/1965 01:00:01 PM").get());
|
||||||
|
assertEquals("1965-08-08", GraphCleaningFunctions.doCleanDate("8/8/1965 01:00 PM").get());
|
||||||
|
assertEquals("1965-08-08", GraphCleaningFunctions.doCleanDate("8/8/1965 1:00 PM").get());
|
||||||
|
assertEquals("1965-08-08", GraphCleaningFunctions.doCleanDate("8/8/1965 12:00 AM").get());
|
||||||
|
assertEquals("2014-02-04", GraphCleaningFunctions.doCleanDate("4/02/2014 03:00:51").get());
|
||||||
|
assertEquals("2012-03-19", GraphCleaningFunctions.doCleanDate("03/19/2012 10:11:59").get());
|
||||||
|
assertEquals("2012-03-19", GraphCleaningFunctions.doCleanDate("03/19/2012 10:11:59.3186369").get());
|
||||||
|
assertEquals("2014-03-31", GraphCleaningFunctions.doCleanDate("2014/3/31").get());
|
||||||
|
assertEquals("2014-03-31", GraphCleaningFunctions.doCleanDate("2014/03/31").get());
|
||||||
|
assertEquals("2014-04-08", GraphCleaningFunctions.doCleanDate("2014/4/8 22:05").get());
|
||||||
|
assertEquals("2014-04-08", GraphCleaningFunctions.doCleanDate("2014/04/08 22:05").get());
|
||||||
|
assertEquals("2014-04-02", GraphCleaningFunctions.doCleanDate("2014/04/2 03:00:51").get());
|
||||||
|
assertEquals("2014-04-02", GraphCleaningFunctions.doCleanDate("2014/4/02 03:00:51").get());
|
||||||
|
assertEquals("2012-03-19", GraphCleaningFunctions.doCleanDate("2012/03/19 10:11:59").get());
|
||||||
|
assertEquals("2012-03-19", GraphCleaningFunctions.doCleanDate("2012/03/19 10:11:59.3186369").get());
|
||||||
|
assertEquals("2014-04-08", GraphCleaningFunctions.doCleanDate("2014年04月08日").get());
|
||||||
|
assertEquals("2006-01-02", GraphCleaningFunctions.doCleanDate("2006-01-02T15:04:05+0000").get());
|
||||||
|
assertEquals("2009-08-13", GraphCleaningFunctions.doCleanDate("2009-08-12T22:15:09-07:00").get());
|
||||||
|
assertEquals("2009-08-12", GraphCleaningFunctions.doCleanDate("2009-08-12T22:15:09").get());
|
||||||
|
assertEquals("2009-08-13", GraphCleaningFunctions.doCleanDate("2009-08-12T22:15:09Z").get());
|
||||||
|
assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26 17:24:37.3186369").get());
|
||||||
|
assertEquals("2012-08-03", GraphCleaningFunctions.doCleanDate("2012-08-03 18:31:59.257000000").get());
|
||||||
|
assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26 17:24:37.123").get());
|
||||||
|
assertEquals("2013-04-01", GraphCleaningFunctions.doCleanDate("2013-04-01 22:43").get());
|
||||||
|
assertEquals("2013-04-01", GraphCleaningFunctions.doCleanDate("2013-04-01 22:43:22").get());
|
||||||
|
assertEquals("2014-12-16", GraphCleaningFunctions.doCleanDate("2014-12-16 06:20:00 UTC").get());
|
||||||
|
assertEquals("2014-12-16", GraphCleaningFunctions.doCleanDate("2014-12-16 06:20:00 GMT").get());
|
||||||
|
assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26 05:24:37 PM").get());
|
||||||
|
assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26 13:13:43 +0800").get());
|
||||||
|
assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26 13:13:43 +0800 +08").get());
|
||||||
|
assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26 13:13:44 +09:00").get());
|
||||||
|
assertEquals("2012-08-03", GraphCleaningFunctions.doCleanDate("2012-08-03 18:31:59.257000000 +0000 UTC").get());
|
||||||
|
assertEquals("2015-09-30", GraphCleaningFunctions.doCleanDate("2015-09-30 18:48:56.35272715 +0000 UTC").get());
|
||||||
|
assertEquals("2015-02-18", GraphCleaningFunctions.doCleanDate("2015-02-18 00:12:00 +0000 GMT").get());
|
||||||
|
assertEquals("2015-02-18", GraphCleaningFunctions.doCleanDate("2015-02-18 00:12:00 +0000 UTC").get());
|
||||||
|
assertEquals(
|
||||||
|
"2015-02-08", GraphCleaningFunctions.doCleanDate("2015-02-08 03:02:00 +0300 MSK m=+0.000000001").get());
|
||||||
|
assertEquals(
|
||||||
|
"2015-02-08", GraphCleaningFunctions.doCleanDate("2015-02-08 03:02:00.001 +0300 MSK m=+0.000000001").get());
|
||||||
|
assertEquals("2017-07-19", GraphCleaningFunctions.doCleanDate("2017-07-19 03:21:51+00:00").get());
|
||||||
|
assertEquals("2014-04-26", GraphCleaningFunctions.doCleanDate("2014-04-26").get());
|
||||||
|
assertEquals("2014-04-01", GraphCleaningFunctions.doCleanDate("2014-04").get());
|
||||||
|
assertEquals("2014-01-01", GraphCleaningFunctions.doCleanDate("2014").get());
|
||||||
|
assertEquals("2014-05-11", GraphCleaningFunctions.doCleanDate("2014-05-11 08:20:13,787").get());
|
||||||
|
assertEquals("2014-03-31", GraphCleaningFunctions.doCleanDate("3.31.2014").get());
|
||||||
|
assertEquals("2014-03-31", GraphCleaningFunctions.doCleanDate("03.31.2014").get());
|
||||||
|
assertEquals("1971-08-21", GraphCleaningFunctions.doCleanDate("08.21.71").get());
|
||||||
|
assertEquals("2014-03-01", GraphCleaningFunctions.doCleanDate("2014.03").get());
|
||||||
|
assertEquals("2014-03-30", GraphCleaningFunctions.doCleanDate("2014.03.30").get());
|
||||||
|
assertEquals("2014-06-01", GraphCleaningFunctions.doCleanDate("20140601").get());
|
||||||
|
assertEquals("2014-07-22", GraphCleaningFunctions.doCleanDate("20140722105203").get());
|
||||||
|
assertEquals("2012-03-19", GraphCleaningFunctions.doCleanDate("1332151919").get());
|
||||||
|
assertEquals("2013-11-12", GraphCleaningFunctions.doCleanDate("1384216367189").get());
|
||||||
|
assertEquals("2013-11-12", GraphCleaningFunctions.doCleanDate("1384216367111222").get());
|
||||||
|
assertEquals("2013-11-12", GraphCleaningFunctions.doCleanDate("1384216367111222333").get());
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testDate() {
|
||||||
|
final String date = GraphCleaningFunctions.cleanDate("23-FEB-1998");
|
||||||
|
assertNotNull(date);
|
||||||
|
System.out.println(date);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void testMergePubs() throws IOException {
|
||||||
|
Publication p1 = read("publication_1.json", Publication.class);
|
||||||
|
Publication p2 = read("publication_2.json", Publication.class);
|
||||||
|
Dataset d1 = read("dataset_1.json", Dataset.class);
|
||||||
|
Dataset d2 = read("dataset_2.json", Dataset.class);
|
||||||
|
|
||||||
|
assertEquals(1, p1.getCollectedfrom().size());
|
||||||
|
assertEquals(ModelConstants.CROSSREF_ID, p1.getCollectedfrom().get(0).getKey());
|
||||||
|
assertEquals(1, d2.getCollectedfrom().size());
|
||||||
|
assertFalse(cfId(d2.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
|
||||||
|
|
||||||
|
assertEquals(
|
||||||
|
ModelConstants.PUBLICATION_RESULTTYPE_CLASSID,
|
||||||
|
OafMapperUtils
|
||||||
|
.mergeResults(p1, d2)
|
||||||
|
.getResulttype()
|
||||||
|
.getClassid());
|
||||||
|
|
||||||
|
assertEquals(1, p2.getCollectedfrom().size());
|
||||||
|
assertFalse(cfId(p2.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
|
||||||
|
assertEquals(1, d1.getCollectedfrom().size());
|
||||||
|
assertTrue(cfId(d1.getCollectedfrom()).contains(ModelConstants.CROSSREF_ID));
|
||||||
|
|
||||||
|
assertEquals(
|
||||||
|
ModelConstants.DATASET_RESULTTYPE_CLASSID,
|
||||||
|
OafMapperUtils
|
||||||
|
.mergeResults(p2, d1)
|
||||||
|
.getResulttype()
|
||||||
|
.getClassid());
|
||||||
|
}
|
||||||
|
|
||||||
|
protected HashSet<String> cfId(List<KeyValue> collectedfrom) {
|
||||||
|
return collectedfrom.stream().map(KeyValue::getKey).collect(Collectors.toCollection(HashSet::new));
|
||||||
|
}
|
||||||
|
|
||||||
|
protected <T extends Result> T read(String filename, Class<T> clazz) throws IOException {
|
||||||
|
final String json = IOUtils.toString(getClass().getResourceAsStream(filename));
|
||||||
|
return OBJECT_MAPPER.readValue(json, clazz);
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -1,51 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.message;
|
|
||||||
|
|
||||||
import static org.junit.jupiter.api.Assertions.*;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.Map;
|
|
||||||
|
|
||||||
import org.junit.jupiter.api.Test;
|
|
||||||
|
|
||||||
public class MessageTest {
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void fromJsonTest() throws IOException {
|
|
||||||
Message m = new Message();
|
|
||||||
m.setWorkflowId("wId");
|
|
||||||
m.setType(MessageType.ONGOING);
|
|
||||||
m.setJobName("Collection");
|
|
||||||
Map<String, String> body = new HashMap<>();
|
|
||||||
body.put("parsedItem", "300");
|
|
||||||
body.put("ExecutionTime", "30s");
|
|
||||||
|
|
||||||
m.setBody(body);
|
|
||||||
System.out.println("m = " + m);
|
|
||||||
Message m1 = Message.fromJson(m.toString());
|
|
||||||
assertEquals(m1.getWorkflowId(), m.getWorkflowId());
|
|
||||||
assertEquals(m1.getType(), m.getType());
|
|
||||||
assertEquals(m1.getJobName(), m.getJobName());
|
|
||||||
|
|
||||||
assertNotNull(m1.getBody());
|
|
||||||
m1.getBody().keySet().forEach(it -> assertEquals(m1.getBody().get(it), m.getBody().get(it)));
|
|
||||||
assertEquals(m1.getJobName(), m.getJobName());
|
|
||||||
}
|
|
||||||
|
|
||||||
@Test
|
|
||||||
public void toStringTest() {
|
|
||||||
final String expectedJson = "{\"workflowId\":\"wId\",\"jobName\":\"Collection\",\"type\":\"ONGOING\",\"body\":{\"ExecutionTime\":\"30s\",\"parsedItem\":\"300\"}}";
|
|
||||||
Message m = new Message();
|
|
||||||
m.setWorkflowId("wId");
|
|
||||||
m.setType(MessageType.ONGOING);
|
|
||||||
m.setJobName("Collection");
|
|
||||||
Map<String, String> body = new HashMap<>();
|
|
||||||
body.put("parsedItem", "300");
|
|
||||||
body.put("ExecutionTime", "30s");
|
|
||||||
|
|
||||||
m.setBody(body);
|
|
||||||
|
|
||||||
assertEquals(expectedJson, m.toString());
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -3,10 +3,10 @@ package eu.dnetlib.scholexplorer.relation;
|
||||||
|
|
||||||
import org.junit.jupiter.api.Test;
|
import org.junit.jupiter.api.Test;
|
||||||
|
|
||||||
public class RelationMapperTest {
|
class RelationMapperTest {
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
public void testLoadRels() throws Exception {
|
void testLoadRels() throws Exception {
|
||||||
|
|
||||||
RelationMapper relationMapper = RelationMapper.load();
|
RelationMapper relationMapper = RelationMapper.load();
|
||||||
relationMapper.keySet().forEach(System.out::println);
|
relationMapper.keySet().forEach(System.out::println);
|
||||||
|
|
File diff suppressed because one or more lines are too long
|
@ -0,0 +1 @@
|
||||||
|
{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1g", "resuttype" : { "classid" : "dataset" }, "pid":[{"qualifier":{"classid":"doi"},"value":"10.1016/j.cmet.2011.03.013"},{"qualifier":{"classid":"urn"},"value":"urn:nbn:nl:ui:29-f3ed5f9e-edf6-457e-8848-61b58a4075e2"},{"qualifier":{"classid":"scp-number"},"value":"79953761260"},{"qualifier":{"classid":"pmc"},"value":"21459329"}], "collectedfrom" : [ { "key" : "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value" : "Crossref"} ]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1g", "resuttype" : { "classid" : "dataset" }, "pid":[{"qualifier":{"classid":"doi"},"value":"10.1016/j.cmet.2011.03.013"},{"qualifier":{"classid":"urn"},"value":"urn:nbn:nl:ui:29-f3ed5f9e-edf6-457e-8848-61b58a4075e2"},{"qualifier":{"classid":"scp-number"},"value":"79953761260"},{"qualifier":{"classid":"pmc"},"value":"21459329"}], "collectedfrom" : [ { "key" : "10|openaire____::081b82f96300b6a6e3d282bad31cb6e3", "value" : "Repository B"} ]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1f", "resuttype" : { "classid" : "publication" }, "pid":[{"qualifier":{"classid":"doi"},"value":"10.1016/j.cmet.2011.03.013"},{"qualifier":{"classid":"urn"},"value":"urn:nbn:nl:ui:29-f3ed5f9e-edf6-457e-8848-61b58a4075e2"},{"qualifier":{"classid":"scp-number"},"value":"79953761260"},{"qualifier":{"classid":"pmc"},"value":"21459329"}], "collectedfrom" : [ { "key" : "10|openaire____::081b82f96300b6a6e3d282bad31cb6e2", "value" : "Crossref"} ]}
|
|
@ -0,0 +1 @@
|
||||||
|
{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1f", "resuttype" : { "classid" : "publication" }, "pid":[{"qualifier":{"classid":"doi"},"value":"10.1016/j.cmet.2011.03.013"},{"qualifier":{"classid":"urn"},"value":"urn:nbn:nl:ui:29-f3ed5f9e-edf6-457e-8848-61b58a4075e2"},{"qualifier":{"classid":"scp-number"},"value":"79953761260"},{"qualifier":{"classid":"pmc"},"value":"21459329"}], "collectedfrom" : [ { "key" : "10|openaire____::081b82f96300b6a6e3d282bad31cb6e3", "value" : "Repository A"} ]}
|
|
@ -1 +0,0 @@
|
||||||
{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1f","pid":[{"qualifier":{"classid":"scp-number"},"value":"79953761260"}]}
|
|
|
@ -1 +0,0 @@
|
||||||
{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1f","pid":[]}
|
|
|
@ -1 +0,0 @@
|
||||||
{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1f"}
|
|
|
@ -1 +0,0 @@
|
||||||
{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1f","pid":[{"qualifier":{"classid":"doi"},"value":"10.1016/j.cmet.2011.03.013"},{"qualifier":{"classid":"urn"},"value":"urn:nbn:nl:ui:29-f3ed5f9e-edf6-457e-8848-61b58a4075e2"},{"qualifier":{"classid":"scp-number"},"value":"79953761260"},{"qualifier":{"classid":"pmc"},"value":"21459329"}]}
|
|
|
@ -1 +0,0 @@
|
||||||
{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1f","pid":[{"qualifier":{"classid":"urn"},"value":"urn:nbn:nl:ui:29-f3ed5f9e-edf6-457e-8848-61b58a4075e2"},{"qualifier":{"classid":"scp-number"},"value":"79953761260"},{"qualifier":{"classid":"pmc"},"value":"21459329"}]}
|
|
|
@ -1 +0,0 @@
|
||||||
{"id":"50|DansKnawCris::0829b5191605bdbea36d6502b8c1ce1f","pid":[{"qualifier":{"classid":"urn"},"value":"urn:nbn:nl:ui:29-f3ed5f9e-edf6-457e-8848-61b58a4075e2"},{"qualifier":{"classid":"scp-number"},"value":"79953761260"},{"qualifier":{"classid":"pmcid"},"value":"21459329"}]}
|
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
|
@ -1,11 +0,0 @@
|
||||||
Description of the project
|
|
||||||
--------------------------
|
|
||||||
This project defines **object schemas** of the OpenAIRE main entities and the relationships that intercur among them.
|
|
||||||
Namely it defines the model for
|
|
||||||
|
|
||||||
- **research product (result)** which subclasses in publication, dataset, other research product, software
|
|
||||||
- **data source** object describing the data provider (institutional repository, aggregators, cris systems)
|
|
||||||
- **organization** research bodies managing a data source or participating to a research project
|
|
||||||
- **project** research project
|
|
||||||
|
|
||||||
Te serialization of such objects (data store files) are used to pass data between workflow nodes in the processing pipeline.
|
|
|
@ -1,73 +0,0 @@
|
||||||
<?xml version="1.0" encoding="UTF-8"?>
|
|
||||||
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
|
||||||
<modelVersion>4.0.0</modelVersion>
|
|
||||||
|
|
||||||
<parent>
|
|
||||||
<groupId>eu.dnetlib.dhp</groupId>
|
|
||||||
<artifactId>dhp</artifactId>
|
|
||||||
<version>1.2.4-SNAPSHOT</version>
|
|
||||||
<relativePath>../</relativePath>
|
|
||||||
</parent>
|
|
||||||
|
|
||||||
<artifactId>dhp-schemas</artifactId>
|
|
||||||
<packaging>jar</packaging>
|
|
||||||
|
|
||||||
<description>This module contains common schema classes meant to be used across the dnet-hadoop submodules</description>
|
|
||||||
|
|
||||||
<build>
|
|
||||||
<plugins>
|
|
||||||
<plugin>
|
|
||||||
<groupId>net.alchim31.maven</groupId>
|
|
||||||
<artifactId>scala-maven-plugin</artifactId>
|
|
||||||
<version>4.0.1</version>
|
|
||||||
<executions>
|
|
||||||
<execution>
|
|
||||||
<id>scala-compile-first</id>
|
|
||||||
<phase>initialize</phase>
|
|
||||||
<goals>
|
|
||||||
<goal>add-source</goal>
|
|
||||||
<goal>compile</goal>
|
|
||||||
</goals>
|
|
||||||
</execution>
|
|
||||||
<execution>
|
|
||||||
<id>scala-test-compile</id>
|
|
||||||
<phase>process-test-resources</phase>
|
|
||||||
<goals>
|
|
||||||
<goal>testCompile</goal>
|
|
||||||
</goals>
|
|
||||||
</execution>
|
|
||||||
</executions>
|
|
||||||
<configuration>
|
|
||||||
<scalaVersion>${scala.version}</scalaVersion>
|
|
||||||
</configuration>
|
|
||||||
</plugin>
|
|
||||||
|
|
||||||
</plugins>
|
|
||||||
</build>
|
|
||||||
|
|
||||||
<dependencies>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>commons-io</groupId>
|
|
||||||
<artifactId>commons-io</artifactId>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>org.apache.commons</groupId>
|
|
||||||
<artifactId>commons-lang3</artifactId>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.fasterxml.jackson.core</groupId>
|
|
||||||
<artifactId>jackson-databind</artifactId>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
<dependency>
|
|
||||||
<groupId>com.google.guava</groupId>
|
|
||||||
<artifactId>guava</artifactId>
|
|
||||||
</dependency>
|
|
||||||
|
|
||||||
</dependencies>
|
|
||||||
|
|
||||||
|
|
||||||
</project>
|
|
|
@ -1,40 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.action;
|
|
||||||
|
|
||||||
import java.io.Serializable;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.Oaf;
|
|
||||||
|
|
||||||
@JsonDeserialize(using = AtomicActionDeserializer.class)
|
|
||||||
public class AtomicAction<T extends Oaf> implements Serializable {
|
|
||||||
|
|
||||||
private Class<T> clazz;
|
|
||||||
|
|
||||||
private T payload;
|
|
||||||
|
|
||||||
public AtomicAction() {
|
|
||||||
}
|
|
||||||
|
|
||||||
public AtomicAction(Class<T> clazz, T payload) {
|
|
||||||
this.clazz = clazz;
|
|
||||||
this.payload = payload;
|
|
||||||
}
|
|
||||||
|
|
||||||
public Class<T> getClazz() {
|
|
||||||
return clazz;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setClazz(Class<T> clazz) {
|
|
||||||
this.clazz = clazz;
|
|
||||||
}
|
|
||||||
|
|
||||||
public T getPayload() {
|
|
||||||
return payload;
|
|
||||||
}
|
|
||||||
|
|
||||||
public void setPayload(T payload) {
|
|
||||||
this.payload = payload;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,32 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.action;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
|
|
||||||
import com.fasterxml.jackson.core.JsonParser;
|
|
||||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
|
||||||
import com.fasterxml.jackson.databind.DeserializationContext;
|
|
||||||
import com.fasterxml.jackson.databind.JsonDeserializer;
|
|
||||||
import com.fasterxml.jackson.databind.JsonNode;
|
|
||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.Oaf;
|
|
||||||
|
|
||||||
public class AtomicActionDeserializer extends JsonDeserializer {
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Object deserialize(JsonParser jp, DeserializationContext ctxt)
|
|
||||||
throws IOException {
|
|
||||||
JsonNode node = jp.getCodec().readTree(jp);
|
|
||||||
String classTag = node.get("clazz").asText();
|
|
||||||
JsonNode payload = node.get("payload");
|
|
||||||
ObjectMapper mapper = new ObjectMapper();
|
|
||||||
|
|
||||||
try {
|
|
||||||
final Class<?> clazz = Class.forName(classTag);
|
|
||||||
return new AtomicAction(clazz, (Oaf) mapper.readValue(payload.toString(), clazz));
|
|
||||||
} catch (ClassNotFoundException e) {
|
|
||||||
throw new IOException(e);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,21 +0,0 @@
|
||||||
|
|
||||||
package eu.dnetlib.dhp.schema.common;
|
|
||||||
|
|
||||||
import eu.dnetlib.dhp.schema.oaf.OafEntity;
|
|
||||||
|
|
||||||
/** Actual entity types in the Graph */
|
|
||||||
public enum EntityType {
|
|
||||||
publication, dataset, otherresearchproduct, software, datasource, organization, project;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Resolves the EntityType, given the relative class name
|
|
||||||
*
|
|
||||||
* @param clazz the given class name
|
|
||||||
* @param <T> actual OafEntity subclass
|
|
||||||
* @return the EntityType associated to the given class
|
|
||||||
*/
|
|
||||||
public static <T extends OafEntity> EntityType fromClass(Class<T> clazz) {
|
|
||||||
|
|
||||||
return EntityType.valueOf(clazz.getSimpleName().toLowerCase());
|
|
||||||
}
|
|
||||||
}
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue