From fe85439e05d8190ab63e745fdf4dee68d2e1a0c0 Mon Sep 17 00:00:00 2001 From: Luca Frosini Date: Wed, 23 Jan 2019 15:24:45 +0000 Subject: [PATCH] Implementing new Solution refs #13207 git-svn-id: https://svn.d4science.research-infrastructures.eu/gcube/trunk/social-networking/social-util-library@176752 82a268e6-3cf1-43bd-a215-b396298e98cf --- .../tokenizer/StringTokenizer.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/java/org/gcube/socialnetworking/tokenizer/StringTokenizer.java b/src/main/java/org/gcube/socialnetworking/tokenizer/StringTokenizer.java index 491e590..b782e05 100644 --- a/src/main/java/org/gcube/socialnetworking/tokenizer/StringTokenizer.java +++ b/src/main/java/org/gcube/socialnetworking/tokenizer/StringTokenizer.java @@ -19,20 +19,25 @@ public class StringTokenizer { this.pattern = Pattern.compile("\\s"); this.matcher = pattern.matcher(originalString); } + + protected Token getToken(int tokenStart) { + int tokenEnd = matcher.start(); + int delimiterStart = tokenEnd; + int delimiterEnd = matcher.end(); + String tokenString = originalString.substring(tokenStart, tokenEnd); + String delimiter = originalString.substring(delimiterStart, delimiterEnd); + Token token = new Token(tokenString, delimiter, tokenStart, tokenEnd); + return token; + } public List getTokens() { if(tokens==null) { tokens = new ArrayList<>(); int tokenStart = 0; while(matcher.find()) { - int tokenEnd = matcher.start(); - String tokenString = originalString.substring(tokenStart, tokenEnd); - int delimiterStart = tokenEnd; - int delimiterEnd = matcher.end(); - String delimiter = originalString.substring(delimiterStart, delimiterEnd); - Token token = new Token(tokenString, delimiter, tokenStart, tokenEnd); + Token token = getToken(tokenStart); tokens.add(token); - tokenStart = delimiterEnd; + tokenStart = matcher.end(); } if(tokenStart!=originalString.length()){ int tokenEnd = originalString.length();