diff --git a/interactive-mining-madoap/madoap/src/madserverv2.py b/interactive-mining-madoap/madoap/src/madserverv2.py index 64a4a17..042864b 100755 --- a/interactive-mining-madoap/madoap/src/madserverv2.py +++ b/interactive-mining-madoap/madoap/src/madserverv2.py @@ -113,6 +113,16 @@ def numberOfDocsUploaded(user_id): return num_lines return 0 +def deleteAllUserFiles(user_id): + if user_id: + file_name = "/tmp/p%s.tsv" % (user_id) + if os.path.isfile(file_name): + os.remove(file_name) + file_name = "/tmp/docs%s.json" % (user_id) + if os.path.isfile(file_name): + os.remove(file_name) + + class BaseHandler(ozhandler.DjangoErrorMixin, ozhandler.BasicAuthMixin, tornado.web.RequestHandler): def __init__(self, *args): tornado.web.RequestHandler.__init__(self, *args) @@ -299,9 +309,6 @@ class createUploadProfileHandler(BaseHandler): def get(self): if 'data' in self.request.arguments: return - elif 'reset' in self.request.arguments: - # TODO RESET everything - print "TODO RESET everything" else: # check if we already gave client a user_id user_id = self.get_secure_cookie('madgikmining') @@ -373,19 +380,23 @@ class uploadCodesHandler(BaseHandler): def get(self): if 'data' in self.request.arguments: return - else: - # check if we already gave client a user_id - user_id = self.get_secure_cookie('madgikmining') - if user_id is None: - return - # check if he already uploaded his grants ids and inform him via a message - self.render('upload_codes.html', settings=msettings) + # check if we gave client a user_id + user_id = self.get_secure_cookie('madgikmining') + if user_id is None: + return + if 'new' in self.request.arguments and self.request.arguments['new'][0] == '1': + msettings.RESET_FIELDS = 1 + # reset everything + deleteAllUserFiles(user_id) + # check if he already uploaded his grants ids and inform him via a message + self.render('upload_codes.html', settings=msettings) def post(self): try: # get user id from cookie. Must have user_id = self.get_secure_cookie('madgikmining') if user_id is None: return + # service to upload a tsv file with the codes. Returns the codes if 'upload' in self.request.files: # get file info and body from post data fileinfo = self.request.files['upload'][0] @@ -398,7 +409,7 @@ class uploadCodesHandler(BaseHandler): codes = {} lines = fileinfo['body'].splitlines() for line in lines: - columns = re.split(r'\t+', line.rstrip('\t')) + columns = re.split(r'\t+', line.rstrip('\t\n\r')) if len(columns) and columns[0] == '': continue elif len(columns) > 1: @@ -415,12 +426,15 @@ class uploadCodesHandler(BaseHandler): self.set_secure_cookie('madgikmining_grantsuploaded', str(len(lines))) self.write(json.dumps(data)) self.finish() + # service to store the final user codes. Returns the number of the codes elif 'concepts' in self.request.arguments and self.request.arguments['concepts'][0] != '{}': # write data to physical file cname = "/tmp/p{0}.tsv".format(user_id) fh = open(cname, 'w') concepts = json.loads(self.request.arguments['concepts'][0]) for key, value in concepts.iteritems(): + if key == '': + continue fh.write("{0}\t{1}\n".format(key,value)) fh.close() # data to be sent @@ -432,6 +446,28 @@ class uploadCodesHandler(BaseHandler): self.set_secure_cookie('madgikmining_grantsuploaded', str(len(concepts))) self.write(json.dumps(data)) self.finish() + # service to return the already uploaded user codes + elif 'already' in self.request.arguments: + data = {} + data['data'] = {} + file_name = "/tmp/p%s.tsv" % (user_id) + if os.path.isfile(file_name): + codes = {} + num_lines = 0 + for line in open(file_name): + columns = re.split(r'\t+', line.rstrip('\t\n\r')) + if len(columns) and columns[0] == '': + continue + elif len(columns) > 1: + codes[columns[0]] = columns[1] + elif len(columns) == 1: + codes[columns[0]] = '' + num_lines += 1 + cookie = self.get_secure_cookie('madgikmining_grantsuploaded') + if cookie and str(num_lines) == cookie: + data['data'] = codes + self.write(json.dumps(data)) + self.finish() except Exception as ints: data = {} @@ -452,9 +488,12 @@ class configureProfileHandler(BaseHandler): # check if we already gave client a user_id user_id = self.get_secure_cookie('madgikmining') if user_id is None: - return - # check if he already uploaded his grants ids and inform him via a message - self.render('configure_profile.html', settings=msettings) + return + # check if he uploaded his codes + if numberOfGrantsUploaded(user_id, self.get_secure_cookie('madgikmining_grantsuploaded')): + self.render('configure_profile.html', settings=msettings) + else: + self.redirect('/upload-codes') def post(self): try: # get user id from cookie. Must have @@ -508,15 +547,17 @@ class configureProfileHandler(BaseHandler): os.remove(cname) print e return - data['respond'] = "File {0} uploaded successfully!".format(fname) + file_name = "/tmp/docs%s.json" % (user_id) + if os.path.isfile(file_name): + data['data'] = sum(1 for line in open(file_name)) self.write(json.dumps(data)) self.finish() # post case where the user selects form preset documents samples elif 'doc_sample' in self.request.arguments and self.request.arguments['doc_sample'][0] != '': sample_file_name = "" - if self.request.arguments['doc_sample'][0] == "nih_sample": - sample_file_name = "static/nih_sample.tsv" + if self.request.arguments['doc_sample'][0] == "egi_sample": + sample_file_name = "static/egi_sample.tsv" elif self.request.arguments['doc_sample'][0] == "rcuk_sample": sample_file_name = "static/rcuk_sample.tsv" elif self.request.arguments['doc_sample'][0] == "arxiv_sample": @@ -524,7 +565,7 @@ class configureProfileHandler(BaseHandler): sample_file = open(sample_file_name, 'r') # write data to physical file - cname = "/tmp/docs{0}/json".format(user_id) + cname = "/tmp/docs{0}.json".format(user_id) fh = open(cname, 'w') while 1: copy_buffer = sample_file.read(1048576) @@ -532,14 +573,27 @@ class configureProfileHandler(BaseHandler): break fh.write(copy_buffer) fh.close() + lines_num = sum(1 for line in open(cname)) # data to be sent data = {} - if len(concepts) == 0: + if lines_num == 0: data['error'] = "You have to provide at least one concept to continue" else: - data['respond'] = "{0} Codes loaded successfully!".format(len(concepts)) - self.set_secure_cookie('madgikmining_grantsuploaded', str(len(concepts))) + data['data'] = lines_num + self.write(json.dumps(data)) + self.finish() + # service to return the already uploaded documents + elif 'already' in self.request.arguments: + data = {} + if msettings.RESET_FIELDS == 1: + data['data'] = -1 + else: + data['data'] = 0 + file_name = "/tmp/docs%s.json" % (user_id) + if os.path.isfile(file_name): + data['data'] = sum(1 for line in open(file_name)) + msettings.RESET_FIELDS = 0 self.write(json.dumps(data)) self.finish() # post case for the actual mining proccess @@ -549,16 +603,20 @@ class configureProfileHandler(BaseHandler): # create positive and negative words weighted regex text pos_set = neg_set = conf = whr_conf = '' if 'poswords' in self.request.arguments and self.request.arguments['poswords'][0] != '{}': + data['poswords'] = [] # construct math string for positive words matching calculation with weights pos_words = json.loads(self.request.arguments['poswords'][0]) for key, value in pos_words.iteritems(): pos_set += r'regexpcountuniquematches("(?:\b)%s(?:\b)",j2s(prev,middle,next))*%s + ' % (key,value) + data['poswords'].append(key) pos_set += "0" if 'negwords' in self.request.arguments and self.request.arguments['negwords'][0] != '{}': + data['negwords'] = [] # construct math string for negative words matching calculation with weights neg_words = json.loads(self.request.arguments['negwords'][0]) for key, value in neg_words.iteritems(): neg_set += r'regexpcountuniquematches("(?:\b)%s(?:\b)",j2s(prev,middle,next))*%s - ' % (key,value) + data['negwords'].append(key) neg_set += "0" if pos_set != '' and neg_set != '': conf = ", ({0} - {1})".format(pos_set, neg_set) @@ -574,8 +632,8 @@ class configureProfileHandler(BaseHandler): cursor=msettings.Connection.cursor() if numberOfDocsUploaded(user_id) != 0: - doc_filters = "regexpr('[\n|\r]',c2,' ')" - ackn_filters = "regexpr(\"\\'\", c2,'')" + doc_filters = "regexpr('[\n|\r]',d2,' ')" + ackn_filters = "regexpr(\"\\'\", p2,'')" if 'punctuation' in self.request.arguments and self.request.arguments['punctuation'][0] == "1": doc_filters = 'keywords('+doc_filters+')' ackn_filters = 'keywords('+ackn_filters+')' @@ -590,10 +648,10 @@ class configureProfileHandler(BaseHandler): doc_filters = 'filterstopwords('+doc_filters+')' ackn_filters = 'filterstopwords('+ackn_filters+')' list(cursor.execute("drop table if exists grantstemp"+user_id, parse=False)) - query_pre_grants = "create temp table grantstemp{0} as select stripchars(c1) as c1, case when c2 is null then null else {1} end as c2 from (setschema 'c1,c2' file '/tmp/p{0}.tsv' dialect:tsv)".format(user_id, ackn_filters) + query_pre_grants = "create temp table grantstemp{0} as select stripchars(p1) as gt1, case when p2 is null then null else {1} end as gt2 from (setschema 'p1,p2' file '/tmp/p{0}.tsv' dialect:tsv)".format(user_id, ackn_filters) cursor.execute(query_pre_grants) list(cursor.execute("drop table if exists docs"+user_id, parse=False)) - query1 = "create temp table docs{0} as select c1, {1} as c2 from (setschema 'c1,c2' select jsonpath(c1, '$.id', '$.text') from (file '/tmp/docs{0}.json'))".format(user_id, doc_filters) + query1 = "create temp table docs{0} as select d1, {1} as d2 from (setschema 'd1,d2' select jsonpath(c1, '$.id', '$.text') from (file '/tmp/docs{0}.json'))".format(user_id, doc_filters) cursor.execute(query1) else: data['error'] = "You have to provide at least one concept to continue" @@ -606,23 +664,33 @@ class configureProfileHandler(BaseHandler): if 'wordssplitnum' in self.request.arguments and self.request.arguments['wordssplitnum'][0] != '': words_split = int(self.request.arguments['wordssplitnum'][0]) if 0 < words_split and words_split <= 10: - acknowledgment_split = r'textwindow2s(regexpr("([\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|])", c2, "\\\1"),0,'+str(words_split)+r',0)' + acknowledgment_split = r'textwindow2s(regexpr("([\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|])", gt2, "\\\1"),0,'+str(words_split)+r',0)' else: - acknowledgment_split = r'"prev" as prev, regexpr("([\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|])", c2, "\\\1") as middle, "next" as next' - query0 = r"create temp table grants"+user_id+r' as select c1 as c3, jmergeregexp(jgroup("(?<=[\s\b])"||middle||"(?=[\s\b])")) as c4 from '+r"(setschema 'c1,prev,middle,next' select c1, "+acknowledgment_split+r' from grantstemp'+user_id+r' where (c1 or c1!="") and c2 not null) group by c1 union all select distinct c1 as c3, "(?!.*)" as c4 from grantstemp'+user_id+r" where (c1 or c1!='') and c2 is null" + acknowledgment_split = r'"prev" as prev, regexpr("([\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|])", gt2, "\\\1") as middle, "next" as next' + # query0 = r"create temp table grants"+user_id+r' as select gt1 as g1, jmergeregexp(jgroup("(?<=[\s\b])"||middle||"(?=[\s\b])")) as g2 from '+r"(setschema 'gt1,prev,middle,next' select gt1, "+acknowledgment_split+r' from grantstemp'+user_id+r' where (gt1 or gt1!="") and gt2 not null) group by gt1 union all select distinct gt1 as g1, "(?!.*)" as g2 from grantstemp'+user_id+r" where (gt1 or gt1!='') and gt2 is null" + query0 = r"create temp table grants"+user_id+r' as select gt1 as g1, jmergeregexp(jgroup(middle)) as g2 from '+r"(setschema 'gt1,prev,middle,next' select gt1, "+acknowledgment_split+r' from grantstemp'+user_id+r' where (gt1 or gt1!="") and gt2 not null) group by gt1 union all select distinct gt1 as g1, "(?!.*)" as g2 from grantstemp'+user_id+r" where (gt1 or gt1!='') and gt2 is null" cursor.execute(query0) + query0get = "select * from grants{0}".format(user_id) + results0get = [r for r in cursor.execute(query0get)] + print results0get - query2 = "select c1, c3, max(confidence) as confidence from (select c1, c3, regexpcountuniquematches(c4, j2s(prev,middle,next)) as confidence {0} from (select c1, textwindow2s(c2,10,1,5) from (select * from docs{1})), (select c3, c4 from grants{1}) T where middle = T.c3 {2}) group by c1".format(conf, user_id, whr_conf) + query2 = "select d1, g1, context, acknmatch, max(confidence) as confidence from (select d1, g1, regexpcountuniquematches(g2, j2s(prev,middle,next)) as confidence, j2s(prev,middle,next) as context, regexprfindall(g2, j2s(prev,middle,next)) as acknmatch {0} from (select d1, textwindow2s(d2,20,1,20) from (select * from docs{1})), (select g1, g2 from grants{1}) T where middle = T.g1 {2}) group by d1".format(conf, user_id, whr_conf) # query2 = "select c1, c3 {0} from (select c1, textwindow2s(c2,10,1,5) from (select * from docs{1})), (select c3 from grants{1}) T where middle = T.c3 {2}".format(conf, user_id, whr_conf) results = [r for r in cursor.execute(query2)] - data['funding_info'] = [{"code": r[1]} for r in results] + print results + doctitles = {} + for r in results: + if r[0] not in doctitles: + doctitles[r[0]] = [] + doctitles[r[0]].append({"match": r[1], "context": r[2], "acknmatch": json.loads(r[3]), "confidence": r[4]}) + data['matches'] = doctitles self.write(json.dumps(data)) self.flush() self.finish() except Exception as ints: data = {} - data['error'] = "File Failed to Upload!" + data['error'] = "Something went very very wrong!" self.write(json.dumps(data)) self.finish() print ints @@ -639,13 +707,104 @@ class saveProfileHandler(BaseHandler): def get(self): if 'data' in self.request.arguments: return + elif 'saveprofile' in self.request.arguments and self.request.arguments['saveprofile'][0] == '1': + user_id = self.get_secure_cookie('madgikmining') + if user_id is None: + return + profile_file_name = "/tmp/OAMiningProfile_{0}.oamp".format(user_id) + buf_size = 4096 + self.set_header('Content-Type', 'application/octet-stream') + self.set_header('Content-Disposition', 'attachment; filename=' + "OAMiningProfile_{0}.oamp".format(user_id)) + self.flush() + with open(profile_file_name, 'r') as f: + while True: + data = f.read(buf_size) + if not data: + break + self.write(data) + self.finish() else: # check if we already gave client a user_id user_id = self.get_secure_cookie('madgikmining') if user_id is None: - return - # check if he already uploaded his grants ids and inform him via a message - self.render('save_profile.html', settings=msettings) + return + # check if he uploaded his codes + if numberOfGrantsUploaded(user_id, self.get_secure_cookie('madgikmining_grantsuploaded')): + self.render('save_profile.html', settings=msettings) + else: + self.redirect('/upload-codes') + def post(self): + try: + # get user id from cookie. Must have + user_id = self.get_secure_cookie('madgikmining') + if user_id is None: + return + # post case where the profile data is uploaded to create the profile file + if 'createprofile' in self.request.arguments and self.request.arguments['createprofile'][0] == '1': + import sys + sys.path.append(msettings.MADIS_PATH) + import madis + # get the database cursor + # profile file name + profile_file_name = "/tmp/OAMiningProfile_{0}.oamp".format(user_id) + cursor=madis.functions.Connection(profile_file_name).cursor() + # Create poswords table + cursor.execute("drop table if exists poswords", parse=False) + cursor.execute("create table poswords(c1,c2)", parse=False) + # Create negwords table + cursor.execute("drop table if exists negwords", parse=False) + cursor.execute("create table negwords(c1,c2)", parse=False) + # Create filters table + cursor.execute("drop table if exists filters", parse=False) + cursor.execute("create table filters(c1,c2)", parse=False) + # Create grants table + cursor.execute("drop table if exists grants", parse=False) + cursor.execute("create table grants(c1)", parse=False) + if 'poswords' in self.request.arguments and self.request.arguments['poswords'][0] != '{}': + # construct math string for positive words matching calculation with weights + pos_words = json.loads(self.request.arguments['poswords'][0]) + cursor.executemany("insert into poswords(c1,c2) values(?,?)", + ( + (key, value,) for key, value in pos_words.iteritems() + ) + ) + if 'negwords' in self.request.arguments and self.request.arguments['negwords'][0] != '{}': + # construct math string for negative words matching calculation with weights + neg_words = json.loads(self.request.arguments['negwords'][0]) + cursor.executemany("insert into negwords(c1,c2) values(?,?)", + ( + (key, value,) for key, value in neg_words.iteritems() + ) + ) + if 'filters' in self.request.arguments and self.request.arguments['filters'][0] != '{}': + # construct math string for negative words matching calculation with weights + filters = json.loads(self.request.arguments['filters'][0]) + cursor.executemany("insert into filters(c1,c2) values(?,?)", + ( + (key, value,) for key, value in filters.iteritems() + ) + ) + if numberOfGrantsUploaded(user_id, self.get_secure_cookie('madgikmining_grantsuploaded')) != 0: + cursor.execute("insert into grants select stripchars(c1) as c1 from (file '/tmp/p{0}.csv')".format(user_id)) + cursor.close() + + data = {} + data['data'] = 1 + self.write(json.dumps(data)) + self.flush() + self.finish() + + except Exception as ints: + data = {} + data['error'] = "Something went very very wrong!" + self.write(json.dumps(data)) + self.finish() + print ints + + try: + cursor.close() + except: + pass class profileCreationHandler(BaseHandler): diff --git a/interactive-mining-madoap/madoap/src/settings.py b/interactive-mining-madoap/madoap/src/settings.py index 3b01a88..bddd8c0 100644 --- a/interactive-mining-madoap/madoap/src/settings.py +++ b/interactive-mining-madoap/madoap/src/settings.py @@ -9,6 +9,7 @@ WEB_SERVER_PORT=8080 APPDIRNAME='openaireplus' MADIS_PATH='../../../interactive-mining-3rdparty-madis/madis/src' DB='static/database.db' +RESET_FIELDS=1 FLOW_PATH='' diff --git a/interactive-mining-madoap/madoap/src/static/animations.css b/interactive-mining-madoap/madoap/src/static/animations.css new file mode 100644 index 0000000..904d84c --- /dev/null +++ b/interactive-mining-madoap/madoap/src/static/animations.css @@ -0,0 +1,102 @@ +/*ANIMATIONS*/ +tr.new-item { + opacity:0; + -webkit-animation:fadeIn .1s linear forwards; + -o-animation:fadeIn .1s linear forwards; + animation:fadeIn .1s linear forwards +} +@keyframes fadeIn { + to { + opacity:1 + } +} + +@keyframes openspace { + to { + height: auto + } +} + +tr.removed-item { + -webkit-animation: removed-item-animation .3s cubic-bezier(.55,-0.04,.91,.94) forwards; + -o-animation: removed-item-animation .3s cubic-bezier(.55,-0.04,.91,.94) forwards; + animation: removed-item-animation .3s cubic-bezier(.55,-0.04,.91,.94) forwards +} + +@keyframes removed-item-animation { + from { + opacity: 1; + } + + to { + opacity: 0 + } +} + +@-webkit-keyframes new-item-animation { + from { + opacity: 0; + -webkit-transform: scale(0); + transform: scale(0) + } + + to { + opacity: 1; + -webkit-transform: scale(1); + transform: scale(1) + } +} + +@-o-keyframes new-item-animation { + from { + opacity: 0; + -o-transform: scale(0); + transform: scale(0) + } + + to { + opacity: 1; + -o-transform: scale(1); + transform: scale(1) + } +} + +@-webkit-keyframes openspace { + to { + height: auto + } +} + +@-o-keyframes openspace { + to { + height: auto + } +} + +@-webkit-keyframes removed-item-animation { + from { + opacity: 1; + -webkit-transform: scale(1); + transform: scale(1) + } + + to { + -webkit-transform: scale(0); + transform: scale(0); + opacity: 0 + } +} + +@-o-keyframes removed-item-animation { + from { + opacity: 1; + -o-transform: scale(1); + transform: scale(1) + } + + to { + -o-transform: scale(0); + transform: scale(0); + opacity: 0 + } +} \ No newline at end of file diff --git a/interactive-mining-madoap/madoap/src/static/configure-profile.js b/interactive-mining-madoap/madoap/src/static/configure-profile.js index dde4994..b284800 100644 --- a/interactive-mining-madoap/madoap/src/static/configure-profile.js +++ b/interactive-mining-madoap/madoap/src/static/configure-profile.js @@ -34,67 +34,107 @@ .on( 'blur', function(){ $input.removeClass( 'has-focus' ); }); }); - var match_level_choice = 0; + var match_level_choice = -1; var advanced_options_open = 0; var handleMatchLevelChoice = function() { $('#1-level').on('click', function( e ) { if (advanced_options_open) { - UIkit.accordion($('.uk-accordion')).toggle(0, true); + UIkit.accordion($('#advanced-opts-toggle')).toggle(0, true); advanced_options_open = 0; } if (match_level_choice != 0) { + deleteAllWords(0,1); + deleteAllWords(0,0); console.log('#1-level'); + // store change to localstorage + localStorage.setItem('matchlevel', "#1-level"); + // Add the positive words + addDataToTable(generateId(1), 'European Grid Infrastructure', 1, 1); + addDataToTable(generateId(1), 'EGI', 1, 1); + addDataToTable(generateId(1), 'European Grid Initiative', 1, 1); } match_level_choice = 0; }); $('#2-level').on('click', function( e ) { if (advanced_options_open) { - UIkit.accordion($('.uk-accordion')).toggle(0, true); + UIkit.accordion($('#advanced-opts-toggle')).toggle(0, true); advanced_options_open = 0; } if (match_level_choice != 1) { + deleteAllWords(0,1); + deleteAllWords(0,0); console.log('#2-level'); + // store change to localstorage + localStorage.setItem('matchlevel', "#2-level"); } match_level_choice = 1; }); $('#3-level').on('click', function( e ) { if (advanced_options_open) { - UIkit.accordion($('.uk-accordion')).toggle(0, true); + UIkit.accordion($('#advanced-opts-toggle')).toggle(0, true); advanced_options_open = 0; } if (match_level_choice != 2) { + deleteAllWords(0,1); + deleteAllWords(0,0); console.log('#3-level'); + // store change to localstorage + localStorage.setItem('matchlevel', "#3-level"); } match_level_choice = 2; }); $('#c-level').on('click', function( e ) { if (advanced_options_open == 0) { - UIkit.accordion($('.uk-accordion')).toggle(0, true); + UIkit.accordion($('#advanced-opts-toggle')).toggle(0, true); advanced_options_open = 1; } if (match_level_choice != 3) { console.log('#c-level'); + // store change to localstorage + localStorage.setItem('matchlevel', "#c-level"); } }); - $('.uk-accordion').on('show', function () { + // $('#advanced-opts-toggle').on('show', function () { + // console.log('#GG-level'); + // UIkit.switcher($('#uk-switcher')).show(3); + // UIkit.switcher($('.uk-switcher')).show(3); + // advanced_options_open = 1; + // }); + // $('#advanced-opts-toggle').on('hide', function () { + // console.log('#BB-level'); + // UIkit.switcher($('#uk-switcher')).show(match_level_choice); + // UIkit.switcher($('.uk-switcher')).show(match_level_choice); + // advanced_options_open = 0; + // }); + + $('#advanced-opts-toggle').on('show', function () { console.log('#GG-level'); - UIkit.switcher($('#uk-switcher')).show(3); - UIkit.switcher($('.uk-switcher')).show(3); advanced_options_open = 1; + $('#c-level').click(); }); - $('.uk-accordion').on('hide', function () { + $('#advanced-opts-toggle').on('hide', function () { console.log('#BB-level'); - UIkit.switcher($('#uk-switcher')).show(match_level_choice); - UIkit.switcher($('.uk-switcher')).show(match_level_choice); advanced_options_open = 0; + if (match_level_choice == 0) { + match_level_choice = 3; + $('#1-level').click(); + } else if (match_level_choice == 1) { + match_level_choice = 3; + $('#2-level').click(); + } else if (match_level_choice == 2) { + match_level_choice = 3; + $('#3-level').click(); + } }); - // UIkit.accordion($('.uk-accordion')).toggle(0, true); + // UIkit.accordion($('#advanced-opts-toggle')).toggle(0, true); // UIkit.switcher($('#uk-switcher')).show(3); // UIkit.switcher($('.uk-switcher')).show(3); } +/////////// LIST FUNCTIONS + var count_pos = 0, count_neg = 0; var updateCounter = function(is_pos){ @@ -128,31 +168,6 @@ } } - var handleNextButton = function() { - $('#next-button').on('click', function( e ) { - console.log(JSON.stringify(wordsDataToArray())); - var formData = new FormData(); - formData.append("concepts", JSON.stringify(wordsDataToArray())); - $.ajax({ - url: "upload-codes", - type: 'POST', - data: formData, - async: false, - success: function (data) { - console.log(JSON.parse(data).respond) - window.location="configure-profile" - }, - error: function (xhr, ajaxOptions, thrownError) { - $('#file-upload-response').html('File Failed to Upload!'+xhr.status) - // $('#file-uploaded')[0].checked = false; - }, - cache: false, - contentType: false, - processData: false - }); - }); - } - var wordsDataToArray = function(is_pos) { // TODO var data = {}; @@ -193,7 +208,7 @@ if (isPhrase) { $(currentEle).html(''); } else { - $(currentEle).html(''); + $(currentEle).html(''); } $(".thVal").focus(); $(".thVal").keyup(function (event) { @@ -202,7 +217,7 @@ var new_val = $(".thVal").val(); $(currentEle).html($(".thVal").val().trim()); if (isPhrase || (new_val != '' && !isNaN(new_val) && new_val >= 0 && new_val <= 100)) { - $(currentEle).html($(".thVal").val().trim()); + $(currentEle).html($(".thVal").val()); } else { $(currentEle).html(1); } @@ -241,16 +256,18 @@ var removeWord = function(id, is_pos){ var item = $('#' + id ); - item.addClass('removed-item') .one('webkitAnimationEnd oanimationend msAnimationEnd animationend', function(e) { $(this).remove(); }); + // remove from localstorage + localStorage.remove('#' + id); if (is_pos === 1) { count_pos--; } else { count_neg--; } + updateCounter(is_pos); }; var count_pos = 0, count_neg = 0; @@ -283,8 +300,8 @@ } } }))); - addDoubleClick($(createdItem).find("div.phrase")); - addDoubleClick($(createdItem).find("div.weight")); + addDoubleClick($(createdItem).find("td.phrase")); + addDoubleClick($(createdItem).find("td.weight")); createdItem.on('keydown', function(ev){ if(ev.keyCode === 13) return false; }); @@ -295,6 +312,17 @@ count_neg++; updateCounter(0); } + // store into localstorage + var obj = {}; + obj["phrase"] = content_word; + obj["weight"] = content_weight; + localStorage.setItem(id, JSON.stringify(obj)); + for(var key in localStorage){ + if (key === null) + continue; + var json_string = localStorage.getItem(key); + console.log(key+' '+json_string); + } } } @@ -325,7 +353,71 @@ input_weight.val('1'); } }); - }; + updateCounter(1); + updateCounter(0); + handleDeleteButtons(); + }; + + var deleteAllWords = function(warnUser = 1, is_pos) { + if(!warnUser || confirm('Are you sure you want to delete all the items in the list? There is no turning back after that.')){ //remove items from DOM + if (is_pos) { + var items = $('tr[id ^= positive]'); + items.addClass('removed-item').one('webkitAnimationEnd oanimationend msAnimationEnd animationend', function(e) { + $(this).remove(); + }); + // Delete all pos from local storage + for(var key in localStorage){ + if (key === null) + continue; + var json_string = localStorage.getItem(key); + if(key.indexOf('positive') === 0){ + localStorage.removeItem(key); + } + } + count_pos = 0; + updateCounter(1); + } else { + var items = $('tr[id ^= negative]'); + items.addClass('removed-item').one('webkitAnimationEnd oanimationend msAnimationEnd animationend', function(e) { + $(this).remove(); + }); + // Delete all neg from local storage + for(var key in localStorage){ + if (key === null) + continue; + var json_string = localStorage.getItem(key); + if(key.indexOf('negative') === 0){ + localStorage.removeItem(key); + } + } + count_neg = 0; + updateCounter(0); + } + } + }; + + //handler for the "delete all" button + var handleDeleteButtons = function(){ + $('#clear-all-pos').on('click', function() { deleteAllWords(1,1);}); + $('#clear-all-neg').on('click', function() { deleteAllWords(1,0)}); + }; + + var handleFiltersInput = function() { + $("#letter-case-select").on('change', function(e) { + localStorage.setItem('lettercase', $("#letter-case-select option:selected").text()); + }); + $("#word-split").on('change', function(e) { + localStorage.setItem('wordssplitnum', $("#word-split").val()); + }); + $("#stop-words-filter").on('change', function(e) { + localStorage.setItem('stopwords', $('#stop-words-filter').prop('checked')===true?1:0); + console.log('stop-words-filter '+localStorage.getItem('stopwords')); + }); + $("#punctuation-filter").on('change', function(e) { + localStorage.setItem('punctuation', $('#punctuation-filter').prop('checked')===true?1:0); + console.log('punctuation-filter '+localStorage.getItem('punctuation')); + }); + } var handleRunMiningButton = function() { $("#run-mining-btn").on('click', function( e ) { @@ -348,7 +440,113 @@ data: formData, async: false, success: function (data) { - console.log(data) + obj = JSON && JSON.parse(data) || $.parseJSON(data); + console.log(obj); + // get poswords + var poswords = []; + if (obj.hasOwnProperty("poswords")) { + poswords = obj["poswords"]; + } + // get poswords + var negwords = []; + if (obj.hasOwnProperty("negwords")) { + negwords = obj["negwords"]; + } + // get matches + var matches = []; + if (obj.hasOwnProperty("matches")) { + doc_matches = obj["matches"]; + for (var docname in doc_matches) { + if (doc_matches.hasOwnProperty(docname)) { + // create document section + var li = $('
  • '+docname+'

  • '); + // create matches section + word_matches = doc_matches[docname]; + var accordion_content = $('
    '); + for (var match in word_matches) { + console.log(word_matches[match]); + var result = word_matches[match]; + var paragraph = $('

    '+result.context.split(' ').map(function(x){return ""+x+"";}).join('')+'

    '); + // find center match string and surrounded text + var matched = paragraph.find(":contains('"+result.match+"')"); + var prev = matched.prev(); + var next = matched.next(); + // get textwindows text as context + var context = []; + var prev_context = []; + var next_context = []; + for (i = 0; prev.text()!=''; i++) { + if (i < 10) { + context.unshift(prev.text()); + } else { + prev_context.unshift(prev.text()); + } + prev = prev.prev(); + } + context.push(matched.text()); + for (i = 0; next.text()!=''; i++) { + if (i < 5) { + context.push(next.text()); + } else { + next_context.push(next.text()); + } + next = next.next(); + } + // hightlight textwindow + context = $(''+context.join(' ')+''); + // hightlight positive words + for (var index in poswords) { + var search_regexp = new RegExp(poswords[index], "g"); + context.html(context.html().replace(search_regexp,""+poswords[index]+"")); + } + // hightlight acknowledgment keywords + if (result.hasOwnProperty("acknmatch")) { + var acknmatches = result["acknmatch"]; + for (var index in acknmatches) { + var search_regexp = new RegExp(acknmatches[index], "g"); + context.html(context.html().replace(search_regexp,""+acknmatches[index]+"")); + } + } + // hightlight negative words + for (var index in negwords) { + var search_regexp = new RegExp(negwords[index], "g"); + context.html(context.html().replace(search_regexp,""+negwords[index]+"")); + } + // hightlight matched phrase + var search_regexp = new RegExp(result.match, "g"); + context.html(context.html().replace(search_regexp,""+result.match+"")); + + // construct results paragraph to show + paragraph = $('

    '+prev_context.join(' ')+' '+context[0].outerHTML+' '+next_context.join(' ')+'

    '); + + li.append(paragraph); + } + $("#docs-results").append(li); + } + } + UIkit.accordion($("#docs-results")); + } + $("#results-section").show(); + // split all paragraphs to word spans + // $(".document-result").each(function() { + // $(this).html($( this ).text().split(' ').map(function(x){return ""+x+" ";}).join('')); + // var search_regexp = new RegExp("dolor", "g"); + // $(this).html($(this).html().replace(search_regexp,""+"AAAAAAAAA"+"")); + // var matched = $(this).find(".highlight").parent(); + // var prev = matched.prev(); + // var next = matched.next(); + // matched.css("background-color", "#fff2ba"); + // // hightlight all prev window words + // for (i = 0; i < 3; i++) { + // prev.css("background-color", "#fff2ba"); + // prev = prev.prev(); + // } + // // hightlight all next window words + // for (i = 0; i < 3; i++) { + // next.css("background-color", "#fff2ba"); + // next = next.next(); + // } + // }); }, error: function (xhr, ajaxOptions, thrownError) { $('#file-upload-response').html('File Failed to Upload!'+xhr.status) @@ -361,17 +559,29 @@ }); } + var uploadedDocs = 0 + var docsUploaded = function(docsNumber) { + uploadedDocs = docsNumber; $("#docs-number").html(docsNumber+" documents uploaded"); + } + + var hideInitialDocsUploadForm = function() { + console.log("AAAAAAAAA") $("#docs-more-btn").show(); - $("#run-mining-btn").removeAttr("disabled"); + $("#run-mining-btn").removeAttr('disabled').removeClass('disabled'); + $('#documents-change-btn').addClass("uk-button"); + $('#initial-docs-upload-form').attr("class", ""); + $('#initial-docs-upload-form').attr("uk-dropdown", "mode: click;"); + $('#initial-docs-upload-form').appendTo("#documents-section"); + UIkit.dropdown($("#initial-docs-upload-form")); + UIkit.dropdown($("#initial-docs-upload-form")).mode = "click"; handleRunMiningButton(); } var handleFileUploadInput = function() { $("#docs-file-input").on('change', function() { if ($('#docs-file-input')[0].value === "") { - window.alert("You must specify a data file to upload."); return false; } // var formData = new FormData(); @@ -383,17 +593,16 @@ data: formData, async: false, success: function (data) { - // TODO check for error - $('#codes-file-upload-response').html(JSON.parse(data).respond) +// TODO TODO TODO TODO TODO TODO check for error obj = JSON && JSON.parse(data).data || $.parseJSON(data).data; // console.log(obj); - for (var key1 in obj) { - if (obj.hasOwnProperty(key1)) { - addDataToTable(generateId(), key1, obj[key1]); + if (obj > 0) { + if (uploadedDocs == 0) { + hideInitialDocsUploadForm(); } + docsUploaded(obj); + UIkit.dropdown($("#initial-docs-upload-form")).hide(); } - hideInitialDocsUploadForm(); - docsUploaded(12); }, error: function (xhr, ajaxOptions, thrownError) { $('#codes-file-upload-response').html('File Failed to Upload!'+xhr.status) @@ -403,6 +612,7 @@ contentType: false, processData: false }); + $("#docs-file-input")[0].value = ""; return false; }); @@ -411,7 +621,7 @@ var handleDocSampleChoice = function(btnIndex) { var formData = new FormData(); if (btnIndex === 0) { - formData.append("doc_sample", "nih_sample"); + formData.append("doc_sample", "egi_sample"); } else if (btnIndex === 1) { formData.append("doc_sample", "rcuk_sample"); } else if (btnIndex === 2) { @@ -425,17 +635,15 @@ data: formData, async: false, success: function (data) { - // TODO check for error - $('#codes-file-upload-response').html(JSON.parse(data).respond) +// TODO TODO TODO TODO TODO TODOcheck for error obj = JSON && JSON.parse(data).data || $.parseJSON(data).data; - // console.log(obj); - for (var key1 in obj) { - if (obj.hasOwnProperty(key1)) { - addDataToTable(generateId(), key1, obj[key1]); + if (obj > 0) { + if (uploadedDocs == 0) { + hideInitialDocsUploadForm(); } + docsUploaded(obj); + UIkit.dropdown($("#initial-docs-upload-form")).hide(); } - hideInitialDocsUploadForm(); - docsUploaded(12); }, error: function (xhr, ajaxOptions, thrownError) { $('#codes-file-upload-response').html('File Failed to Upload!'+xhr.status) @@ -450,7 +658,7 @@ } var handleDocSampleButtons = function() { - $('#nih-sample').on('click', function( e ) { + $('#egi-sample').on('click', function( e ) { handleDocSampleChoice(0); }); $('#rcuk-sample').on('click', function( e ) { @@ -461,20 +669,137 @@ }); } - var hideInitialDocsUploadForm = function() { - $('#initial-docs-upload-form').hide(); + var handleSaveProfileInfoSend = function() { + var formData = new FormData(); + formData.append("createprofile", "1") + formData.append("poswords", JSON.stringify(wordsDataToArray(1))); + formData.append("negwords", JSON.stringify(wordsDataToArray(0))); + filters_list = {}; + filters_list["lettercase"] = $("#letter-case-select option:selected").text(); + filters_list["wordssplitnum"] = $("#word-split").val(); + filters_list["stopwords"] = $('#stop-words-filter').prop('checked')===true?1:0; + filters_list["punctuation"] = $('#punctuation-filter').prop('checked')===true?1:0; + formData.append("filters", JSON.stringify(filters_list)); + $.ajax({ + url: "save-profile", + type: 'POST', + data: formData, + async: false, + success: function (data) { + console.log(data) + // if (data.indexOf('successfully!') != -1) { + // $('#file-uploaded')[0].checked = true; + // } + window.location="save-profile" + }, + error: function (xhr, ajaxOptions, thrownError) { + $('#file-upload-response').html('File Failed to Upload!'+xhr.status) + // $('#file-uploaded')[0].checked = false; + }, + cache: false, + contentType: false, + processData: false + }); + } + + var handleSaveProfileButtons = function() { + $("#next-button").on('click', function(e) { + handleSaveProfileInfoSend(); + }); + $("#save-profile-option").on('click', function(e) { + handleSaveProfileInfoSend(); + }); + } + + var checkAlreadyUploadedDocs = function() { + var formData = new FormData(); + formData.append("already", ""); + $.ajax({ + url: "configure-profile", + type: 'POST', + data: formData, + async: false, + success: function (data) { + obj = JSON && JSON.parse(data).data || $.parseJSON(data).data; + console.log(data); + if (obj > 0) { + hideInitialDocsUploadForm(); + docsUploaded(obj); + } else if (obj == -1) { + localStorage.clear(); + } + init(); + }, + error: function (xhr, ajaxOptions, thrownError) { + $('#codes-file-upload-response').html('File Failed to Upload!'+xhr.status) + // $('#file-uploaded')[0].checked = false; + }, + cache: false, + contentType: false, + processData: false + }); + } + + var checkAlreadyMiningSettings = function() { + for (var key in localStorage) { + if (key === null) + continue; + var value = localStorage.getItem(key); + console.log(key+' '+value); + if(key.indexOf('positive') === 0){ + data = JSON.parse(value); + addDataToTable(key, data.phrase, data.weight, 1); + } else if(key.indexOf('negative') === 0) { + data = JSON.parse(value); + addDataToTable(key, data.phrase, data.weight, 0); + } else if (key === 'matchlevel') { + console.log(key+' '+value); + $(value).click(); + } else if (key === 'lettercase') { + $("#letter-case-select").val(value); + } else if (key === 'wordssplitnum') { + $("#word-split").val(value) + } else if (key === 'stopwords') { + if (value == 1) { + $("#stop-words-filter").prop('checked', true); + $("#stop-words-filter").attr('checked', true); + $('#stop-words-filter')[0].checked = true; + } else if (value == 0) { + $("#stop-words-filter").prop('checked', false); + $("#stop-words-filter").attr('checked', false); + $('#stop-words-filter')[0].checked = false; + $('#stop-words-filter').removeAttr('checked'); + } + } else if (key === 'punctuation') { + if (value == 1) { + $("#punctuation-filter").prop('checked', true); + $("#punctuation-filter").attr('checked', true); + $('#punctuation-filter')[0].checked = true; + } else if (value == 0) { + $("#punctuation-filter").prop('checked', false); + $("#punctuation-filter").attr('checked', false); + $('#punctuation-filter')[0].checked = false; + $('#punctuation-filter').removeAttr('checked'); + } + } + } + if (localStorage.getItem('matchlevel') === null) { + // Click the by default option + $('#1-level').click(); + } } var init = function(){ - localStorage.clear(); handleMatchLevelChoice(); + checkAlreadyMiningSettings(); handleWordsInput(); + handleFiltersInput(); handleDocSampleButtons(); handleFileUploadInput(); - handleNextButton(); + handleSaveProfileButtons(); }; //start all - init(); + checkAlreadyUploadedDocs(); })(); diff --git a/interactive-mining-madoap/madoap/src/static/egi_sample.tsv b/interactive-mining-madoap/madoap/src/static/egi_sample.tsv new file mode 100644 index 0000000..d47942c --- /dev/null +++ b/interactive-mining-madoap/madoap/src/static/egi_sample.tsv @@ -0,0 +1 @@ +{"id":"atlasDoc", "text":"The crucial computing support from all WLCG partners is acknowledged gratefully, in particular from CERN and the ATLAS Collaboration."} \ No newline at end of file diff --git a/interactive-mining-madoap/madoap/src/static/save-profile.js b/interactive-mining-madoap/madoap/src/static/save-profile.js new file mode 100644 index 0000000..e3fcbe1 --- /dev/null +++ b/interactive-mining-madoap/madoap/src/static/save-profile.js @@ -0,0 +1,15 @@ +(function(){ + + function getCookie(name) { + var value = "; " + document.cookie; + var parts = value.split("; " + name + "="); + if (parts.length == 2) return parts.pop().split(";").shift(); + } + + var init = function(){ + }; + + //start all + init(); + +})(); diff --git a/interactive-mining-madoap/madoap/src/static/upload-codes.js b/interactive-mining-madoap/madoap/src/static/upload-codes.js index 3e1d9d3..772f79a 100644 --- a/interactive-mining-madoap/madoap/src/static/upload-codes.js +++ b/interactive-mining-madoap/madoap/src/static/upload-codes.js @@ -41,7 +41,7 @@ var onUserStartTyping = function() { $("#initial-type-input").on( 'keypress', function( e ){ - $(this).css('display', 'none'); + triggerDataTable(); var value = $(this).val(); var newId = generateId(); addDataToTable(newId, value, ""); @@ -52,7 +52,10 @@ var handleAddRowButton = function() { $('#add-row-below').on('click', function( e ) { - addDataToTable(generateId(), "", ""); + var newId = generateId(); + addDataToTable(newId, "", ""); + var currentEle = $('#'+ newId).find("td.code"); + editValue(currentEle, " ", 0); }); } @@ -88,6 +91,8 @@ // $(this).children('td').each(function(ii, vv){ // data[i][ii] = $(this).text(); // }); + if ($(v).find("td.code").text() === '') + return true; data[$(v).find("td.code").text()] = $(v).find("td.acknowl").text(); }) return data @@ -148,17 +153,27 @@ var removeDataRow = function(id){ var item = $('#' + id ); - + item.addClass('removed-item') .one('webkitAnimationEnd oanimationend msAnimationEnd animationend', function(e) { $(this).remove(); + // fix # column numbering + count_table_rows = 1; + $("#data-table tbody tr").each(function(i, v){ + // data[i] = Array(); + // $(this).children('td').each(function(ii, vv){ + // data[i][ii] = $(this).text(); + // }); + $(v).find("td.num").text(count_table_rows); + count_table_rows++; + }) }); }; var count_table_rows = 1; var addDataToTable = function(id, code, acknowledgment) { - var row = ''+count_table_rows+'' + code + '' + acknowledgment +'' + var row = ''+count_table_rows+'' + code + '' + acknowledgment +'' table = $('#data-table tbody'); // if content is correct and not empty append to table @@ -187,8 +202,13 @@ }); } + var triggerDataTable = function() { + $("#initial-type-input").remove(); + $("#data-table").show(); + } + var handleFileUploadButton = function() { - $("form#codes-file-input-form").submit(function(){ + $("form#codes-file-input-form").on('change', function(){ if ($('#codes-file-input')[0].value === "") { window.alert("You must specify a data file to import."); return false; @@ -209,6 +229,7 @@ addDataToTable(generateId(), key1, obj[key1]); } } + triggerDataTable(); }, error: function (xhr, ajaxOptions, thrownError) { $('#codes-file-upload-response').html('File Failed to Upload!'+xhr.status) @@ -223,8 +244,40 @@ }); } + var checkAlreadyUserState = function() { + var formData = new FormData(); + formData.append("already", ""); + $.ajax({ + url: "upload-codes", + type: 'POST', + data: formData, + async: false, + success: function (data) { + obj = JSON && JSON.parse(data).data || $.parseJSON(data).data; + // console.log(obj); + var numOfCodes = 0; + for (var key1 in obj) { + if (obj.hasOwnProperty(key1)) { + addDataToTable(generateId(), key1, obj[key1]); + } + numOfCodes++ + } + if (numOfCodes != 0) { + triggerDataTable(); + } + }, + error: function (xhr, ajaxOptions, thrownError) { + $('#codes-file-upload-response').html('File Failed to Upload!'+xhr.status) + // $('#file-uploaded')[0].checked = false; + }, + cache: false, + contentType: false, + processData: false + }); + } + var init = function(){ - localStorage.clear(); + checkAlreadyUserState(); handleFileUploadButton(); onUserStartTyping(); handleAddRowButton(); diff --git a/interactive-mining-madoap/madoap/src/templates/configure_profile.html b/interactive-mining-madoap/madoap/src/templates/configure_profile.html index d146837..8db2f3d 100644 --- a/interactive-mining-madoap/madoap/src/templates/configure_profile.html +++ b/interactive-mining-madoap/madoap/src/templates/configure_profile.html @@ -2,17 +2,18 @@ {% block configure_profile %} class="current" {% end %} {% block content %} Configure profile + - +

    -
    @@ -104,8 +105,8 @@ @@ -122,12 +123,21 @@
    Clean matching area's text
    -
    - - - -
    - + +
    + +
    + +
    +
    +
    + +
    +
    + +
    +
    +
    - - -
    +
    +
    @@ -150,8 +159,8 @@

    Matching results

    -
    -
    +
    +
    0 documents uploaded
    @@ -159,43 +168,40 @@
    -
    -
    -
    - -

    No documents to test matching accuracy.

    -
    -

    Choose a sample of documents to load

    -
    - - - -
    -

    Upload your project codes you want to match

    -
    -

    -

    - Attach documents file by dropping them here or -

    -
    - -
    - browse documents file -
    -

    -

    - JSONPDFTXT - file types -

    -

    Maximum 10MB upload

    -
    +
    +

    Choose a sample of documents to load

    +
    + + + +
    +

    Upload your project codes you want to match

    +
    +

    +

    + Attach documents file by dropping them here or +

    +
    + +
    + browse documents file +
    +

    +

    + JSONPDFTXT + file types +

    +

    Maximum 10MB upload

    diff --git a/interactive-mining-madoap/madoap/src/templates/save_profile.html b/interactive-mining-madoap/madoap/src/templates/save_profile.html index 4e33ac5..71ac3c8 100644 --- a/interactive-mining-madoap/madoap/src/templates/save_profile.html +++ b/interactive-mining-madoap/madoap/src/templates/save_profile.html @@ -3,16 +3,16 @@ {% block content %} Save profile - +
    @@ -23,10 +23,10 @@
    - +
    or
    + {% end %} \ No newline at end of file diff --git a/interactive-mining-madoap/madoap/src/templates/upload_codes.html b/interactive-mining-madoap/madoap/src/templates/upload_codes.html index 42bea92..c7f23a3 100644 --- a/interactive-mining-madoap/madoap/src/templates/upload_codes.html +++ b/interactive-mining-madoap/madoap/src/templates/upload_codes.html @@ -2,17 +2,18 @@ {% block upload_codes %} class="current" {% end %} {% block content %} Upload project codes + - +
    @@ -23,19 +24,14 @@

    Upload your project codes you want to match

    -
    -
    -
    + +
    +
    - - -
    -
    - - -
    -
    -
    + Choose a file… +
    +
    +