From 8db72a90338995daa1ef30242ed3fadce7051f6d Mon Sep 17 00:00:00 2001 From: deva Date: Thu, 27 Jan 2011 10:34:07 +0000 Subject: New data extraction tool. --- server/src/macrotool/Makefile.am | 12 +- server/src/macrotool/export.cc | 303 +++++++++++++++++++++++++++++++ server/src/macrotool/export.h | 36 ++++ server/src/macrotool/fieldnamescanner.cc | 158 ++++++++++++++++ server/src/macrotool/fieldnamescanner.h | 44 +++++ server/src/macrotool/macrotool.cc | 26 +-- 6 files changed, 562 insertions(+), 17 deletions(-) create mode 100644 server/src/macrotool/export.cc create mode 100644 server/src/macrotool/export.h create mode 100644 server/src/macrotool/fieldnamescanner.cc create mode 100644 server/src/macrotool/fieldnamescanner.h (limited to 'server/src') diff --git a/server/src/macrotool/Makefile.am b/server/src/macrotool/Makefile.am index b8a42cb..061b904 100644 --- a/server/src/macrotool/Makefile.am +++ b/server/src/macrotool/Makefile.am @@ -10,7 +10,9 @@ macrotool_CXXFLAGS = -I.. $(PQXX_CXXFLAGS) $(CONFIG_CXXFLAGS) \ macrotool_SOURCES = \ macrotool.cc \ dump.cc \ + export.cc \ fieldnames.cc \ + fieldnamescanner.cc \ filehandler.cc \ util.cc \ ../debug.cc \ @@ -34,7 +36,9 @@ macrotool_SOURCES = \ EXTRA_DIST = \ dump.h \ + export.h \ fieldnames.h \ + fieldnamescanner.h \ filehandler.h \ util.h @@ -42,9 +46,9 @@ EXTRA_DIST = \ # Test Section # ################ -#TEST_SOURCE_DEPS = ${macrotool_SOURCES} ${EXTRA_DIST} -#TEST_SCRIPT_DIR = $(top_srcdir)/../../tools +TEST_SOURCE_DEPS = ${macrotool_SOURCES} ${EXTRA_DIST} +TEST_SCRIPT_DIR = $(top_srcdir)/../tools -#include ${TEST_SCRIPT_DIR}/Makefile.am.test +include ${TEST_SCRIPT_DIR}/Makefile.am.test -#include Makefile.am.test \ No newline at end of file +include Makefile.am.test \ No newline at end of file diff --git a/server/src/macrotool/export.cc b/server/src/macrotool/export.cc new file mode 100644 index 0000000..150a85c --- /dev/null +++ b/server/src/macrotool/export.cc @@ -0,0 +1,303 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set et sw=2 ts=2: */ +/*************************************************************************** + * export.cc + * + * Wed Jan 26 11:44:12 CET 2011 + * Copyright 2011 Bent Bisballe Nyeng + * deva@aasimon.org + ****************************************************************************/ + +/* + * This file is part of Pracro. + * + * Pracro is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * Pracro is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pracro; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + */ +#include "export.h" + +#include +#include +#include + +#include "debug.h" +#include "fieldnamescanner.h" +#include "configuration.h" + +#define SEP "\t" + +static std::string escape(std::string &str) +{ + std::string out = "\""; + std::string::iterator i = str.begin(); + while(i != str.end()) { + if(*i == '\"') out += "''"; + else out += *i; + i++; + } + out += "\""; + return out; +} + +class File { +public: + File(std::string n, fieldnames_t &f, pqxx::work &w) + : work(w), name(n), fieldnames(f) + { + name += ".csf"; + fp = fopen(name.c_str(), "w"); + + pos["id"] = 0; + pos["patientid"] = 1; + pos["time"] = 2; + pos["template"] = 3; + + printf("%s\n", n.c_str()); + + size_t idx = 4; + fieldnames_t::iterator i = f.begin(); + while(i != f.end()) { + printf("%s ", i->c_str()); + pos[*i] = idx; + idx++; + i++; + } + printf("\n"); + + output_header(); + } + + ~File() + { + if(fp) fclose(fp); + } + + void output_header() + { + beginrow(); + addcell("id", "ID"); + addcell("patientid", "Patient"); + addcell("time", "Time"); + addcell("template", "Template"); + + { + pqxx::result result = + work.exec("SELECT DISTINCT ON(name) name, caption FROM fieldnames" + " WHERE extract='true';"); + pqxx::result::const_iterator ri = result.begin(); + for(unsigned int r = 0; r < result.size(); r++) { + pqxx::result::tuple tuple = result.at(r); + std::string name = tuple.at(0).c_str(); + std::string caption = tuple.at(1).c_str(); + addcell(name, caption); + } + } + + endrow(); + } + + void beginrow() + { + cells.clear(); + cells.insert(cells.begin(), pos.size(), ""); + } + + void addcell(std::string key, std::string value) + { + if(pos.find(key) == pos.end()) return; + cells[pos[key]] = value; + } + + void endrow() + { + std::vector::iterator i = cells.begin(); + while(i != cells.end()) { + fprintf(fp, "%s"SEP, escape(*i).c_str()); + i++; + } + fprintf(fp, "\n"); + } + +private: + pqxx::work &work; + std::map pos; + std::vector cells; + std::string name; + fieldnames_t &fieldnames; + FILE *fp; +}; + + +static void export_prefix(std::string prefix) +{ + + std::map files; + + if(Conf::database_backend != "pgsql") { + printf("ERROR: Export only available for the pgsql database backend.\n"); + return; + } + + std::string host = Conf::database_addr; + std::string port = "";//Conf::database_port; + std::string user = Conf::database_user;; + std::string passwd = Conf::database_passwd; + std::string dbname = "";//Conf::database_database; + + std::string cs; + if(host.size()) cs += " host=" + host; + if(port.size()) cs += " port=" + port; + if(user.size()) cs += " user=" + user; + if(passwd.size()) cs += " password=" + passwd; + cs += " dbname=" + (dbname.size() ? dbname : "pracro"); + + pqxx::connection conn(cs); + pqxx::work work(conn); + + std::set filter; + + { + pqxx::result result = + work.exec("SELECT DISTINCT name FROM fieldnames" + " WHERE extract='true';"); + pqxx::result::const_iterator ri = result.begin(); + for(unsigned int r = 0; r < result.size(); r++) { + pqxx::result::tuple tuple = result.at(r); + std::string name = tuple.at(0).c_str(); + filter.insert(name); + //printf("filter: '%s'\n", name.c_str()); + } + } + + templates_t t = scanfieldnames(filter); + + templates_t::iterator ti = t.begin(); + while(ti != t.end()) { + printf("%s\n", ti->first.c_str()); + fieldnames_t::iterator fi = ti->second.begin(); + while(fi != ti->second.end()) { + printf("\t%s\n", (*fi).c_str()); + fi++; + } + ti++; + } + + + { + pqxx::result result = + work.exec("SELECT * FROM commits WHERE template LIKE '"+prefix+"%'" + " AND status='committed' ORDER BY patientid, timestamp;"); + pqxx::result::const_iterator ri = result.begin(); + for(unsigned int r = 0; r < result.size(); r++) { + pqxx::result::tuple tuple = result.at(r); + std::string patientid = tuple.at(0).c_str(); + std::string templ = tuple.at(1).c_str(); + std::string version = tuple.at(2).c_str(); + std::string timestamp = tuple.at(3).c_str(); + std::string uid = tuple.at(4).c_str(); + std::string status = tuple.at(5).c_str(); + + if(files.find(templ) == files.end()) { + files[templ] = new File(templ, t[templ], work); + } + + files[templ]->beginrow(); + files[templ]->addcell("id", uid); + files[templ]->addcell("patientid", patientid); + files[templ]->addcell("template", templ); + time_t t = atol(timestamp.c_str()); + files[templ]->addcell("time", ctime(&t)); + + printf("%s %s %s\n", timestamp.c_str(), patientid.c_str(), templ.c_str()); + + { + pqxx::result result = + work.exec("SELECT f.name, f.value" + " FROM transactions t, fields f, fieldnames n" + " WHERE t.cid='"+uid+"' AND f.transaction=t.uid" + " AND f.name=n.name AND n.extract='true'" + " ORDER BY t.timestamp;"); + pqxx::result::const_iterator ri = result.begin(); + for(unsigned int r = 0; r < result.size(); r++) { + pqxx::result::tuple tuple = result.at(r); + std::string name = tuple.at(0).c_str(); + std::string value = tuple.at(1).c_str(); + + files[templ]->addcell(name, value); + + } + } + + files[templ]->endrow(); + } + } + + std::map::iterator i = files.begin(); + while(i != files.end()) { + delete i->second; + i++; + } +} + +static const char usage_str[] = +" help Prints this helptext.\n" +" prefix p Export all templates matching the prefix p.\n" +; + +void macrotool_export(std::vector params) +{ + if(params.size() < 1) { + printf("%s", usage_str); + return; + } + + DEBUG(export, "export: %s\n", params[0].c_str()); + + if(params[0] == "prefix") { + if(params.size() != 2) { + printf("The command 'prefix' needs a parameter.\n"); + printf("%s", usage_str); + return; + } + export_prefix(params[1]); + return; + } + + if(params[0] == "help") { + printf("%s", usage_str); + return; + } + + printf("Unknown command '%s'\n", params[0].c_str()); + printf("%s", usage_str); + return; +} + + +#ifdef TEST_EXPORT +//Additional dependency files +//deps: +//Required cflags (autoconf vars may be used) +//cflags: +//Required link options (autoconf vars may be used) +//libs: +#include "test.h" + +TEST_BEGIN; + +// TODO: Put some testcode here (see test.h for usable macros). + +TEST_END; + +#endif/*TEST_EXPORT*/ diff --git a/server/src/macrotool/export.h b/server/src/macrotool/export.h new file mode 100644 index 0000000..5da8c7c --- /dev/null +++ b/server/src/macrotool/export.h @@ -0,0 +1,36 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set et sw=2 ts=2: */ +/*************************************************************************** + * export.h + * + * Wed Jan 26 11:44:12 CET 2011 + * Copyright 2011 Bent Bisballe Nyeng + * deva@aasimon.org + ****************************************************************************/ + +/* + * This file is part of Pracro. + * + * Pracro is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * Pracro is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pracro; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + */ +#ifndef __PRACRO_EXPORT_H__ +#define __PRACRO_EXPORT_H__ + +#include +#include + +void macrotool_export(std::vector params); + +#endif/*__PRACRO_EXPORT_H__*/ diff --git a/server/src/macrotool/fieldnamescanner.cc b/server/src/macrotool/fieldnamescanner.cc new file mode 100644 index 0000000..9c436fb --- /dev/null +++ b/server/src/macrotool/fieldnamescanner.cc @@ -0,0 +1,158 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set et sw=2 ts=2: */ +/*************************************************************************** + * fieldnamescanner.cc + * + * Wed Jan 26 09:21:58 CET 2011 + * Copyright 2011 Bent Bisballe Nyeng + * deva@aasimon.org + ****************************************************************************/ + +/* + * This file is part of Pracro. + * + * Pracro is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * Pracro is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pracro; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + */ +#include "fieldnamescanner.h" + +#include + +#include "util.h" + +#include "macroparser.h" +#include "templateparser.h" +#include "configuration.h" +#include "exception.h" +#include "macrolist.h" + +#include "debug.h" + +fieldnames_t getFields(Widget &widget) +{ + fieldnames_t fieldnames; + + if(widget.attributes.find("name") != widget.attributes.end()) { + fieldnames.push_back(widget.attributes["name"]); + //printf("\t\t%s\n", widget.attributes["name"].c_str()); + } + + std::vector< Widget >::iterator i = widget.widgets.begin(); + while(i != widget.widgets.end()) { + fieldnames_t f = getFields(*i); + fieldnames.insert(fieldnames.end(), f.begin(), f.end()); + i++; + } + + return fieldnames; +} + +templates_t scanfieldnames(std::set &filter) +{ + templates_t templates; + + // TemplateList templatelist(Conf::xml_basedir + "/templates"); + MacroList macrolist(Conf::xml_basedir + "/macros"); + + // Iterate templates: + std::vector templatefiles = getTemplates(); + std::vector::iterator tfs = templatefiles.begin(); + while(tfs != templatefiles.end()) { + std::string templ = tfs->substr(0, tfs->length() - 4); + fieldnames_t fieldnames; + DEBUG(scanner, "%s:\n", tfs->c_str()); + TemplateParser parser(Conf::xml_basedir + "/templates/" + *tfs); + try { + parser.parse(); + Template *t = parser.getTemplate(); + if(!t) ERR(scanner, "Missing template!\n"); + // Iterate macros: + std::vector::iterator ms = t->macros.begin(); + while(ms != t->macros.end()) { + if(ms->isHeader == false) { + std::string macro = ms->attributes["name"]; + DEBUG(scanner, "Name '%s'\n", macro.c_str()); + std::string macrofile = macrolist.getLatestVersion(macro); + DEBUG(scanner, "File '%s'\n", macrofile.c_str()); + + // Iterate fields: + MacroParser parser(macrofile); + try { + parser.parse(); + Macro *m = parser.getMacro(); + fieldnames_t f = getFields(m->widgets); + fieldnames.insert(fieldnames.end(), f.begin(), f.end()); + } catch( Exception &e ) { + ERR(scanner, "Error reading macro: %s: %s\n", + macrofile.c_str(), e.what()); + } + } + + ms++; + } + } catch( Exception &e ) { + ERR(scanner, "Error reading template: %s: %s\n", tfs->c_str(), e.what()); + } + + fieldnames_t nodubs; + fieldnames_t::iterator fi = fieldnames.begin(); + while(fi != fieldnames.end()) { + bool hasdub = false; + fieldnames_t::iterator di = nodubs.begin(); + while(di != nodubs.end()) { + if(*di == *fi) { + hasdub = true; + break; + } + di++; + } + + if(!hasdub && filter.find(*fi) != filter.end()) nodubs.push_back(*fi); + fi++; + } + + templates[templ] = nodubs; + tfs++; + } + + return templates; +} + +#ifdef TEST_FIELDNAMESCANNER +//deps: util.cc ../configuration.cc ../debug.cc ../log.cc ../templateparser.cc ../exception.cc ../saxparser.cc ../macrolist.cc ../entitylist.cc ../inotify.cc ../mutex.cc ../macroheaderparser.cc ../versionstr.cc ../macroparser.cc +//cflags: -I.. -I../.. $(EXPAT_CFLAGS) $(PTHREAD_CFLAGS) +//libs: $(EXPAT_LIBS) $(PTHREAD_LIBS) +#include "test.h" + +TEST_BEGIN; +Conf::xml_basedir = "../../xml"; +std::set filter; +filter.insert("dims"); +filter.insert("test1"); +templates_t t = scanfieldnames(filter); + +templates_t::iterator ti = t.begin(); +while(ti != t.end()) { + printf("%s\n", ti->first.c_str()); + fieldnames_t::iterator fi = ti->second.begin(); + while(fi != ti->second.end()) { + printf("\t%s\n", (*fi).c_str()); + fi++; + } + ti++; +} + +TEST_END; + +#endif/*TEST_FIELDNAMESCANNER*/ diff --git a/server/src/macrotool/fieldnamescanner.h b/server/src/macrotool/fieldnamescanner.h new file mode 100644 index 0000000..c766ba1 --- /dev/null +++ b/server/src/macrotool/fieldnamescanner.h @@ -0,0 +1,44 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set et sw=2 ts=2: */ +/*************************************************************************** + * fieldnamescanner.h + * + * Wed Jan 26 09:21:58 CET 2011 + * Copyright 2011 Bent Bisballe Nyeng + * deva@aasimon.org + ****************************************************************************/ + +/* + * This file is part of Pracro. + * + * Pracro is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * Pracro is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Pracro; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + */ +#ifndef __PRACRO_FIELDNAMESCANNER_H__ +#define __PRACRO_FIELDNAMESCANNER_H__ + +#include +#include +#include +#include + +typedef std::string fieldname_t; +typedef std::vector< fieldname_t > fieldnames_t; + +typedef std::string template_name_t; +typedef std::map< template_name_t, fieldnames_t > templates_t; + +templates_t scanfieldnames(std::set &filter); + +#endif/*__PRACRO_FIELDNAMESCANNER_H__*/ diff --git a/server/src/macrotool/macrotool.cc b/server/src/macrotool/macrotool.cc index 6d484ad..b811c29 100644 --- a/server/src/macrotool/macrotool.cc +++ b/server/src/macrotool/macrotool.cc @@ -45,6 +45,7 @@ #include "dump.h" #include "fieldnames.h" #include "filehandler.h" +#include "export.h" static const char version_str[] = "Pracro server v" VERSION "\n" @@ -69,10 +70,11 @@ static const char usage_str[] = " -D, --debug ddd Enable debug messages on 'ddd'; see documentation for details\n" "\n" "Commands:\n" -" dump entity Dumps 'entity' to screen ('dump help' to see list of entities).\n" -" fieldnames entity Add/delete/update entries in the fieldnames database\n" +" dump entity Dumps 'entity' to screen ('dump help' to see list of entities).\n" +" fieldnames entity Add/delete/update entries in the fieldnames database\n" " ('fieldnames help' to see list of entities).\n" -" filehandler entity Handle macro files ('macrohandler help' to see list of entities).\n" +" filehandler entity Handle macro files ('filehandler help' to see list of entities).\n" +" export entity Export data from database to comma separated file ('export help' to see list of entities)\n" ; ConfigurationParser *configparser = NULL; @@ -80,8 +82,8 @@ ConfigurationParser *configparser = NULL; int main(int argc, char *argv[]) { int c; - char *configfile = NULL; - char *xml_basedir = NULL; + std::string configfile; + std::string xml_basedir; char *debugstr = NULL; debug_init(stderr); @@ -104,11 +106,11 @@ int main(int argc, char *argv[]) switch(c) { case 'c': - configfile = strdup(optarg); + configfile = optarg; break; case 'x': - xml_basedir = strdup(optarg); + xml_basedir = optarg; break; case 'D': @@ -142,10 +144,10 @@ int main(int argc, char *argv[]) } // Load config - if(configfile) configparser = new ConfigurationParser(configfile); - else configparser = new ConfigurationParser(ETC"/pracrod.conf"); + if(configfile == "") configfile = ETC"/pracrod.conf"; + configparser = new ConfigurationParser(configfile); - if(xml_basedir) { + if(xml_basedir != "") { Conf::xml_basedir = xml_basedir; } @@ -158,9 +160,7 @@ int main(int argc, char *argv[]) if(command == "dump") macrotool_dump(params); if(command == "fieldnames") macrotool_fieldnames(params); if(command == "filehandler") macrotool_filehandler(params); - - // Clean up - if(configfile) free(configfile); + if(command == "export") macrotool_export(params); return 0; } -- cgit v1.2.3