New data extraction tool.

6 files changed, 562 insertions, 17 deletions

diff --git a/server/src/macrotool/Makefile.am b/server/src/macrotool/Makefile.am
index b8a42cb..061b904 100644
--- a/server/src/macrotool/Makefile.am
+++ b/server/src/macrotool/Makefile.am
@@ -10,7 +10,9 @@ macrotool_CXXFLAGS = -I.. $(PQXX_CXXFLAGS) $(CONFIG_CXXFLAGS) \
 macrotool_SOURCES = \
 	macrotool.cc \
 	dump.cc \
+	export.cc \
 	fieldnames.cc \
+	fieldnamescanner.cc \
 	filehandler.cc \
 	util.cc \
 	../debug.cc \
@@ -34,7 +36,9 @@ macrotool_SOURCES = \
 
 EXTRA_DIST = \
 	dump.h \
+	export.h \
 	fieldnames.h \
+	fieldnamescanner.h \
 	filehandler.h \
 	util.h
 
@@ -42,9 +46,9 @@ EXTRA_DIST = \
 # Test Section #
 ################
 
-#TEST_SOURCE_DEPS = ${macrotool_SOURCES} ${EXTRA_DIST}
-#TEST_SCRIPT_DIR = $(top_srcdir)/../../tools
+TEST_SOURCE_DEPS = ${macrotool_SOURCES} ${EXTRA_DIST}
+TEST_SCRIPT_DIR = $(top_srcdir)/../tools
 
-#include ${TEST_SCRIPT_DIR}/Makefile.am.test
+include ${TEST_SCRIPT_DIR}/Makefile.am.test
 
-#include Makefile.am.test
\ No newline at end of file
+include Makefile.am.test
\ No newline at end of file
diff --git a/server/src/macrotool/export.cc b/server/src/macrotool/export.cc
new file mode 100644
index 0000000..150a85c
--- /dev/null
+++ b/server/src/macrotool/export.cc
@@ -0,0 +1,303 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set et sw=2 ts=2: */
+/***************************************************************************
+ *            export.cc
+ *
+ *  Wed Jan 26 11:44:12 CET 2011
+ *  Copyright 2011 Bent Bisballe Nyeng
+ *  deva@aasimon.org
+ ****************************************************************************/
+
+/*
+ *  This file is part of Pracro.
+ *
+ *  Pracro is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  Pracro is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with Pracro; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
+ */
+#include "export.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <pqxx/pqxx>
+
+#include "debug.h"
+#include "fieldnamescanner.h"
+#include "configuration.h"
+
+#define SEP "\t"
+
+static std::string escape(std::string &str)
+{
+  std::string out = "\"";
+  std::string::iterator i = str.begin();
+  while(i != str.end()) {
+    if(*i == '\"') out += "''";
+    else out += *i;
+    i++;
+  }
+  out += "\"";
+  return out;
+}
+
+class File {
+public:
+  File(std::string n, fieldnames_t &f, pqxx::work &w)
+    : work(w), name(n), fieldnames(f)
+  {
+    name += ".csf";
+    fp = fopen(name.c_str(), "w");
+
+    pos["id"] = 0;
+    pos["patientid"] = 1;
+    pos["time"] = 2;
+    pos["template"] = 3;
+
+    printf("%s\n", n.c_str());
+
+    size_t idx = 4;
+    fieldnames_t::iterator i = f.begin();
+    while(i != f.end()) {
+      printf("%s ", i->c_str());
+      pos[*i] = idx;
+      idx++;
+      i++;
+    }
+    printf("\n");
+
+    output_header();
+  }
+
+  ~File()
+  {
+    if(fp) fclose(fp);
+  }
+
+  void output_header()
+  {
+    beginrow();
+    addcell("id", "ID");
+    addcell("patientid", "Patient");
+    addcell("time", "Time");
+    addcell("template", "Template");
+
+    {
+      pqxx::result result =
+        work.exec("SELECT DISTINCT ON(name) name, caption FROM fieldnames"
+                  " WHERE extract='true';");
+      pqxx::result::const_iterator ri = result.begin();
+      for(unsigned int r = 0; r < result.size(); r++) {
+        pqxx::result::tuple tuple = result.at(r); 
+        std::string name = tuple.at(0).c_str();
+        std::string caption = tuple.at(1).c_str();
+        addcell(name, caption);
+      }
+    }
+
+    endrow();
+  }
+
+  void beginrow()
+  {
+    cells.clear();
+    cells.insert(cells.begin(), pos.size(), "");
+  }
+
+  void addcell(std::string key, std::string value)
+  {
+    if(pos.find(key) == pos.end()) return;
+    cells[pos[key]] = value;
+  }
+
+  void endrow()
+  {
+    std::vector<std::string>::iterator i = cells.begin();
+    while(i != cells.end()) {
+      fprintf(fp, "%s"SEP, escape(*i).c_str());
+      i++;
+    }
+    fprintf(fp, "\n");
+  }
+
+private:
+  pqxx::work &work;
+  std::map<std::string, int> pos;
+  std::vector<std::string> cells;
+  std::string name;
+  fieldnames_t &fieldnames;
+  FILE *fp;
+};
+
+
+static void export_prefix(std::string prefix)
+{
+
+  std::map<std::string, File *> files;
+
+  if(Conf::database_backend != "pgsql") {
+    printf("ERROR: Export only available for the pgsql database backend.\n");
+    return;
+  }
+
+  std::string host = Conf::database_addr;
+  std::string port = "";//Conf::database_port;
+  std::string user = Conf::database_user;;
+  std::string passwd = Conf::database_passwd;
+  std::string dbname = "";//Conf::database_database;
+
+  std::string cs;
+  if(host.size()) cs += " host=" + host;
+  if(port.size()) cs += " port=" + port;
+  if(user.size()) cs += " user=" + user;
+  if(passwd.size()) cs += " password=" + passwd;
+  cs += " dbname=" + (dbname.size() ? dbname : "pracro");
+
+  pqxx::connection conn(cs);
+  pqxx::work work(conn);
+
+  std::set<std::string> filter;
+
+  {
+    pqxx::result result =
+      work.exec("SELECT DISTINCT name FROM fieldnames"
+                " WHERE extract='true';");
+    pqxx::result::const_iterator ri = result.begin();
+    for(unsigned int r = 0; r < result.size(); r++) {
+      pqxx::result::tuple tuple = result.at(r); 
+      std::string name = tuple.at(0).c_str();
+      filter.insert(name);
+      //printf("filter: '%s'\n", name.c_str());
+    }
+  }
+
+  templates_t t = scanfieldnames(filter);
+  
+  templates_t::iterator ti = t.begin();
+  while(ti != t.end()) {
+    printf("%s\n", ti->first.c_str());
+    fieldnames_t::iterator fi = ti->second.begin();
+    while(fi != ti->second.end()) {
+      printf("\t%s\n", (*fi).c_str());
+      fi++;
+    }
+    ti++;
+  }
+
+
+  {
+    pqxx::result result =
+      work.exec("SELECT * FROM commits WHERE template LIKE '"+prefix+"%'"
+                " AND status='committed' ORDER BY patientid, timestamp;");
+    pqxx::result::const_iterator ri = result.begin();
+    for(unsigned int r = 0; r < result.size(); r++) {
+      pqxx::result::tuple tuple = result.at(r); 
+      std::string patientid = tuple.at(0).c_str();
+      std::string templ = tuple.at(1).c_str();
+      std::string version = tuple.at(2).c_str();
+      std::string timestamp = tuple.at(3).c_str();
+      std::string uid = tuple.at(4).c_str();
+      std::string status = tuple.at(5).c_str();
+
+      if(files.find(templ) == files.end()) {
+        files[templ] = new File(templ, t[templ], work);
+      }
+
+      files[templ]->beginrow();
+      files[templ]->addcell("id", uid);
+      files[templ]->addcell("patientid", patientid);
+      files[templ]->addcell("template", templ);
+      time_t t = atol(timestamp.c_str());
+      files[templ]->addcell("time", ctime(&t));
+
+      printf("%s %s %s\n", timestamp.c_str(), patientid.c_str(), templ.c_str());
+
+      {
+        pqxx::result result =
+          work.exec("SELECT f.name, f.value"
+                    " FROM transactions t, fields f, fieldnames n"
+                    " WHERE t.cid='"+uid+"' AND f.transaction=t.uid"
+                    "  AND f.name=n.name AND n.extract='true'"
+                    " ORDER BY t.timestamp;");
+        pqxx::result::const_iterator ri = result.begin();
+        for(unsigned int r = 0; r < result.size(); r++) {
+          pqxx::result::tuple tuple = result.at(r); 
+          std::string name = tuple.at(0).c_str();
+          std::string value = tuple.at(1).c_str();
+
+          files[templ]->addcell(name, value);
+
+        }
+      }
+
+      files[templ]->endrow();
+    }
+  }
+  
+  std::map<std::string, File *>::iterator i = files.begin();
+  while(i != files.end()) {
+    delete i->second;
+    i++;
+  }
+}
+
+static const char usage_str[] =
+"  help        Prints this helptext.\n"
+"  prefix p    Export all templates matching the prefix p.\n"
+;
+
+void macrotool_export(std::vector<std::string> params)
+{
+  if(params.size() < 1) {
+    printf("%s", usage_str);
+    return;
+  }
+
+  DEBUG(export, "export: %s\n", params[0].c_str());
+
+  if(params[0] == "prefix") {
+    if(params.size() != 2) {
+      printf("The command 'prefix' needs a parameter.\n");
+      printf("%s", usage_str);
+      return;
+    }
+    export_prefix(params[1]);
+    return;
+  }
+
+  if(params[0] == "help") {
+    printf("%s", usage_str);
+    return;
+  }
+
+  printf("Unknown command '%s'\n", params[0].c_str());
+  printf("%s", usage_str);
+  return;
+}
+
+
+#ifdef TEST_EXPORT
+//Additional dependency files
+//deps:
+//Required cflags (autoconf vars may be used)
+//cflags:
+//Required link options (autoconf vars may be used)
+//libs:
+#include "test.h"
+
+TEST_BEGIN;
+
+// TODO: Put some testcode here (see test.h for usable macros).
+
+TEST_END;
+
+#endif/*TEST_EXPORT*/
diff --git a/server/src/macrotool/export.h b/server/src/macrotool/export.h
new file mode 100644
index 0000000..5da8c7c
--- /dev/null
+++ b/server/src/macrotool/export.h
@@ -0,0 +1,36 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set et sw=2 ts=2: */
+/***************************************************************************
+ *            export.h
+ *
+ *  Wed Jan 26 11:44:12 CET 2011
+ *  Copyright 2011 Bent Bisballe Nyeng
+ *  deva@aasimon.org
+ ****************************************************************************/
+
+/*
+ *  This file is part of Pracro.
+ *
+ *  Pracro is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  Pracro is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with Pracro; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
+ */
+#ifndef __PRACRO_EXPORT_H__
+#define __PRACRO_EXPORT_H__
+
+#include <vector>
+#include <string>
+
+void macrotool_export(std::vector<std::string> params);
+
+#endif/*__PRACRO_EXPORT_H__*/
diff --git a/server/src/macrotool/fieldnamescanner.cc b/server/src/macrotool/fieldnamescanner.cc
new file mode 100644
index 0000000..9c436fb
--- /dev/null
+++ b/server/src/macrotool/fieldnamescanner.cc
@@ -0,0 +1,158 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set et sw=2 ts=2: */
+/***************************************************************************
+ *            fieldnamescanner.cc
+ *
+ *  Wed Jan 26 09:21:58 CET 2011
+ *  Copyright 2011 Bent Bisballe Nyeng
+ *  deva@aasimon.org
+ ****************************************************************************/
+
+/*
+ *  This file is part of Pracro.
+ *
+ *  Pracro is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  Pracro is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with Pracro; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
+ */
+#include "fieldnamescanner.h"
+
+#include <stdio.h>
+
+#include "util.h"
+
+#include "macroparser.h"
+#include "templateparser.h"
+#include "configuration.h"
+#include "exception.h"
+#include "macrolist.h"
+
+#include "debug.h"
+
+fieldnames_t getFields(Widget &widget)
+{
+  fieldnames_t fieldnames;
+  
+  if(widget.attributes.find("name") != widget.attributes.end()) {
+    fieldnames.push_back(widget.attributes["name"]);
+    //printf("\t\t%s\n", widget.attributes["name"].c_str());
+  }
+
+  std::vector< Widget >::iterator i = widget.widgets.begin();
+  while(i != widget.widgets.end()) {
+    fieldnames_t f = getFields(*i);
+    fieldnames.insert(fieldnames.end(), f.begin(), f.end());
+    i++;
+  }
+
+  return fieldnames;
+}
+
+templates_t scanfieldnames(std::set<std::string> &filter)
+{
+  templates_t templates;
+
+  //  TemplateList templatelist(Conf::xml_basedir + "/templates");
+  MacroList macrolist(Conf::xml_basedir + "/macros");
+
+  // Iterate templates:
+  std::vector<std::string> templatefiles = getTemplates();
+  std::vector<std::string>::iterator tfs = templatefiles.begin();
+  while(tfs != templatefiles.end()) {
+    std::string templ = tfs->substr(0, tfs->length() - 4);
+    fieldnames_t fieldnames;
+    DEBUG(scanner, "%s:\n", tfs->c_str());
+    TemplateParser parser(Conf::xml_basedir + "/templates/" + *tfs);
+    try {
+      parser.parse();
+      Template *t = parser.getTemplate();
+      if(!t) ERR(scanner, "Missing template!\n");
+      // Iterate macros:
+      std::vector<Macro>::iterator ms = t->macros.begin();
+      while(ms != t->macros.end()) {
+        if(ms->isHeader == false) {
+          std::string macro = ms->attributes["name"];
+          DEBUG(scanner, "Name '%s'\n", macro.c_str());
+          std::string macrofile = macrolist.getLatestVersion(macro);
+          DEBUG(scanner, "File '%s'\n", macrofile.c_str());
+
+          // Iterate fields:
+          MacroParser parser(macrofile);
+          try {
+            parser.parse();
+            Macro *m = parser.getMacro();
+            fieldnames_t f = getFields(m->widgets);
+            fieldnames.insert(fieldnames.end(), f.begin(), f.end());
+          } catch( Exception &e ) {
+            ERR(scanner, "Error reading macro: %s: %s\n",
+                macrofile.c_str(), e.what());
+          }
+        }
+          
+        ms++;
+      }
+    } catch( Exception &e ) {
+      ERR(scanner, "Error reading template: %s: %s\n", tfs->c_str(), e.what());
+    }
+
+    fieldnames_t nodubs;
+    fieldnames_t::iterator fi = fieldnames.begin();
+    while(fi != fieldnames.end()) {
+      bool hasdub = false;
+      fieldnames_t::iterator di = nodubs.begin();
+      while(di != nodubs.end()) {
+        if(*di == *fi) {
+          hasdub = true;
+          break;
+        }
+        di++;
+      }
+
+      if(!hasdub && filter.find(*fi) != filter.end()) nodubs.push_back(*fi);
+      fi++;
+    }
+    
+    templates[templ] = nodubs;
+    tfs++;
+  }
+
+  return templates;
+}
+
+#ifdef TEST_FIELDNAMESCANNER
+//deps: util.cc ../configuration.cc ../debug.cc ../log.cc ../templateparser.cc ../exception.cc ../saxparser.cc ../macrolist.cc ../entitylist.cc ../inotify.cc ../mutex.cc ../macroheaderparser.cc ../versionstr.cc ../macroparser.cc
+//cflags: -I.. -I../.. $(EXPAT_CFLAGS) $(PTHREAD_CFLAGS)
+//libs: $(EXPAT_LIBS) $(PTHREAD_LIBS)
+#include "test.h"
+
+TEST_BEGIN;
+Conf::xml_basedir = "../../xml";
+std::set<std::string> filter;
+filter.insert("dims");
+filter.insert("test1");
+templates_t t = scanfieldnames(filter);
+
+templates_t::iterator ti = t.begin();
+while(ti != t.end()) {
+  printf("%s\n", ti->first.c_str());
+  fieldnames_t::iterator fi = ti->second.begin();
+  while(fi != ti->second.end()) {
+    printf("\t%s\n", (*fi).c_str());
+    fi++;
+  }
+  ti++;
+}
+
+TEST_END;
+
+#endif/*TEST_FIELDNAMESCANNER*/
diff --git a/server/src/macrotool/fieldnamescanner.h b/server/src/macrotool/fieldnamescanner.h
new file mode 100644
index 0000000..c766ba1
--- /dev/null
+++ b/server/src/macrotool/fieldnamescanner.h
@@ -0,0 +1,44 @@
+/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set et sw=2 ts=2: */
+/***************************************************************************
+ *            fieldnamescanner.h
+ *
+ *  Wed Jan 26 09:21:58 CET 2011
+ *  Copyright 2011 Bent Bisballe Nyeng
+ *  deva@aasimon.org
+ ****************************************************************************/
+
+/*
+ *  This file is part of Pracro.
+ *
+ *  Pracro is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  Pracro is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with Pracro; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA.
+ */
+#ifndef __PRACRO_FIELDNAMESCANNER_H__
+#define __PRACRO_FIELDNAMESCANNER_H__
+
+#include <map>
+#include <vector>
+#include <string>
+#include <set>
+
+typedef std::string fieldname_t;
+typedef std::vector< fieldname_t > fieldnames_t;
+
+typedef std::string template_name_t;
+typedef std::map< template_name_t, fieldnames_t > templates_t;
+
+templates_t scanfieldnames(std::set<std::string> &filter);
+
+#endif/*__PRACRO_FIELDNAMESCANNER_H__*/
diff --git a/server/src/macrotool/macrotool.cc b/server/src/macrotool/macrotool.cc
index 6d484ad..b811c29 100644
--- a/server/src/macrotool/macrotool.cc
+++ b/server/src/macrotool/macrotool.cc
@@ -45,6 +45,7 @@
 #include "dump.h"
 #include "fieldnames.h"
 #include "filehandler.h"
+#include "export.h"
 
 static const char version_str[] =
 "Pracro server v" VERSION "\n"
@@ -69,10 +70,11 @@ static const char usage_str[] =
 "  -D, --debug ddd     Enable debug messages on 'ddd'; see documentation for details\n"
 "\n"
 "Commands:\n"
-"  dump entity          Dumps 'entity' to screen ('dump help' to see list of entities).\n"
-"  fieldnames entity    Add/delete/update entries in the fieldnames database\n"
+"  dump entity         Dumps 'entity' to screen ('dump help' to see list of entities).\n"
+"  fieldnames entity   Add/delete/update entries in the fieldnames database\n"
 "                       ('fieldnames help' to see list of entities).\n"
-"  filehandler entity  Handle macro files ('macrohandler help' to see list of entities).\n"
+"  filehandler entity  Handle macro files ('filehandler help' to see list of entities).\n"
+"  export entity       Export data from database to comma separated file ('export help' to see list of entities)\n"
 ;
 
 ConfigurationParser *configparser = NULL;
@@ -80,8 +82,8 @@ ConfigurationParser *configparser = NULL;
 int main(int argc, char *argv[])
 {
   int c;
-  char *configfile = NULL;
-  char *xml_basedir = NULL;
+  std::string configfile;
+  std::string xml_basedir;
   char *debugstr = NULL;
 
   debug_init(stderr);
@@ -104,11 +106,11 @@ int main(int argc, char *argv[])
 
     switch(c) {
     case 'c':
-      configfile = strdup(optarg);
+      configfile = optarg;
       break;
 
     case 'x':
-      xml_basedir = strdup(optarg);
+      xml_basedir = optarg;
       break;
 
     case 'D':
@@ -142,10 +144,10 @@ int main(int argc, char *argv[])
   }
 
   // Load config
-  if(configfile) configparser = new ConfigurationParser(configfile);
-  else configparser = new ConfigurationParser(ETC"/pracrod.conf");
+  if(configfile == "") configfile = ETC"/pracrod.conf";
+  configparser = new ConfigurationParser(configfile);
 
-  if(xml_basedir) {
+  if(xml_basedir != "") {
     Conf::xml_basedir = xml_basedir;
   }
 
@@ -158,9 +160,7 @@ int main(int argc, char *argv[])
   if(command == "dump") macrotool_dump(params);
   if(command == "fieldnames") macrotool_fieldnames(params);
   if(command == "filehandler") macrotool_filehandler(params);
-
-  // Clean up
-  if(configfile) free(configfile);
+  if(command == "export") macrotool_export(params);
 
   return 0;
 }