#!/usr/bin/tclsh # lex2me - training data generation script # Copyright (C) 2004-2007 Richard Moot (Richard.Moot@labri.fr) # Copyright (C) 2004-2007 CNRS (http://www.cnrs.fr) # Copyright (C) 2004-2007 INRIA (http://www.inria.fr) # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Based on a series of .lex files as produced by the cgntoform script, # this script will create a file maxentdata.txt where every line is # a sequence of word/pos/supertag triples for a sentence in the corpus. set total 0 set forms 1 source "formula.tcl" set fo [open "maxentdata.txt" w] set fb [open "untaggeddata.txt" w] foreach file [lrange $argv 0 end] { set fh [open $file r] puts stderr "$file" while {[gets $fh line] >= 0} { if {[regexp {^lex\(\"(.*)\"\, (.*)\, (.*)\, (.*)\, (.*)\, (.*)\, (.*)\, (.*)\, (.*)\)} $line all pros form sem file sent wrdn trig big uni]} { set sentence($wrdn) [list $pros $uni $formula($form)] } if {[regexp {^\%= sentence ([0-9]+) =} $line]} { set s "" set s2 "" set list [lsort -integer [array names sentence]] if {$list !={}} { foreach i $list { incr total set x $sentence($i) unset sentence($i) set s "$s [string trim [lindex $x 0]]/[string trim [lindex $x 1]]/[string trim [lindex $x 2]]" set s2 "$s2 [string trim [lindex $x 0]]/[string trim [lindex $x 1]]" } puts $fo [string trim $s] puts $fb [string trim $s2] } } } close $fh } close $fo close $fb