#!/usr/bin/tclsh # filtercgn corpus filtration script # Copyright (C) 2004-2007 Richard Moot (Richard.Moot@labri.fr) # Copyright (C) 2004-2007 CNRS (http://www.cnrs.fr) # Copyright (C) 2004-2007 INRIA (http://www.inria.fr) # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2.1 of the License, or (at your option) any later version. # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # You should have received a copy of the GNU Lesser General Public # License along with this library; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # This script takes a CGN .syn file as input and removes all isolated # nodes (parentless nodes which are nonetheless part of a phrase, like # is typically the case for hesitations like "uh") and tags with the # type TSW (like "ja" in "een ja goed idee"). # If you are interested in just one of these behaviours, set the flag # of the option you're not interested in to 0 below. # options set debug 0 set isolated 1 set notags 1 # counters set passed 0 set d_iso 0 set d_tag 0 proc loadcgn {file} { global sentence isolated notags passed d_iso d_tag set f [open $file r] set of [open "[file rootname $file].out" w] set df [open "[file rootname $file].del" w] set sentno 0 # read the cgn output file and convert it to an array while {[gets $f line] >= 0} { if {[regexp {^#} $line]} { # parent node, keep it puts $of $line } elseif {$isolated == 1 && [regexp {\-\-[[:space:]]+0} $line x sentno]} { puts $df $line incr d_iso } elseif {$notags == 1 && [regexp {T001[[:space:]]+TAG} $line x sentno]} { puts $df $line incr d_tag } else { puts $of $line incr passed } } close $f close $of close $df } set files [lrange $argv 0 end] foreach f $files { if {[file readable $f]} { puts stderr "File: $f" loadcgn $f } else { puts stderr "File $f does not exist or is unreadable" exit 1 } } puts stderr "Total words: [expr $passed+$d_tag+$d_iso]" puts stderr "Isolated : -$d_iso" puts stderr "Tags : -$d_tag" puts stderr "Remaining : $passed"