← Back
Editing: munchlist
#!/bin/sh # # $Id: munchlist.X,v 1.70 2015-02-08 00:35:41-08 geoff Exp $ # # Copyright 1987, 1988, 1989, 1992, 1993, 1999, 2001, 2005, Geoff Kuenning, # Claremont, CA. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: # # 1. Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. # 2. Redistributions in binary form must reproduce the above copyright # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # 3. All modifications to the source code must be clearly marked as # such. Binary redistributions based on modified source code # must be clearly marked as modified versions in the documentation # and/or other materials provided with the distribution. # 4. The code that causes the 'ispell -v' command to display a prominent # link to the official ispell Web site may not be removed. # 5. The name of Geoff Kuenning may not be used to endorse or promote # products derived from this software without specific prior # written permission. # # THIS SOFTWARE IS PROVIDED BY GEOFF KUENNING AND CONTRIBUTORS ``AS IS'' AND # ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL GEOFF KUENNING OR CONTRIBUTORS BE LIABLE # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS # OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY # OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF # SUCH DAMAGE. # # Given a list of words for ispell, generate a reduced list # in which all possible affixes have been collapsed. The reduced # list will match the same list as the original. # # Usage: # # munchlist [-l lang] [-c lang] [-s hashfile] [-D] [-w chars] [-v] \ # [file] ... # # Options: # # -l lang Specifies the language table to be used. The default # is "$LIBDIR/default.aff". # -c lang Specifies "conversion" language table. If this option is # given, the input file(s) will be assumed to be described by # this table, rather than the table given in the -l option. # This may be used to convert between incompatible language # tables. (When in doubt, use this option -- it doesn't # hurt, and it may save you from creating a dictionary that has # illegal words in it). The default is no conversion. # -T suff Specifies that the source word lists are in the format # of a "suff"-suffixed file, rather than in the # canonical form. For example, "-T tex" specifies that # string characters in the word lists are in TeX format. # The string character conversions are taken from the language # table specified by the "-l" switch. # -s Remove any words that are already covered by the # dictionary in 'hashfile'. The words will be removed # only if all affixes are covered. This option should not be # specified when the main dictionary is being munched. # 'Hashfile' must have been created with the language # table given in the -l option, but this is not checked. # -D Leave temporary files for debugging purposes # -w Passed on to ispell (specify chars that are part of a word) # Unfortunately, special characters must be quoted twice # rather than once when invoking this script. Also, since # buildhash doesn't accept this option, the final ispell -l # step ignores it, making it somewhat less than useful. # -v Report progress to stderr. # # The given input files are merged, then processed by 'ispell -c' # to generate possible affix lists; these are then combined # and reduced. The final result is written to standard output. # # For portability to older systems, I have avoided getopt. # # Geoff Kuenning # 2/28/87 # # $Log: munchlist.X,v $ # Revision 1.70 2015-02-08 00:35:41-08 geoff # Be a bit more paranoid about creating temporary files. Fix a problem # with detecting a new-style sort that refuses to be backwards # compatible (and yes, it's still cretinism to break backwards # compatibility--but I have to put up with the cretins). # # Revision 1.69 2005/04/28 14:46:51 geoff # Remove references to the now-obsolete count file. # # Revision 1.68 2005/04/27 01:18:34 geoff # Work around idiotic POSIX incompatibilities in sort. Add secure # temp-file handling. # # Revision 1.67 2005/04/14 23:11:36 geoff # Pass the -w switch to icombine. # # Revision 1.66 2005/04/14 21:25:52 geoff # Make the temporary-file handling safer (using mktemp, if it exists). # # Revision 1.65 2005/04/14 14:39:33 geoff # Use /tmp as the default temp directory # # Revision 1.64 2005/04/14 14:38:23 geoff # Update license. Protect against modernized (i.e., incompatible) and # internationalized sort commands. Change the debugging names of the # minimal-affixes count and stat files. # # Revision 1.63 2002/06/20 23:46:16 geoff # Add yet more locale definitions so that we won't run into bugs caused # by sorting inconsistencies. # # Revision 1.62 2001/09/06 00:30:28 geoff # Many changes from Eli Zaretskii to support DJGPP compilation. # # Revision 1.61 2001/07/25 21:51:46 geoff # Minor license update. # # Revision 1.60 2001/07/23 20:24:04 geoff # Update the copyright and the license. # # Revision 1.59 2001/06/07 08:02:18 geoff # Fix a copule of typos in comments. # # Revision 1.58 2000/11/14 07:27:04 geoff # Don't generate an extra dot when attempting to preserve the count # files in -D mode. # # Revision 1.57 2000/10/06 23:59:48 geoff # Don't assume dot is in the path # # Revision 1.56 1999/01/07 01:22:42 geoff # Update the copyright. # # Revision 1.55 1997/12/02 06:25:01 geoff # Start the cross-expansions loop count at 1, not zero. # # Revision 1.54 1997/12/01 00:53:52 geoff # Abort the munchlist cross-product loop if it goes over 100 passes. # # Revision 1.53 1995/01/08 23:23:36 geoff # Support variable hashfile suffixes for DOS purposes. # # Revision 1.52 1994/12/27 23:08:46 geoff # Dynamically determine how to pass backslashes to 'tr' so that it'll # work on any machine. Define LC_CTYPE to work around yet more # internationalized sort programs. Work around a bug in GNU uniq that # uses the wrong separator between counts and duplicated lines. # # Revision 1.51 1994/11/21 07:02:54 geoff # Correctly quote the arguments to 'tr' when detecting systems with # unsigned sorts. Be sure to provide a zero exit status on all systems, # even if MUNCHDEBUG is not set. # # Revision 1.50 1994/10/25 05:46:05 geoff # Export values for LANG and LOCALE in an attempt to override some # stupidly-internationalized sort programs. # # Revision 1.49 1994/10/04 03:51:30 geoff # Add the MUNCHMAIL feature. If the MUNCHMAIL environment variable is # set to an email address, debugging information about the munchlist run # will automatically be collected and mailed to that address. # # Revision 1.48 1994/05/17 06:32:06 geoff # Don't look for affix tables in LIBDIR if the name contains a slash # # Revision 1.47 1994/04/27 02:50:48 geoff # Fix some cosmetic flaws in the verbose-mode messages. # # Revision 1.46 1994/01/25 07:11:59 geoff # Get rid of all old RCS log lines in preparation for the 3.1 release. # # LIBDIR=/usr/lib/ispell TDIR=${TMPDIR-/tmp} MUNCHDIR=`mktemp -d ${TDIR}/munchXXXXXXXXXX 2>/dev/null` || { echo "$0: Failed to create temporary directory, exiting..." 1>&2; exit 1; } TMP=${MUNCHDIR}/munch. MAILDEBUGDIR=${MUNCHDIR-/tmp} if [ "X$MUNCHMAIL" != X ] then exec 2> ${MAILDEBUGDIR}/munchlist.mail echo "munchlist $*" 1>&2 set -vx fi SORTTMP="-T ${TDIR}" # !!SORTTMP!! DBDIR=${MUNCHDEBUGDIR-$MAILDEBUGDIR} # Detect MS-DOS systems and arrange to use their silly suffix system if [ -z "$COMSPEC$ComSpec" ] then EXE="" else EXE=".exe" fi # # Set up some program names. This prefers the versions that are in # the same directory as munchlist was run from; if that can't be # figured out, it prefers local versions and finally ones chosen from # $PATH. # # This code could be simplified by using the dirname command, but it's # not available everywhere. For the same reason, we use -r rather than # -x to test for executable files. # case "$0" in */*) bindir=`expr "$0" : '\(.*\)/[^/]*'` ;; *) bindir='.' ;; esac if [ -r $bindir/buildhash$EXE ] then BUILDHASH=$bindir/buildhash$EXE elif [ -r ./buildhash$EXE ] then BUILDHASH=./buildhash$EXE else BUILDHASH=buildhash fi if [ -r $bindir/icombine$EXE ] then COMBINE=$bindir/icombine$EXE elif [ -r ./icombine$EXE ] then COMBINE=./icombine$EXE else COMBINE=icombine fi if [ -r $bindir/ijoin$EXE ] then JOIN=$bindir/ijoin$EXE elif [ -r ./ijoin$EXE ] then JOIN=./ijoin$EXE else JOIN=ijoin fi if [ -r $bindir/ispell$EXE ] then ISPELL=$bindir/ispell$EXE elif [ -r ./ispell$EXE ] then ISPELL=./ispell$EXE else ISPELL=ispell fi # In one of the most incredibly stupid decisions of all time, some # genius decided to break backwards compatibility by "deprecating" the # old-style sort switches even though it was trivial to recognize both # styles. The result is that that thousands of people (like me) will # have to rewrite shell scripts to tolerate that stupidity. (It's not # that the new syntax is bad--it's definitely easier to understand. # But that doesn't excuse breaking compatibility.) # CRETIN_SORT=true # # The following is necessary so that some internationalized versions of # sort(1) don't confuse things by sorting into a nonstandard order. # LANG=C LOCALE=C LC_ALL=C LC_COLLATE=C LC_CTYPE=C export LANG LOCALE LC_COLLATE LC_CTYPE # # The following aren't strictly necessary, but I've been made paranoid # by problems with the stuff above. It can't hurt to set them to a # sensible value. LC_MESSAGES=C LC_MONETARY=C LC_NUMERIC=C LC_TIME=C export LC_MESSAGES LC_MONETARY LC_NUMERIC LC_TIME debug=no dictopt= langtabs=${LIBDIR}/default.aff convtabs= strip=no icflags= verbose=false # The following value of "wchars" is necessary to prevent ispell from # receiving a null argument if -w is not specified. As long as "A" is # a member of the existing character set, ispell will ignore the argument. wchars=-wA while [ $# != 0 ] do case "$1" in -l) case "$2" in */*) langtabs=$2 ;; *) if [ -r "$2" ] then langtabs="$2" else langtabs="${LIBDIR}/$2" fi ;; esac if [ ! -r "$langtabs" ] then echo "Can't open language table '$2'" 1>&2 rm -rf $MUNCHDIR exit 1 fi shift ;; -c) if [ -r "$2" ] then convtabs="$2" elif [ -r "${LIBDIR}/$2" ] then convtabs="${LIBDIR}/$2" else echo "Can't open conversion language table '$2'" 1>&2 rm -rf $MUNCHDIR exit 1 fi shift ;; -s) dictopt="-d $2" strip=yes shift ;; -D) debug=yes ;; -T) icflags="-T $2" shift ;; -v) verbose=true ;; -w) wchars="-w$2" shift ;; --) shift break ;; -) break ;; -*) echo 'Usage: munchlist [-l lang] [-c lang] [-T suff] [-s hashfile] [-D] [-w chars] [-v] [file] ...' \ 1>&2 rm -rf $MUNCHDIR exit 2 ;; *) break ;; esac shift done if [ "X$MUNCHMAIL" != X ] then verbose=true debug=yes fi trap "rm -rf $MUNCHDIR; exit 1" 1 2 13 15 # # Names of temporary files. This is just to make the code a little easier # to read. # EXPANDEDINPUT=${TMP}a STRIPPEDINPUT=${TMP}b CRUNCHEDINPUT=${TMP}c PRODUCTLIST=${TMP}d EXPANDEDPAIRS=${TMP}e LEGALFLAGLIST=${TMP}f JOINEDPAIRS=${TMP}g MINIMALAFFIXES=${TMP}h CROSSROOTS=${TMP}i CROSSEXPANDED=${TMP}j CROSSPAIRS=${TMP}k CROSSILLEGAL=${TMP}l ILLEGALCOMBOS=${TMP}m FAKEDICT=${TMP}n # Ispell insists that hash files have a ".hash" suffix FAKEHASH=${TMP}o.hash AWKSCRIPT=${TMP}p # If the file exists than we should exit with error ERRORFLAGFILE=${TMP}z if [ "$debug" = yes ] then touch $EXPANDEDINPUT $STRIPPEDINPUT $CRUNCHEDINPUT $PRODUCTLIST \ $EXPANDEDPAIRS $LEGALFLAGLIST $JOINEDPAIRS $MINIMALAFFIXES \ $CROSSROOTS $CROSSEXPANDED $CROSSPAIRS $CROSSILLEGAL $ILLEGALCOMBOS \ $FAKEDICT $FAKEHASH $AWKSCRIPT rm -f ${DBDIR}/EXPANDEDINPUT ${DBDIR}/STRIPPEDINPUT \ ${DBDIR}/CRUNCHEDINPUT ${DBDIR}/PRODUCTLIST ${DBDIR}/EXPANDEDPAIRS \ ${DBDIR}/LEGALFLAGLIST ${DBDIR}/JOINEDPAIRS ${DBDIR}/MINIMALAFFIXES \ ${DBDIR}/CROSSROOTS ${DBDIR}/CROSSEXPANDED ${DBDIR}/CROSSPAIRS \ ${DBDIR}/CROSSILLEGAL ${DBDIR}/ILLEGALCOMBOS ${DBDIR}/FAKEDICT \ ${DBDIR}/FAKEHASH.hash ${DBDIR}/AWKSCRIPT \ ${DBDIR}/CROSSROOTS.[0-9]* ${DBDIR}/CROSSEXP.[0-9]* \ ${DBDIR}/CROSSPAIRS.[0-9]* ${DBDIR}/CROSSILLEGAL.[0-9]* ln $EXPANDEDINPUT ${DBDIR}/EXPANDEDINPUT ln $STRIPPEDINPUT ${DBDIR}/STRIPPEDINPUT ln $CRUNCHEDINPUT ${DBDIR}/CRUNCHEDINPUT ln $PRODUCTLIST ${DBDIR}/PRODUCTLIST ln $EXPANDEDPAIRS ${DBDIR}/EXPANDEDPAIRS ln $LEGALFLAGLIST ${DBDIR}/LEGALFLAGLIST ln $JOINEDPAIRS ${DBDIR}/JOINEDPAIRS ln $MINIMALAFFIXES ${DBDIR}/MINIMALAFFIXES ln $CROSSROOTS ${DBDIR}/CROSSROOTS ln $CROSSEXPANDED ${DBDIR}/CROSSEXPANDED ln $CROSSPAIRS ${DBDIR}/CROSSPAIRS ln $CROSSILLEGAL ${DBDIR}/CROSSILLEGAL ln $ILLEGALCOMBOS ${DBDIR}/ILLEGALCOMBOS ln $FAKEDICT ${DBDIR}/FAKEDICT ln $FAKEHASH ${DBDIR}/FAKEHASH.hash ln $AWKSCRIPT ${DBDIR}/AWKSCRIPT fi run() { "$@" || { status=$? touch "$ERRORFLAGFILE" echo "$@ failed with $status" > "$ERRORFLAGFILE" exit $status } } checkerrorflagfile() { [ ! -e "$ERRORFLAGFILE" ] || { cat "$ERRORFLAGFILE" 1>&2 rm -rf $MUNCHDIR exit 1 } } rm -f "$ERRORFLAGFILE" JOIN="run $JOIN" COMBINE="run $COMBINE" ISPELL="run $ISPELL" set -e # # Create a dummy dictionary to hold a compiled copy of the language # table. Initially, it holds the conversion table, if it exists. # case "X$convtabs" in X) convtabs="$langtabs" ;; esac echo 'QQQQQQQQ' > $FAKEDICT $BUILDHASH -s $FAKEDICT $convtabs $FAKEHASH \ || (echo "Couldn't create fake hash file" 1>&2; rm -rf $MUNCHDIR; exit 1) \ || exit 1 # # Figure out how 'sort' sorts signed fields, for arguments to ijoin. # This is a little bit of a tricky pipe, but the result is that SIGNED # is set to "-s" if characters with the top bit set sort before those # without, and "-u" if the reverse is true. How does it work? The # first "tr" step generates two lines, one containing "-u", the other # with the same but with the high-order bit set. The second "tr" # changes the high-bit "-u" back to "-s". If the high-bit "-u" was # sorted first, the sed step will select "-s" for SIGNED; otherwise # it'll pick "-u". We have to be careful about backslash quoting # conventions, because some systems differ. # backslash=\\ for i in 0 1 2 3 do if [ `echo a | tr "${backslash}141" b` = b ] then break fi backslash="$backslash$backslash" done SIGNED=`echo '-s -u' | tr s "${backslash}365" | sort | tr "${backslash}365" s | sed -e 1q` # # Collect all the input and expand all the affix options ($ISPELL -e), # and preserve (sorted) for later joining in EXPANDEDINPUT. The icombine # step is to make sure that unneeded capitalizations (e.g., Farmer and farmer) # are weeded out. The first sort must be folded for icombine; the second # must be unfolded for join. # $verbose && echo "Collecting input." 1>&2 if $CRETIN_SORT then sortopts='-k 1f,1 -k 1' else sortopts='+0f -1 +0' fi if [ $# -eq 0 ] then $ISPELL "$wchars" -e1 -d $FAKEHASH -p /dev/null | tr " " ' ' else cat "$@" | $ISPELL "$wchars" -e1 -d $FAKEHASH -p /dev/null | tr " " ' ' fi \ | sort $SORTTMP -u $sortopts \ | $COMBINE $icflags "$wchars" $langtabs \ | sort $SORTTMP -u > $EXPANDEDINPUT checkerrorflagfile # # If a conversion table existed, recreate the fake hash file with the # "real" language table. # case "$convtabs" in $langtabs) ;; *) $BUILDHASH -s $FAKEDICT $langtabs $FAKEHASH \ || (echo "Couldn't create fake hash file" 1>&2; \ rm -rf $MUNCHDIR; exit 1) \ || exit 1 ;; esac rm -f ${FAKEDICT}* # # If the -s (strip) option was specified, remove all # expanded words that are covered by the dictionary. This produces # the final list of expanded words that this dictionary must cover. # Leave the list in STRIPPEDINPUT. # if [ "X$strip" = "Xno" ] then rm -f $STRIPPEDINPUT ln $EXPANDEDINPUT $STRIPPEDINPUT if [ "$debug" = yes ] then rm -f ${DBDIR}/STRIPPEDINPUT ln $STRIPPEDINPUT ${DBDIR}/STRIPPEDINPUT fi else $verbose && echo "Stripping words already in the dictionary." 1>&2 $ISPELL "$wchars" -l $dictopt -p /dev/null < $EXPANDEDINPUT \ > $STRIPPEDINPUT checkerrorflagfile fi # # Figure out what the flag-marking character is. # $verbose && echo "Finding flag marker." 1>&2 flagmarker=`$ISPELL -D -d $FAKEHASH \ | sed -n -e '/^flagmarker/s/flagmarker //p'` case "$flagmarker" in \\*) flagmarker=`expr "$flagmarker" : '.\(.\)'` ;; esac checkerrorflagfile # # Munch the input to generate roots and affixes ($ISPELL -c). We are # only interested in words that have at least one affix (grep -E $flagmarker); # the next step will pick up the rest. Some of the roots are illegal. We # use join to restrict the output to those root words that are found # in the original dictionary. # $verbose && echo "Generating roots and affixes." 1>&2 if $CRETIN_SORT then sortopts='-k 1,1 -k 2' else sortopts='+0 -1 +1' fi $ISPELL "$wchars" -c -W0 -d $FAKEHASH -p /dev/null < $STRIPPEDINPUT \ | tr " " ' ' \ | grep -a -E "$flagmarker" | sort $SORTTMP -u "-t$flagmarker" $sortopts \ | $JOIN $SIGNED "-t$flagmarker" - $EXPANDEDINPUT > $CRUNCHEDINPUT checkerrorflagfile # # We now have a list of legal roots, and of affixes that apply to the # root words. However, it is possible for some affix flags to generate more # than one output word. For example, with the flag table entry # # flag R: . > ER # . > ERS # # the input "BOTHER" will generate an entry "BOTH/R" in CRUNCHEDINPUT. But # this will accept "BOTHER" and "BOTHERS" in the dictionary, which is # wrong (in this case, though it's good English). # # To cure this problem, we first have to know which flags generate which # expansions. We use $ISPELL -e3 to expand the flags (the second e causes # the root and flag to be included in the output), and get pairs # suitable for joining. In the example above, we would get # # BOTH/R BOTHER # BOTH/R BOTHERS # # We save this in EXPANDEDPAIRS for the next step. # $verbose && echo 'Expanding dictionary into EXPANDEDPAIRS.' 1>&2 if $CRETIN_SORT then sortopts='-k 2' else sortopts='+1' fi $ISPELL "$wchars" -e3 -d $FAKEHASH -p /dev/null < $CRUNCHEDINPUT \ | sort $SORTTMP $sortopts > $EXPANDEDPAIRS checkerrorflagfile # # Now we want to extract the lines in EXPANDEDPAIRS in which the second field # is *not* listed in the original dictionary EXPANDEDINPUT; these illegal # lines contain the flags we cannot include without accepting illegal words. # It is somewhat easier to extract those which actually are listed (with # join), and then use comm to strip these from EXPANDEDPAIRS to get the # illegal expansions, together with the flags that generate them (we must # re-sort EXPANDEDPAIRS before running comm). Sed # gets rid of the expansion and uniq gets rid of duplicates. Comm then # selects the remainder of the list from CRUNCHEDINPUT and puts it in # LEGALFLAGLIST. The final step is to use a sort and icombine to put # the list into a one-entry-per-root format. # # BTW, I thought of using cut for the sed step (on systems that have it), # but it turns out that sed is faster! # $JOIN -j1 2 -o 1.1 1.2 $SIGNED $EXPANDEDPAIRS $EXPANDEDINPUT \ | sort $SORTTMP -u > $JOINEDPAIRS sort $SORTTMP -o $EXPANDEDPAIRS $EXPANDEDPAIRS sort $SORTTMP -o $CRUNCHEDINPUT $CRUNCHEDINPUT $verbose && echo 'Creating list of legal roots/flags.' 1>&2 if $CRETIN_SORT then sortopts='-k 1f,1 -k 1' else sortopts='+0f -1 +0' fi comm -13 $JOINEDPAIRS $EXPANDEDPAIRS \ | (sed -e 's; .*$;;' ; rm -f $JOINEDPAIRS $EXPANDEDPAIRS) \ | uniq \ | (comm -13 - $CRUNCHEDINPUT ; rm -f $CRUNCHEDINPUT) \ | sort $SORTTMP -u "-t$flagmarker" $sortopts \ | $COMBINE "$wchars" $langtabs > $LEGALFLAGLIST checkerrorflagfile # # LEGALFLAGLIST now contains root/flag combinations that, when expanded, # produce only words from EXPANDEDPAIRS. However, there is still a # problem if the language tables have any cross-product flags. A legal # root may appear in LEGALFLAGLIST with two flags that participate # in cross-products. When such a dictionary entry is expanded, # the cross-products will generate some extra words that may not # be in EXPANDEDPAIRS. We need to remove these from LEGALFLAGLIST. # # The first step is to collect the names of the flags that participate # in cross-products. Ispell will dump the language tables for us, and # sed is a pretty handy way to strip out extra information. We use # uniq -c and a numerical sort to put the flags in approximate order of how # "productive" they are (in terms of how likely they are to generate a lot # of output words). The least-productive flags are given last and will # be removed first. # $verbose \ && echo 'Creating list of flags that participate in cross-products.' 1>&2 if $CRETIN_SORT then sortopts='-k 1rn,1 -k 3' else sortopts='+0rn -1 +2' fi $ISPELL -D -d $FAKEHASH \ | sed -n -e '1,$s/:.*$// /^flagmarker/d /^prefixes/,/^suffixes/s/^ flag \*/p /p /^suffixes/,$s/^ flag \*/s /p' \ | sort $SORTTMP \ | uniq -c \ | tr ' ' ' ' \ | sort $SORTTMP $sortopts > $PRODUCTLIST checkerrorflagfile if [ `grep -a -F -c ' p ' $PRODUCTLIST` -gt 0 \ -a `grep -a -F -c ' s ' $PRODUCTLIST` -gt 0 ] then # # The language tables allow cross products. See if LEGALFLAGLIST has # any roots with multiple cross-product flags. Put them in CROSSROOTS. # $verbose && echo 'Finding prefix and suffix flags.' 1>&2 preflags=`sed -n -e 's/^[ 0-9]*p //p' $PRODUCTLIST | tr -d ' '` sufflags=`sed -n -e 's/^[ 0-9]*s //p' $PRODUCTLIST | tr -d ' '` grep -a -E "$flagmarker.*[$preflags].*[$sufflags]|$flagmarker.*[$sufflags].*[$preflags]" \ $LEGALFLAGLIST \ > $CROSSROOTS || : # # We will need an awk script; it's so big that it core-dumps my shell # under certain conditions. The rationale behind the script is commented # where the script is used. Note that you may want to change this # script for languages other than English. # case "$flagmarker" in /) sedchar=: ;; *) sedchar=/ ;; esac $verbose && echo 'Creating awk script.' 1>&2 sed -e "s/PREFLAGS/$preflags/" -e "s/SUFFLAGS/$sufflags/" \ -e "s;ILLEGALCOMBOS;$ILLEGALCOMBOS;" \ -e "s${sedchar}FLAGMARKER${sedchar}$flagmarker${sedchar}" \ > $AWKSCRIPT << 'ENDOFAWKSCRIPT' BEGIN \ { preflags = "PREFLAGS" sufflags = "SUFFLAGS" illegalcombos = "ILLEGALCOMBOS" flagmarker = "FLAGMARKER" pflaglen = length (preflags) for (i = 1; i <= pflaglen; i++) pflags[i] = substr (preflags, i, 1); sflaglen = length (sufflags) for (i = 1; i <= sflaglen; i++) sflags[i] = substr (sufflags, i, 1); } { len = length ($2) pnew2 = "" snew2 = "" pbad = "" sbad = "" sufs = 0 pres = 0 for (i = 1; i <= len; i++) { curflag = substr ($2, i, 1) for (j = 1; j <= pflaglen; j++) { if (pflags[j] == curflag) { pres++ pnew2 = substr ($2, 1, i - 1) substr ($2, i + 1) pbad = curflag } } for (j = 1; j <= sflaglen; j++) { if (sflags[j] == curflag) { sufs++ snew2 = substr ($2, 1, i - 1) substr ($2, i + 1) sbad = curflag } } } if (pres == 1) { print $1 flagmarker pnew2 print $1 flagmarker pbad >> illegalcombos } else if (sufs == 1) { print $1 flagmarker snew2 print $1 flagmarker sbad >> illegalcombos } else if (pres > 0) { print $1 flagmarker pnew2 print $1 flagmarker pbad >> illegalcombos } else { print $1 flagmarker snew2 print $1 flagmarker sbad >> illegalcombos } } ENDOFAWKSCRIPT : > $ILLEGALCOMBOS dbnum=1 while [ -s $CROSSROOTS ] do # # CROSSROOTS contains the roots whose cross-product expansions # might be illegal. We now need to locate the actual illegal ones. # We do this in much the same way we created LEGALFLAGLIST from # CRUNCHEDINPUT. First we make CROSSEXPANDED, which is analogous # to EXPANDEDPAIRS. # $verbose && echo "Creating cross expansions (pass $dbnum)." 1>&2 if $CRETIN_SORT then sortopts='-k 2' else sortopts='+1' fi $ISPELL "$wchars" -e3 -d $FAKEHASH -p /dev/null < $CROSSROOTS \ | sort $SORTTMP $sortopts > $CROSSEXPANDED checkerrorflagfile # # Now we join CROSSEXPANDED against EXPANDEDINPUT to produce # CROSSPAIRS, and then comm that against CROSSEXPANDED to # get CROSSILLEGAL, the list of illegal cross-product flag # combinations. # $JOIN -j1 2 -o 1.1 1.2 $SIGNED $CROSSEXPANDED $EXPANDEDINPUT \ | sort $SORTTMP -u > $CROSSPAIRS checkerrorflagfile sort $SORTTMP -u -o $CROSSEXPANDED $CROSSEXPANDED $verbose \ && echo "Finding illegal cross expansions (pass $dbnum)." 1>&2 comm -13 $CROSSPAIRS $CROSSEXPANDED \ | sed -e 's; .*$;;' \ | uniq > $CROSSILLEGAL if [ "$debug" = yes ] then mv $CROSSROOTS $DBDIR/CROSSROOTS.$dbnum ln $CROSSEXPANDED $DBDIR/CROSSEXP.$dbnum ln $CROSSPAIRS $DBDIR/CROSSPAIRS.$dbnum ln $CROSSILLEGAL $DBDIR/CROSSILLEGAL.$dbnum fi # # Now it is time to try to clear up the illegalities. For # each word in the illegal list, remove one of the cross-product # flags. The flag chosen is selected in an attempt to cure the # problem quickly, as follows: (1) if there is only one suffix # flag or only one prefix flag, we remove that. (2) If there is # a prefix flag, we remove the "least desirable" (according to # the order of preflags). (This may be pro-English prejudice, # and you might want to change this if your language is prefix-heavy). # (3) Otherwise we remove the least-desirable suffix flag # # The output of the awk script becomes the new CROSSROOTS. In # addition, we add the rejected flags to ILLEGALCOMBOS (this is done # inside the awk script) so they can be removed from LEGALFLAGLIST # later. # awk "-F$flagmarker" -f $AWKSCRIPT $CROSSILLEGAL > $CROSSROOTS if [ "$debug" = yes ] then rm -f $CROSSEXPANDED $CROSSPAIRS $CROSSILLEGAL fi dbnum=`expr $dbnum + 1` if [ $dbnum -gt 100 ] then echo "Too many passes, aborting cross-product loop. Munchlist failed." 1>&2 if [ "X$MUNCHMAIL" != X ] then ( ls -ld ${DBDIR}/[A-Z]* cat ${MAILDEBUGDIR}/munchlist.mail ) | mail -s 'Munchlist debug output' "$MUNCHMAIL" rm -f ${MAILDEBUGDIR}/munchlist.mail fi rm -rf $MUNCHDIR exit 1 fi done rm -f $CROSSEXPANDED $CROSSPAIRS $CROSSILLEGAL $AWKSCRIPT # # Now we have, in ILLEGALCOMBOS, a list of root/flag combinations # that must be removed from LEGALFLAGLIST to get the final list # of truly legal flags. ILLEGALCOMBOS has one flag per line, so # by turning LEGALFLAGLIST into this form (sed), it's an # easy task for comm. We have to recombine flags again after the # extraction, to get all flags for a given root on the same line so that # cross-products will come out right. # if [ -s $ILLEGALCOMBOS ] then sort $SORTTMP -u -o $ILLEGALCOMBOS $ILLEGALCOMBOS $verbose && echo 'Finding roots of cross expansions.' 1>&2 if $CRETIN_SORT then sortopts='-k 1f,1 -k 1' else sortopts='+0f -1 +0' fi sort $SORTTMP $LEGALFLAGLIST \ | sed -e '/\/../{ s;^\(.*\)/\(.\)\(.*\);\1/\2\ \1/\3; P D }' \ | comm -23 - $ILLEGALCOMBOS \ | sort $SORTTMP -u "-t$flagmarker" $sortopts \ | $COMBINE "$wchars" $langtabs > $CROSSROOTS checkerrorflagfile mv $CROSSROOTS $LEGALFLAGLIST if [ "$debug" = yes ] then rm -f ${DBDIR}/LEGALFLAGLIST1 ln $LEGALFLAGLIST ${DBDIR}/LEGALFLAGLIST1 fi fi fi rm -f $PRODUCTLIST $CROSSROOTS $ILLEGALCOMBOS $EXPANDEDINPUT # # We now have (in LEGALFLAGLIST) a list of roots and flags which will # accept words taken from EXPANDEDINPUT and no others (though some of # EXPANDEDINPUT is not covered by this list). However, many of the # expanded words can be generated in more than one way. For example, # "bather" can be generated from "bath/R" and "bathe/R". This wastes # unnecessary space in the raw dictionary and, in some cases, in the # hash file as well. The solution is to list the various ways of # getting a given word and choose exactly one. All other things being # equal, we want to choose the one with the highest expansion length # to root length ratio. The $ISPELL -e4 option takes care of this by # providing us with a field to sort on. # # The ispell/awk combination is similar to the ispell/sed pipe used to # generate EXPANDEDPAIRS, except that ispell adds an extra field # giving the sort order. The first sort gets things in order so the # first root listed is the one we want, and the second sort (-um) then # selects that first root. Sed strips the expansion from the root, # and a final sort -u generates MINIMALAFFIXES, the final list of # affixes that (more or less) minimally covers what it can from # EXPANDEDINPUT. # $verbose && echo 'Eliminating non-optimal affixes.' 1>&2 if $CRETIN_SORT then sortopts1='-k 2,2 -k 2rn,3 -k 1,1' sortopts2='-k 2,2' sortopts3='-k 1f,1 -k 1' else sortopts1='+1 -2 +2rn -3 +0 -1' sortopts2='+1 -2' sortopts3='+0f -1 +0' fi $ISPELL "$wchars" -e4 -d $FAKEHASH -p /dev/null < $LEGALFLAGLIST \ | sort $SORTTMP $sortopts1 \ | sort $SORTTMP -um $sortopts2 \ | sed -e 's; .*$;;' \ | sort $SORTTMP -u "-t$flagmarker" $sortopts3 > $MINIMALAFFIXES rm -f $LEGALFLAGLIST # # Now we're almost done. MINIMALAFFIXES covers some (with luck, most) # of the words in STRIPPEDINPUT. Now we must create a list of the remaining # words (those omitted by MINIMALAFFIXES) and add it to MINIMALAFFIXES. # The best way to do this is to actually build a partial dictionary from # MINIMALAFFIXES in FAKEHASH, and then use $ISPELL -l to list the words that # are not covered by this dictionary. This must then be combined with the # reduced version of MINIMALAFFIXES and sorted to produce the final result. # $verbose && echo "Generating output word list." 1>&2 if $CRETIN_SORT then sortopts='-k 1f,1 -k 1' else sortopts='+0f -1 +0' fi if [ -s $MINIMALAFFIXES ] then $BUILDHASH -s $MINIMALAFFIXES $langtabs $FAKEHASH > /dev/null \ || (echo "Couldn't create intermediate hash file" 1>&2; rm -rf $MUNCHDIR; exit 1) \ || exit 1 if [ "$debug" = yes ] then rm -f ${DBDIR}/MINIMALAFFIXES.stat ln $MINIMALAFFIXES.stat ${DBDIR}/MINIMALAFFIXES..stat fi ($ISPELL "$wchars" -l -d $FAKEHASH -p /dev/null < $STRIPPEDINPUT; \ $COMBINE "$wchars" $langtabs < $MINIMALAFFIXES) \ | sort $SORTTMP "-t$flagmarker" -u $sortopts else # MINIMALAFFIXES is empty; just produce a sorted version of STRIPPEDINPUT sort $SORTTMP "-t$flagmarker" -u $sortopts $STRIPPEDINPUT fi checkerrorflagfile if [ "X$MUNCHMAIL" != X ] then ( ls -ld ${DBDIR}/[A-Z]* cat ${MAILDEBUGDIR}/munchlist.mail ) | mail -s 'Munchlist debug output' "$MUNCHMAIL" rm -f ${MAILDEBUGDIR}/munchlist.mail fi rm -rf $MUNCHDIR exit 0
Save File
Cancel