/*++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
#+
#+     Information Storage and Retrieval Kit
#+
#+     Copyright (C) 2021 by Kevin C. O'Kane
#+
#+     Kevin C. O'Kane
#+     kc.okane@gmail.com
#+     https://www.cs.uni.edu/~okane
#+     https://threadsafebooks.com/
#+
#+ This program is free software; you can redistribute it and/or modify
#+ it under the terms of the GNU General Public License as published by
#+ the Free Software Foundation; either version 2 of the License, or
#+ (at your option) any later version.
#+
#+ This program is distributed in the hope that it will be useful,
#+ but WITHOUT ANY WARRANTY; without even the implied warranty of
#+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#+ GNU General Public License for more details.
#+
#+ You should have received a copy of the GNU General Public License
#+ along with this program; if not, write to the Free Software
#+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
#+
#++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++*/

//      May 8, 2022

#include "definitions.h"

void    FrequencyFilter(STATE *SP) {

#include "globals.h"

	mstring line;
	int min, max, i;

        min = atoi(SP->min_word_count);
	max = atoi(SP->max_word_count);

	printf("\nBegin FrequencyFilter\n");
	printf("	Add to stoplist words with frequency less than %d\n", min);
	printf("	or greater than %d\n",max);

	system("sort -nr < stemmed-vocabulary.tmp > stemmed-vocabulary-sorted.tmp");

        ifstream f1("stemmed-vocabulary.tmp", ios_base::in);
        ofstream f2("stop.words");

	while(1) {
		if (!line.ReadLine(f1)) break;
		line = line.zBlanks();
		i = line.Piece(" ",1);
		if (i < min || i > max) {
			f2 << line.Piece(" ",2) << endl;
			}
		}

	f1.close();
	f2.close();

	file_size("stop.words");

	return;
	}
