#!/bin/bash

# index.script
# Copyright 2014 - 2026 Kevin C. O'Kane
# May 11, 2026

# $1 is number of docs to index

# tabs 10

segtabs () {

for (( i=0 ; i<$task ; i++ ));
        do
	printf "\t" "" 1>&2
	done
}

tabprint () {
	printf "%-9s\r" "$1" 1>&2
	}

echo

cat /dev/null > nohup.out	# erase any prior nohup.out

echo
echo "index.script `date`"
echo

cpu_count=`grep processor /proc/cpuinfo | wc -l`

make clean
make -B -j $cpu_count

DB="ohsu"

if [ -z $1 ]
        then
                max_docs=1000
                echo "No doc count given."
                echo "1000 used."
        else
                max_docs=$1
                echo "Nbr docs = $in_docs"
        fi

starttime.mps

export max_docs

echo
echo "max_docs = $max_docs"

rm -f DocCount key.dat data.dat
rm -f *.txt *.sorted *.tmp tmp tmp1	# delete old results

# create sql database in case needed

echo "Creating Sqlite3 database where needed or not"

echo "y" | mumps-sql-db-create 

echo $DB > DBPREFIX

FILE="../$DB.converted"
TITLES="../titles.list"
CONVERTED="../$DB.converted"

        rm -f $FILE
        rm -f $TITLES

        echo
        echo "***********************"
        echo "Processing input file"
        echo "***********************"

        rm -f titles.list ../titles.list

                reformat < ../$DB.medline > ../$DB.converted
                mv titles.list ..

export DB
ln -s ../titles.list titles.list
ln -s $FILE $DB.converted

echo
echo "***************************************"
echo "Docs converted: `wc -l ../$DB.converted`"
echo "***************************************"

task="1"
export task

cluster_tt="yes"
export cluster_tt
echo "	cluster_tt: $cluster_tt"

ttt="yes"
export ttt
echo "	ttt: $ttt"

jaccard_tt="yes"
export jaccard_tt
echo "	jaccard_tt: $jaccard_tt"

tt_corelations="yes"
export tt_corelations
echo "	tt_corelations: $tt_corelations"

cohesion_tt="yes" # requires tt_corelations
export cohesion_tt
echo "	cohesion_tt: $chhesion_tt"

discrim="yes"
export discrim
echo "	discrim: $discrim"

proximity="yes"
export proximity
echo "	proximity: $proximity"

doc_doc_matrix="yes"
export doc_doc_matrix
echo "	doc doc matrix: $doc_doc_matrix"

document_cluster="yes" # requires doc_doc_matrix
echo "	document cluster: $document_cluster"
export document_cluster

./starttime.mps			# record starting time

wc -l titles.list > MAXDOCS
echo "	Documents found based on title count: `cat MAXDOCS`"

./endtime.mps

SQL="no"
SQLite="no"

zipf="no"
export zipf

kwic="no"
export kwic

discrim="no"
export discrim

proximity="yes"
export proximity

tt_correlations="yes"
export tt_correlations

export DB

#----------------------------------------------------------------------------------
#	DocumentTermMarix settings"
#
#	Max Number of Docs: max_docs
#
#	min_word_length: Words must be longer than: min_word_length
#
#	min_doc_vector_word_count: Kill docs with fewer than 
#		min_doc_vector_word_count words
#
#	min_doc_word_count: A word must appear in at least 
#		min_doc_word_count separate docs
#
#	max_collection_word_count: A word may appear in no more 
#		than: max_collection_word_count docs
#
#	min_doc_word_count: delete documents with fewer than: min_doc_word_count
#----------------------------------------------------------------------------------

	min_word_length=4
	min_doc_vector_word_count=20
	min_word_count=5
	max_word_count=$((max_docs/5))
	min_doc_word_count=5

	export min_word_length
	export min_doc_vector_word_count
	export min_word_count
	export max_word_count
	export min_doc_word_count

#-------------------------------------------------------------------------
#	Inverse Document Frequency settings"
#
#	Delete docs after IDF with fewer than: min_idf_vector_count words
#-------------------------------------------------------------------------

	min_idf_vector_count=$min_doc_word_count
	export min_idf_vector_count

#-------------------------------------------------------------
#	Document weights
#
#	delete words with weight less than: min_doc_weight
#	delete documents with fewer than: min_doc_word_count
#-------------------------------------------------------------

	min_doc_weight=10
	export min_doc_weight

#-----------------------------------------------------------------------
#	Term-Term matrix settings
#
#	Related terms must have this correlation: min_tt_correlation
#	Min co-occurances for term-term matrix: min_tt_cooccur
#-----------------------------------------------------------------------

	min_tt_correlation=0.250
	min_tt_cooccur=3

	export min_tt_correlation
	export min_tt_cooccur

#----------------------------------------------------------------
#	Proximity matrix settings
#
#	Proximity matrix min count: proximity_min_count
#	Proximity distance factor: prox_dist_factor
#----------------------------------------------------------------

	proximity_min_count=5
	prox_dist_factor=30

	export proximity_min_count
	export prox_dist_factor

#-----------------------------------------------------------------
#	Document-document matrix settings
#
#	Doc Words in common for inclusion: min_doc_doc_cooccur
#	Min doc-doc matrix cosine: min_dd_cosine
#	Min doc-doc matrix cosine: min_dd_cohesion
#-----------------------------------------------------------------

	min_doc_doc_cooccur=5                    	
	min_dd_cosine=0.25
	min_dd_cohesion=.10

	export min_doc_doc_cooccur                  	
	export min_dd_cosine
	export min_dd_cohesion

#-------------------------------------
#	Jaccard term-term settings
#`
#	Jaccard minimum: jaccard_min
#-------------------------------------

	jaccard_min=0.001
	export jaccard_min

#----------------------------------------------------
#	Clusterdd settings
#
#	Minimum doc-doc cosine: min_cluster_cosine
#----------------------------------------------------

	min_cluster_cosine=$min_dd_cosine
	export min_cluster_cosine

#-------------------------------------------
#	Hypercluster settings
#
#	Hyper cluser min cosine: hyper_min
#-------------------------------------------

	hyper_min=.3
	hyper_wgt=.3

	export hyper_min
	export hyper_wgt

#----------------------------------------------------------
#	Term-term cluster settings
#
#	Min term-term cluster correlation: min_tt_cluster
#----------------------------------------------------------

	min_tt_cluster=.1
	export min_tt_cluster

# set -v
# set -x
# $1 is DB

echo "max_docs $max_docs"
echo

echo
echo "Start isr "
echo

./isr

rm -f *.tmp
rm -f *.stemmed
rm -f tmp tmp1

