Minsize

.gitignore

@@ -1,4 +1,6 @@
 mincard
+minsize
+test_rmqueue
 test/*.eds
 test/*.gfa
 test/*.fai
@@ -25,4 +27,12 @@ experiments/pbwt/mincard-naive
 experiments/pbwt/mincard-recursive
 experiments/pbwt/mincard-rmq
 
-experiments/transpose/output
+experiments/minsize/input/*
+!experiments/minsize/input/.gitkeep
+experiments/minsize/output
+experiments/minsize/mincard-ring
+experiments/minsize/mincard-tree
+experiments/minsize/mincard-queue
+
+experiments/transpose/output
+experiments/card_vs_size/output

Makefile

@@ -9,8 +9,14 @@ HTSLIB_FLAGS=-L $(HTSLIB_LIB) -lhts -Wl,-rpath $(HTSLIB_LIB)
 OTHER_INCLUDE=ext/
 SDSL_INCLUDE=ext/sdsl-lite/include/
 
-mincard: src/mincard.cpp src/segment.hpp src/RMaxQTree.h src/RMaxQTree.cpp src/msa_chunker.hpp src/trie.hpp src/pbwt.hpp src/algo.hpp
-	${CXX} $(CXX_FLAGS) -DVERSION="\"$(VERSION)\"" -o mincard src/mincard.cpp src/RMaxQTree.cpp -I $(HTSLIB_INCLUDE) -I $(OTHER_INCLUDE) -I $(SDSL_INCLUDE) $(HTSLIB_FLAGS)
+mincard: src/mincard.cpp src/segment.hpp src/rmqueue.h src/rmqueue.cpp src/RMaxQTree.h src/RMaxQTree.cpp src/msa_chunker.hpp src/trie.hpp src/pbwt.h src/pbwt.cpp src/algo.hpp src/minsize.hpp
+	${CXX} $(CXX_FLAGS) -DVERSION="\"$(VERSION)\"" -o mincard src/mincard.cpp src/rmqueue.cpp src/RMaxQTree.cpp src/pbwt.cpp -I $(HTSLIB_INCLUDE) -I $(OTHER_INCLUDE) -I $(SDSL_INCLUDE) $(HTSLIB_FLAGS)
+
+minsize: src/mincard.cpp src/segment.hpp src/rmqueue.h src/rmqueue.cpp src/RMaxQTree.h src/RMaxQTree.cpp src/msa_chunker.hpp src/trie.hpp src/pbwt.h src/pbwt.cpp src/algo.hpp src/minsize.hpp
+	${CXX} $(CXX_FLAGS) -DMETRIC=SIZE -DVERSION="\"$(VERSION)\"" -o minsize src/mincard.cpp src/rmqueue.cpp src/RMaxQTree.cpp src/pbwt.cpp -I $(HTSLIB_INCLUDE) -I $(OTHER_INCLUDE) -I $(SDSL_INCLUDE) $(HTSLIB_FLAGS)
 
 clean:
-	rm -f mincard
+	rm -f mincard && rm -f minsize && rm -f test_rmqueue
+
+test_rmqueue: src/test_rmqueue.cpp  src/rmqueue.cpp
+	${CXX} $(CXX_FLAGS) -o test_rmqueue src/test_rmqueue.cpp src/rmqueue.cpp

experiments/card_vs_size/README.md

@@ -0,0 +1,13 @@
+# cardinality vs size experiment
+
+This experiment compares the resulting eds stats and running times for all 4 algorithms (mincard/minsize x gaps/gapless) with various bounds (L and U) on all 3 datasets.
+
+Compile `mincard` and `minsize`:
+```
+make -C ../../
+make minsize -C ../../
+```
+
+Get the ecoli, covid, and chr datasets in their respective experiments. Then, simply run the experiment. 
+
+This experiment does not stream the eds-es, though it could with a minor change in the script. The results are collected automatically in output/results.csv

experiments/card_vs_size/run_experiment.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+set -euo pipefail
+thisfolder=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cd "$thisfolder"
+export LC_NUMERIC="en_US.UTF-8"
+
+mincard="$thisfolder/../../mincard"
+minsize="$thisfolder/../../minsize"
+
+# MSA fasta files from the 3 datasets
+input_ecoli_msa="$thisfolder/../e_coli_sim/input/msa.fa"
+input_covid_msa="$thisfolder/../covid/input/covid_100000.fa.gz"
+input_chr19_msa="$thisfolder/../chr19/input/chr19_100.aligned.uppercase.fa"
+
+# Create links to msa files
+outdir="output"
+mkdir -p "$outdir"
+rm -f "$outdir"/*
+cd "$outdir"
+
+ln -s "$input_ecoli_msa" ecoli_msa.fa
+ln -s "$input_covid_msa" covid_msa.fa
+ln -s "$input_chr19_msa" chr19_msa.fa
+
+# Keep them in an array for easy iteration
+declare -A msa
+
+datasets=(
+    ecoli
+    covid
+    chr19
+)
+
+msa[ecoli]=ecoli_msa.fa
+msa[covid]=covid_msa.fa
+msa[chr19]=chr19_msa.fa
+
+# Store algorithm flags
+declare -A flags
+declare -A alg
+
+algos=(
+	mincard_gaps
+	mincard_gapless
+	minsize_gaps
+	minsize_gapless
+)
+
+flags[mincard_gaps]="--gaps"
+flags[minsize_gaps]="--gaps"
+flags[mincard_gapless]=""
+flags[minsize_gapless]=""
+
+alg[mincard_gaps]="$mincard"
+alg[mincard_gapless]="$mincard"
+alg[minsize_gaps]="$minsize"
+alg[minsize_gapless]="$minsize"
+
+# Create temporary files for collecting results
+outfile=$(mktemp)
+timefile=$(mktemp)
+trap 'rm -f "$outfile"' EXIT
+trap 'rm -f "$timefile"' EXIT
+
+usrbintimeformat="%e"
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+BLUE='\033[0;34m'
+PURPLE='\033[0;35m'
+NC='\033[0m'
+
+# Iterate through different [L, U] bounds
+
+values=("1 16" "16 32" "32 64")
+printf "Running mincard/minsize with gaps/gapless strategies on MSAs with LU bounds"
+for pair in "${values[@]}"; do
+	printf " [%s]" "${pair/ /, }"
+done
+printf "\n"
+
+
+# Write results to csv table
+echo -e "Dataset,Algorithm,L,U,Cardinality,Size,Avg. Segment,Time" > results.csv
+
+for dataset in "${datasets[@]}"; do
+	input_msa="${msa[$dataset]}"
+
+	for pair in "${values[@]}"; do
+		read lower upper <<< "$pair"
+		printf "\nDataset: ${BLUE}${dataset}${NC} |  L = ${BLUE}${lower}${NC} U = ${BLUE}${upper}${NC}\n"
+
+		for algo in "${algos[@]}"; do
+		  /usr/bin/time -f"$usrbintimeformat" -o "$timefile" \
+				${alg[$algo]} "$input_msa" \
+   		  -L "$lower" -U "$upper" \
+  		  -q --stats \
+   		  ${flags[$algo]} > "$outfile" 2>&1;
+
+			time=$(cat "$timefile")
+			read card size <<< "$(head -n1 "$outfile")"
+			read min max avg <<< "$(tail -n1 "$outfile")"
+
+			printf "Algorithm: ${GREEN}${algo}${NC} | cardinality = ${PURPLE}${card}${NC} | gap-aware size = ${PURPLE}${size}${NC} | segment sizes: min = ${PURPLE}${min}${NC}, max = ${PURPLE}${max}${NC}, avg = ${PURPLE}${avg}${NC} | ${BLUE}${time}s${NC}\n"
+	    echo -e "${dataset},${algo},${lower},${upper},${card},${size},${avg},${time}" >> results.csv
+		done
+	done
+done
+
+printf "\n${GREEN}Finished running the experiment!\n"
+printf "${BLUE}Wrote algorithm times and stats to results.csv${NC}\n"

experiments/chr19/bench_pbwt.sh

@@ -36,11 +36,11 @@ for U in "${U_values[@]}"
 do
     printf "\n${BLUE}Comparison for U = $U${NC}\n"
 	# trie run
-	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard" msa.fa -v --gaps-as-symbols -U $U -o mincard_U${U}.eds
+	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard" msa.fa -v --trie -U $U -o mincard_U${U}.eds
 	t1=$(<"$tmpfile")
 	trie_times+=("$t1")
 	# pbwt run
-	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard" msa.fa --gaps-as-symbols --pbwt -U $U -o mincard_U${U}_pbwt.eds
+	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard" msa.fa -v -U $U -o mincard_U${U}_pbwt.eds
 	t2=$(<"$tmpfile")
 	pbwt_times+=("$t2")
 

experiments/chr19/run_experiment.sh

@@ -17,20 +17,20 @@ ln -s $inputmsa msa.fa
 # mincard
 for U in 4 8 16
 do
-	/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v -U $U -o mincard_U${U}.eds
+	/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v --gaps -U $U -o mincard_U${U}.eds
 done
 
 # mincard with perfect segments
 for U in 4 8 16
 do
-	/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v -U $U --perfect-segments -o mincard_U${U}_p.eds
+	/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v --gaps -U $U --perfect-segments -o mincard_U${U}_p.eds
 done
 
 # mincard trivial S^|||
-/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v --trivial-vertical -o mincard_t.eds
+/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v --gaps --trivial-vertical -o mincard_t.eds
 
 # mincard trivial S^≡ with perfect segments
-/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v --trivial-horizontal --perfect-segments -o mincard_np.eds
+/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v --gaps --trivial-horizontal --perfect-segments -o mincard_np.eds
 
 exit
 # msatoeds heuristics, they require >= 100GB RAM

experiments/covid/bench_pbwt.sh

@@ -35,11 +35,11 @@ for U in "${U_values[@]}"
 do
     printf "\n${BLUE}Comparison for U = $U${NC}\n"
 	# trie run
-	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard" msa.fa --gaps-as-symbols -U $U -o mincard_U${U}.eds
+	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard" msa.fa --trie -U $U -o mincard_U${U}.eds
 	t1=$(<"$tmpfile")
 	trie_times+=("$t1")
 	# pbwt run
-	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard" msa.fa --gaps-as-symbols --pbwt -U $U -o mincard_U${U}_pbwt.eds
+	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard" msa.fa -U $U -o mincard_U${U}_pbwt.eds
 	t2=$(<"$tmpfile")
 	pbwt_times+=("$t2")
 

experiments/covid/run_experiment.sh

@@ -20,21 +20,21 @@ do
 	# mincard
 	for U in 4 8 16 32 64 128 256 512
 	do
-		/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa.gz -v -U $U -o ${base}_mincard_U${U}.eds # plain
+		/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa.gz -v --gaps -U $U -o ${base}_mincard_U${U}.eds # plain
 		rm msa.fa.gz.fai msa.fa.gz.gzi
-		/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa.gz -v -U $U --perfect-segments -o ${base}_mincard_U${U}_p.eds # perfect segments
+		/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa.gz -v --gaps -U $U --perfect-segments -o ${base}_mincard_U${U}_p.eds # perfect segments
 		rm msa.fa.gz.fai msa.fa.gz.gzi
-		/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa.gz --preprocess -v -U $U -o ${base}_mincard_U${U}.eds # preprocess
+		/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa.gz --gaps --preprocess -v -U $U -o ${base}_mincard_U${U}.eds # preprocess
 		rm msa.fa.gz.fai msa.fa.gz.gzi
-		/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa.gz -v -U $U --perfect-segments --preprocess -o ${base}_mincard_U${U}_p.eds
+		/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa.gz -v --gaps -U $U --perfect-segments --preprocess -o ${base}_mincard_U${U}_p.eds
 		rm msa.fa.gz.fai msa.fa.gz.gzi
 	done
 
 	# mincard trivial S^|||
-	/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa.gz -v --trivial-vertical -o ${base}_mincard_t.eds
+	/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa.gz -v --gaps --trivial-vertical -o ${base}_mincard_t.eds
 
 	# mincard trivial S^≡ with perfect segments
-	/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa.gz -v --trivial-horizontal --perfect-segments -o ${base}_mincard_np.eds
+	/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa.gz -v --gaps --trivial-horizontal --perfect-segments -o ${base}_mincard_np.eds
 
 	# msatoeds heuristics
 	for strat in trivial greedy double-greedy

experiments/e_coli_sim/bench_pbwt.sh

@@ -35,11 +35,11 @@ for U in "${U_values[@]}"
 do
     printf "\n${BLUE}Comparison for U = $U${NC}\n"
 	# trie run
-	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard" msa.fa --gaps-as-symbols -U $U -o mincard_U${U}.eds
+	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard" msa.fa --trie -U $U -o mincard_U${U}.eds
 	t1=$(<"$tmpfile")
 	trie_times+=("$t1")
 	# pbwt run
-	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard" msa.fa --gaps-as-symbols --pbwt -U $U -o mincard_U${U}_pbwt.eds
+	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard" msa.fa -U $U -o mincard_U${U}_pbwt.eds
 	t2=$(<"$tmpfile")
 	pbwt_times+=("$t2")
 

experiments/e_coli_sim/run_experiment.sh

@@ -16,20 +16,20 @@ ln -s $inputmsa msa.fa
 # mincard
 for U in 4 8 16 32 64
 do
-	/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v -U $U -o mincard_U${U}.eds
+	/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v --gaps -U $U -o mincard_U${U}.eds
 done
 
 # mincard with perfect segments
 for U in 4 8 16 32 64
 do
-	/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v -U $U --perfect-segments -o mincard_U${U}_p.eds
+	/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v --gaps -U $U --perfect-segments -o mincard_U${U}_p.eds
 done
 
 # mincard trivial S^|||
-/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v --trivial-vertical -o mincard_t.eds
+/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v --gaps --trivial-vertical -o mincard_t.eds
 
 # mincard trivial S^≡ with perfect segments
-/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v --trivial-horizontal --perfect-segments -o mincard_np.eds
+/usr/bin/time -f"$usrbintimeformat" $mincard msa.fa -v --gaps --trivial-horizontal --perfect-segments -o mincard_np.eds
 
 # msatoeds heuristics
 for strat in trivial greedy double-greedy

experiments/minsize/README.md

@@ -0,0 +1,18 @@
+# min range experiment
+
+Simple minsize experiment to compare the different ways of calculating the range minimum query for the M matrix rows.
+
+Get covid dataset as described [here](../covid/README.md)
+
+Compile `minsize` with each algorithm (ring buffer, rmaxtree, rmqueue) by changing [this line](../../src/minsize.hpp#L25) and then running
+
+```sh
+ make minsize -C ../../
+ mv ../../minsize ./minsize-ring
+```
+
+Do this with the names [ring, tree, queue]. Finally run the experiment script (running times will be collected in output/results.csv)
+
+```sh
+ ./run_experiment.sh
+```

experiments/minsize/run_experiment.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+set -euo pipefail
+thisfolder=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) # https://stackoverflow.com/questions/59895/how-do-i-get-the-directory-where-a-bash-script-is-located-from-within-the-script
+cd "$thisfolder"
+
+minsize_ring="$thisfolder/minsize-ring"
+minsize_tree="$thisfolder/minsize-tree"
+minsize_queue="$thisfolder/minsize-queue"
+inputmsa="$thisfolder/input/covid_100000.fa.gz"
+usrbintimeformat="%e"
+
+outdir="output"
+mkdir -p "$outdir"
+rm -f "$outdir"/*
+cd "$outdir"
+ln -s "$inputmsa" msa.fa
+
+L_values=(1 2 4 8 16 32 64 128 256 512)
+printf "Running minsize with pBWT and different range min strategies (ring, rmaxtree, rmqueue) on different L values %s \n" "${L_values[*]}"
+
+tmpfile=$(mktemp)
+trap 'rm -f "$tmpfile"' EXIT
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+BLUE='\033[0;34m'
+PURPLE='\033[0;35m'
+NC='\033[0m'
+
+# Write results to table
+echo -e "L value,Ring Buffer,RMaxTree,RMQueue" > results.csv
+
+# minsize with pbwt (gaps as symbols strategy)
+for L in "${L_values[@]}"
+do
+	printf "\n${BLUE}Comparison for L = $L${NC}\n"
+	# ring buffer
+	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$minsize_ring" msa.fa -v -L $L --min-size -o minsize_ring_L${L}.eds
+	t1=$(<"$tmpfile")
+	# rmaxtree
+	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$minsize_tree" msa.fa -v -L $L --min-size -o minsize_tree_L${L}.eds
+	t2=$(<"$tmpfile")
+	# rmqueue
+	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$minsize_queue" msa.fa -v -L $L --min-size -o minsize_queue_L${L}.eds
+	t3=$(<"$tmpfile")
+
+	if cmp -s minsize_ring_L${L}.eds minsize_tree_L${L}.eds &&
+			cmp -s minsize_ring_L${L}.eds minsize_queue_L${L}.eds
+	then
+		printf "${GREEN}Outputs are identical for L = ${L}!${NC}\n"
+		printf "${PURPLE}ring buffer %.3fs vs. rmaxtree suffix trie %.3fs vs. rmqueue %.3fs${NC}\n" "$t1" "$t2" "$t3"
+	else 
+		printf "${RED}Outputs are different for ${name} on U = ${U}!${NC}\n"
+		exit 1
+	fi
+
+	echo -e "${L},${t1},${t2},${t3}" >> results.csv
+done
+
+printf "\n${GREEN}Finished running the experiment!\n${BLUE}Wrote algorithm times to results.csv${NC}\n"

experiments/pbwt/run_experiment.sh

@@ -36,15 +36,15 @@ for U in "${U_values[@]}"
 do
     printf "\n${BLUE}Comparison for U = $U${NC}\n"
 	# naive
-	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard_naive" msa.fa --gaps-as-symbols --pbwt -U $U -o mincard_naive_U${U}.eds
+	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard_naive" msa.fa -U $U -o mincard_naive_U${U}.eds
 	t1=$(<"$tmpfile")
 	naive_times+=("$t1")
 	# recursive
-	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard_recursive" msa.fa --gaps-as-symbols --pbwt -U $U -o mincard_recursive_U${U}.eds
+	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard_recursive" msa.fa -U $U -o mincard_recursive_U${U}.eds
 	t2=$(<"$tmpfile")
 	recursive_times+=("$t2")
 	# rmq
-	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard_rmq" msa.fa --gaps-as-symbols --pbwt -U $U -o mincard_rmq_U${U}.eds
+	/usr/bin/time -f"$usrbintimeformat" -o "$tmpfile" "$mincard_rmq" msa.fa -U $U -o mincard_rmq_U${U}.eds
 	t3=$(<"$tmpfile")
 	rmq_times+=("$t3")