From b94d3db60ee33f550078c313560af3718d304ee6 Mon Sep 17 00:00:00 2001 From: "Steinar H. Gunderson" Date: Sat, 13 Dec 2014 02:02:59 +0100 Subject: [PATCH] Set parallel merges to a value different from parallel loads. --- build-book.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build-book.sh b/build-book.sh index e6e8c09..60d5710 100755 --- a/build-book.sh +++ b/build-book.sh @@ -3,7 +3,8 @@ set -e export SHARDS=40 -export PARALLEL_LOADS=20 # Reduce if you have problems with OOM +export PARALLEL_LOADS=15 # Reduce if you have problems with OOM +export PARALLEL_MERGES=40 rm -f pgnnames.txt part-*.bin part-*.mtbl part-*.mtbl.part???? open.mtbl.new open.mtbl.part???? open.mtbl.part????.new 2>/dev/null @@ -22,7 +23,7 @@ parallel -i -j ${PARALLEL_LOADS} bash -c 'FILE="{}"; set -x; ./binloader "$FILE" rm -f part-*.bin -parallel -i -j ${PARALLEL_LOADS} bash -c 'X={}; set -x; ./binmerger part-*.mtbl.part$( printf %04d $X ) open.mtbl.part$( printf %04d $X ).new' -- $( seq 0 $(( SHARDS - 1 )) ) +parallel -i -j ${PARALLEL_MERGES} bash -c 'X={}; set -x; ./binmerger part-*.mtbl.part$( printf %04d $X ) open.mtbl.part$( printf %04d $X ).new' -- $( seq 0 $(( SHARDS - 1 )) ) for X in $( seq 0 $(( SHARDS - 1 )) ); do mv open.mtbl.part$( printf %04d $X ).new open.mtbl.part$( printf %04d $X ) -- 2.39.2